Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.1: the identity of the outbox sequence.
97const OUTBOX_ID_KEY: &str = "outboxId";
98const CLIENT_ID_KEY: &str = "clientId";
99const LEASE_STATE_KEY: &str = "leaseState";
100const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
101pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
102/// Persisted fail-closed quarantine marker: present while a security preflight
103/// is pending and `activateSecurity` has yet to run. Storing it in the replica
104/// keeps the gate with the data it protects, so a rebuilt host handle reopening
105/// the same file returns in preflight even though its in-memory state is fresh.
106const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
107const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
108const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
109/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
110/// the new schema after a bump. Never a wire code (§10.3).
111const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
112/// §5.11 client-local code: an encode at the push seam resolved no usable key
113/// id or named an unknown key. Never a wire code (§10.3).
114const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
115const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
116/// §7.4.4: static text for a persisted commit the current codec refuses.
117const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
118    "the persisted commit carries values the current codec refuses";
119/// §7.4.4: bounded reason on every intrinsic value drop.
120const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
121
122#[derive(Debug, Serialize, Deserialize)]
123#[serde(rename_all = "camelCase", deny_unknown_fields)]
124struct PersistedLocalDataRebootstrapReceipt {
125    version: u8,
126    retained_commits: u64,
127    reset_subscriptions: u64,
128}
129
130fn invalid_local_rebootstrap_receipt() -> String {
131    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
132}
133
134fn encode_local_rebootstrap_receipt(
135    retained_commits: usize,
136    reset_subscriptions: usize,
137) -> Result<String, String> {
138    let retained_commits =
139        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
140    let reset_subscriptions =
141        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
143        return Err(invalid_local_rebootstrap_receipt());
144    }
145    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
146        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
147        retained_commits,
148        reset_subscriptions,
149    })
150    .map_err(|_| invalid_local_rebootstrap_receipt())
151}
152
153fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
154    // Pre-0.15.36 markers proved application but did not retain the receipt.
155    if value == "v1" {
156        return Ok((0, 0));
157    }
158    let receipt: PersistedLocalDataRebootstrapReceipt =
159        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
160    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
161        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
162        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
163    {
164        return Err(invalid_local_rebootstrap_receipt());
165    }
166    Ok((
167        usize::try_from(receipt.retained_commits)
168            .map_err(|_| invalid_local_rebootstrap_receipt())?,
169        usize::try_from(receipt.reset_subscriptions)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171    ))
172}
173
174#[cfg(test)]
175mod observation_tests;
176
177/// Internal state for hosts that perform network I/O outside the client owner.
178#[doc(hidden)]
179pub struct CurrentBasePreparation {
180    client_commit_id: String,
181    sequence: i64,
182    units: Vec<UnitRecord>,
183    transport_epoch: u64,
184    registration_epoch: u64,
185    security_epoch: u64,
186}
187
188#[cfg(test)]
189mod units_tests;
190
191#[derive(Debug, Clone)]
192struct OutboxOp {
193    upsert: bool,
194    table: String,
195    row_id: String,
196    base_version: Option<i64>,
197    /// Schema-agnostic local form (§0): driver JSON values, encoded with
198    /// the current codec at send time. A key's presence IS the §6.1 sparse
199    /// presence set — both the overlay and the wire payload derive from it.
200    values: Option<Map<String, Value>>,
201    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
202    /// absent from the visible replica. Replay and send-time encoding present
203    /// every current-schema column; a column `values` lacks is NULL.
204    insert: bool,
205}
206
207impl OutboxOp {
208    /// The values this upsert presents under the current `table` (§7.1). An
209    /// insert-intent operation presents every column, a missing one as NULL;
210    /// `None` when a missing column is non-nullable, which leaves the commit
211    /// unencodable (§7.4.4), or when the operation carries no values.
212    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
213        let values = self.values.as_ref()?;
214        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
215            return Some(Cow::Borrowed(values));
216        }
217        full_row_values(table, values.clone()).ok().map(Cow::Owned)
218    }
219}
220
221impl From<CommitOperation> for OutboxOp {
222    fn from(operation: CommitOperation) -> Self {
223        Self {
224            upsert: operation.op == "upsert",
225            table: operation.table,
226            row_id: operation.row_id,
227            base_version: operation.base_version,
228            values: operation.values,
229            insert: operation.insert,
230        }
231    }
232}
233
234/// §7.5 static authoring refusal; the table and column travel in details.
235fn invalid_authoring(message: &str, details: Value) -> ClientError {
236    ClientError {
237        code: "sync.invalid_request".to_owned(),
238        message: message.to_owned(),
239        details: Some(details),
240        retryable: false,
241    }
242}
243
244/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
245/// row and presents only columns `earlier` presents.
246fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
247    // Only an upsert carries values, and it always carries its primary key.
248    earlier.row_id == later.row_id
249        && earlier.table == later.table
250        && later.base_version.is_some_and(|base| base > 0)
251        && earlier.base_version == later.base_version
252        && later.values.as_ref().is_some_and(|values| {
253            values.keys().all(|column| {
254                earlier
255                    .values
256                    .as_ref()
257                    .is_some_and(|earlier| earlier.contains_key(column))
258            })
259        })
260}
261
262impl From<&OutboxOp> for CommitOperation {
263    fn from(operation: &OutboxOp) -> Self {
264        Self {
265            table: operation.table.clone(),
266            row_id: operation.row_id.clone(),
267            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
268            base_version: operation.base_version,
269            values: operation.values.clone(),
270            insert: operation.insert,
271        }
272    }
273}
274
275#[derive(Debug, Clone)]
276struct OutboxCommit {
277    client_commit_id: String,
278    ops: Vec<OutboxOp>,
279}
280
281/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
282/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
283/// can never disagree about which commits cannot re-encode.
284fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
285    commit
286        .ops
287        .iter()
288        .filter(|operation| operation.upsert)
289        .map(|operation| {
290            (
291                operation.table.as_str(),
292                operation
293                    .values
294                    .as_ref()
295                    .map(|values| values.keys().map(String::as_str).collect())
296                    .unwrap_or_default(),
297                operation.insert,
298            )
299        })
300        .collect()
301}
302
303type FailedCommit = (OutboxCommit, Map<String, Value>);
304
305#[derive(Debug, Clone)]
306struct CompiledLocalDataPurgeTarget {
307    table: String,
308    selectors: Vec<(String, Vec<String>)>,
309}
310
311struct StoredCommitOutcomeRow {
312    sequence: i64,
313    client_commit_id: String,
314    status: String,
315    recorded_at_ms: i64,
316    results_json: String,
317    operations_json: Option<String>,
318    resolution: String,
319    resolved_at_ms: Option<i64>,
320    replacement_client_commit_id: Option<String>,
321}
322
323impl StoredCommitOutcomeRow {
324    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
325        Ok(Self {
326            sequence: row.get(0)?,
327            client_commit_id: row.get(1)?,
328            status: row.get(2)?,
329            recorded_at_ms: row.get(3)?,
330            results_json: row.get(4)?,
331            operations_json: row.get(5)?,
332            resolution: row.get(6)?,
333            resolved_at_ms: row.get(7)?,
334            replacement_client_commit_id: row.get(8)?,
335        })
336    }
337}
338
339/// Section outcome distinguishing the §5.6 unit-local fail-closed
340/// path from a round-aborting failure (§1.4 rule 5).
341/// §4.4 a unit section's `SUB_START`.
342struct SectionStart {
343    id: String,
344    status: SubStatus,
345    reason_code: String,
346    bootstrap: bool,
347    fenced: Vec<FencedRecord>,
348}
349
350enum SectionError {
351    FailClosed,
352    /// §5.5: the server refused a segment with `sync.forbidden`.
353    Refused,
354    Abort(String, String),
355    Transport(TransportError),
356}
357
358#[derive(Debug)]
359pub(crate) struct RequestMeta {
360    pushed_ids: Vec<String>,
361    /// Hex of the registration digest this pull carried; the server stores
362    /// that list once the round completes (§4.1).
363    registration_digest: String,
364    pub(crate) accept: u8,
365    /// §6.1 splitBatch: outbox commits held back from THIS request because
366    /// the running operation count reached the push cap — the next round
367    /// pushes them (`sync_needed` stays set while any remain).
368    deferred_commits: usize,
369}
370
371#[derive(Default)]
372struct ChangeAccumulator {
373    /// None means table-wide; Some is the exact scoped domain.
374    tables: BTreeMap<String, Option<BTreeSet<String>>>,
375    windows: BTreeMap<String, BTreeSet<String>>,
376    status: bool,
377    conflicts: bool,
378    rejections: bool,
379    outcomes: bool,
380}
381
382impl ChangeAccumulator {
383    fn table(&mut self, table: &str) {
384        self.tables.insert(table.to_owned(), None);
385    }
386
387    fn scope(&mut self, table: &str, key: String) {
388        match self.tables.get_mut(table) {
389            Some(None) => {}
390            Some(Some(keys)) => {
391                keys.insert(key);
392            }
393            None => {
394                self.tables
395                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
396            }
397        }
398    }
399
400    fn window(&mut self, base_key: &str, value: &str) {
401        self.windows
402            .entry(base_key.to_owned())
403            .or_default()
404            .insert(value.to_owned());
405    }
406
407    fn touched(&self) -> bool {
408        !self.tables.is_empty()
409            || !self.windows.is_empty()
410            || self.status
411            || self.conflicts
412            || self.rejections
413            || self.outcomes
414    }
415}
416
417/// §6.1: the server caps total operations per request (reference default
418/// 500) and rejects the whole batch with `sync.too_many_operations`; the
419/// client "splits and retries". Splitting happens at build time: commits are
420/// included IN ORDER until the operation budget is spent, the rest wait for
421/// the next round.
422const PUSH_OPS_PER_REQUEST: usize = 500;
423const MAX_LOCAL_PURGE_TARGETS: usize = 64;
424const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
425const MAX_LOCAL_PURGE_VALUES: usize = 128;
426const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
427pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
428
429/// Dirty visible tables; `None` means a whole-replica recovery/reset.
430struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
431
432impl OverlayDirty {
433    fn clean() -> Self {
434        Self(RefCell::new(Some(BTreeSet::new())))
435    }
436    fn get(&self) -> bool {
437        self.0
438            .borrow()
439            .as_ref()
440            .is_none_or(|tables| !tables.is_empty())
441    }
442    fn set(&self, dirty: bool) {
443        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
444    }
445    fn table(&self, table: &str) {
446        if let Some(tables) = self.0.borrow_mut().as_mut() {
447            tables.insert(table.to_owned());
448        }
449    }
450    fn snapshot(&self) -> Option<BTreeSet<String>> {
451        self.0.borrow().clone()
452    }
453    fn restore(&self, tables: Option<BTreeSet<String>>) {
454        *self.0.borrow_mut() = tables;
455    }
456}
457
458/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
459/// write after a capture copies it once.
460#[derive(Debug, Clone, Default)]
461struct Shared<T>(Arc<T>);
462
463impl<T> std::ops::Deref for Shared<T> {
464    type Target = T;
465    fn deref(&self) -> &T {
466        &self.0
467    }
468}
469
470impl<T: Clone> std::ops::DerefMut for Shared<T> {
471    fn deref_mut(&mut self) -> &mut T {
472        Arc::make_mut(&mut self.0)
473    }
474}
475
476impl<T> From<T> for Shared<T> {
477    fn from(value: T) -> Self {
478        Self(Arc::new(value))
479    }
480}
481
482/// The in-memory mirrors of durable state, captured before a local
483/// transaction and restored when it rolls back. Queues only grow inside a
484/// transaction, so their lengths suffice.
485#[derive(Debug, Clone)]
486struct Rollback {
487    outbox: Shared<Vec<Arc<OutboxCommit>>>,
488    failed_commits: Shared<Vec<FailedCommit>>,
489    conflicts: Shared<Vec<ConflictRecord>>,
490    rejections: Shared<Vec<RejectionRecord>>,
491    schema_floor: Option<SchemaFloor>,
492    lease_state: Option<LeaseState>,
493    stopped: bool,
494    upgrading: bool,
495    sync_needed: bool,
496    active_round: Option<uuid::Uuid>,
497    overlay_dirty: Option<BTreeSet<String>>,
498    change_queue: usize,
499    sync_intent_queue: usize,
500    last_change: Option<DiagnosticLastChange>,
501}
502
503pub struct SyncClient {
504    #[cfg(feature = "bench-internals")]
505    benchmark_phases: Recorder,
506    conn: Connection,
507    schema: ClientSchema,
508    client_id: String,
509    limits: ClientLimits,
510    outbox: Shared<Vec<Arc<OutboxCommit>>>,
511    failed_commits: Shared<Vec<FailedCommit>>,
512    retain_failed_commits: bool,
513    conflicts: Shared<Vec<ConflictRecord>>,
514    rejections: Shared<Vec<RejectionRecord>>,
515    schema_floor: Option<SchemaFloor>,
516    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
517    lease_state: Option<LeaseState>,
518    /// §1.6: the schema-floor response stops syncing until an upgrade.
519    stopped: bool,
520    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
521    upgrading: bool,
522    /// §8.4 coalesced sync-needed signal.
523    sync_needed: bool,
524    /// §8.8: how sync rounds treat the realtime binding, and its exact
525    /// availability state with the evidence behind a lost or refused one.
526    realtime_policy: RealtimePolicy,
527    realtime_state: RealtimeState,
528    realtime_reason_code: Option<String>,
529    realtime_retry_delay_ms: Option<u64>,
530    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
531    presence: HashMap<String, HashMap<String, PresencePeer>>,
532    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check; the
533    /// host may pin it (conformance runs on a virtual clock).
534    now_ms: Option<i64>,
535    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
536    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
537    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
538    encryption: crate::values::EncryptionConfig,
539    /// Fail-closed host bootstrap gate. While set, command hosts permit only
540    /// status/lifecycle inspection and an exact authorized local purge.
541    security_preflight: bool,
542    security_epoch: u64,
543    transport_enabled: bool,
544    transport_epoch: u64,
545    /// Raised by every registration change; a current-base proof spanning
546    /// one reports missing coverage.
547    registration_epoch: u64,
548    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
549    /// means the feature is off; the descriptor write and the container orphan
550    /// sweep still happen.
551    previous_version: Option<PreviousVersionContextConfig>,
552    /// Per-table primary-key upsert SQL, built once per (full table name) —
553    /// the row write path runs per row during bootstrap (§5.6), so the SQL
554    /// string (and, via `prepare_cached`, its compiled statement) is reused
555    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
556    /// schema reset (the column lists may have changed).
557    insert_sql: RefCell<HashMap<String, String>>,
558    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
559    /// reset and restart explicitly invalidate every table.
560    overlay_dirty: OverlayDirty,
561    // First SQLite failure of the current operation, before savepoint cleanup.
562    storage_failure: RefCell<Option<QueryReadFailure>>,
563    /// Test-only structural performance signal: response processing must not
564    /// turn a batch of acknowledgements into one full overlay rebuild each.
565    #[cfg(test)]
566    overlay_rebuild_count: Cell<usize>,
567    #[cfg(test)]
568    acknowledged_replay_count: Cell<usize>,
569    /// Test-only structural performance signal: outcome retention is enforced
570    /// once per response rather than once per acknowledgement.
571    #[cfg(test)]
572    outcome_prune_count: Cell<usize>,
573    /// Test-only pinned duration of every §5.3 image chunk, in ms.
574    #[cfg(test)]
575    image_chunk_ms: Cell<Option<f64>>,
576    /// Exact observer-transaction output drained by command/FFI hosts.
577    progress: ProgressObserver,
578    change_queue: VecDeque<ClientChangeBatch>,
579    sync_intent_queue: VecDeque<SyncIntent>,
580    /// Explicit exponential retry policy for transient transport failures.
581    retry_delay_ms: u64,
582    /// §7.7: the background retry the current round scheduled, if any.
583    round_retry_delay_ms: Option<u64>,
584    /// A round whose response never completed (§1.5): the client keeps its
585    /// code and reports it on every following request until a round receives a
586    /// complete, decodable response. The server already recorded the round, so
587    /// nothing else can tell it the round failed on this side.
588    round_failure: Option<String>,
589    active_round: Option<uuid::Uuid>,
590    last_round: Option<DiagnosticLastRound>,
591    last_change: Option<DiagnosticLastChange>,
592    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
593    query_failures: Vec<DiagnosticQueryFailure>,
594    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
595    coalesced: Option<CoalescedApply>,
596    /// A coalesced apply transaction of the current response failed; every
597    /// later observation of that response fails until it ends.
598    coalesced_failure: Option<String>,
599}
600
601/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
602/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
603/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
604struct CoalescedApply {
605    rows: usize,
606    rollback: Rollback,
607}
608
609const COALESCED_APPLY_ROWS: usize = 1024;
610
611pub(crate) fn quote_ident(name: &str) -> String {
612    format!("\"{}\"", name.replace('"', "\"\""))
613}
614/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
615/// previous-version module reads and writes the SAME records as the client
616/// instead of growing a parallel meta layer.
617pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
618    conn.query_row(
619        "SELECT value FROM _syncular_meta WHERE key = ?1",
620        rusqlite::params![key],
621        |row| row.get::<_, String>(0),
622    )
623    .optional()
624}
625
626pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
627    conn.execute(
628        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
629        rusqlite::params![key, value],
630    )
631    .map(|_| ())
632}
633
634pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
635    conn.execute(
636        "DELETE FROM _syncular_meta WHERE key = ?1",
637        rusqlite::params![key],
638    )
639    .map(|_| ())
640}
641
642/// Seek by the existing key while retaining the exact legacy text match.
643fn row_id_predicate(table: &TableSchema) -> String {
644    let key = quote_ident(&table.primary_key);
645    let text_match = format!("CAST({key} AS TEXT) = ?1");
646    match table.columns[table.pk_index].ty {
647        ColumnType::String | ColumnType::Json => {
648            format!("{key} = ?1 AND {text_match}")
649        }
650        // Native columns have no declared SQLite affinity. Match the stored
651        // integer bind type explicitly. Unary + removes expression affinity so
652        // SQLite can seek the typeless key; the text check preserves exact IDs.
653        ColumnType::Integer | ColumnType::Boolean => {
654            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
655        }
656        // REAL-to-text conversion can round distinct values to the same text.
657        // Narrowing those matches by numeric equality would change behavior.
658        _ => text_match,
659    }
660}
661
662/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
663/// `effective` and that no entry of `owners` matches. `owners` holds the
664/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
665/// Scope values bind by the column's declared type, as `row_id_predicate`
666/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
667/// is a missing local scope-column mapping (the fail-closed case).
668fn local_scope_predicate(
669    table: &TableSchema,
670    effective: &[(String, Vec<String>)],
671    owners: &[Vec<(String, Vec<String>)>],
672) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
673    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
674        return Ok(None);
675    }
676    let column = |variable: &str| {
677        let name = table.scope_column(variable).ok_or(())?;
678        let column = table
679            .columns
680            .iter()
681            .find(|column| column.name == name)
682            .ok_or(())?;
683        Ok((quote_ident(name), column.ty == ColumnType::Integer))
684    };
685    let mut clauses = Vec::new();
686    let mut params = Vec::new();
687    for (variable, values) in effective {
688        let (key, integer) = column(variable)?;
689        let holes = vec!["?"; values.len()].join(", ");
690        clauses.push(if integer {
691            params.extend(values.iter().cloned().map(SqlValue::Text));
692            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
693            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
694        } else {
695            format!("{key} IN ({holes})")
696        });
697        params.extend(values.iter().cloned().map(SqlValue::Text));
698    }
699    // Owners bind as one JSON parameter per variable set, so the statement
700    // keeps its size at any owner count; owners of one variable bind the
701    // union of their values. An owner holding other values of a variable
702    // `effective` names owns none of its rows and drops out. Values compare
703    // as text, the exact semantics of the effective clauses.
704    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
705    for owner in owners {
706        if owner.is_empty()
707            || owner.iter().any(|(variable, values)| {
708                values.is_empty()
709                    || effective.iter().any(|(name, wanted)| {
710                        name == variable && wanted.iter().all(|value| !values.contains(value))
711                    })
712            })
713        {
714            continue;
715        }
716        let mut owner: Vec<_> = owner.iter().collect();
717        owner.sort_by(|a, b| a.0.cmp(&b.0));
718        groups
719            .entry(
720                owner
721                    .iter()
722                    .map(|(variable, _)| variable.as_str())
723                    .collect(),
724            )
725            .or_default()
726            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
727    }
728    let within = |variable: &str, list: &str| {
729        let (key, integer) = column(variable)?;
730        let key = if integer {
731            format!("CAST({key} AS TEXT)")
732        } else {
733            key
734        };
735        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
736    };
737    for (variables, tuples) in groups {
738        if let [variable] = variables[..] {
739            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
740            params.push(SqlValue::Text(
741                Value::from_iter(values.into_iter().cloned()).to_string(),
742            ));
743            clauses.push(format!("NOT {}", within(variable, "?")?));
744        } else {
745            params.push(SqlValue::Text(
746                serde_json::to_string(&tuples).expect("owner tuples serialize"),
747            ));
748            let matches = variables
749                .iter()
750                .enumerate()
751                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
752                .collect::<Result<Vec<_>, ()>>()?;
753            clauses.push(format!(
754                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
755                matches.join(" AND ")
756            ));
757        }
758    }
759    Ok(Some((clauses.join(" AND "), params)))
760}
761
762/// [`local_scope_predicate`] over decoded row values (retained rows).
763fn owns_local_row(
764    table: &TableSchema,
765    effective: &[(String, Vec<String>)],
766    owners: &[Vec<(String, Vec<String>)>],
767    values: &Map<String, Value>,
768) -> bool {
769    let matches = |scopes: &[(String, Vec<String>)]| {
770        !scopes.is_empty()
771            && scopes.iter().all(|(variable, allowed)| {
772                table
773                    .scope_column(variable)
774                    .and_then(|column| values.get(column))
775                    .filter(|value| !value.is_null())
776                    .is_some_and(|value| {
777                        allowed.contains(
778                            &value
779                                .as_str()
780                                .map_or_else(|| value.to_string(), str::to_owned),
781                        )
782                    })
783            })
784    };
785    matches(effective) && !owners.iter().any(|owner| matches(owner))
786}
787
788fn is_local_operation_code_like(value: &str) -> bool {
789    let bytes = value.as_bytes();
790    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
791        && bytes
792            .iter()
793            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
794}
795
796fn base_table(name: &str) -> String {
797    quote_ident(&format!("_syncular_base_{name}"))
798}
799
800/// §4.8 deferred eviction record: (unit id, table, effective scope map).
801type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
802
803/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
804fn window_base_key(base: &WindowBase) -> String {
805    format!(
806        "{}\0{}",
807        base.variable,
808        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
809    )
810}
811
812/// The unit of one window value: the fixed scopes plus the value.
813fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
814    let mut scopes = base.fixed_scopes.clone();
815    scopes.insert(base.variable.clone(), value.to_owned());
816    scopes
817}
818
819/// A unit's scope map as the codec and the digests take it.
820fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
821    let mut pairs: Vec<(String, String)> = scopes
822        .iter()
823        .map(|(variable, value)| (variable.clone(), value.clone()))
824        .collect();
825    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
826    pairs
827}
828
829/// §4.1: the unit covers every table declaring all of its variables.
830fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
831    scopes
832        .keys()
833        .all(|variable| table.scope_column(variable).is_some())
834}
835
836/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
837/// scope map as one-element lists plus the fenced lists. `None` when the
838/// fence revoked the pair.
839fn pair_scopes(
840    scopes: &UnitScopes,
841    table: &str,
842    echo: Option<&[EchoRecord]>,
843) -> Option<Vec<(String, Vec<String>)>> {
844    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
845    if fenced.is_some_and(|record| record.status == "revoked") {
846        return None;
847    }
848    let mut effective: BTreeMap<String, Vec<String>> = fenced
849        .map(|record| record.scopes.clone())
850        .unwrap_or_default();
851    for (variable, value) in scopes {
852        effective.insert(variable.clone(), vec![value.clone()]);
853    }
854    Some(effective.into_iter().collect())
855}
856
857/// §3.3 row ownership: the pair scopes of every owner of `table` except
858/// `except_id`. An owner is active with a persisted echo and live or
859/// post-bootstrap, or bootstrapping with a segment of the table in its
860/// current bootstrap.
861fn unit_owners(
862    units: &[UnitRecord],
863    table: &TableSchema,
864    except_id: &str,
865) -> Vec<Vec<(String, Vec<String>)>> {
866    units
867        .iter()
868        .filter(|unit| {
869            unit.id != except_id
870                && unit.status == UnitStatus::Active
871                && unit.echo.is_some()
872                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
873                && covers(table, &unit.scopes)
874        })
875        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
876        .collect()
877}
878
879/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
880fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
881    scopes
882        .iter()
883        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
884        .collect()
885}
886
887/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
888/// It joins through the (tbl, id) index: a scan per acknowledgement would
889/// make draining a batch quadratic.
890const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
891
892const UNIT_COLUMNS: &str =
893    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
894
895/// A unit row; a field that does not parse is a conversion failure, which
896/// both read paths classify as `sync.local_corrupt`.
897fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
898    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
899        serde_json::from_str(raw).map_err(|error| {
900            rusqlite::Error::FromSqlConversionFailure(
901                index,
902                rusqlite::types::Type::Text,
903                Box::new(error),
904            )
905        })
906    }
907    Ok(UnitRecord {
908        id: row.get(0)?,
909        scopes: json(1, &row.get::<_, String>(1)?)?,
910        explicit: row.get::<_, i64>(2)? == 1,
911        status: json(
912            3,
913            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
914        )?,
915        reason_code: row.get(4)?,
916        cursor: row.get(5)?,
917        bootstrap_state: row.get(6)?,
918        seeded: json(7, &row.get::<_, String>(7)?)?,
919        echo: row
920            .get::<_, Option<String>>(8)?
921            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
922            .transpose()?
923            .map(|echo| {
924                if echo
925                    .iter()
926                    .all(|record| record.status == "active" || record.status == "revoked")
927                {
928                    Ok(echo)
929                } else {
930                    Err(rusqlite::Error::FromSqlConversionFailure(
931                        8,
932                        rusqlite::types::Type::Text,
933                        "echo status is neither active nor revoked".into(),
934                    ))
935                }
936            })
937            .transpose()?,
938    })
939}
940
941/// Every unit in registration order (§4.1).
942fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
943    conn.prepare_cached(&format!(
944        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
945    ))?
946    .query_map([], unit_from_row)?
947    .collect()
948}
949
950fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
951    conn.query_row(
952        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
953        [id],
954        unit_from_row,
955    )
956    .optional()
957}
958
959/// Insert or update in place: an update keeps the registration order.
960fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
961    conn.prepare_cached(&format!(
962        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
963         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
964           reason_code=excluded.reason_code, cursor=excluded.cursor,
965           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo"
966    ))?
967    .execute(rusqlite::params![
968        unit.id,
969        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
970        i64::from(unit.explicit),
971        unit.status.name(),
972        unit.reason_code,
973        unit.cursor,
974        unit.bootstrap_state,
975        serde_json::to_string(&unit.seeded).expect("table list serializes"),
976        unit.echo
977            .as_ref()
978            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
979    ])
980    .map(|_| ())
981}
982
983/// §4.1 references: every window registry entry `(base, value)` holding a unit.
984fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
985    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
986        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
987        .collect()
988}
989
990/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
991/// registrations and discard every synced state, so the next round sends the
992/// list and fresh-bootstraps every unit. The caller owns the transaction.
993fn reset_units(conn: &Connection) -> rusqlite::Result<()> {
994    conn.execute_batch(
995        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
996           bootstrap_state=NULL, seeded='[]', echo=NULL;
997         DELETE FROM _syncular_meta WHERE key IN ('replicaCursor','echoDigest','registrationDigest');",
998    )
999}
1000
1001/// §4.5: the replica cursor, `-1` while no unit is live.
1002fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1003    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1004        raw.parse().map_err(|error| {
1005            rusqlite::Error::FromSqlConversionFailure(
1006                0,
1007                rusqlite::types::Type::Text,
1008                Box::new(error),
1009            )
1010        })
1011    })
1012}
1013
1014fn visible_table(name: &str) -> String {
1015    quote_ident(name)
1016}
1017
1018const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1019
1020/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1021/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1022fn is_synced_table_name(name: &str) -> bool {
1023    if name.starts_with("sqlite_") {
1024        return false;
1025    }
1026    if name.starts_with("_syncular_base_") {
1027        return true; // the base half of a synced table pair
1028    }
1029    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1030    !name.starts_with("_syncular_")
1031}
1032
1033/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1034fn blob_id_for(bytes: &[u8]) -> String {
1035    let digest = Sha256::digest(bytes);
1036    format!("sha256:{}", bytes_to_hex(&digest))
1037}
1038
1039/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1040/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1041/// per row on the §5.6 bootstrap path), scalars bind owned.
1042enum RowParam<'a> {
1043    Cell(&'a Option<ColumnValue>),
1044    Version(i64),
1045}
1046
1047impl rusqlite::ToSql for RowParam<'_> {
1048    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1049        Ok(match self {
1050            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1051            RowParam::Cell(cell) => match cell {
1052                None => ToSqlOutput::Owned(SqlValue::Null),
1053                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1054                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1055                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1056                Some(ColumnValue::Boolean(b)) => {
1057                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1058                }
1059                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1060                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1061                }
1062                // §5.10: crdt bytes store as BLOB, like bytes.
1063                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1064                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1065                }
1066            },
1067        })
1068    }
1069}
1070
1071// Retain a typed primary key across the current import chunk without copying row payloads.
1072fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1073    match param.to_sql().map_err(|error| error.to_string())? {
1074        ToSqlOutput::Owned(value) => Ok(value),
1075        ToSqlOutput::Borrowed(value) => {
1076            SqlValue::try_from(value).map_err(|error| error.to_string())
1077        }
1078        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1079    }
1080}
1081
1082/// §5.3 image cell → bind parameter, strict per the declared column type
1083/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1084/// nullable). Mismatches are image-producer violations. Returns the cell
1085/// borrowed when the stored representation already matches what the row
1086/// codec would write, or the normalized scalar (boolean → 0/1, float from
1087/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1088/// old convert-then-insert path without allocating per cell.
1089fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1090    use ssp2::segment::ColumnType;
1091    let mismatch = || {
1092        Err(format!(
1093            "image column {:?} holds a value of the wrong type",
1094            column.name
1095        ))
1096    };
1097    match value {
1098        ValueRef::Null => {
1099            if !column.nullable {
1100                return Err(format!(
1101                    "image column {:?} is NULL but not nullable",
1102                    column.name
1103                ));
1104            }
1105            Ok(ToSqlOutput::Owned(SqlValue::Null))
1106        }
1107        ValueRef::Integer(i) => match column.ty {
1108            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1109            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1110            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1111            _ => mismatch(),
1112        },
1113        ValueRef::Real(_) => match column.ty {
1114            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1115            _ => mismatch(),
1116        },
1117        ValueRef::Text(t) => {
1118            std::str::from_utf8(t)
1119                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1120            match column.ty {
1121                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1122                    Ok(ToSqlOutput::Borrowed(value))
1123                }
1124                _ => mismatch(),
1125            }
1126        }
1127        ValueRef::Blob(_) => match column.ty {
1128            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1129            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1130            _ => mismatch(),
1131        },
1132    }
1133}
1134
1135pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1136    use rusqlite::types::ValueRef;
1137    match value {
1138        ValueRef::Null => Value::Null,
1139        ValueRef::Integer(i) => match column.ty {
1140            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1141            ssp2::segment::ColumnType::Float => {
1142                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1143            }
1144            _ => Value::from(i),
1145        },
1146        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1147        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1148        ValueRef::Blob(b) => {
1149            let mut map = Map::new();
1150            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1151            Value::Object(map)
1152        }
1153    }
1154}
1155
1156/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1157/// Objects are accepted in lossless `{"$bytes": hex}` and
1158/// `{"$bigint": decimal}` envelope forms.
1159fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1160    Ok(match value {
1161        Value::Null => SqlValue::Null,
1162        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1163        Value::Number(n) => {
1164            if let Some(i) = n.as_i64() {
1165                SqlValue::Integer(i)
1166            } else if let Some(f) = n.as_f64() {
1167                SqlValue::Real(f)
1168            } else {
1169                return Err(format!("query param number {n} is out of range"));
1170            }
1171        }
1172        Value::String(s) => SqlValue::Text(s.clone()),
1173        Value::Object(_) => {
1174            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1175                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1176            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1177                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1178                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1179                })?)
1180            } else {
1181                return Err(
1182                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1183                        .to_owned(),
1184                );
1185            }
1186        }
1187        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1188    })
1189}
1190
1191/// Map a rusqlite value with no schema column to consult (arbitrary query
1192/// output): integers/reals/text pass through by stored affinity, blobs ride
1193/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1194/// schema column type to recover booleans/floats/json.
1195fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1196    use rusqlite::types::ValueRef;
1197    match value {
1198        ValueRef::Null => Value::Null,
1199        ValueRef::Integer(i) => {
1200            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1201            // ordinary UI-sized integers ergonomic and envelope only values
1202            // beyond JavaScript's safe range.
1203            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1204            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1205                Value::from(i)
1206            } else {
1207                let mut map = Map::new();
1208                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1209                Value::Object(map)
1210            }
1211        }
1212        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1213        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1214        ValueRef::Blob(b) => {
1215            let mut map = Map::new();
1216            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1217            Value::Object(map)
1218        }
1219    }
1220}
1221
1222/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1223fn query_connection(
1224    conn: &Connection,
1225    sql: &str,
1226    params: &[QueryValue],
1227) -> Result<Vec<QueryRow>, QueryReadFailure> {
1228    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1229    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1230    let bound: Vec<SqlValue> = params
1231        .iter()
1232        .map(json_param_to_sql)
1233        .collect::<Result<_, _>>()?;
1234    // §2.4: an unaliased physical version column stays engine-internal, so a
1235    // `SELECT *` row feeds straight back into mutation values. An explicit
1236    // alias (`_sync_version AS v`) returns it as ordinary query data.
1237    let columns: Vec<(usize, String)> = stmt
1238        .column_names()
1239        .enumerate()
1240        .filter(|(_, name)| *name != "_syncular_version")
1241        .map(|(i, name)| (i, name.to_owned()))
1242        .collect();
1243    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1244        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1245    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1246    let mut out = Vec::new();
1247    while let Some(row) = sql_rows.next()? {
1248        let mut record = Map::new();
1249        for (i, name) in &columns {
1250            let value = row.get_ref(*i)?;
1251            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1252        }
1253        out.push(record);
1254    }
1255    Ok(out)
1256}
1257
1258fn persisted_window_state(
1259    conn: &Connection,
1260    base: &WindowBase,
1261) -> Result<WindowState, QueryReadFailure> {
1262    let mut stmt = conn.prepare(
1263        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1264           FROM _syncular_windows AS windows
1265           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1266          WHERE windows.base = ?1
1267          ORDER BY windows.unit ASC",
1268    )?;
1269    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1270        Ok((
1271            row.get::<_, String>(0)?,
1272            row.get::<_, Option<String>>(1)?,
1273            row.get::<_, Option<i64>>(2)?,
1274            row.get::<_, Option<String>>(3)?,
1275        ))
1276    })?;
1277    let mut units = Vec::new();
1278    let mut pending = Vec::new();
1279    for row in rows {
1280        let (unit, status, cursor, bootstrap_state) = row?;
1281        // §4.8: complete iff registered, active and not bootstrapping.
1282        if status.as_deref() != Some("active")
1283            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1284        {
1285            pending.push(unit.clone());
1286        }
1287        units.push(unit);
1288    }
1289    Ok(WindowState { units, pending })
1290}
1291
1292/// One read-only statement in a §7.5 snapshot read.
1293#[derive(Debug, Clone, Copy)]
1294pub struct SnapshotStatement<'a> {
1295    pub sql: &'a str,
1296    pub params: &'a [Value],
1297}
1298
1299/// A §7.5 atomic read request: read-only statements, window coverage, unit
1300/// catch-up states, and commit deliveries, all resolved from one SQLite read
1301/// transaction and one revision.
1302#[derive(Debug, Clone, Default)]
1303pub struct SnapshotReadRequest<'a> {
1304    pub statements: Vec<SnapshotStatement<'a>>,
1305    pub coverage: &'a [WindowCoverage],
1306    pub units: Vec<String>,
1307    pub commit_ids: Vec<String>,
1308}
1309
1310#[cfg(test)]
1311thread_local! {
1312    /// Test-only rendezvous that runs between the statements of one snapshot
1313    /// read. A test uses it to commit on another connection mid-read and prove
1314    /// the read transaction's snapshot and revision stay stable.
1315    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1316        std::cell::RefCell::new(None);
1317}
1318
1319#[cfg(test)]
1320fn run_snapshot_read_hook() {
1321    SNAPSHOT_READ_HOOK.with(|slot| {
1322        if let Some(hook) = slot.borrow_mut().take() {
1323            hook();
1324        }
1325    });
1326}
1327
1328/// §7.5: resolve a request over one SQLite read transaction. The revision,
1329/// every statement's rows, coverage, catch-up states, and deliveries are
1330/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1331/// read back and releases the connection.
1332fn snapshot_read_connection(
1333    conn: &Connection,
1334    request: &SnapshotReadRequest<'_>,
1335) -> Result<SnapshotRead, QueryReadFailure> {
1336    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1337    let corrupt = |message: &str| QueryReadFailure {
1338        code: Some("sync.local_corrupt"),
1339        sqlite_code: None,
1340        sqlite_message: None,
1341        rollback_failure: None,
1342        message: message.to_owned(),
1343    };
1344    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1345        rusqlite::Error::Utf8Error(..)
1346        | rusqlite::Error::InvalidColumnType(..)
1347        | rusqlite::Error::FromSqlConversionFailure(..)
1348        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1349        _ => QueryReadFailure::from(error),
1350    };
1351    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1352        // The durable revision as a canonical decimal `u64`; a missing marker
1353        // is the legacy zero, and any other non-canonical value is corrupt.
1354        let revision: Option<String> = conn
1355            .query_row(
1356                "SELECT value FROM _syncular_meta WHERE key = ?1",
1357                rusqlite::params![LOCAL_REVISION_KEY],
1358                |row| row.get(0),
1359            )
1360            .optional()
1361            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1362        let revision = match revision {
1363            None => "0".to_owned(),
1364            Some(value)
1365                if value == "0"
1366                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1367                        && value.bytes().all(|b| b.is_ascii_digit())
1368                        && value.parse::<u64>().is_ok()) =>
1369            {
1370                value
1371            }
1372            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1373        };
1374        let mut queries = Vec::with_capacity(request.statements.len());
1375        for statement in &request.statements {
1376            queries.push(query_connection(conn, statement.sql, statement.params)?);
1377            #[cfg(test)]
1378            run_snapshot_read_hook();
1379        }
1380        let mut pending = Vec::new();
1381        let mut missing = Vec::new();
1382        for requested in request.coverage {
1383            let base_key = window_base_key(&requested.base);
1384            let state = persisted_window_state(conn, &requested.base)?;
1385            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1386                let reference = WindowUnitRef {
1387                    base_key: base_key.clone(),
1388                    unit: unit.clone(),
1389                };
1390                if !state.units.iter().any(|held| held == &unit) {
1391                    missing.push(reference);
1392                } else if state.pending.iter().any(|held| held == &unit) {
1393                    pending.push(reference);
1394                }
1395            }
1396        }
1397        let mut units = Vec::with_capacity(request.units.len());
1398        for id in &request.units {
1399            let exists: bool = conn.query_row(
1400                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1401                [id],
1402                |row| row.get(0),
1403            )?;
1404            if !exists {
1405                units.push(UnitCatchup::Unknown { id: id.clone() });
1406                continue;
1407            }
1408            // Every persisted field is required to be well typed; a corrupt
1409            // record fails typed instead of inventing a state.
1410            let unit = get_unit(conn, id)
1411                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1412                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1413            let cursor = match unit.cursor {
1414                Some(cursor) => cursor,
1415                None => replica_cursor(conn)
1416                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1417            };
1418            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1419                return Err(corrupt("persisted unit state is invalid"));
1420            }
1421            let active = unit.status == UnitStatus::Active;
1422            let pending = unit.bootstrapping();
1423            units.push(UnitCatchup::Known {
1424                id: id.clone(),
1425                status: unit.status,
1426                live: unit.cursor.is_none(),
1427                cursor,
1428                has_resume_token: unit.bootstrap_state.is_some(),
1429                bootstrap_complete: active && !pending,
1430                known_pending_pages: active && pending,
1431                reason_code: unit.reason_code,
1432            });
1433        }
1434        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1435        for id in &request.commit_ids {
1436            let pending: bool = conn.query_row(
1437                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1438                rusqlite::params![id],
1439                |row| row.get(0),
1440            )?;
1441            if pending {
1442                deliveries.push(CommitDelivery::Pending {
1443                    client_commit_id: id.clone(),
1444                });
1445                continue;
1446            }
1447            let row = conn
1448                .query_row(
1449                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1450                            resolution, resolved_at_ms, replacement_client_commit_id
1451                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1452                    rusqlite::params![id],
1453                    StoredCommitOutcomeRow::from_row,
1454                )
1455                .optional()
1456                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1457            deliveries.push(match row {
1458                Some(row) => CommitDelivery::Known {
1459                    client_commit_id: id.clone(),
1460                    outcome: SyncClient::outcome_from_row(row)
1461                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1462                },
1463                None => CommitDelivery::Unknown {
1464                    client_commit_id: id.clone(),
1465                },
1466            });
1467        }
1468        Ok(SnapshotRead {
1469            revision,
1470            queries,
1471            coverage: CoverageSnapshot {
1472                complete: pending.is_empty() && missing.is_empty(),
1473                pending,
1474                missing,
1475            },
1476            units,
1477            deliveries,
1478        })
1479    })();
1480    match result {
1481        Ok(read) => {
1482            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1483            Ok(read)
1484        }
1485        Err(mut error) => {
1486            if let Err(rollback) = conn
1487                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1488            {
1489                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1490            }
1491            Err(error)
1492        }
1493    }
1494}
1495
1496fn snapshot_connection(
1497    conn: &Connection,
1498    sql: &str,
1499    params: &[Value],
1500    coverage: &[WindowCoverage],
1501) -> Result<QuerySnapshot, QueryReadFailure> {
1502    let read = snapshot_read_connection(
1503        conn,
1504        &SnapshotReadRequest {
1505            statements: vec![SnapshotStatement { sql, params }],
1506            coverage,
1507            units: Vec::new(),
1508            commit_ids: Vec::new(),
1509        },
1510    )?;
1511    Ok(QuerySnapshot {
1512        revision: read.revision,
1513        rows: read
1514            .queries
1515            .into_iter()
1516            .next()
1517            .expect("single-query snapshot returns one result"),
1518        coverage: read.coverage,
1519    })
1520}
1521
1522/// A long-lived read-only SQLite sidecar for latency-critical native views.
1523/// Network rounds stay serialized on the mutable core owner, while atomic
1524/// query snapshots use this independent connection and therefore never queue
1525/// behind HTTP/WebSocket latency.
1526pub struct FileQuerySnapshotReader {
1527    path: String,
1528    conn: Option<Connection>,
1529}
1530
1531impl FileQuerySnapshotReader {
1532    #[must_use]
1533    pub fn new(path: impl Into<String>) -> Self {
1534        Self {
1535            path: path.into(),
1536            conn: None,
1537        }
1538    }
1539
1540    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1541        if self.conn.is_none() {
1542            let conn = Connection::open_with_flags(
1543                &self.path,
1544                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1545            )
1546            .map_err(QueryReadFailure::from)?;
1547            conn.busy_timeout(std::time::Duration::from_millis(250))
1548                .map_err(QueryReadFailure::from)?;
1549            self.conn = Some(conn);
1550        }
1551        self.conn
1552            .as_ref()
1553            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1554    }
1555
1556    /// One atomic read over the shared primitive: every statement, coverage,
1557    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1558    pub fn snapshot_read(
1559        &mut self,
1560        request: &SnapshotReadRequest<'_>,
1561    ) -> Result<SnapshotRead, QueryReadFailure> {
1562        snapshot_read_connection(self.connection()?, request)
1563    }
1564
1565    /// The current durable local revision. A host can skip a read when it is
1566    /// unchanged; a corrupt or unreadable marker fails typed.
1567    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1568        Ok(self
1569            .snapshot_read(&SnapshotReadRequest::default())?
1570            .revision)
1571    }
1572
1573    /// One unit's persisted catch-up state (see the §7.5 contract).
1574    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1575        let read = self.snapshot_read(&SnapshotReadRequest {
1576            units: vec![id.to_owned()],
1577            ..Default::default()
1578        })?;
1579        Ok(read
1580            .units
1581            .into_iter()
1582            .next()
1583            .expect("one requested catch-up"))
1584    }
1585
1586    /// One client commit's delivery status: pending, its retained outcome, or
1587    /// an explicit unknown (never held, or pruned by retention).
1588    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1589        let read = self.snapshot_read(&SnapshotReadRequest {
1590            commit_ids: vec![id.to_owned()],
1591            ..Default::default()
1592        })?;
1593        Ok(read
1594            .deliveries
1595            .into_iter()
1596            .next()
1597            .expect("one requested delivery"))
1598    }
1599
1600    /// Hosts report an owned failure, and the first success after it, to the
1601    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1602    pub fn query_snapshot(
1603        &mut self,
1604        sql: &str,
1605        params: &[Value],
1606        coverage: &[WindowCoverage],
1607    ) -> Result<QuerySnapshot, QueryReadFailure> {
1608        snapshot_connection(self.connection()?, sql, params, coverage)
1609    }
1610}
1611
1612fn validate_authority_reads(
1613    schema: &ClientSchema,
1614    declarations: &[crate::api::AuthorityReadDeclaration],
1615) -> Result<(), String> {
1616    let mut seen = BTreeSet::new();
1617    for read in declarations {
1618        let table = schema
1619            .table(&read.table)
1620            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1621        let plain = |name: &str| {
1622            table.columns.iter().enumerate().any(|(i, c)| {
1623                c.name == name
1624                    && name != "authority_version"
1625                    && !name.starts_with("_sync")
1626                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1627                    && matches!(
1628                        c.ty,
1629                        ssp2::segment::ColumnType::String
1630                            | ssp2::segment::ColumnType::Integer
1631                            | ssp2::segment::ColumnType::Float
1632                            | ssp2::segment::ColumnType::Boolean
1633                            | ssp2::segment::ColumnType::Json
1634                    )
1635            })
1636        };
1637        if !seen.insert(&read.table)
1638            || read.columns.is_empty()
1639            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1640            || !read.columns.contains(&table.primary_key)
1641            || !read.columns.iter().all(|c| plain(c))
1642            || read.scopes.is_empty()
1643            || !read.scopes.iter().all(|(v, values)| {
1644                table
1645                    .scope_column(v)
1646                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1647                    && !values.is_empty()
1648                    && values.iter().all(|v| !v.is_empty() && v != "*")
1649            })
1650        {
1651            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1652        }
1653    }
1654    Ok(())
1655}
1656
1657// §7.4.2: every open and recreation checks before any persistent writes.
1658/// A marker read that hit a recognized SQLite storage failure keeps that
1659/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1660/// failure is the invalid-marker `sync.local_corrupt`.
1661fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1662    let failure = QueryReadFailure::from(error);
1663    if failure.code.is_some() {
1664        failure.to_string()
1665    } else {
1666        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1667    }
1668}
1669
1670/// §7.1: reject an invalid configured client limit before any storage is
1671/// opened or written. Zero would wedge the push queue and an out-of-range
1672/// value would differ per target; the TS core enforces the same bounds.
1673fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1674    if limits.outcome_retention_max_entries == Some(0) {
1675        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1676    }
1677    if [
1678        limits.max_push_commits_per_request,
1679        limits.max_push_operations_per_request,
1680        limits.max_push_request_bytes,
1681    ]
1682    .into_iter()
1683    .flatten()
1684    .any(|value| value == 0 || value > u32::MAX as usize)
1685    {
1686        return Err(
1687            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1688        );
1689    }
1690    Ok(())
1691}
1692
1693fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1694    let corrupt =
1695        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1696    let table_type: Option<String> = conn
1697        .query_row(
1698            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1699            [],
1700            |row| row.get(0),
1701        )
1702        .optional()
1703        .map_err(schema_marker_read_failure)?;
1704    let Some(table_type) = table_type else {
1705        return Ok(None);
1706    };
1707    if table_type != "table" {
1708        return Err(corrupt());
1709    }
1710    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1711    // whichever row SQLite returns first, and an older one would drive a
1712    // destructive reset over a newer replica.
1713    let mut statement = conn
1714        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1715        .map_err(schema_marker_read_failure)?;
1716    let mut rows = statement
1717        .query([LOCAL_SCHEMA_VERSION_KEY])
1718        .map_err(schema_marker_read_failure)?;
1719    let value: Option<String> = rows
1720        .next()
1721        .map_err(schema_marker_read_failure)?
1722        .map(|row| row.get(0))
1723        .transpose()
1724        .map_err(schema_marker_read_failure)?;
1725    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1726        return Err(corrupt());
1727    }
1728    let Some(value) = value else {
1729        let has_descriptor = conn
1730            .query_row(
1731                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1732                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1733                |_| Ok(()),
1734            )
1735            .optional()
1736            .map_err(schema_marker_read_failure)?
1737            .is_some();
1738        if has_descriptor {
1739            return Err(corrupt());
1740        }
1741        return Ok(None);
1742    };
1743    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1744        || !value.bytes().all(|b| b.is_ascii_digit())
1745    {
1746        return Err(corrupt());
1747    }
1748    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1749    if version > requested {
1750        return Err(
1751            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1752                .to_owned(),
1753        );
1754    }
1755    Ok(Some(version))
1756}
1757
1758/// A file replica on a browser worker must recover a hot journal before any
1759/// statement reads it. `wasm_vfs` replaces the installed OPFS VFS's reserved
1760/// lock callback, so install it before the connection opens its file.
1761#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1762fn prepare_wasm_replica(path: &str) -> Result<(), String> {
1763    if path.is_empty() || path == ":memory:" {
1764        return Ok(());
1765    }
1766    crate::wasm_vfs::disable_foreign_lock_check()
1767}
1768
1769#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
1770fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
1771    Ok(())
1772}
1773
1774impl SyncClient {
1775    pub fn new_with_identity(
1776        client_id: Option<String>,
1777        schema_json: &Value,
1778        limits: ClientLimits,
1779    ) -> Result<Self, String> {
1780        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1781        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1782        Self::with_connection(resolved, schema_json, limits, conn)
1783    }
1784
1785    pub fn new(
1786        client_id: String,
1787        schema_json: &Value,
1788        limits: ClientLimits,
1789    ) -> Result<Self, String> {
1790        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1791        Self::with_connection(client_id, schema_json, limits, conn)
1792    }
1793
1794    /// Build a client backed by an on-disk SQLite database at `path` — the
1795    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1796    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1797    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1798    /// of the command router's dependency set (the router only holds a path).
1799    pub fn open_path(
1800        client_id: String,
1801        schema_json: &Value,
1802        limits: ClientLimits,
1803        path: &str,
1804    ) -> Result<Self, String> {
1805        validate_client_limits(&limits)?;
1806        parse_schema_json(schema_json)?;
1807        prepare_wasm_replica(path)?;
1808        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1809        Self::with_connection(client_id, schema_json, limits, conn)
1810    }
1811
1812    /// Open a file replica and resolve its client id. Native targets require
1813    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1814    /// accepts the sqlite-wasm worker journal instead.
1815    pub fn open_path_with_identity(
1816        client_id: Option<String>,
1817        schema_json: &Value,
1818        limits: ClientLimits,
1819        path: &str,
1820    ) -> Result<Self, String> {
1821        Self::open_path_with_identity_journal(
1822            client_id,
1823            schema_json,
1824            limits,
1825            path,
1826            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1827        )
1828    }
1829
1830    fn open_path_with_identity_journal(
1831        client_id: Option<String>,
1832        schema_json: &Value,
1833        limits: ClientLimits,
1834        path: &str,
1835        require_wal: bool,
1836    ) -> Result<Self, String> {
1837        validate_client_limits(&limits)?;
1838        // §7.4.2: an invalid requested version is refused before the replica is
1839        // even created, so a rejected constructor leaves no file behind.
1840        parse_schema_json(schema_json)?;
1841        prepare_wasm_replica(path)?;
1842        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1843        // A file-backed native client uses an independent read connection for
1844        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1845        // concurrency mode: a view read never holds a rollback-journal lock
1846        // that delays the client's next commit. The busy timeout must precede
1847        // this replica's own startup transaction; WAL is applied only after
1848        // startup succeeded, because a refused open must leave the replica's
1849        // journal mode and contents untouched (§7.4.2).
1850        conn.busy_timeout(std::time::Duration::from_millis(250))
1851            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1852        // Durability is the caller's contract here, not the VFS default: a
1853        // rollback-journal commit needs FULL before it writes the database
1854        // pages, or a killed worker can leave a half-written image behind.
1855        // WAL below inherits the same setting.
1856        conn.pragma_update(None, "synchronous", "FULL")
1857            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1858        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1859        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1860        // host has no independent snapshot reader. The WAL requirement exists
1861        // for native file replicas, where it is the reader/writer boundary.
1862        if require_wal {
1863            let journal_mode: String = client
1864                .conn
1865                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1866                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1867            if journal_mode != "wal" {
1868                return Err(
1869                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1870                );
1871            }
1872        }
1873        Ok(client)
1874    }
1875
1876    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1877    /// `-shm` sidecars, through the VFS that owns it. Use this after
1878    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1879    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1880    /// pool's slot accounting.
1881    ///
1882    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1883    /// a VFS cannot delete a file another handle still owns. A file that does
1884    /// not exist counts as deleted, so a cleanly closed replica without
1885    /// sidecars resets. The next open
1886    /// creates a fresh replica (new client id) and re-bootstraps from the
1887    /// server. A memory replica has nothing to delete and fails with
1888    /// `sync.invalid_request`.
1889    pub fn reset_replica(path: &str) -> Result<(), String> {
1890        if path.is_empty() || path == ":memory:" {
1891            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1892        }
1893        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1894        // the same one `Connection::open(path)` above selects. `xDelete` is
1895        // that VFS's own function pointer, and `c_name` outlives the call.
1896        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1897        if vfs.is_null() {
1898            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1899        }
1900        let delete = unsafe { (*vfs).xDelete }
1901            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1902        for suffix in ["-wal", "-shm", "-journal", ""] {
1903            let file_name = format!("{path}{suffix}");
1904            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1905                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1906            })?;
1907            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1908            // SQLite library; `c_name` is a valid NUL-terminated path.
1909            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1910            // A cleanly closed WAL replica has no sidecars; a missing file is
1911            // already deleted.
1912            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1913            {
1914                return Err(format!(
1915                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1916                ));
1917            }
1918        }
1919        Ok(())
1920    }
1921
1922    #[must_use]
1923    pub fn client_id(&self) -> &str {
1924        &self.client_id
1925    }
1926
1927    /// Rotate the caller's client id (§1.5) and return the new one. Use it
1928    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
1929    /// resetClientId`): the next round carries the new id. A live realtime
1930    /// socket is closed so the next connection registers with the new id.
1931    /// The replica and its outbox stay in place; wipe them yourself when the
1932    /// stored data belongs to the account that signed out.
1933    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
1934        self.disconnect_realtime(transport);
1935        let client_id = uuid::Uuid::new_v4().to_string();
1936        self.set_meta(CLIENT_ID_KEY, &client_id)?;
1937        self.client_id.clone_from(&client_id);
1938        self.round_failure = None;
1939        Ok(client_id)
1940    }
1941
1942    /// Build a client over a caller-supplied rusqlite connection — the seam a
1943    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
1944    /// with an on-disk database (`Connection::open(path)`) rather than the
1945    /// default `:memory:`. The connection MUST be fresh (no pre-existing
1946    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
1947    /// the same file across process restarts reuses the persisted rows.
1948    pub fn with_connection(
1949        client_id: String,
1950        schema_json: &Value,
1951        limits: ClientLimits,
1952        conn: Connection,
1953    ) -> Result<Self, String> {
1954        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
1955    }
1956
1957    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
1958    /// means a durable replica's persisted `clientId` wins over a fresh one.
1959    fn with_connection_identity(
1960        client_id: Option<String>,
1961        schema_json: &Value,
1962        limits: ClientLimits,
1963        conn: Connection,
1964    ) -> Result<Self, String> {
1965        validate_client_limits(&limits)?;
1966        let schema = parse_schema_json(schema_json)?;
1967        validate_authority_reads(&schema, &limits.authority_reads)?;
1968        // RFC 0005 D8: resolve the previous-version config before the opening
1969        // reset runs — the capture happens inside it. Bad bounds fail loud.
1970        let previous_version = match limits.previous_version_context {
1971            Some(config) => {
1972                config.validate()?;
1973                Some(config)
1974            }
1975            None => None,
1976        };
1977        let mut client = SyncClient {
1978            #[cfg(feature = "bench-internals")]
1979            benchmark_phases: Recorder::default(),
1980            conn,
1981            schema,
1982            client_id: client_id
1983                .clone()
1984                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
1985            limits,
1986            outbox: Shared::default(),
1987            failed_commits: Shared::default(),
1988            retain_failed_commits: false,
1989            conflicts: Shared::default(),
1990            rejections: Shared::default(),
1991            schema_floor: None,
1992            lease_state: None,
1993            stopped: false,
1994            upgrading: false,
1995            sync_needed: false,
1996            realtime_policy: RealtimePolicy::Optional,
1997            realtime_state: RealtimeState::Disconnected,
1998            realtime_reason_code: None,
1999            realtime_retry_delay_ms: None,
2000            presence: HashMap::new(),
2001            now_ms: None,
2002            encryption: crate::values::EncryptionConfig::default(),
2003            security_preflight: false,
2004            security_epoch: 0,
2005            transport_enabled: true,
2006            transport_epoch: 0,
2007            registration_epoch: 0,
2008            previous_version,
2009            insert_sql: RefCell::new(HashMap::new()),
2010            overlay_dirty: OverlayDirty::clean(),
2011            storage_failure: RefCell::new(None),
2012            #[cfg(test)]
2013            overlay_rebuild_count: Cell::new(0),
2014            #[cfg(test)]
2015            acknowledged_replay_count: Cell::new(0),
2016            #[cfg(test)]
2017            outcome_prune_count: Cell::new(0),
2018            #[cfg(test)]
2019            image_chunk_ms: Cell::new(None),
2020            progress: ProgressObserver::default(),
2021            change_queue: VecDeque::new(),
2022            sync_intent_queue: VecDeque::new(),
2023            retry_delay_ms: 250,
2024            round_retry_delay_ms: None,
2025            round_failure: None,
2026            active_round: None,
2027            last_round: None,
2028            last_change: None,
2029            query_failures: Vec::new(),
2030            coalesced: None,
2031            coalesced_failure: None,
2032        };
2033        // The row write path leans on the prepared-statement cache (two
2034        // insert statements per synced table, plus the bookkeeping
2035        // statements); size it so a multi-table schema never thrashes.
2036        client
2037            .conn
2038            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2039        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2040        // writer) covers the marker read, the identity check, and every
2041        // bookkeeping/schema write, so a concurrent open that upgraded the
2042        // replica cannot interleave between the guard and the writes, and a
2043        // refusal leaves the replica unchanged.
2044        client
2045            .conn
2046            .execute_batch("BEGIN IMMEDIATE")
2047            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2048        let startup = (|| -> Result<(), String> {
2049            // The marker guard runs inside the write transaction, so the version
2050            // it validates is the version every write below commits under.
2051            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2052            // New app indexes may reference columns that only exist after a
2053            // version-bump reset.
2054            client.create_bookkeeping_tables()?;
2055            match client.get_meta(CLIENT_ID_KEY)? {
2056                Some(existing) => match client_id {
2057                    Some(requested) if existing != requested => {
2058                        return Err(format!(
2059                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2060                        ))
2061                    }
2062                    _ => client.client_id = existing,
2063                },
2064                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2065            }
2066            client.restore_persisted_state()?;
2067            client.failed_commits = client.load_failed_commits()?.into();
2068            match marker {
2069                None => {
2070                    client.create_synced_tables()?;
2071                    client.persist_schema_version()?;
2072                    client.save_unit_scope_schema()?;
2073                }
2074                Some(version) if version == client.schema.version => {
2075                    client.create_synced_tables()?;
2076                    // RFC 0005 D1: the same-version open backfills the descriptor
2077                    // for a database first opened by an unaware binary, so the
2078                    // NEXT bump can capture without needing a schema bump first.
2079                    set_local_schema_descriptor(&client.conn, &client.schema)
2080                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2081                    client.save_unit_scope_schema()?;
2082                }
2083                Some(_) => client.run_schema_reset()?,
2084            }
2085            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2086            // committed in its OWN file while the replica's savepoint never
2087            // released — a crash at that seam leaves a container beside a matching
2088            // OLD marker, and the next open at that version runs no reset. Discard
2089            // it before anything can read it. No-op when no container exists.
2090            client.reconcile_previous_version_at_boot()?;
2091            client.clear_satisfied_persisted_schema_floor()?;
2092            client.prune_unknown_units(false)?;
2093            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2094            // the full startup/reset reconciliation boundary, inside this open
2095            // transaction and after the schema and identity guards, so a commit
2096            // an earlier version accepted is dropped with durable rejection
2097            // evidence before any replay. A table the current schema removed
2098            // keeps the §7.4.4 send-time classification, and an unresolved
2099            // encryption key stays with the send seam because hosts configure
2100            // keys after open.
2101            client.drop_unencodable_outbox(false, None)?;
2102            if marker == Some(client.schema.version)
2103                && (!client.outbox.is_empty()
2104                    || !client.failed_commits.is_empty()
2105                    || client.has_acknowledged_rows()?)
2106            {
2107                // Reconstruct the visible optimistic overlay from the durable
2108                // base plus outbox instead of trusting a process-interrupted
2109                // mirror.
2110                client.overlay_dirty.set(true);
2111                client.rebuild_overlay()?;
2112            }
2113            // Every persisted active unit needs one catch-up pull on
2114            // open: realtime only covers changes after connection, while an
2115            // idempotent setWindow correctly creates no fresh command effect.
2116            // Pending outbox work has the same restart requirement. The core
2117            // owns this intent so native hosts never poll or require an
2118            // application-issued sync().
2119            client.enqueue_startup_sync_if_needed()?;
2120            Ok(())
2121        })();
2122        match startup {
2123            Ok(()) => match client.conn.execute_batch("COMMIT") {
2124                // §4.8: an eviction a crash interrupted finishes before any
2125                // read, with or without a transport; a replica reopened in
2126                // preflight finishes it at activation.
2127                Ok(()) => {
2128                    if !client.security_preflight {
2129                        client.drain_pending_evictions()?;
2130                    }
2131                    Ok(client)
2132                }
2133                Err(error) => {
2134                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2135                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2136                        // Keep a failed rollback's typed storage classification.
2137                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2138                    }
2139                    Err(failure)
2140                }
2141            },
2142            Err(error) => {
2143                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2144                    // Keep a failed rollback's typed storage classification.
2145                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2146                }
2147                Err(error)
2148            }
2149        }
2150    }
2151
2152    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2153    /// against this instead of system time (conformance virtual clock).
2154    pub fn set_now_ms(&mut self, now_ms: i64) {
2155        self.now_ms = Some(now_ms);
2156    }
2157
2158    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2159    /// The command router parses these from the `create` command's
2160    /// `encryption` config (keys as `{$bytes: hex}`).
2161    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2162        self.encryption = encryption;
2163    }
2164
2165    #[must_use]
2166    pub fn security_lifecycle(&self) -> &'static str {
2167        if self.security_preflight {
2168            "preflight"
2169        } else {
2170            "active"
2171        }
2172    }
2173
2174    #[must_use]
2175    pub fn security_preflight(&self) -> bool {
2176        self.security_preflight
2177    }
2178
2179    /// Host-owned, non-persisted network gate. Local operations remain available.
2180    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2181        if self.transport_enabled != enabled {
2182            self.transport_enabled = enabled;
2183            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2184            self.sync_intent_queue.clear();
2185            self.sync_intent_queue
2186                .push_back(if enabled && !self.security_preflight {
2187                    SyncIntent::Interactive
2188                } else {
2189                    SyncIntent::None
2190                });
2191        }
2192        // A captured reply keeps its normal apply/revocation semantics.
2193        if !enabled && self.active_round.is_none() {
2194            self.disconnect_realtime(transport);
2195        }
2196    }
2197
2198    pub fn transport_enabled(&self) -> bool {
2199        self.transport_enabled
2200    }
2201
2202    /// Quarantine this replica: enter the fail-closed gate, release all
2203    /// core-owned key material, and record the gate in the database so
2204    /// reopening re-enters preflight until `activate_security` clears it.
2205    ///
2206    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2207    /// which must NOT leave that durable mark.
2208    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2209        let entering = !self.security_preflight;
2210        self.seal_security_on_teardown();
2211        // Persist the quarantine so it survives handle teardown and restart:
2212        // reopening this replica re-enters preflight until activation clears it.
2213        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2214        if entering {
2215            self.emit_status_batch()?;
2216        }
2217        Ok(())
2218    }
2219
2220    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2221    fn emit_status_batch(&mut self) -> Result<(), String> {
2222        self.begin_observation("syncular_status")?;
2223        let batch = ChangeAccumulator {
2224            status: true,
2225            ..ChangeAccumulator::default()
2226        };
2227        self.finish_observation("syncular_status", batch)
2228            .inspect_err(|_| self.rollback_observation("syncular_status"))
2229    }
2230
2231    /// Release core-owned key material as a host tears the client down,
2232    /// WITHOUT recording a quarantine.
2233    ///
2234    /// Shutting an activated client down is not a quarantine event. Persisting
2235    /// the gate here marks every cleanly closed replica as pending, and the
2236    /// next plain `create` is then refused permanently: the reopen path
2237    /// restores the flag from the marker and the create guard rejects it. The
2238    /// in-memory flag still closes the gate for anything still holding this
2239    /// instance, which is all a teardown barrier needs.
2240    pub fn seal_security_on_teardown(&mut self) {
2241        self.cancel_sync_round();
2242        self.security_preflight = true;
2243        self.security_epoch = self.security_epoch.wrapping_add(1);
2244        self.encryption = crate::values::EncryptionConfig::default();
2245        self.sync_intent_queue.clear();
2246    }
2247
2248    /// Install the post-authentication keyring and release the host loop.
2249    pub fn activate_security(
2250        &mut self,
2251        encryption: crate::values::EncryptionConfig,
2252    ) -> Result<(), String> {
2253        if !self.security_preflight {
2254            return Err(
2255                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2256            );
2257        }
2258        // §4.8: an interrupted eviction finishes before protected reads open.
2259        self.drain_pending_evictions()?;
2260        self.enqueue_startup_sync_if_needed()?;
2261        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2262        self.encryption = encryption;
2263        self.security_preflight = false;
2264        self.emit_status_batch()
2265    }
2266
2267    fn clock_now_ms(&self) -> i64 {
2268        self.now_ms.unwrap_or_else(crate::wall_clock_ms)
2269    }
2270
2271    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2272        let exists = |name: &str| {
2273            self.conn
2274                .query_row(
2275                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2276                    [name],
2277                    |row| row.get::<_, bool>(0),
2278                )
2279                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2280        };
2281        let outbox_existed = exists("_syncular_outbox")?;
2282        // Durable client bookkeeping (outbox + units + meta).
2283        self.conn
2284            .execute_batch(
2285                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2286                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2287                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2288                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2289                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2290                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2291                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2292                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2293                   PRIMARY KEY(commit_id, idx));
2294                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2295                   ON _syncular_acked_rows(tbl, id);
2296                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2297                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2298                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2299                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2300                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2301                   client_commit_id TEXT NOT NULL UNIQUE,
2302                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2303                   recorded_at_ms INTEGER NOT NULL,
2304                   results_json TEXT NOT NULL,
2305                   operations_json TEXT,
2306                   resolution TEXT NOT NULL DEFAULT 'active'
2307                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2308                   resolved_at_ms INTEGER,
2309                   replacement_client_commit_id TEXT);
2310                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2311                   ON _syncular_commit_outcomes(resolution, seq);
2312                 CREATE TABLE IF NOT EXISTS _syncular_units (
2313                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2314                   explicit INTEGER NOT NULL DEFAULT 0,
2315                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2316                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2317                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT);
2318                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2319                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2320                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2321                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2322                   PRIMARY KEY (base, unit));
2323                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2324                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2325                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2326                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2327                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2328                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2329                   PRIMARY KEY(commit_id, blob_id));
2330                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2331                   ON _syncular_blob_commit_refs(blob_id);",
2332            )
2333            .map_err(|e| e.to_string())?;
2334        // Migrate an outcome journal created before failed aggregate
2335        // envelopes were retained. Historical rows intentionally stay NULL.
2336        let _ = self
2337            .conn
2338            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2339        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2340            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2341        }
2342        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2343        // a new sequence, so it replaces the id in the same transaction.
2344        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2345            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2346        }
2347        // §9: a replica written under wire version 3 keeps its outbox, journal,
2348        // rows and pending evictions and registers its units fresh. Its
2349        // subscriptions, cursors, echoes, resume tokens and window entries go;
2350        // units left from an earlier version 4 life restart.
2351        if exists("_syncular_window_pending_evict")? {
2352            let invalid: bool = self
2353                .conn
2354                .query_row(
2355                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2356                       WHERE NOT json_valid(effective_scopes)
2357                          OR json_type(effective_scopes) <> 'object'
2358                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2359                               WHERE type <> 'array'
2360                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2361                                       WHERE v.type <> 'text')))",
2362                    [],
2363                    |row| row.get(0),
2364                )
2365                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2366            if invalid {
2367                return Err(
2368                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2369                        .to_owned(),
2370                );
2371            }
2372            self.conn
2373                .execute_batch(
2374                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2375                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2376                     DROP TABLE _syncular_window_pending_evict;
2377                     DROP TABLE IF EXISTS _syncular_subscriptions;
2378                     DELETE FROM _syncular_windows;",
2379                )
2380                .and_then(|()| reset_units(&self.conn))
2381                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2382        }
2383        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2384        // completing a large upload does not rewrite its BLOB record.
2385        if self.schema_has_blobs() {
2386            if exists("_syncular_blobs")? {
2387                let mut statement = self
2388                    .conn
2389                    .prepare("PRAGMA table_info(_syncular_blobs)")
2390                    .map_err(|e| e.to_string())?;
2391                let mut columns = statement
2392                    .query_map([], |row| row.get::<_, String>(1))
2393                    .map_err(|e| e.to_string())?
2394                    .collect::<Result<Vec<_>, _>>()
2395                    .map_err(|e| e.to_string())?;
2396                columns.sort();
2397                let mut expected = vec![
2398                    "blob_id",
2399                    "byte_length",
2400                    "bytes",
2401                    "created_at_ms",
2402                    "media_type",
2403                ];
2404                expected.sort_unstable();
2405                if columns != expected {
2406                    return Err(
2407                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2408                    );
2409                }
2410            } else {
2411                self.conn
2412                    .execute_batch(
2413                        "CREATE TABLE _syncular_blobs(
2414                           blob_id TEXT PRIMARY KEY,
2415                           bytes BLOB NOT NULL,
2416                           byte_length INTEGER NOT NULL,
2417                           media_type TEXT,
2418                           created_at_ms INTEGER NOT NULL);",
2419                    )
2420                    .map_err(|e| e.to_string())?;
2421            }
2422            self.conn
2423                .execute_batch(
2424                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2425                       blob_id TEXT PRIMARY KEY,
2426                       media_type TEXT,
2427                       created_at_ms INTEGER NOT NULL);",
2428                )
2429                .map_err(|e| e.to_string())?;
2430        }
2431        Ok(())
2432    }
2433
2434    /// True iff any synced table declares a `blob_ref` column (§5.9).
2435    fn schema_has_blobs(&self) -> bool {
2436        self.schema
2437            .tables
2438            .iter()
2439            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2440    }
2441
2442    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2443
2444    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2445        meta_get(&self.conn, key)
2446            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2447    }
2448
2449    // The caller holds the startup/reset transaction. Either both schema
2450    // records persist or the enclosing operation fails and rolls back.
2451    fn persist_schema_version(&self) -> Result<(), String> {
2452        self.conn
2453            .execute(
2454                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2455                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2456            )
2457            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2458        set_local_schema_descriptor(&self.conn, &self.schema)
2459            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2460    }
2461
2462    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2463        meta_set(&self.conn, key, value)
2464            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2465    }
2466
2467    fn delete_meta(&self, key: &str) -> Result<(), String> {
2468        meta_delete(&self.conn, key)
2469            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2470    }
2471
2472    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2473    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2474        let mut stmt = self
2475            .conn
2476            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2477            .map_err(|error| error.to_string())?;
2478        let rows = stmt
2479            .query_map([], |row| {
2480                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2481            })
2482            .map_err(|error| error.to_string())?;
2483        let mut commits = Vec::new();
2484        for row in rows {
2485            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2486            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2487                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2488            })?;
2489            let mut ops = Vec::with_capacity(entries.len());
2490            for entry in entries {
2491                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2492                ops.push(OutboxOp {
2493                    upsert: op == "upsert",
2494                    table: entry
2495                        .get("table")
2496                        .and_then(Value::as_str)
2497                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2498                        .to_owned(),
2499                    row_id: entry
2500                        .get("rowId")
2501                        .and_then(Value::as_str)
2502                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2503                        .to_owned(),
2504                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2505                    // A legacy entry's `changedFields` key is ignored:
2506                    // the values map is the presence set (§6.1).
2507                    values: entry.get("values").and_then(Value::as_object).cloned(),
2508                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2509                });
2510            }
2511            commits.push(Arc::new(OutboxCommit {
2512                client_commit_id,
2513                ops,
2514            }));
2515        }
2516        Ok(commits)
2517    }
2518
2519    fn restore_persisted_state(&mut self) -> Result<(), String> {
2520        self.outbox = self.load_outbox()?.into();
2521
2522        self.prune_commit_outcomes()?;
2523        let active = self.commit_outcomes(CommitOutcomeQuery {
2524            active_only: true,
2525            ..CommitOutcomeQuery::default()
2526        })?;
2527        self.conflicts = active
2528            .iter()
2529            .flat_map(|outcome| outcome.results.iter())
2530            .filter_map(|result| match result {
2531                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2532                _ => None,
2533            })
2534            .collect::<Vec<_>>()
2535            .into();
2536        self.rejections = active
2537            .iter()
2538            .flat_map(|outcome| outcome.results.iter())
2539            .filter_map(|result| match result {
2540                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2541                _ => None,
2542            })
2543            .collect::<Vec<_>>()
2544            .into();
2545
2546        self.lease_state = self
2547            .get_meta(LEASE_STATE_KEY)?
2548            .map(|raw| serde_json::from_str(&raw))
2549            .transpose()
2550            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2551        self.schema_floor = self
2552            .get_meta(SCHEMA_FLOOR_KEY)?
2553            .map(|raw| serde_json::from_str(&raw))
2554            .transpose()
2555            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2556        self.stopped = self.schema_floor.is_some();
2557        // A replica left in unactivated preflight reopens gated: the quarantine
2558        // decision persists with the data across handle teardown and restart.
2559        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2560            self.security_preflight = true;
2561        }
2562        Ok(())
2563    }
2564
2565    fn save_unit_scope_schema(&self) -> Result<(), String> {
2566        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2567            .schema
2568            .tables
2569            .iter()
2570            .map(|table| {
2571                (
2572                    table.name.clone(),
2573                    table
2574                        .scope_variables
2575                        .iter()
2576                        .map(|scope| {
2577                            (
2578                                scope.variable.clone(),
2579                                [scope.prefix.clone(), scope.column.clone()],
2580                            )
2581                        })
2582                        .collect(),
2583                )
2584            })
2585            .collect();
2586        self.set_meta(
2587            "subscriptionScopeSchema",
2588            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2589        )
2590    }
2591
2592    /// Drop units that cover no table of the running schema, with their window
2593    /// registry rows and pending evictions; never translate scopes. At a bump
2594    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2595    /// on every table it covered before.
2596    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2597        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2598            self.get_meta("subscriptionScopeSchema")?
2599                .map(|raw| serde_json::from_str(&raw))
2600                .transpose()
2601                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2602        } else {
2603            None
2604        };
2605        // A table keeps the variables' meaning when each has the same prefix
2606        // and column as before. An old binary stored no declaration evidence;
2607        // matching names cannot prove unchanged meaning, so the app registers
2608        // again.
2609        let unchanged = |name: &str, variables: &[&String]| {
2610            let (Some(old), Some(table)) = (
2611                previous.as_ref().and_then(|previous| previous.get(name)),
2612                self.schema.table(name),
2613            ) else {
2614                return false;
2615            };
2616            variables.iter().all(|variable| {
2617                table
2618                    .scope_variables
2619                    .iter()
2620                    .find(|scope| &scope.variable == *variable)
2621                    .is_some_and(|scope| {
2622                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2623                    })
2624            })
2625        };
2626        let compatible = |scopes: &UnitScopes| {
2627            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2628                return false;
2629            }
2630            if !bump {
2631                return true;
2632            }
2633            let Some(previous) = &previous else {
2634                return false;
2635            };
2636            previous.iter().all(|(name, old)| {
2637                !scopes.keys().all(|variable| old.contains_key(variable))
2638                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2639            })
2640        };
2641        let mut stale: BTreeSet<String> = self
2642            .stored(load_units(&self.conn))?
2643            .into_iter()
2644            .filter(|unit| !compatible(&unit.scopes))
2645            .map(|unit| unit.id)
2646            .collect();
2647        // A shrunk window's deferred eviction outlives its unit row and keeps
2648        // its per-table scopes. A table the schema no longer has cannot be
2649        // evicted, and at a bump a table whose scope declarations changed must
2650        // not be evicted by the outdated rule.
2651        for (id, table, effective) in self.load_pending_evictions()? {
2652            if self.schema.table(&table).is_none()
2653                || (bump
2654                    && !unchanged(
2655                        &table,
2656                        &effective
2657                            .iter()
2658                            .map(|(variable, _)| variable)
2659                            .collect::<Vec<_>>(),
2660                    ))
2661            {
2662                stale.insert(id);
2663            }
2664        }
2665        for id in &stale {
2666            for sql in [
2667                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2668                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2669                "DELETE FROM _syncular_units WHERE id = ?1",
2670            ] {
2671                self.conn
2672                    .execute(sql, rusqlite::params![id])
2673                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2674            }
2675        }
2676        Ok(())
2677    }
2678
2679    pub fn local_revision(&self) -> Result<u64, String> {
2680        Ok(self
2681            .get_meta(LOCAL_REVISION_KEY)?
2682            .and_then(|value| value.parse().ok())
2683            .unwrap_or(0))
2684    }
2685
2686    /// §7.5: resolve a snapshot read on the mutable owner connection through
2687    /// the same primitive the read-only sidecar uses, so the command surface
2688    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2689    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2690    /// `owner` records its failure (and its next success) in diagnostics.
2691    pub fn snapshot_read(
2692        &mut self,
2693        request: &SnapshotReadRequest<'_>,
2694        owner: Option<QueryOwner<'_>>,
2695    ) -> Result<SnapshotRead, QueryReadFailure> {
2696        let result = snapshot_read_connection(&self.conn, request);
2697        if let Some(owner) = owner {
2698            self.record_query_read(owner, result.as_ref().err());
2699        }
2700        result
2701    }
2702
2703    #[must_use]
2704    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2705        SyncStatusSnapshot {
2706            current_schema_version: self.schema.version,
2707            outbox: self.outbox.len(),
2708            upgrading: self.upgrading,
2709            lease_state: self.lease_state.clone(),
2710            schema_floor: self.schema_floor.clone(),
2711            sync_needed: self.sync_needed,
2712            previous_version_context: self.previous_version_status(),
2713            security_lifecycle: self.security_lifecycle(),
2714        }
2715    }
2716
2717    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2718    /// rollback precondition), read without creating the file.
2719    fn previous_version_status(&self) -> PreviousVersionStatus {
2720        let Some(replica_path) = self.previous_version_replica_path() else {
2721            return PreviousVersionStatus {
2722                present: false,
2723                created_at_ms: None,
2724            };
2725        };
2726        match crate::previous_version::read_previous_version_record(&replica_path) {
2727            Ok(Some(record)) => PreviousVersionStatus {
2728                present: true,
2729                created_at_ms: Some(record.created_at_ms),
2730            },
2731            _ => PreviousVersionStatus {
2732                present: std::path::Path::new(
2733                    &crate::previous_version::previous_version_container_path(&replica_path),
2734                )
2735                .exists(),
2736                created_at_ms: None,
2737            },
2738        }
2739    }
2740
2741    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2742    /// is always `previousVersion`; the container never makes
2743    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2744    /// revocation, coverage completion and the TTL DISCARD the container and
2745    /// close the read with their own reason.
2746    pub fn previous_version_snapshot(
2747        &self,
2748        spec: &PreviousVersionReadSpec,
2749    ) -> Result<PreviousVersionSnapshot, String> {
2750        let now_ms = self.clock_now_ms();
2751        crate::previous_version::previous_version_snapshot(
2752            &self.conn,
2753            self.previous_version_replica_path().as_deref(),
2754            self.previous_version.as_ref(),
2755            PreviousVersionLifecycle {
2756                lease_inactive: self.previous_version_lease_inactive(now_ms),
2757                scope_revoked: self
2758                    .stored(load_units(&self.conn))?
2759                    .iter()
2760                    .any(|unit| unit.status == UnitStatus::Revoked),
2761                coverage_complete: self.previous_version_coverage_complete()?,
2762            },
2763            now_ms,
2764            self.schema.version,
2765            spec,
2766        )
2767    }
2768
2769    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2770    pub fn previous_version_audit(
2771        &self,
2772    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2773        crate::previous_version::stored_previous_version_audit(&self.conn)
2774            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2775    }
2776
2777    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2778    /// and both metadata records, and report whether anything was present.
2779    /// Idempotent: absence is a successful no-op.
2780    pub fn previous_version_discard(
2781        &self,
2782    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2783        self.drop_previous_version()
2784    }
2785
2786    fn drop_previous_version(
2787        &self,
2788    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2789        let Some(replica_path) = self.previous_version_replica_path() else {
2790            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2791                present: false,
2792                discarded: false,
2793            });
2794        };
2795        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2796    }
2797
2798    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2799    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2800        let Some(lease) = &self.lease_state else {
2801            return false;
2802        };
2803        if lease.error_code.is_some() {
2804            return true;
2805        }
2806        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2807    }
2808
2809    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2810    /// bootstrapping. An empty active set is not completion (mirrors the TS
2811    /// predicate exactly).
2812    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2813        let units = self.stored(load_units(&self.conn))?;
2814        let mut active = units
2815            .iter()
2816            .filter(|unit| unit.status == UnitStatus::Active)
2817            .peekable();
2818        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2819    }
2820
2821    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2822    /// apply. A container that no longer has a reason to exist is discarded so
2823    /// the rows do not outlive the window they were kept for.
2824    fn previous_version_lifetime_check(&self) {
2825        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2826            return;
2827        };
2828        let now_ms = self.clock_now_ms();
2829        if self.previous_version_lease_inactive(now_ms) {
2830            let _ = self.drop_previous_version();
2831            return;
2832        }
2833        let Some(replica_path) = self.previous_version_replica_path() else {
2834            return;
2835        };
2836        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2837        else {
2838            return;
2839        };
2840        let Ok(units) = load_units(&self.conn) else {
2841            return;
2842        };
2843        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2844            let _ = self.drop_previous_version();
2845            return;
2846        }
2847        if now_ms - record.created_at_ms > config.max_age_ms {
2848            let _ = self.drop_previous_version();
2849            return;
2850        }
2851        if self.previous_version_coverage_complete() == Ok(true) {
2852            let _ = self.drop_previous_version();
2853        }
2854    }
2855
2856    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2857        let registered = self.stored(load_units(&self.conn))?;
2858        // Both meta values in one statement: the FFI host recomputes
2859        // diagnostics after every command.
2860        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2861            self.conn
2862                .prepare_cached(
2863                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2864                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2865                )
2866                .and_then(|mut statement| {
2867                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2868                        Ok((row.get(0)?, row.get(1)?))
2869                    })
2870                }),
2871        )?;
2872        let mut units = DiagnosticUnits::default();
2873        for unit in &registered {
2874            *match unit.status {
2875                UnitStatus::Revoked => &mut units.revoked,
2876                UnitStatus::Failed => &mut units.failed,
2877                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2878                UnitStatus::Active
2879                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2880                {
2881                    &mut units.reset
2882                }
2883                UnitStatus::Active => &mut units.bootstrapping,
2884            } += 1;
2885        }
2886        units.reason_codes = registered
2887            .iter()
2888            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
2889            .collect::<BTreeSet<_>>()
2890            .into_iter()
2891            .collect();
2892        let captured_at_ms = self.clock_now_ms();
2893        let lease = if let Some(error_code) = self
2894            .lease_state
2895            .as_ref()
2896            .and_then(|state| state.error_code.clone())
2897        {
2898            ClientDiagnosticsLease {
2899                state: "stopped".to_owned(),
2900                expires_at_ms: self
2901                    .lease_state
2902                    .as_ref()
2903                    .and_then(|state| state.expires_at_ms),
2904                error_code: Some(Self::diagnostic_code(&error_code)),
2905            }
2906        } else if let Some(expires_at_ms) = self
2907            .lease_state
2908            .as_ref()
2909            .and_then(|state| state.expires_at_ms)
2910        {
2911            ClientDiagnosticsLease {
2912                state: if expires_at_ms <= captured_at_ms {
2913                    "expired".to_owned()
2914                } else {
2915                    "active".to_owned()
2916                },
2917                expires_at_ms: Some(expires_at_ms),
2918                error_code: None,
2919            }
2920        } else {
2921            ClientDiagnosticsLease {
2922                state: "none".to_owned(),
2923                expires_at_ms: None,
2924                error_code: None,
2925            }
2926        };
2927        let connectivity = match self.last_round.as_ref() {
2928            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
2929            Some(round) if round.status == "succeeded" => "online",
2930            Some(round)
2931                if round.status == "failed"
2932                    && matches!(
2933                        round.cause_kind,
2934                        Some(
2935                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
2936                        )
2937                    ) =>
2938            {
2939                "offline"
2940            }
2941            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
2942            _ => "unknown",
2943        };
2944        Ok(ClientDiagnosticsSnapshot {
2945            version: CLIENT_DIAGNOSTICS_VERSION,
2946            captured_at_ms,
2947            host: ClientDiagnosticsHost {
2948                kind: "direct".to_owned(),
2949                role: "single".to_owned(),
2950                connectivity: connectivity.to_owned(),
2951                realtime: self.realtime_state,
2952                realtime_policy: self.realtime_policy,
2953                realtime_reason_code: self.realtime_reason_code.clone(),
2954                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
2955            },
2956            security_lifecycle: self.security_lifecycle().to_owned(),
2957            schema: ClientDiagnosticsSchema {
2958                current_version: self.schema.version,
2959                upgrading: self.upgrading,
2960                required_version: self
2961                    .schema_floor
2962                    .as_ref()
2963                    .and_then(|floor| floor.required_schema_version),
2964                latest_version: self
2965                    .schema_floor
2966                    .as_ref()
2967                    .and_then(|floor| floor.latest_schema_version),
2968            },
2969            replica: ClientDiagnosticsReplica {
2970                local_revision: local_revision
2971                    .and_then(|value| value.parse::<u64>().ok())
2972                    .unwrap_or(0)
2973                    .to_string(),
2974                sync_needed: self.sync_needed,
2975                pending_outbox: self.outbox.len(),
2976            },
2977            lease,
2978            active_units: registered
2979                .iter()
2980                .filter(|unit| unit.status == UnitStatus::Active)
2981                .count(),
2982            units,
2983            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
2984                raw.parse()
2985                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
2986            })?,
2987            last_round: self.last_round.clone(),
2988            last_change: self.last_change.clone(),
2989            storage: self.diagnostics_storage(),
2990            query_failures: self.query_failures.clone(),
2991        })
2992    }
2993
2994    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
2995        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
2996            let page_count: i64 = self
2997                .conn
2998                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
2999            let page_size: i64 = self
3000                .conn
3001                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3002            // One cached aggregate, since the FFI host recomputes diagnostics
3003            // after every command; every invocation still reads current storage.
3004            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3005                .conn
3006                .prepare_cached(
3007                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3008                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3009                       FROM _syncular_commit_outcomes",
3010                )?
3011                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3012            let blob_bytes = if self.schema_has_blobs() {
3013                self.conn
3014                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3015                    .query_row([], |row| row.get(0))?
3016            } else {
3017                0
3018            };
3019            let pressure = self
3020                .limits
3021                .blob_cache_max_bytes
3022                .is_some_and(|limit| blob_bytes > limit);
3023            Ok(ClientDiagnosticsStorage {
3024                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3025                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3026                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3027                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3028                retained_outcome_entries: Some(outcome_entries.max(0)),
3029                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3030                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3031            })
3032        };
3033        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3034            status: "unreadable".to_owned(),
3035            database_bytes_approx: None,
3036            pending_outbox_bytes_approx: None,
3037            retained_outcome_bytes_approx: None,
3038            retained_outcome_entries: None,
3039            blob_cache_bytes_approx: None,
3040            pressure_reason_code: None,
3041        })
3042    }
3043
3044    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3045        self.change_queue.drain(..).collect()
3046    }
3047
3048    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3049        let intents = self.sync_intent_queue.drain(..).collect();
3050        if self.transport_enabled {
3051            intents
3052        } else {
3053            vec![SyncIntent::None]
3054        }
3055    }
3056
3057    fn schedule_background_retry(&mut self) -> u64 {
3058        let delay_ms = self.retry_delay_ms;
3059        self.sync_intent_queue
3060            .push_back(SyncIntent::Background { delay_ms });
3061        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3062        self.round_retry_delay_ms = Some(delay_ms);
3063        delay_ms
3064    }
3065
3066    fn reset_background_retry(&mut self) {
3067        self.retry_delay_ms = 250;
3068    }
3069
3070    fn retryable_transport_code(code: &str) -> bool {
3071        code == "transport.failed"
3072            || code == "transport.timeout"
3073            || code == "transport.unavailable"
3074            || code == "sync.transport_failed"
3075    }
3076
3077    /// §1.8: the transport reported a refused or unreachable endpoint. The
3078    /// native transport says `connect` (DNS failure, connection refused,
3079    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3080    /// the request on a backoff repairs neither, so no retry is scheduled.
3081    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3082        matches!(
3083            details
3084                .as_ref()
3085                .and_then(|details| details.get("causeKind"))
3086                .and_then(serde_json::Value::as_str),
3087            Some("connect" | "network")
3088        )
3089    }
3090
3091    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3092    /// invisible to the server (the round response arrived whole), so its code
3093    /// rides the following requests like a socket-round failure. A code the
3094    /// frame grammar rejects is not reportable and the previous code stays.
3095    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3096        if ssp2::model::valid_round_failure_code(&error.code) {
3097            self.round_failure = Some(error.code.clone());
3098        }
3099        SectionError::Transport(error)
3100    }
3101
3102    /// A failed round worth a background retry: a transport failure, or a
3103    /// §10.2 code the catalog marks retryable (clients may hardcode the
3104    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3105    /// on the server's `retryable` flag.
3106    fn retryable_failure_code(code: &str) -> bool {
3107        Self::retryable_transport_code(code)
3108            || matches!(
3109                code,
3110                "sync.auth_required"
3111                    | "sync.auth_lease_required"
3112                    | "sync.auth_lease_revoked"
3113                    | "sync.internal_error"
3114                    | "sync.idempotency_cache_miss"
3115                    | "sync.schema_not_ready"
3116                    | "sync.segment_expired"
3117                    | "sync.rate_limited"
3118                    | "sync.websocket_connection_limit"
3119            )
3120    }
3121
3122    fn diagnostic_code(code: &str) -> String {
3123        let valid = !code.is_empty()
3124            && code.len() <= 96
3125            && code.contains('.')
3126            && code
3127                .bytes()
3128                .next()
3129                .is_some_and(|byte| byte.is_ascii_lowercase())
3130            && code.bytes().all(|byte| {
3131                byte.is_ascii_lowercase()
3132                    || byte.is_ascii_digit()
3133                    || matches!(byte, b'.' | b'_' | b'-')
3134            });
3135        if valid {
3136            code.to_owned()
3137        } else {
3138            "client.unknown_failure".to_owned()
3139        }
3140    }
3141
3142    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3143        if self.sync_needed != value {
3144            if self.begin_observation("syncular_status").is_ok() {
3145                self.sync_needed = value;
3146                let batch = ChangeAccumulator {
3147                    status: true,
3148                    ..ChangeAccumulator::default()
3149                };
3150                if self.finish_observation("syncular_status", batch).is_err() {
3151                    self.rollback_observation("syncular_status");
3152                }
3153            } else {
3154                self.sync_needed = value;
3155            }
3156        }
3157        if value && interactive {
3158            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3159        }
3160    }
3161
3162    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3163    /// every other SQLite error keeps its storage classification.
3164    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3165        result.map_err(|error| match error {
3166            rusqlite::Error::FromSqlConversionFailure(..) => {
3167                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3168            }
3169            error => Self::sqlite_failure(&self.storage_failure, error),
3170        })
3171    }
3172
3173    fn sqlite_failure(
3174        storage_failure: &RefCell<Option<QueryReadFailure>>,
3175        error: rusqlite::Error,
3176    ) -> String {
3177        let message = error.to_string();
3178        let failure = QueryReadFailure::from(error);
3179        let code = failure.code;
3180        if code.is_some() && storage_failure.borrow().is_none() {
3181            *storage_failure.borrow_mut() = Some(failure);
3182        }
3183        match code {
3184            Some(code) => format!("{code}: {message}"),
3185            None => message,
3186        }
3187    }
3188
3189    fn begin_observation(&self, name: &str) -> Result<(), String> {
3190        if let Some(failure) = &self.coalesced_failure {
3191            return Err(failure.clone());
3192        }
3193        // SQLite rolls a whole transaction back on some I/O and capacity
3194        // errors; later frames must not commit on their own after that.
3195        if self.coalesced.is_some() && self.conn.is_autocommit() {
3196            return Err("the coalesced apply transaction was rolled back".to_owned());
3197        }
3198        self.conn
3199            .execute_batch(&format!("SAVEPOINT {name}"))
3200            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3201    }
3202
3203    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3204    /// already owns one.
3205    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3206        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3207            return Ok(());
3208        }
3209        self.conn
3210            .execute_batch("BEGIN")
3211            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3212        self.coalesced = Some(CoalescedApply {
3213            rows: 0,
3214            rollback: self.capture(),
3215        });
3216        Ok(())
3217    }
3218
3219    fn capture(&self) -> Rollback {
3220        Rollback {
3221            outbox: self.outbox.clone(),
3222            failed_commits: self.failed_commits.clone(),
3223            conflicts: self.conflicts.clone(),
3224            rejections: self.rejections.clone(),
3225            schema_floor: self.schema_floor.clone(),
3226            lease_state: self.lease_state.clone(),
3227            stopped: self.stopped,
3228            upgrading: self.upgrading,
3229            sync_needed: self.sync_needed,
3230            active_round: self.active_round,
3231            overlay_dirty: self.overlay_dirty.snapshot(),
3232            change_queue: self.change_queue.len(),
3233            sync_intent_queue: self.sync_intent_queue.len(),
3234            last_change: self.last_change.clone(),
3235        }
3236    }
3237
3238    fn restore(&mut self, rollback: Rollback) {
3239        self.outbox = rollback.outbox;
3240        self.failed_commits = rollback.failed_commits;
3241        self.conflicts = rollback.conflicts;
3242        self.rejections = rollback.rejections;
3243        self.schema_floor = rollback.schema_floor;
3244        self.lease_state = rollback.lease_state;
3245        self.stopped = rollback.stopped;
3246        self.upgrading = rollback.upgrading;
3247        self.sync_needed = rollback.sync_needed;
3248        self.active_round = rollback.active_round;
3249        self.overlay_dirty.restore(rollback.overlay_dirty);
3250        self.change_queue.truncate(rollback.change_queue);
3251        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3252        self.last_change = rollback.last_change;
3253    }
3254
3255    /// Account the rows of the next frame or block. A unit that would take the
3256    /// coalesced transaction past the bound commits the earlier units first.
3257    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3258        let Some(open) = &self.coalesced else {
3259            return Ok(());
3260        };
3261        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3262            self.commit_coalesced_apply()?;
3263            self.begin_coalesced_apply()?;
3264        }
3265        if let Some(open) = &mut self.coalesced {
3266            open.rows += rows;
3267        }
3268        Ok(())
3269    }
3270
3271    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3272    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3273    /// change batches and sync intents queued since, and fails the rest of the
3274    /// response.
3275    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3276        let Some(open) = self.coalesced.take() else {
3277            return Ok(());
3278        };
3279        let committed = if self.conn.is_autocommit() {
3280            Err("the coalesced apply transaction was rolled back".to_owned())
3281        } else {
3282            self.conn
3283                .execute_batch("COMMIT")
3284                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3285        };
3286        let Err(error) = committed else {
3287            return Ok(());
3288        };
3289        if !self.conn.is_autocommit() {
3290            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3291                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3292            }
3293        }
3294        self.restore(open.rollback);
3295        self.coalesced_failure = Some(error.clone());
3296        Err(error)
3297    }
3298
3299    /// End the response's coalescing: COMMIT what is open and clear a failure.
3300    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3301        let committed = self.commit_coalesced_apply();
3302        match self.coalesced_failure.take() {
3303            Some(failure) => Err(failure),
3304            None => committed,
3305        }
3306    }
3307
3308    fn rollback_observation(&self, name: &str) {
3309        if let Err(error) = self
3310            .conn
3311            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3312        {
3313            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3314                if failure.rollback_failure.is_none() {
3315                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3316                }
3317            }
3318        }
3319    }
3320
3321    fn finish_observation(
3322        &mut self,
3323        name: &str,
3324        mut batch: ChangeAccumulator,
3325    ) -> Result<(), String> {
3326        #[cfg(feature = "bench-internals")]
3327        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3328        if self.failed_commits.iter().any(|(commit, _)| {
3329            commit
3330                .ops
3331                .iter()
3332                .any(|operation| batch.tables.contains_key(&operation.table))
3333        }) {
3334            batch.outcomes = true;
3335        }
3336        if !batch.touched() {
3337            self.conn
3338                .execute_batch(&format!("RELEASE {name}"))
3339                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3340            return Ok(());
3341        }
3342        let revision = self
3343            .local_revision()?
3344            .checked_add(1)
3345            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3346        self.conn
3347            .execute(
3348                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3349                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3350            )
3351            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3352        let status = batch.status.then(|| self.status_snapshot());
3353        let event = ClientChangeBatch {
3354            revision: revision.to_string(),
3355            tables: batch
3356                .tables
3357                .into_iter()
3358                .map(|(table, scope_keys)| TableChange {
3359                    table,
3360                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3361                })
3362                .collect(),
3363            windows: batch
3364                .windows
3365                .into_iter()
3366                .map(|(base_key, units)| WindowChange {
3367                    base_key,
3368                    units: units.into_iter().collect(),
3369                })
3370                .collect(),
3371            status,
3372            conflicts_changed: batch.conflicts,
3373            rejections_changed: batch.rejections,
3374            outcomes_changed: batch.outcomes,
3375            applied_commits: Vec::new(),
3376        };
3377        self.conn
3378            .execute_batch(&format!("RELEASE {name}"))
3379            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3380        let mut diagnostic_tables = event
3381            .tables
3382            .iter()
3383            .map(|entry| entry.table.clone())
3384            .collect::<BTreeSet<_>>()
3385            .into_iter()
3386            .collect::<Vec<_>>();
3387        // §7.6: the selected variables of the changed window bases.
3388        let mut diagnostic_windows = event
3389            .windows
3390            .iter()
3391            .map(|entry| {
3392                entry
3393                    .base_key
3394                    .split('\0')
3395                    .next()
3396                    .unwrap_or_default()
3397                    .to_owned()
3398            })
3399            .collect::<BTreeSet<_>>()
3400            .into_iter()
3401            .collect::<Vec<_>>();
3402        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3403            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3404        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3405        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3406        self.last_change = Some(DiagnosticLastChange {
3407            revision: event.revision.clone(),
3408            recorded_at_ms: self.clock_now_ms(),
3409            tables: diagnostic_tables,
3410            windows: diagnostic_windows,
3411            domains_truncated,
3412            status_changed: event.status.is_some(),
3413            conflicts_changed: event.conflicts_changed,
3414            rejections_changed: event.rejections_changed,
3415            outcomes_changed: event.outcomes_changed,
3416        });
3417        self.change_queue.push_back(event);
3418        Ok(())
3419    }
3420
3421    fn record_scope_map(
3422        &self,
3423        batch: &mut ChangeAccumulator,
3424        table_name: &str,
3425        scopes: &[(String, Vec<String>)],
3426    ) {
3427        let Some(table) = self.schema.table(table_name) else {
3428            return;
3429        };
3430        // A bulk scoped delete whose map leaves a declared variable unnamed
3431        // cannot list the deleted rows' keys for it: the change is table-wide.
3432        if table.scope_variables.iter().any(|scope| {
3433            !scopes
3434                .iter()
3435                .any(|(variable, _)| *variable == scope.variable)
3436        }) {
3437            batch.table(table_name);
3438            return;
3439        }
3440        for (variable, values) in scopes {
3441            let Some(scope) = table
3442                .scope_variables
3443                .iter()
3444                .find(|scope| &scope.variable == variable)
3445            else {
3446                continue;
3447            };
3448            for value in values {
3449                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3450            }
3451        }
3452    }
3453
3454    /// Record a row's current scope keys from the base or visible table.
3455    fn record_row_scopes(
3456        &self,
3457        batch: &mut ChangeAccumulator,
3458        table_name: &str,
3459        row_id: &str,
3460        base: bool,
3461    ) -> bool {
3462        let Some(table) = self.schema.table(table_name) else {
3463            return false;
3464        };
3465        if table.scope_variables.is_empty() {
3466            return false;
3467        }
3468        let columns = table
3469            .scope_variables
3470            .iter()
3471            .map(|scope| quote_ident(&scope.column))
3472            .collect::<Vec<_>>()
3473            .join(", ");
3474        let full_table = if base {
3475            base_table(table_name)
3476        } else {
3477            visible_table(table_name)
3478        };
3479        let sql = format!(
3480            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3481            row_id_predicate(table)
3482        );
3483        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3484            return false;
3485        };
3486        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3487            let mut values = Vec::with_capacity(table.scope_variables.len());
3488            for index in 0..table.scope_variables.len() {
3489                values.push(row.get::<_, Option<String>>(index)?);
3490            }
3491            Ok(values)
3492        });
3493        let Ok(values) = values else {
3494            return false;
3495        };
3496        let mut recorded = false;
3497        for (scope, value) in table.scope_variables.iter().zip(values) {
3498            if let Some(value) = value {
3499                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3500                recorded = true;
3501            }
3502        }
3503        recorded
3504    }
3505
3506    fn record_commit_changes(
3507        &self,
3508        batch: &mut ChangeAccumulator,
3509        tables: &[String],
3510        changes: &[ssp2::model::Change],
3511    ) {
3512        #[cfg(feature = "bench-internals")]
3513        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3514        for change in changes {
3515            let Some(table_name) = tables.get(change.table_index as usize) else {
3516                continue;
3517            };
3518            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3519            if let Some(table) = self.schema.table(table_name) {
3520                for (variable, value) in &change.scopes {
3521                    if let Some(scope) = table
3522                        .scope_variables
3523                        .iter()
3524                        .find(|scope| &scope.variable == variable)
3525                    {
3526                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3527                        precise = true;
3528                    }
3529                }
3530            }
3531            if !precise {
3532                batch.table(table_name);
3533            }
3534        }
3535    }
3536
3537    /// Whether a base or visible row matches the scope predicate; `false`
3538    /// without a local scope-column mapping. A SQLite failure is an error.
3539    fn scoped_rows_exist(
3540        &self,
3541        table_name: &str,
3542        effective: &[(String, Vec<String>)],
3543        owners: &[Vec<(String, Vec<String>)>],
3544    ) -> Result<bool, String> {
3545        let Some(table) = self.schema.table(table_name) else {
3546            return Ok(false);
3547        };
3548        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3549            return Ok(false);
3550        };
3551        for source in [base_table(table_name), visible_table(table_name)] {
3552            if self
3553                .conn
3554                .query_row(
3555                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3556                    rusqlite::params_from_iter(&params),
3557                    |row| row.get::<_, bool>(0),
3558                )
3559                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3560            {
3561                return Ok(true);
3562            }
3563        }
3564        Ok(false)
3565    }
3566
3567    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3568    /// those another owner covers (§3.3), failing closed without a local
3569    /// scope-column mapping. Owners are read only when a row matches, inside
3570    /// the caller's transaction. Returns whether rows matched.
3571    fn clear_scope(
3572        &mut self,
3573        table: &TableSchema,
3574        effective: &[(String, Vec<String>)],
3575        unit_id: &str,
3576    ) -> Result<bool, SectionError> {
3577        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3578        let matched = self
3579            .scoped_rows_exist(&table.name, effective, &[])
3580            .map_err(storage)?;
3581        let owners = if matched {
3582            unit_owners(
3583                &self.stored(load_units(&self.conn)).map_err(storage)?,
3584                table,
3585                unit_id,
3586            )
3587        } else {
3588            Vec::new()
3589        };
3590        let changed = matched
3591            && (owners.is_empty()
3592                || self
3593                    .scoped_rows_exist(&table.name, effective, &owners)
3594                    .map_err(storage)?);
3595        if !self
3596            .purge_scope_rows(&table.name, effective, &owners)
3597            .map_err(storage)?
3598        {
3599            return Err(SectionError::FailClosed);
3600        }
3601        Ok(changed)
3602    }
3603
3604    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3605    pub fn upgrading(&self) -> bool {
3606        self.upgrading
3607    }
3608
3609    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3610    /// keeping this client's local database (identity, outbox, tables). The
3611    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3612    /// version increases. Recreation runs the same marker guard as opening
3613    /// a durable replica, holds the guard and every write it authorizes under
3614    /// one write transaction, and preserves the current client (compiled
3615    /// schema, outbox, tables) on refusal.
3616    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3617        let new_schema = parse_schema_json(schema_json)?;
3618        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3619        self.conn
3620            .execute_batch("BEGIN IMMEDIATE")
3621            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3622        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3623        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3624        // state (dropped outbox commits, rejections, upgrade and floor flags,
3625        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3626        // fails after the reset succeeded rolls the database back, so every
3627        // one of those fields is captured here and restored with it.
3628        let rollback = self.capture();
3629        let guarded = (|| -> Result<(), String> {
3630            // §7.4.2: the marker is re-read under the write lock the caller
3631            // holds, so a replica another process upgraded meanwhile refuses
3632            // this recreation instead of having its marker and tables
3633            // downgraded.
3634            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3635            self.create_bookkeeping_tables()?;
3636            match marker {
3637                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3638                None => {
3639                    self.create_synced_tables()?;
3640                    self.persist_schema_version()?;
3641                }
3642                _ => {}
3643            }
3644            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3645            // durable outbox the same-version path is about to replay. Only the
3646            // outbox is reloaded: the recovery below reads it, and the caller's
3647            // rollback snapshot already covers this field.
3648            self.outbox = self.load_outbox()?.into();
3649            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3650            // this guarded boundary once the schema and marker agree. A bump went
3651            // through `run_schema_reset`, which performs the same recovery after
3652            // it creates the new physical tables and before its final replay.
3653            self.drop_unencodable_outbox(false, None)?;
3654            if marker == Some(self.schema.version)
3655                && (!self.outbox.is_empty()
3656                    || !self.failed_commits.is_empty()
3657                    || self.has_acknowledged_rows()?)
3658            {
3659                self.overlay_dirty.set(true);
3660                self.rebuild_overlay()?;
3661            }
3662            self.prune_unknown_units(false)?;
3663            self.save_unit_scope_schema()?;
3664            // The conformance recreate is the in-memory equivalent of reopening
3665            // a durable client. Apply the same startup catch-up contract even
3666            // when the schema itself did not change.
3667            self.enqueue_startup_sync_if_needed()?;
3668            Ok(())
3669        })();
3670        let outcome = match guarded {
3671            Ok(()) => self
3672                .conn
3673                .execute_batch("COMMIT")
3674                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3675            Err(error) => Err(error),
3676        };
3677        if let Err(error) = outcome {
3678            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3679                // Keep a failed rollback's typed storage classification.
3680                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3681            }
3682            self.schema = prior_schema;
3683            self.restore(rollback);
3684            // Prepared insert statements were derived from the reverted schema.
3685            self.insert_sql.borrow_mut().clear();
3686            return Err(error);
3687        }
3688        if !self.security_preflight {
3689            self.drain_pending_evictions()?;
3690        }
3691        Ok(())
3692    }
3693
3694    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3695        let startup_work = !self.stopped
3696            && (!self.outbox.is_empty()
3697                || self.has_acknowledged_rows()?
3698                || !self.load_pending_evictions()?.is_empty()
3699                || self
3700                    .stored(load_units(&self.conn))?
3701                    .iter()
3702                    .any(|unit| unit.status == UnitStatus::Active));
3703        if startup_work {
3704            self.sync_needed = true;
3705            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3706        }
3707        Ok(())
3708    }
3709
3710    /// A native client persists schema-floor stops across process restarts.
3711    /// Once the running app already satisfies that floor, the persisted stop
3712    /// is only stale evidence from an older server round (for example, the
3713    /// server was restarted after catching up to an app that was ahead).
3714    /// Clear it and let the normal startup pull re-negotiate. If the server is
3715    /// still incompatible it will return the floor again in that first round.
3716    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3717        let satisfied = self
3718            .schema_floor
3719            .as_ref()
3720            .and_then(|floor| floor.required_schema_version)
3721            .is_some_and(|required| self.schema.version >= required);
3722        if !satisfied {
3723            return Ok(());
3724        }
3725        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3726        self.schema_floor = None;
3727        self.stopped = false;
3728        Ok(())
3729    }
3730
3731    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3732    /// path already swept it; this covers a same-version or fresh-install open,
3733    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3734    /// an aware binary can apply.
3735    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3736        let Some(replica_path) = self.previous_version_replica_path() else {
3737            return Ok(());
3738        };
3739        let max_age_ms = self
3740            .previous_version
3741            .filter(|config| config.enabled)
3742            .map(|config| config.max_age_ms);
3743        reconcile_previous_version_at_boot(
3744            &self.conn,
3745            &replica_path,
3746            self.schema.version,
3747            max_age_ms,
3748            self.clock_now_ms(),
3749        )
3750    }
3751
3752    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3753    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3754    /// has no plain sibling path — so the feature resolves as off there.
3755    fn previous_version_replica_path(&self) -> Option<String> {
3756        let path = self.conn.path()?;
3757        if path.is_empty() || path.starts_with("file:") {
3758            return None;
3759        }
3760        Some(path.to_owned())
3761    }
3762
3763    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3764    /// and leaseState. Drops/recreates every synced table from the new
3765    /// schema, resets unit sync state (keeping registrations), clears
3766    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3767    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3768    fn run_schema_reset(&mut self) -> Result<(), String> {
3769        self.begin_observation("syncular_schema_reset")?;
3770        let rollback = self.capture();
3771        let mut batch = ChangeAccumulator::default();
3772        let result = self
3773            .run_schema_reset_observed(&mut batch, true)
3774            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3775        if let Err(error) = result {
3776            self.rollback_observation("syncular_schema_reset");
3777            self.restore(rollback);
3778            return Err(error);
3779        }
3780        Ok(())
3781    }
3782
3783    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3784        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3785            self.begin_observation("syncular_epoch_acquisition")?;
3786            let rollback = self.capture();
3787            if let Err(error) = self.conn.execute(
3788                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3789                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3790            ) {
3791                self.rollback_observation("syncular_epoch_acquisition");
3792                return Err(Self::sqlite_failure(&self.storage_failure, error));
3793            }
3794            self.sync_needed = true;
3795            let batch = ChangeAccumulator {
3796                status: true,
3797                ..ChangeAccumulator::default()
3798            };
3799            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3800                self.rollback_observation("syncular_epoch_acquisition");
3801                self.restore(rollback);
3802                return Err(error);
3803            }
3804            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3805            return Ok(Vec::new());
3806        }
3807        let resets = self
3808            .stored(load_units(&self.conn))?
3809            .into_iter()
3810            .map(|unit| unit.id)
3811            .collect();
3812        self.begin_observation("syncular_log_epoch_reset")?;
3813        let rollback = self.capture();
3814        let mut batch = ChangeAccumulator::default();
3815        let result = self
3816            .run_schema_reset_observed(&mut batch, false)
3817            .and_then(|()| {
3818                self.conn
3819                    .execute(
3820                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3821                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3822                    )
3823                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3824                self.sync_needed = true;
3825                batch.status = true;
3826                self.finish_observation("syncular_log_epoch_reset", batch)
3827            });
3828        if let Err(error) = result {
3829            self.rollback_observation("syncular_log_epoch_reset");
3830            self.restore(rollback);
3831            return Err(error);
3832        }
3833        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3834        Ok(resets)
3835    }
3836
3837    fn run_schema_reset_observed(
3838        &mut self,
3839        batch: &mut ChangeAccumulator,
3840        capture: bool,
3841    ) -> Result<(), String> {
3842        // §7.4.2: the reset is the only path that wipes local tables for a
3843        // version bump, and it rewrites the marker last. Re-read the marker
3844        // under this reset's transaction: a replica another process upgraded
3845        // meanwhile refuses here instead of being reset to this client's
3846        // older schema.
3847        read_local_schema_version(&self.conn, self.schema.version)?;
3848        self.conn
3849            .execute_batch(
3850                "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
3851            )
3852            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3853        self.cancel_sync_round();
3854        self.upgrading = true;
3855        batch.status = true;
3856        // RFC 0005 D6: classify the pending outbox against the NEW compiled
3857        // schema BEFORE the wipe, and record the advisory audit. This pass
3858        // drops nothing, so the audit sees the original pending intent; the
3859        // §7.4.4 send-time drop and the intrinsic value recovery later in this
3860        // reset are the paths that remove a commit. The classification is
3861        // shared with the send-time drop so the two cannot disagree about what
3862        // is incompatible.
3863        if self.previous_version.is_some_and(|config| config.enabled) {
3864            let previous_version = self
3865                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
3866                .and_then(|value| value.parse::<i32>().ok())
3867                .unwrap_or(self.schema.version);
3868            let pending = self
3869                .outbox
3870                .iter()
3871                .map(|commit| PendingCommitAudit {
3872                    commit_id: &commit.client_commit_id,
3873                    operations: commit_audit_operations(commit),
3874                })
3875                .collect::<Vec<_>>();
3876            crate::previous_version::write_previous_version_audit(
3877                &self.conn,
3878                &crate::previous_version::build_previous_version_audit(
3879                    &self.schema,
3880                    &pending,
3881                    previous_version,
3882                    self.schema.version,
3883                    self.clock_now_ms(),
3884                ),
3885            )?;
3886        }
3887        // RFC 0005 D5: the container lives in its own FILE, so the reset can
3888        // never reach it. Sweep first (unconditional — also on the log-epoch
3889        // reset, which captures nothing), then capture BEFORE the wipe drops
3890        // any row. A refused capture records its reason and leaves no file.
3891        if let Some(replica_path) = self.previous_version_replica_path() {
3892            sweep_previous_version_container(&self.conn, &replica_path)?;
3893            if capture {
3894                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
3895                    let previous_version = self
3896                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
3897                        .and_then(|value| value.parse::<i32>().ok())
3898                        .unwrap_or(self.schema.version);
3899                    capture_previous_version_from_replica(
3900                        &self.conn,
3901                        &replica_path,
3902                        previous_version,
3903                        self.schema.version,
3904                        &config,
3905                        self.clock_now_ms(),
3906                    )?;
3907                }
3908            }
3909        }
3910        for table in &self.schema.tables {
3911            batch.table(&table.name);
3912        }
3913        for (base_key, value) in self.load_window_registry()? {
3914            batch.window(&base_key, &value);
3915        }
3916        if capture {
3917            self.prune_unknown_units(true)?;
3918        }
3919        // The per-table insert SQL is derived from the OLD column lists.
3920        self.insert_sql.borrow_mut().clear();
3921        self.overlay_dirty.set(true);
3922        // Drop every synced table (base + visible) that currently exists —
3923        // discovered from sqlite_master so a bump that adds/removes tables is
3924        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
3925        // and the blob cache) are preserved; base tables are `_syncular_base_*`
3926        // so they are matched explicitly, not by the bookkeeping filter.
3927        // Drop virtual tables first so SQLite removes every FTS shadow table
3928        // atomically instead of the generic discovery tearing it apart.
3929        let virtual_tables: Vec<String> = {
3930            let mut stmt = self
3931                .conn
3932                .prepare(
3933                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
3934                )
3935                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3936            let rows = stmt
3937                .query_map([], |row| row.get::<_, String>(0))
3938                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3939            rows.filter_map(Result::ok)
3940                .filter(|name| is_synced_table_name(name))
3941                .collect()
3942        };
3943        for name in virtual_tables {
3944            self.conn
3945                .execute_batch(&format!(
3946                    "DROP TABLE IF EXISTS {}",
3947                    quote_ident(&format!("_syncular_fts_{name}"))
3948                ))
3949                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3950            self.conn
3951                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3952                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3953        }
3954        let existing: Vec<String> = {
3955            let mut stmt = self
3956                .conn
3957                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
3958                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3959            let rows = stmt
3960                .query_map([], |row| row.get::<_, String>(0))
3961                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3962            rows.filter_map(Result::ok)
3963                .filter(|name| is_synced_table_name(name))
3964                .collect()
3965        };
3966        for name in &existing {
3967            if let Some(table) = name.strip_prefix("_syncular_base_") {
3968                batch.table(table);
3969            } else if !name.starts_with("_syncular_") {
3970                batch.table(name);
3971            }
3972        }
3973        for name in existing {
3974            self.conn
3975                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3976                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3977        }
3978        // Recreate the synced tables from the NEW schema.
3979        self.create_synced_tables()?;
3980        // Reset every unit's sync state, keeping the registration.
3981        reset_units(&self.conn)
3982            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3983        // The stop state is over: this client now ships a servable schema.
3984        self.stopped = false;
3985        self.schema_floor = None;
3986        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3987        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
3988        self.persist_schema_version()?;
3989        self.save_unit_scope_schema()?;
3990        // §7.1/§7.4.4: the previous-version capture and audit above saw the
3991        // original pending intent; the synced tables now carry the new schema, so
3992        // recover intrinsically codec-invalid persisted values here, before the
3993        // final replay. A pending upsert that names a column the new schema
3994        // removed stays pending for the send-time `drop_incompatible_outbox`
3995        // prepass. The reset wrapper snapshots and restores the outbox,
3996        // rejections, and change queues, so a failed commit rolls this recovery
3997        // back with the rest of the reset.
3998        // §7.4.4: the enclosing reset observation owns the publication, so the
3999        // recovery contributes its rejection, outcome, and table domains to it.
4000        self.drop_unencodable_outbox(false, Some(batch))?;
4001        // Re-apply the surviving outbox optimistically over the empty tables.
4002        self.rebuild_overlay()?;
4003        Ok(())
4004    }
4005
4006    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4007    /// column the current schema lacks (or a removed table) cannot be encoded
4008    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4009    /// encodable and is validated by the server), nor can an insert-intent
4010    /// upsert that lacks a non-nullable column (§7.1, the audit's
4011    /// `missing-required-column`). Drop the commit, undo
4012    /// its optimistic projection, and raise a client-local
4013    /// `sync.outbox_incompatible` rejection.
4014    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4015        let incompatible = self
4016            .outbox
4017            .iter()
4018            .filter(|commit| {
4019                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4020            })
4021            .cloned()
4022            .collect::<Vec<_>>();
4023        if incompatible.is_empty() {
4024            return Ok(false);
4025        }
4026        self.drop_outbox_commits(
4027            &incompatible,
4028            OUTBOX_INCOMPATIBLE_CODE,
4029            "the persisted commit cannot encode under the current schema",
4030        )?;
4031        Ok(true)
4032    }
4033
4034    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4035    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4036    /// remove them from the outbox, and undo their optimistic rows.
4037    fn drop_outbox_commits(
4038        &mut self,
4039        incompatible: &[Arc<OutboxCommit>],
4040        code: &str,
4041        message: &str,
4042    ) -> Result<(), String> {
4043        // §7.4.4: one observation covers the durable outcome writes, the outbox
4044        // removal, the in-memory drop, and the overlay rebuild, so a storage
4045        // fault leaves the replica and the client unchanged and publishes no
4046        // change; a successful drop publishes exactly one batch carrying the
4047        // rejections, outcomes, status, and the tables whose projection changed.
4048        let rollback = self.capture();
4049        let owns_transaction = self.conn.is_autocommit();
4050        self.begin_observation("syncular_outbox_drop")?;
4051        let mut batch = ChangeAccumulator::default();
4052        let dropped = (|| -> Result<(), String> {
4053            let mut rejections = Vec::new();
4054            for commit in incompatible {
4055                let results = commit
4056                    .ops
4057                    .iter()
4058                    .enumerate()
4059                    .map(|(op_index, operation)| {
4060                        let rejection = RejectionRecord {
4061                            client_commit_id: commit.client_commit_id.clone(),
4062                            op_index: op_index as i32,
4063                            code: code.to_owned(),
4064                            message: message.to_owned(),
4065                            retryable: false,
4066                            details: None,
4067                            operation: Some(CommitOperation::from(operation)),
4068                        };
4069                        rejections.push(rejection.clone());
4070                        CommitOperationOutcome::Error { rejection }
4071                    })
4072                    .collect::<Vec<_>>();
4073                self.persist_commit_outcome(
4074                    &commit.client_commit_id,
4075                    CommitOutcomeStatus::Rejected,
4076                    &results,
4077                    Some(&commit.ops),
4078                )?;
4079                self.delete_outbox_persisted(&commit.client_commit_id)?;
4080                // A removed table has no local projection to invalidate; every
4081                // surviving table the dropped commit touched does.
4082                for operation in &commit.ops {
4083                    if self.schema.table(&operation.table).is_some() {
4084                        batch.table(&operation.table);
4085                    }
4086                }
4087            }
4088            self.prune_commit_outcomes()?;
4089            let incompatible_ids = incompatible
4090                .iter()
4091                .map(|commit| commit.client_commit_id.as_str())
4092                .collect::<BTreeSet<_>>();
4093            self.outbox
4094                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4095            self.rejections.extend(rejections);
4096            // The dropped commits' purely-optimistic rows are undone: the
4097            // visible projection is re-derived from the surviving base + outbox.
4098            self.overlay_dirty.set(true);
4099            self.rebuild_overlay()?;
4100            batch.status = true;
4101            batch.rejections = true;
4102            batch.outcomes = true;
4103            Ok(())
4104        })();
4105        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4106        if let Err(error) = result {
4107            // An outer RELEASE is the COMMIT. Roll back that transaction
4108            // directly: retrying RELEASE after ROLLBACK TO still needs the
4109            // reader's lock. A nested observation only rolls back its savepoint.
4110            if owns_transaction {
4111                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4112                    let mut failure = self.storage_failure.borrow_mut();
4113                    let failure =
4114                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4115                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4116                }
4117            } else {
4118                self.rollback_observation("syncular_outbox_drop");
4119            }
4120            self.restore(rollback);
4121            return Err(error);
4122        }
4123        Ok(())
4124    }
4125
4126    /// §7.4.3: (re)create the base + visible table pair for every synced
4127    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4128    fn create_synced_tables(&self) -> Result<(), String> {
4129        for table in &self.schema.tables {
4130            // The base half + the visible half form the synced-table pair. An
4131            // index name is global in SQLite, so the base half's indexes are
4132            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4133            for (full, index_prefix) in [
4134                (base_table(&table.name), "_syncular_base_"),
4135                (visible_table(&table.name), ""),
4136            ] {
4137                let mut cols: Vec<String> =
4138                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4139                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4140                let sql = format!(
4141                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4142                    cols.join(", "),
4143                    quote_ident(&table.primary_key)
4144                );
4145                self.conn
4146                    .execute(&sql, [])
4147                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4148                // Local secondary indexes (CREATE INDEX subset). Created on
4149                // both halves so mirror reads hit an index on either. Runs on
4150                // both the initial create and the §7.4.3 reset recreate path.
4151                for index in &table.indexes {
4152                    let unique = if index.unique { "UNIQUE " } else { "" };
4153                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4154                    let cols_sql = index
4155                        .columns
4156                        .iter()
4157                        .map(|c| quote_ident(c))
4158                        .collect::<Vec<_>>()
4159                        .join(", ");
4160                    let index_sql = format!(
4161                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4162                    );
4163                    self.conn
4164                        .execute(&index_sql, [])
4165                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4166                }
4167            }
4168        }
4169        // FTS exists only on the visible half. It is a local search
4170        // projection, never a synced/base table or a wire-schema member.
4171        for table in &self.schema.tables {
4172            for index in &table.fts_indexes {
4173                self.create_fts_projection(table, index)?;
4174            }
4175        }
4176        Ok(())
4177    }
4178
4179    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4180        let count: i64 = self
4181            .conn
4182            .query_row(
4183                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4184                rusqlite::params![index.name],
4185                |row| row.get(0),
4186            )
4187            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4188        Ok(count > 0)
4189    }
4190
4191    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4192        for suffix in ["ai", "ad", "au"] {
4193            self.conn
4194                .execute(
4195                    &format!(
4196                        "DROP TRIGGER IF EXISTS {}",
4197                        quote_ident(&format!("{}_{suffix}", index.name))
4198                    ),
4199                    [],
4200                )
4201                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4202        }
4203        Ok(())
4204    }
4205
4206    fn create_fts_triggers(
4207        &self,
4208        table: &TableSchema,
4209        index: &FtsIndexSchema,
4210    ) -> Result<(), String> {
4211        let fts = quote_ident(&index.name);
4212        let source = visible_table(&table.name);
4213        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4214        let pk = quote_ident(&table.primary_key);
4215        let projection_columns = std::iter::once(source_id.clone())
4216            .chain(index.columns.iter().map(|column| quote_ident(column)))
4217            .collect::<Vec<_>>()
4218            .join(", ");
4219        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4220            .chain(
4221                index
4222                    .columns
4223                    .iter()
4224                    .map(|column| format!("new.{}", quote_ident(column))),
4225            )
4226            .collect::<Vec<_>>()
4227            .join(", ");
4228        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4229        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4230        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4231        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4232        let bi = quote_ident(&format!("{}_bi", index.name));
4233        let ai = quote_ident(&format!("{}_ai", index.name));
4234        let ad = quote_ident(&format!("{}_ad", index.name));
4235        let au = quote_ident(&format!("{}_au", index.name));
4236        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4237        let sql = format!(
4238            "DROP TRIGGER IF EXISTS {bi};
4239             DROP TRIGGER IF EXISTS {ai};
4240             DROP TRIGGER IF EXISTS {ad};
4241             DROP TRIGGER IF EXISTS {au};
4242             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4243             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4244             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4245             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4246        );
4247        self.conn
4248            .execute_batch(&sql)
4249            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4250    }
4251
4252    fn rebuild_fts_projection(
4253        &self,
4254        table: &TableSchema,
4255        index: &FtsIndexSchema,
4256    ) -> Result<(), String> {
4257        let fts = quote_ident(&index.name);
4258        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4259        let pk = quote_ident(&table.primary_key);
4260        let indexed_columns = index
4261            .columns
4262            .iter()
4263            .map(|column| quote_ident(column))
4264            .collect::<Vec<_>>();
4265        let projection_columns = std::iter::once(source_id.clone())
4266            .chain(indexed_columns.iter().cloned())
4267            .collect::<Vec<_>>()
4268            .join(", ");
4269        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4270        let sql = format!(
4271            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4272            columns = indexed_columns.join(", "),
4273            source = visible_table(&table.name),
4274        );
4275        self.conn
4276            .execute_batch(&sql)
4277            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4278    }
4279
4280    fn create_fts_projection(
4281        &self,
4282        table: &TableSchema,
4283        index: &FtsIndexSchema,
4284    ) -> Result<(), String> {
4285        self.begin_observation("syncular_fts_schema")?;
4286        let result = (|| {
4287            let existed = self.fts_projection_exists(index)?;
4288            let tokenizer = index.tokenize.replace('\'', "''");
4289            let columns = index
4290                .columns
4291                .iter()
4292                .map(|column| quote_ident(column))
4293                .collect::<Vec<_>>()
4294                .join(", ");
4295            let sql = format!(
4296            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4297            fts = quote_ident(&index.name),
4298            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4299        );
4300            self.conn.execute(&sql, []).map_err(|error| {
4301                format!(
4302                    "cannot create local FTS5 projection {:?}: {error}",
4303                    index.name
4304                )
4305            })?;
4306            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4307            let mapped: bool = self
4308                .conn
4309                .query_row(
4310                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4311                    [format!("_syncular_fts_{}", index.name)],
4312                    |row| row.get(0),
4313                )
4314                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4315            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4316            if !mapped && existed {
4317                self.conn
4318                    .execute_batch(&format!(
4319                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4320                        quote_ident(FTS_SOURCE_ID_COLUMN),
4321                        quote_ident(&index.name)
4322                    ))
4323                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4324            }
4325            self.create_fts_triggers(table, index)?;
4326            if !existed {
4327                self.rebuild_fts_projection(table, index)?;
4328            }
4329            Ok(())
4330        })();
4331        if let Err(error) = result {
4332            self.rollback_observation("syncular_fts_schema");
4333            return Err(error);
4334        }
4335        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4336            self.rollback_observation("syncular_fts_schema");
4337            return Err(error.to_string());
4338        }
4339        Ok(())
4340    }
4341
4342    // -- persistence write-through --------------------------------------------
4343
4344    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4345        let ops: Vec<Value> = commit
4346            .ops
4347            .iter()
4348            .map(|op| {
4349                let mut entry = serde_json::json!({
4350                    "op": if op.upsert { "upsert" } else { "delete" },
4351                    "table": op.table,
4352                    "rowId": op.row_id,
4353                    "baseVersion": op.base_version,
4354                    "values": op.values.clone().map(Value::Object),
4355                });
4356                if op.insert {
4357                    entry["insert"] = Value::Bool(true);
4358                }
4359                entry
4360            })
4361            .collect();
4362        self.conn
4363            .execute(
4364                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4365                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4366            )
4367            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4368        for operation in &commit.ops {
4369            let Some(values) = &operation.values else {
4370                continue;
4371            };
4372            let Some(table) = self.schema.table(&operation.table) else {
4373                continue;
4374            };
4375            for column in table
4376                .columns
4377                .iter()
4378                .filter(|column| column.ty == ColumnType::BlobRef)
4379            {
4380                let Some(Value::String(raw)) = values.get(&column.name) else {
4381                    continue;
4382                };
4383                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4384                    continue;
4385                };
4386                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4387                    continue;
4388                };
4389                self.conn
4390                    .execute(
4391                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4392                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4393                        rusqlite::params![commit.client_commit_id, blob_id],
4394                    )
4395                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4396            }
4397        }
4398        Ok(())
4399    }
4400
4401    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4402        self.conn
4403            .execute(
4404                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4405                rusqlite::params![client_commit_id],
4406            )
4407            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4408        self.conn
4409            .execute(
4410                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4411                rusqlite::params![client_commit_id],
4412            )
4413            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4414        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4415            .map(|_| ())
4416            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4417    }
4418
4419    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4420        match status {
4421            CommitOutcomeStatus::Applied => "applied",
4422            CommitOutcomeStatus::Cached => "cached",
4423            CommitOutcomeStatus::Conflict => "conflict",
4424            CommitOutcomeStatus::Rejected => "rejected",
4425        }
4426    }
4427
4428    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4429        match resolution {
4430            CommitOutcomeResolution::Active => "active",
4431            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4432            CommitOutcomeResolution::Superseded => "superseded",
4433            CommitOutcomeResolution::Dismissed => "dismissed",
4434        }
4435    }
4436
4437    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4438        match value {
4439            "applied" => Ok(CommitOutcomeStatus::Applied),
4440            "cached" => Ok(CommitOutcomeStatus::Cached),
4441            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4442            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4443            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4444        }
4445    }
4446
4447    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4448        match value {
4449            "active" => Ok(CommitOutcomeResolution::Active),
4450            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4451            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4452            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4453            _ => Err(format!(
4454                "invalid persisted commit outcome resolution {value:?}"
4455            )),
4456        }
4457    }
4458
4459    fn persist_commit_outcome(
4460        &self,
4461        client_commit_id: &str,
4462        status: CommitOutcomeStatus,
4463        results: &[CommitOperationOutcome],
4464        operations: Option<&[OutboxOp]>,
4465    ) -> Result<(), String> {
4466        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4467        let operations_json = operations
4468            .map(|items| {
4469                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4470            })
4471            .transpose()
4472            .map_err(|error| error.to_string())?;
4473        self.conn
4474            .execute(
4475                "INSERT INTO _syncular_commit_outcomes (
4476                   client_commit_id, status, recorded_at_ms, results_json,
4477                   operations_json, resolution
4478                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4479                rusqlite::params![
4480                    client_commit_id,
4481                    Self::outcome_status_name(status),
4482                    self.clock_now_ms(),
4483                    results_json,
4484                    operations_json
4485                ],
4486            )
4487            .map(|_| ())
4488            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4489    }
4490
4491    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4492    /// the byte-envelope rule refuses cannot be journaled verbatim.
4493    fn stored_cell_is_representable(cell: &Value) -> bool {
4494        cell.is_null()
4495            || cell.is_string()
4496            || cell.is_boolean()
4497            || cell.is_number()
4498            || is_canonical_bytes_envelope(cell)
4499    }
4500
4501    fn stored_row_is_representable(value: &Value) -> bool {
4502        value
4503            .as_object()
4504            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4505    }
4506
4507    fn stored_operation_is_representable(value: &Value) -> bool {
4508        value.get("table").is_some_and(Value::is_string)
4509            && value.get("rowId").is_some_and(Value::is_string)
4510            && value.get("baseVersion").is_none_or(|v| {
4511                v.as_i64()
4512                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4513            })
4514            && match value.get("op").and_then(Value::as_str) {
4515                Some("upsert") => {
4516                    value
4517                        .get("values")
4518                        .is_some_and(Self::stored_row_is_representable)
4519                        && value
4520                            .get("insert")
4521                            .is_none_or(|insert| *insert == Value::Bool(true))
4522                }
4523                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4524                _ => false,
4525            }
4526    }
4527
4528    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4529    /// only when the journal reader accepts the serialized operation, which is the
4530    /// rule `outcome_from_row` applies to this same row later.
4531    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4532        serde_json::to_value(CommitOperation::from(operation))
4533            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4534    }
4535
4536    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4537        let StoredCommitOutcomeRow {
4538            sequence,
4539            client_commit_id,
4540            status,
4541            recorded_at_ms,
4542            results_json,
4543            operations_json,
4544            resolution,
4545            resolved_at_ms,
4546            replacement_client_commit_id,
4547        } = row;
4548        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4549            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4550            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4551        {
4552            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4553        }
4554        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4555        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4556        for result in &stored {
4557            let nested = match result.get("status").and_then(Value::as_str) {
4558                Some("conflict") => result.get("conflict"),
4559                Some("error") => result.get("rejection"),
4560                _ => None,
4561            };
4562            if nested
4563                .and_then(|record| record.get("operation"))
4564                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4565                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4566                    && nested
4567                        .and_then(|record| record.get("serverRow"))
4568                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4569            {
4570                return Err(invalid());
4571            }
4572        }
4573        let results: Vec<CommitOperationOutcome> =
4574            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4575        let has_conflict = results
4576            .iter()
4577            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4578        let has_error = results
4579            .iter()
4580            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4581        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4582        let redacted = results.is_empty() && operations_json.is_none();
4583        if match status {
4584            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4585            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4586            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4587        } {
4588            return Err(invalid());
4589        }
4590        let operations: Option<Vec<CommitOperation>> = operations_json
4591            .map(|raw| {
4592                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4593                if !stored.iter().all(Self::stored_operation_is_representable) {
4594                    return Err(invalid());
4595                }
4596                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4597            })
4598            .transpose()?;
4599        for result in &results {
4600            match result {
4601                CommitOperationOutcome::Conflict { conflict }
4602                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4603                {
4604                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4605                }
4606                CommitOperationOutcome::Error { rejection } => {
4607                    if let Some(details) = &rejection.details {
4608                        let encoded = serde_json::to_string(details).map_err(|_| {
4609                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4610                        })?;
4611                        RejectionDetails::parse(&encoded).map_err(|_| {
4612                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4613                        })?;
4614                    }
4615                }
4616                _ => {}
4617            }
4618        }
4619        Ok(CommitOutcome {
4620            sequence,
4621            client_commit_id,
4622            status,
4623            recorded_at_ms,
4624            results,
4625            operations,
4626            retained_rows: None,
4627            resolution: Self::parse_outcome_resolution(&resolution)?,
4628            resolved_at_ms,
4629            replacement_client_commit_id,
4630        })
4631    }
4632
4633    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4634    fn unique_conflicts(
4635        &self,
4636        table: &TableSchema,
4637        values: &Map<String, Value>,
4638        base: bool,
4639    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4640        let source = if base {
4641            base_table(&table.name)
4642        } else {
4643            visible_table(&table.name)
4644        };
4645        let mut conflicts = Vec::new();
4646        for index in table.indexes.iter().filter(|index| index.unique) {
4647            let mut params = vec![json_param_to_sql(
4648                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4649            )?];
4650            for name in &index.columns {
4651                let column = table
4652                    .columns
4653                    .iter()
4654                    .find(|column| column.name == *name)
4655                    .ok_or("sync.local_corrupt")?;
4656                let cell = json_to_column_value(column, values.get(name))?;
4657                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4658            }
4659            let sql = format!(
4660                "SELECT * FROM {source} WHERE {} != ? AND {}",
4661                quote_ident(&table.primary_key),
4662                index
4663                    .columns
4664                    .iter()
4665                    .map(|column| format!("{} = ?", quote_ident(column)))
4666                    .collect::<Vec<_>>()
4667                    .join(" AND ")
4668            );
4669            let mut statement = self
4670                .conn
4671                .prepare_cached(&sql)
4672                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4673            let records = statement
4674                .query_map(rusqlite::params_from_iter(params), |record| {
4675                    let values = table
4676                        .columns
4677                        .iter()
4678                        .enumerate()
4679                        .map(|(index, column)| {
4680                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4681                            json_to_column_value(column, Some(&value)).map_err(|message| {
4682                                rusqlite::Error::ToSqlConversionFailure(message.into())
4683                            })
4684                        })
4685                        .collect::<Result<Row, _>>()?;
4686                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4687                })
4688                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4689            for record in records {
4690                let (row, version) =
4691                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4692                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4693                conflicts.push(RetainedUniqueConflict {
4694                    index: index.name.clone(),
4695                    columns: index.columns.clone(),
4696                    row_id,
4697                    server_row: table
4698                        .columns
4699                        .iter()
4700                        .zip(&row)
4701                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4702                        .collect(),
4703                    server_version: version,
4704                });
4705            }
4706        }
4707        Ok(conflicts)
4708    }
4709
4710    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4711        if let Some((commit, initial)) = self
4712            .failed_commits
4713            .iter()
4714            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4715        {
4716            let mut rows = Vec::new();
4717            for operation in &commit.ops {
4718                // A table the current schema removed has no rows to retain.
4719                let Some(table) = self.schema.table(&operation.table) else {
4720                    continue;
4721                };
4722                let key =
4723                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4724                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4725                let base = self.stored_row(table, &operation.row_id, true)?;
4726                rows.push(RetainedCommitRow {
4727                    table: operation.table.clone(),
4728                    row_id: operation.row_id.clone(),
4729                    unique_conflicts: if operation.upsert {
4730                        self.unique_conflicts(
4731                            table,
4732                            initial
4733                                .get(&key)
4734                                .and_then(Value::as_object)
4735                                .or(operation.values.as_ref())
4736                                .ok_or("sync.local_corrupt")?,
4737                            true,
4738                        )?
4739                    } else {
4740                        Vec::new()
4741                    },
4742                    local_row: if operation.upsert {
4743                        initial
4744                            .get(&key)
4745                            .and_then(Value::as_object)
4746                            .cloned()
4747                            .or_else(|| operation.values.clone())
4748                    } else {
4749                        None
4750                    },
4751                    server_row: base.as_ref().map(|(values, _)| {
4752                        table
4753                            .columns
4754                            .iter()
4755                            .zip(values)
4756                            .map(|(column, value)| {
4757                                (column.name.clone(), column_value_to_json(value))
4758                            })
4759                            .collect()
4760                    }),
4761                    server_version: base.map(|(_, version)| version),
4762                });
4763            }
4764            outcome.retained_rows = Some(rows);
4765        }
4766        Ok(outcome)
4767    }
4768
4769    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
4770        let row = self
4771            .conn
4772            .query_row(
4773                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
4774                        resolution, resolved_at_ms, replacement_client_commit_id
4775                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
4776                rusqlite::params![client_commit_id],
4777                |row| {
4778                    Ok(StoredCommitOutcomeRow {
4779                        sequence: row.get(0)?,
4780                        client_commit_id: row.get(1)?,
4781                        status: row.get(2)?,
4782                        recorded_at_ms: row.get(3)?,
4783                        results_json: row.get(4)?,
4784                        operations_json: row.get(5)?,
4785                        resolution: row.get(6)?,
4786                        resolved_at_ms: row.get(7)?,
4787                        replacement_client_commit_id: row.get(8)?,
4788                    })
4789                },
4790            )
4791            .optional()
4792            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4793        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
4794            .transpose()
4795    }
4796
4797    /// Internal host step. The host must run its idle drive before finishing.
4798    #[doc(hidden)]
4799    pub fn prepare_current_base(
4800        &self,
4801        client_commit_id: &str,
4802        sequence: i64,
4803    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
4804        let outcome = self
4805            .commit_outcome(client_commit_id)
4806            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
4807        let valid = outcome.as_ref().is_some_and(|outcome| {
4808            outcome.sequence == sequence
4809                && outcome.resolution == CommitOutcomeResolution::Active
4810                && matches!(
4811                    outcome.status,
4812                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
4813                )
4814                && outcome
4815                    .operations
4816                    .as_ref()
4817                    .zip(outcome.retained_rows.as_ref())
4818                    .is_some_and(|(operations, rows)| {
4819                        !rows.is_empty() && operations.len() == rows.len()
4820                    })
4821        });
4822        if !valid {
4823            return Err(CurrentBaseProof::failure(
4824                CurrentBaseProofReason::OutcomeUnavailable,
4825            ));
4826        }
4827        if !self.transport_enabled {
4828            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
4829        }
4830        Ok(CurrentBasePreparation {
4831            client_commit_id: client_commit_id.to_owned(),
4832            sequence,
4833            units: load_units(&self.conn)
4834                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
4835            transport_epoch: self.transport_epoch,
4836            registration_epoch: self.registration_epoch,
4837            security_epoch: self.security_epoch,
4838        })
4839    }
4840
4841    /// Internal host step after the completed post-rejection idle drive.
4842    #[doc(hidden)]
4843    pub fn finish_current_base(
4844        &mut self,
4845        prepared: CurrentBasePreparation,
4846        drive: SyncOutcome,
4847    ) -> CurrentBaseProof {
4848        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
4849            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
4850        }
4851        if self.registration_epoch != prepared.registration_epoch {
4852            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
4853        }
4854        if self.security_preflight || self.security_epoch != prepared.security_epoch {
4855            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
4856        }
4857        let report = match drive {
4858            SyncOutcome::Ok(report) => report,
4859            SyncOutcome::BudgetExhausted(_) => {
4860                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
4861            }
4862            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
4863        };
4864        if report.schema_floor.is_some() {
4865            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
4866        }
4867        if self
4868            .conn
4869            .execute_batch("SAVEPOINT syncular_base_proof")
4870            .is_err()
4871        {
4872            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
4873        }
4874        let proof = (|| {
4875            let snapshot = snapshot_read_connection(
4876                &self.conn,
4877                &SnapshotReadRequest {
4878                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
4879                    commit_ids: vec![prepared.client_commit_id.clone()],
4880                    ..Default::default()
4881                },
4882            )
4883            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
4884            let current = self
4885                .commit_outcome(&prepared.client_commit_id)
4886                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
4887            let unavailable = if report.revoked.is_empty() {
4888                CurrentBaseProofReason::OutcomeUnavailable
4889            } else {
4890                CurrentBaseProofReason::Revoked
4891            };
4892            let Some(outcome) = current else {
4893                return Err(unavailable);
4894            };
4895            if !matches!(
4896                snapshot.deliveries.first(),
4897                Some(CommitDelivery::Known { .. })
4898            ) || outcome.sequence != prepared.sequence
4899                || outcome.resolution != CommitOutcomeResolution::Active
4900                || !matches!(
4901                    outcome.status,
4902                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
4903                )
4904            {
4905                return Err(unavailable);
4906            }
4907            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
4908                return Err(unavailable);
4909            };
4910            if rows.is_empty() || operations.len() != rows.len() {
4911                return Err(unavailable);
4912            }
4913            let after =
4914                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
4915            let complete: HashSet<&str> = snapshot
4916                .units
4917                .iter()
4918                .filter_map(|unit| match unit {
4919                    UnitCatchup::Known {
4920                        id,
4921                        bootstrap_complete: true,
4922                        ..
4923                    } => Some(id.as_str()),
4924                    _ => None,
4925                })
4926                .collect();
4927            // A unit id names one scope map (§4.1), so the same id before and
4928            // after the drive is the same registration.
4929            let candidates: Vec<&UnitRecord> = after
4930                .iter()
4931                .filter(|unit| {
4932                    complete.contains(unit.id.as_str())
4933                        && prepared.units.iter().any(|before| before.id == unit.id)
4934                })
4935                .collect();
4936            let covered = |table_name: &str, row: &Map<String, Value>| {
4937                let Some(table) = self.schema.table(table_name) else {
4938                    return false;
4939                };
4940                candidates.iter().any(|unit| {
4941                    covers(table, &unit.scopes)
4942                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
4943                            |scopes| {
4944                                scopes.iter().all(|(variable, allowed)| {
4945                                    table
4946                                        .scope_column(variable)
4947                                        .and_then(|column| match row.get(column) {
4948                                            Some(Value::String(value)) => Some(value.clone()),
4949                                            Some(Value::Bool(value)) => Some(value.to_string()),
4950                                            Some(Value::Number(value)) => {
4951                                                value.as_f64().map(|number| {
4952                                                    ryu_js::Buffer::new()
4953                                                        .format_finite(number)
4954                                                        .to_owned()
4955                                                })
4956                                            }
4957                                            _ => None,
4958                                        })
4959                                        .is_some_and(|value| {
4960                                            !value.is_empty() && allowed.contains(&value)
4961                                        })
4962                                })
4963                            },
4964                        )
4965                })
4966            };
4967            for row in &rows {
4968                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
4969                    return Err(if report.revoked.is_empty() {
4970                        CurrentBaseProofReason::MissingCoverage
4971                    } else {
4972                        CurrentBaseProofReason::Revoked
4973                    });
4974                };
4975                if !covered(&row.table, base)
4976                    || row
4977                        .unique_conflicts
4978                        .iter()
4979                        .any(|conflict| !covered(&row.table, &conflict.server_row))
4980                {
4981                    return Err(if report.revoked.is_empty() {
4982                        CurrentBaseProofReason::MissingCoverage
4983                    } else {
4984                        CurrentBaseProofReason::Revoked
4985                    });
4986                }
4987            }
4988            Ok(rows)
4989        })();
4990        if self
4991            .conn
4992            .execute_batch("RELEASE syncular_base_proof")
4993            .is_err()
4994        {
4995            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
4996        }
4997        match proof {
4998            Ok(rows) => CurrentBaseProof {
4999                proven: true,
5000                rows: Some(rows),
5001                reason: None,
5002            },
5003            Err(reason) => CurrentBaseProof::failure(reason),
5004        }
5005    }
5006
5007    pub fn prove_current_base(
5008        &mut self,
5009        transport: &mut dyn Transport,
5010        client_commit_id: &str,
5011        sequence: i64,
5012        max_rounds: Option<u32>,
5013    ) -> CurrentBaseProof {
5014        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5015            Ok(prepared) => prepared,
5016            Err(proof) => return proof,
5017        };
5018        let drive = self.sync_until_idle(transport, max_rounds);
5019        self.finish_current_base(prepared, drive)
5020    }
5021
5022    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5023        if query.limit == Some(0) {
5024            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5025        }
5026        let mut sql = String::from(
5027            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5028                    resolution, resolved_at_ms, replacement_client_commit_id
5029               FROM _syncular_commit_outcomes",
5030        );
5031        if query.active_only {
5032            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5033        }
5034        sql.push_str(" ORDER BY seq DESC");
5035        if let Some(limit) = query.limit {
5036            sql.push_str(&format!(" LIMIT {limit}"));
5037        }
5038        let mut stmt = self
5039            .conn
5040            .prepare(&sql)
5041            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5042        let rows = stmt
5043            .query_map([], |row| {
5044                Ok(StoredCommitOutcomeRow {
5045                    sequence: row.get(0)?,
5046                    client_commit_id: row.get(1)?,
5047                    status: row.get(2)?,
5048                    recorded_at_ms: row.get(3)?,
5049                    results_json: row.get(4)?,
5050                    operations_json: row.get(5)?,
5051                    resolution: row.get(6)?,
5052                    resolved_at_ms: row.get(7)?,
5053                    replacement_client_commit_id: row.get(8)?,
5054                })
5055            })
5056            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5057        let mut outcomes = Vec::new();
5058        for row in rows {
5059            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5060                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5061            )?)?);
5062        }
5063        Ok(outcomes)
5064    }
5065
5066    fn prune_commit_outcomes(&self) -> Result<(), String> {
5067        #[cfg(test)]
5068        self.outcome_prune_count
5069            .set(self.outcome_prune_count.get() + 1);
5070        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5071        let count = self
5072            .conn
5073            .query_row(
5074                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5075                [],
5076                |row| row.get::<_, i64>(0),
5077            )
5078            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5079            as usize;
5080        let excess = count.saturating_sub(max_entries);
5081        if excess == 0 {
5082            return Ok(());
5083        }
5084        let mut stmt = self
5085            .conn
5086            .prepare(
5087                "SELECT seq FROM _syncular_commit_outcomes
5088                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5089                  ORDER BY seq ASC LIMIT ?1",
5090            )
5091            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5092        let rows = stmt
5093            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5094            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5095        let sequences = rows
5096            .collect::<Result<Vec<_>, _>>()
5097            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5098        drop(stmt);
5099        for sequence in sequences {
5100            self.conn
5101                .execute(
5102                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5103                    rusqlite::params![sequence],
5104                )
5105                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5106        }
5107        Ok(())
5108    }
5109
5110    // -- units (§4.1) ------------------------------------------------------------
5111
5112    /// Every registered unit in registration order (§4.1).
5113    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5114        self.stored(load_units(&self.conn))
5115    }
5116
5117    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5118        self.stored(get_unit(&self.conn, id))
5119    }
5120
5121    /// Register one unit (§4.1) and return its id.
5122    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5123        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5124    }
5125
5126    /// Register several units in one local transaction and return their ids:
5127    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5128    /// cold start with hundreds of units pays it once). An error rolls back
5129    /// every registration of the call. Registering an active unit again
5130    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5131    /// re-arms a unit raises sync-needed with an interactive intent.
5132    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5133        self.begin_observation("syncular_subscribe")?;
5134        let mut changed = false;
5135        let registered = (|| {
5136            let mut ids = Vec::with_capacity(units.len());
5137            for scopes in units {
5138                let (id, held) = self.hold_unit(scopes, true)?;
5139                changed |= held;
5140                ids.push(id);
5141            }
5142            self.conn
5143                .execute_batch("RELEASE syncular_subscribe")
5144                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5145            Ok(ids)
5146        })();
5147        let ids = match registered {
5148            Ok(ids) => ids,
5149            Err(error) => {
5150                self.rollback_observation("syncular_subscribe");
5151                return Err(error);
5152            }
5153        };
5154        if changed {
5155            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5156            self.set_sync_needed(true, true);
5157        }
5158        Ok(ids)
5159    }
5160
5161    /// Release the explicit reference to a unit. The unit leaves the
5162    /// registration list when no window base holds it (§4.1); its rows stay.
5163    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5164        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5165            return Ok(());
5166        };
5167        if !unit.explicit {
5168            return Ok(());
5169        }
5170        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5171            return self.stored(save_unit(
5172                &self.conn,
5173                &UnitRecord {
5174                    explicit: false,
5175                    ..unit
5176                },
5177            ));
5178        }
5179        self.conn
5180            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5181            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5182        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5183        if unit.status == UnitStatus::Active {
5184            self.set_sync_needed(true, true);
5185        }
5186        Ok(())
5187    }
5188
5189    /// Take one reference on a unit (§4.1): validate the scope map, register a
5190    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5191    /// Returns the id and whether the registration list changed.
5192    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5193        if scopes.is_empty()
5194            || scopes.values().any(|value| value == "*")
5195            || !self
5196                .schema
5197                .tables
5198                .iter()
5199                .any(|table| covers(table, &scopes))
5200        {
5201            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5202        }
5203        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5204        let existing = self.stored(get_unit(&self.conn, &id))?;
5205        if let Some(existing) = existing
5206            .as_ref()
5207            .filter(|unit| unit.status != UnitStatus::Revoked)
5208        {
5209            if explicit && !existing.explicit {
5210                self.stored(save_unit(
5211                    &self.conn,
5212                    &UnitRecord {
5213                        explicit,
5214                        ..existing.clone()
5215                    },
5216                ))?;
5217            }
5218            return Ok((id, false));
5219        }
5220        self.delete_pending_evict(&id)?;
5221        self.stored(save_unit(
5222            &self.conn,
5223            &UnitRecord {
5224                id: id.clone(),
5225                scopes,
5226                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5227                status: UnitStatus::Active,
5228                reason_code: None,
5229                cursor: Some(-1),
5230                bootstrap_state: None,
5231                seeded: Vec::new(),
5232                echo: None,
5233            },
5234        ))?;
5235        Ok((id, true))
5236    }
5237
5238    // -- windows (§4.8) -----------------------------------------------------------
5239
5240    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5241    /// values take a reference on their unit (a new unit bootstraps on the
5242    /// next sync); removed values release theirs, and a unit no reference
5243    /// holds leaves the registration list fused with its first eviction chunk
5244    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5245    pub fn set_window(
5246        &mut self,
5247        base: &WindowBase,
5248        units: &[String],
5249    ) -> Result<CommandEffects, String> {
5250        let base_key = window_base_key(base);
5251        let live = self.load_window_units(&base_key)?;
5252        let mut changed = false;
5253        for value in units {
5254            if live.iter().any(|(held, _)| held == value) {
5255                continue;
5256            }
5257            let rollback = self.capture();
5258            self.begin_observation("syncular_window")?;
5259            let widened = (|| {
5260                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5261                self.sync_needed |= registered;
5262                self.conn
5263                    .execute(
5264                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5265                        rusqlite::params![base_key, value, id],
5266                    )
5267                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5268                let mut batch = ChangeAccumulator::default();
5269                batch.window(&base_key, value);
5270                batch.status = true;
5271                self.finish_observation("syncular_window", batch)
5272            })();
5273            if let Err(error) = widened {
5274                self.rollback_observation("syncular_window");
5275                self.restore(rollback);
5276                return Err(error);
5277            }
5278            changed = true;
5279            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5280        }
5281        for (value, id) in live {
5282            if units.contains(&value) {
5283                continue;
5284            }
5285            self.rebuild_overlay_if_dirty()?;
5286            let rollback = self.capture();
5287            self.begin_observation("syncular_window")?;
5288            let remaining = self.release_window_unit(&base_key, &value, &id);
5289            let remaining = match remaining {
5290                Ok(remaining) => remaining,
5291                Err(error) => {
5292                    self.rollback_observation("syncular_window");
5293                    self.restore(rollback);
5294                    return Err(error);
5295                }
5296            };
5297            changed = true;
5298            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5299            if remaining {
5300                self.drain_pending_evictions()?;
5301            }
5302        }
5303        Ok(if changed && self.transport_enabled {
5304            CommandEffects::interactive()
5305        } else {
5306            CommandEffects::none()
5307        })
5308    }
5309
5310    /// §4.8 completeness oracle: the windowed-in values of a base and those
5311    /// whose unit is not complete. A value is complete once its unit is active
5312    /// and no longer bootstrapping; registration alone is not completeness.
5313    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5314        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5315    }
5316
5317    /// Registry rows for a base, ordered by value: (value, unit id).
5318    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5319        let mut statement = self
5320            .conn
5321            .prepare_cached(
5322                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5323            )
5324            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5325        let rows = statement
5326            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5327            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5328        rows.collect::<Result<_, _>>()
5329            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5330    }
5331
5332    /// Every window registry row: (base key, value).
5333    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5334        let mut statement = self
5335            .conn
5336            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5337            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5338        let rows = statement
5339            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5340            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5341        rows.collect::<Result<_, _>>()
5342            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5343    }
5344
5345    /// §4.8: release a window value's reference inside the caller's
5346    /// observation. When no reference holds the unit, unregister it fused with
5347    /// its first eviction chunk (E3): delete the rows of every covered table
5348    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5349    /// owned by another unit (§3.3), and park the per-table scopes of what
5350    /// remains in the pending-eviction registry. Fails closed without a local
5351    /// scope-column mapping. Returns whether unpinned rows remain.
5352    fn release_window_unit(
5353        &mut self,
5354        base_key: &str,
5355        value: &str,
5356        id: &str,
5357    ) -> Result<bool, String> {
5358        self.conn
5359            .execute(
5360                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5361                rusqlite::params![base_key, value],
5362            )
5363            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5364        let mut batch = ChangeAccumulator::default();
5365        batch.window(base_key, value);
5366        batch.status = true;
5367        let unit = self.stored(get_unit(&self.conn, id))?;
5368        let mut remaining = false;
5369        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5370            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5371                let units = self.stored(load_units(&self.conn))?;
5372                self.conn
5373                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5374                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5375                self.sync_needed = true;
5376                for table in self.schema.tables.clone() {
5377                    let Some(effective) = unit
5378                        .echo
5379                        .as_deref()
5380                        .filter(|_| covers(&table, &unit.scopes))
5381                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5382                    else {
5383                        continue;
5384                    };
5385                    let pinned = self.pinned_row_ids(&table.name);
5386                    let owners = unit_owners(&units, &table, id);
5387                    let (more, deferred) =
5388                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5389                    remaining |= more;
5390                    if deferred {
5391                        self.save_pending_evict(id, &table.name, &effective)?;
5392                    }
5393                    self.record_scope_map(&mut batch, &table.name, &effective);
5394                }
5395            }
5396        }
5397        self.finish_observation("syncular_window", batch)?;
5398        Ok(remaining)
5399    }
5400
5401    /// §4.8 E1: delete base rows owned by the effective scopes
5402    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5403    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5404    /// fails without deleting rows.
5405    fn evict_scope_rows(
5406        &self,
5407        table_name: &str,
5408        effective: &[(String, Vec<String>)],
5409        owners: &[Vec<(String, Vec<String>)>],
5410        pinned: &std::collections::HashSet<String>,
5411    ) -> Result<(bool, bool), String> {
5412        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5413        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5414            .map_err(|()| "sync.scope_revoked".to_owned())?
5415        else {
5416            return Ok((false, false));
5417        };
5418        let scope_params = params.len();
5419        let pk = quote_ident(&table.primary_key);
5420        // Pinned ids are rowId text; the text form matches every key type.
5421        let predicate = if pinned.is_empty() {
5422            scope.clone()
5423        } else {
5424            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5425            let holes = vec!["?"; pinned.len()].join(", ");
5426            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5427        };
5428        let base = base_table(table_name);
5429        let visible = visible_table(table_name);
5430        let mut ids: Vec<SqlValue> = Vec::new();
5431        for source in [&base, &visible] {
5432            let mut stmt = self
5433                .conn
5434                .prepare_cached(&format!(
5435                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5436                ))
5437                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5438            ids = stmt
5439                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5440                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5441                .collect::<Result<_, _>>()
5442                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5443            if !ids.is_empty() {
5444                break;
5445            }
5446        }
5447        if !ids.is_empty() {
5448            for id in &ids {
5449                self.conn
5450                    .execute(
5451                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5452                        rusqlite::params![table_name, id],
5453                    )
5454                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5455            }
5456            let holes = vec!["?"; ids.len()].join(", ");
5457            for source in [&base, &visible] {
5458                self.conn
5459                    .execute(
5460                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5461                        rusqlite::params_from_iter(&ids),
5462                    )
5463                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5464            }
5465        }
5466        let mut remaining = false;
5467        let mut deferred = false;
5468        for source in [&base, &visible] {
5469            remaining |= self
5470                .conn
5471                .query_row(
5472                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5473                    rusqlite::params_from_iter(&params),
5474                    |row| row.get::<_, bool>(0),
5475                )
5476                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5477            deferred |= self
5478                .conn
5479                .query_row(
5480                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5481                    rusqlite::params_from_iter(&params[..scope_params]),
5482                    |row| row.get::<_, bool>(0),
5483                )
5484                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5485        }
5486        Ok((remaining, deferred))
5487    }
5488
5489    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5490    /// chunks, retaining outbox pins and their durable marker. A unit that
5491    /// re-entered the window has no pending entry left.
5492    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5493        let pending = self.load_pending_evictions()?;
5494        if pending.is_empty() {
5495            return Ok(());
5496        }
5497        self.rebuild_overlay_if_dirty()?;
5498        for (unit_id, table_name, effective) in pending {
5499            let Some(table) = self.schema.table(&table_name).cloned() else {
5500                self.delete_pending_evict_table(&unit_id, &table_name)?;
5501                continue;
5502            };
5503            loop {
5504                std::thread::yield_now();
5505                self.begin_observation("syncular_evict")?;
5506                let chunk = (|| {
5507                    let pinned = self.pinned_row_ids(&table_name);
5508                    let owners =
5509                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5510                    let (remaining, deferred) =
5511                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5512                    if !deferred {
5513                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5514                    }
5515                    let mut batch = ChangeAccumulator::default();
5516                    self.record_scope_map(&mut batch, &table_name, &effective);
5517                    self.finish_observation("syncular_evict", batch)?;
5518                    Ok(remaining)
5519                })();
5520                match chunk {
5521                    Ok(true) => {}
5522                    Ok(false) => break,
5523                    Err(error) => {
5524                        self.rollback_observation("syncular_evict");
5525                        return Err(error);
5526                    }
5527                }
5528            }
5529        }
5530        Ok(())
5531    }
5532
5533    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5534    /// commit — rows that MUST NOT be evicted until the commit drains.
5535    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5536        let mut pinned = std::collections::HashSet::new();
5537        for commit in self
5538            .outbox
5539            .iter()
5540            .map(|commit| &**commit)
5541            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5542        {
5543            for op in &commit.ops {
5544                if op.table == table {
5545                    pinned.insert(op.row_id.clone());
5546                }
5547            }
5548        }
5549        pinned
5550    }
5551
5552    fn save_pending_evict(
5553        &self,
5554        unit_id: &str,
5555        table: &str,
5556        effective: &[(String, Vec<String>)],
5557    ) -> Result<(), String> {
5558        self.conn.execute(
5559            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5560            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5561        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5562    }
5563
5564    /// Re-entry cancels every pending eviction of the unit.
5565    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5566        self.conn
5567            .execute(
5568                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5569                [unit_id],
5570            )
5571            .map(|_| ())
5572            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5573    }
5574
5575    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5576        self.conn
5577            .execute(
5578                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5579                [unit_id, table],
5580            )
5581            .map(|_| ())
5582            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5583    }
5584
5585    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5586        let mut stmt = self
5587            .conn
5588            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5589            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5590        let rows = stmt
5591            .query_map([], |row| {
5592                Ok((
5593                    row.get::<_, String>(0)?,
5594                    row.get::<_, String>(1)?,
5595                    row.get::<_, String>(2)?,
5596                ))
5597            })
5598            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5599        rows.map(|row| {
5600            let (unit_id, table, json) =
5601                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5602            let value: Value = serde_json::from_str(&json)
5603                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5604            let effective = json_to_scope_map(&value)
5605                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5606            Ok((unit_id, table, effective))
5607        })
5608        .collect()
5609    }
5610
5611    /// Select failed-intent retention; reload durable retained aggregates.
5612    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5613        self.retain_failed_commits = enabled;
5614        for (commit, _) in self.failed_commits.iter() {
5615            for operation in &commit.ops {
5616                self.overlay_dirty.table(&operation.table);
5617            }
5618        }
5619        self.failed_commits = self.load_failed_commits()?.into();
5620        for (commit, _) in self.failed_commits.iter() {
5621            for operation in &commit.ops {
5622                self.overlay_dirty.table(&operation.table);
5623            }
5624        }
5625        Ok(())
5626    }
5627
5628    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5629        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5630            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5631        let rows = statement
5632            .query_map([], |row| {
5633                Ok((
5634                    row.get::<_, String>(0)?,
5635                    row.get::<_, String>(1)?,
5636                    row.get::<_, String>(2)?,
5637                ))
5638            })
5639            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5640        rows.map(|row| {
5641            let (id, operations, initial) =
5642                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5643            let operations: Vec<CommitOperation> =
5644                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5645            let initial =
5646                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5647            Ok((
5648                OutboxCommit {
5649                    client_commit_id: id,
5650                    ops: operations
5651                        .into_iter()
5652                        .map(|op| OutboxOp {
5653                            upsert: op.op == "upsert",
5654                            table: op.table,
5655                            row_id: op.row_id,
5656                            base_version: op.base_version,
5657                            values: op.values,
5658                            insert: op.insert,
5659                        })
5660                        .collect(),
5661                },
5662                initial,
5663            ))
5664        })
5665        .collect()
5666    }
5667
5668    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5669        let mut initial = Map::new();
5670        for operation in operations {
5671            // A table the current schema removed has no rows to retain.
5672            let Some(table) = self.schema.table(&operation.table) else {
5673                continue;
5674            };
5675            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5676                Some(row) => Some(row),
5677                None => self.visible_row(table, &operation.row_id)?,
5678            };
5679            if let Some((values, _)) = initial_row {
5680                let mut intended: Map<String, Value> = table
5681                    .columns
5682                    .iter()
5683                    .zip(values.iter())
5684                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5685                    .collect();
5686                if let Some(patch) = &operation.values {
5687                    intended.extend(patch.clone());
5688                }
5689                initial.insert(
5690                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5691                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5692                    Value::Object(intended),
5693                );
5694            }
5695        }
5696        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5697            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5698            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5699            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5700        self.failed_commits.push((
5701            OutboxCommit {
5702                client_commit_id: id.to_owned(),
5703                ops: operations.to_vec(),
5704            },
5705            initial,
5706        ));
5707        Ok(())
5708    }
5709
5710    fn drop_failed_commits(
5711        &mut self,
5712        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5713    ) -> Result<Vec<OutboxCommit>, String> {
5714        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5715        for (commit, initial) in self.failed_commits.iter() {
5716            for operation in &commit.ops {
5717                let key =
5718                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5719                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5720                if initial
5721                    .get(&key)
5722                    .and_then(Value::as_object)
5723                    .is_some_and(|values| matches(&operation.table, values))
5724                {
5725                    doomed.push(commit.clone());
5726                    break;
5727                }
5728            }
5729        }
5730        for commit in &doomed {
5731            self.conn
5732                .execute(
5733                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5734                    rusqlite::params![commit.client_commit_id],
5735                )
5736                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5737        }
5738        for commit in &doomed {
5739            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5740        }
5741        self.conflicts.retain(|record| {
5742            !doomed
5743                .iter()
5744                .any(|commit| commit.client_commit_id == record.client_commit_id)
5745        });
5746        self.rejections.retain(|record| {
5747            !doomed
5748                .iter()
5749                .any(|commit| commit.client_commit_id == record.client_commit_id)
5750        });
5751        self.failed_commits.retain(|(commit, _)| {
5752            !doomed
5753                .iter()
5754                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
5755        });
5756        for commit in &doomed {
5757            for operation in &commit.ops {
5758                self.overlay_dirty.table(&operation.table);
5759            }
5760        }
5761        Ok(doomed)
5762    }
5763
5764    /// Record one commit of local mutations. The returned commit ID is the
5765    /// §2.3 idempotency key. Failures are structured so callers read
5766    /// `code`/`message`/`details` instead of parsing a string prefix.
5767    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
5768        self.mutate_with_commit_id(mutations, None)
5769    }
5770
5771    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
5772    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
5773    /// one this replica still holds fails with `client.commit_id_reused`.
5774    pub fn mutate_with_commit_id(
5775        &mut self,
5776        mutations: Vec<Mutation>,
5777        client_commit_id: Option<String>,
5778    ) -> Result<String, ClientError> {
5779        let prior = self.storage_failure.borrow_mut().take();
5780        if let Some(id) = &client_commit_id {
5781            let held = if id.is_empty() {
5782                Err("sync.invalid_request: clientCommitId is empty".to_owned())
5783            } else {
5784                self.conn
5785                    .query_row(
5786                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
5787                        [id],
5788                        |row| row.get(0),
5789                    )
5790                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5791            };
5792            match held {
5793                Ok(false) => {}
5794                Ok(true) => {
5795                    return self.finish_authoring(
5796                        prior,
5797                        Err(ClientError {
5798                            code: "client.commit_id_reused".to_owned(),
5799                            message: "clientCommitId is held".to_owned(),
5800                            details: Some(json!({ "clientCommitId": id })),
5801                            retryable: false,
5802                        }),
5803                    )
5804                }
5805                Err(message) => return self.finish_authoring(prior, Err(message.into())),
5806            }
5807        }
5808        let outcome = (|| -> Result<String, ClientError> {
5809            if mutations.is_empty() {
5810                return Err(
5811                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
5812                        .to_owned()
5813                        .into(),
5814                );
5815            }
5816            let mut ops = Vec::with_capacity(mutations.len());
5817            for mutation in mutations {
5818                match mutation {
5819                    Mutation::Patch {
5820                        table,
5821                        values,
5822                        base_version,
5823                    } => {
5824                        let schema_table = self
5825                            .schema
5826                            .table(&table)
5827                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
5828                        let values = normalize_values_casing(schema_table, values)?;
5829                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
5830                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
5831                    }
5832                    Mutation::Upsert {
5833                        table,
5834                        values,
5835                        base_version,
5836                    } => {
5837                        let schema_table = self.schema.table(&table).ok_or_else(|| {
5838                            format!("sync.unknown_table: unknown table {table:?}")
5839                        })?;
5840                        // §5: value keys are accepted in snake_case AND the
5841                        // generated row types' camelCase; normalize to SQL truth
5842                        // before the pk lookup / codec see them.
5843                        let values = normalize_values_casing(schema_table, values)?;
5844                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
5845                        // §6.7 `mutate` marks every column present: fill missing
5846                        // nullable columns with NULL before the payload is built.
5847                        let values = full_row_values(schema_table, values)?;
5848                        // §5.11: validate the payload encodes with the current
5849                        // codec. A key-selection or unknown-key failure is a
5850                        // durable rejection at the push seam (§10.3), never an
5851                        // author-time error. Full-row `mutate` presents every
5852                        // column, so no stored fallback is needed.
5853                        self.validate_author_encode(schema_table, &row_id, &values)?;
5854                        // §7.1 insert intent: the row is absent from the
5855                        // visible replica at call time, before this batch
5856                        // applies anything.
5857                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
5858                        ops.push(OutboxOp {
5859                            upsert: true,
5860                            table,
5861                            row_id,
5862                            base_version,
5863                            values: Some(values),
5864                            insert,
5865                        });
5866                    }
5867                    Mutation::Delete {
5868                        table,
5869                        row_id,
5870                        base_version,
5871                    } => {
5872                        if self.schema.table(&table).is_none() {
5873                            return Err(
5874                                format!("sync.unknown_table: unknown table {table:?}").into()
5875                            );
5876                        }
5877                        // §2.4: the server rejects a rowId holding U+0000.
5878                        if row_id.contains('\0') {
5879                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
5880                                .to_owned()
5881                                .into());
5882                        }
5883                        ops.push(OutboxOp {
5884                            upsert: false,
5885                            table,
5886                            row_id,
5887                            base_version,
5888                            values: None,
5889                            insert: false,
5890                        });
5891                    }
5892                }
5893            }
5894            Ok(self.record_outbox_commit(ops, client_commit_id)?)
5895        })();
5896        self.finish_authoring(prior, outcome)
5897    }
5898
5899    /// Scope the retained `storage_failure` diagnostic to this call so an
5900    /// earlier operation's failure cannot classify a new one, and surface the
5901    /// call's own classified failure with its details. A fresh classified
5902    /// failure replaces the retained diagnostic; otherwise the earlier one is
5903    /// kept for the next sync.
5904    fn finish_authoring(
5905        &self,
5906        prior: Option<QueryReadFailure>,
5907        outcome: Result<String, ClientError>,
5908    ) -> Result<String, ClientError> {
5909        let fresh = self.storage_failure.borrow_mut().take();
5910        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
5911        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
5912    }
5913
5914    fn record_outbox_commit(
5915        &mut self,
5916        ops: Vec<OutboxOp>,
5917        client_commit_id: Option<String>,
5918    ) -> Result<String, String> {
5919        let commit = OutboxCommit {
5920            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
5921            ops,
5922        };
5923        self.begin_observation("syncular_mutation")?;
5924        let mut batch = ChangeAccumulator::default();
5925        for op in &commit.ops {
5926            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
5927            if let Some(values) = &op.values {
5928                if let Some(table) = self.schema.table(&op.table) {
5929                    for scope in &table.scope_variables {
5930                        if let Some(Value::String(value)) = values.get(&scope.column) {
5931                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
5932                            precise = true;
5933                        }
5934                    }
5935                }
5936            }
5937            if !precise {
5938                batch.table(&op.table);
5939            }
5940        }
5941        if let Err(error) = self.persist_outbox_insert(&commit) {
5942            self.rollback_observation("syncular_mutation");
5943            return Err(error);
5944        }
5945        // A clean overlay contains the FIFO fold of every previous commit, so
5946        // the new commit applies over it. §7.1: an operation that does not
5947        // apply (a secondary unique collision) fails the whole commit; the
5948        // savepoint rollback removes the outbox entry and every visible write.
5949        // The append undoes itself: a rollback capture would copy the outbox.
5950        let was_dirty = self.overlay_dirty.snapshot();
5951        let id = commit.client_commit_id.clone();
5952        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
5953            commit
5954                .ops
5955                .iter()
5956                .try_for_each(|op| self.apply_outbox_op(op))?;
5957            self.outbox.push(Arc::new(commit));
5958            batch.status = true;
5959            self.finish_observation("syncular_mutation", batch)
5960                .inspect_err(|_| drop(self.outbox.pop()))
5961        });
5962        if let Err(error) = appended {
5963            self.rollback_observation("syncular_mutation");
5964            self.overlay_dirty.restore(was_dirty);
5965            return Err(error);
5966        }
5967        Ok(id)
5968    }
5969
5970    /// Record one §6.1 sparse upsert: the primary key plus the supplied
5971    /// non-scope columns are present; every other column is absent and stays
5972    /// untouched on the server and in the local overlay. An absent local base
5973    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
5974    /// Failures are structured so callers read `code`/`message`/`details`.
5975    pub fn patch(
5976        &mut self,
5977        table: &str,
5978        row_id: &str,
5979        partial: Map<String, Value>,
5980        base_version: Option<i64>,
5981    ) -> Result<String, ClientError> {
5982        let prior = self.storage_failure.borrow_mut().take();
5983        let outcome = self.perform_patch(table, row_id, partial, base_version);
5984        self.finish_authoring(prior, outcome)
5985    }
5986
5987    /// The unscoped patch body. Public authoring entry points scope
5988    /// `storage_failure` around the whole operation, including any pre-read.
5989    fn perform_patch(
5990        &mut self,
5991        table: &str,
5992        row_id: &str,
5993        partial: Map<String, Value>,
5994        base_version: Option<i64>,
5995    ) -> Result<String, ClientError> {
5996        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
5997        Ok(self.record_outbox_commit(vec![operation], None)?)
5998    }
5999
6000    fn prepare_patch(
6001        &self,
6002        table: &str,
6003        row_id: &str,
6004        partial: Map<String, Value>,
6005        base_version: Option<i64>,
6006    ) -> Result<OutboxOp, ClientError> {
6007        let schema_table = self
6008            .schema
6009            .table(table)
6010            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6011        let partial = normalize_values_casing(schema_table, partial)?;
6012        if let Some(pk) = partial.get(&schema_table.primary_key) {
6013            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6014                return Err(invalid_authoring(
6015                    "patch cannot change the primary key",
6016                    json!({ "table": table }),
6017                ));
6018            }
6019        }
6020        if !self.visible_row_exists(schema_table, row_id)? {
6021            return Err("sync.row_missing: a sparse patch requires a local row"
6022                .to_owned()
6023                .into());
6024        }
6025        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6026        // server accepts a present scope column whose value equals the
6027        // stored row and applies it as a no-op, so a decoded envelope
6028        // round-trips. The local row is the only value the client can prove
6029        // equality against; when the row is absent locally the client cannot
6030        // prove equality against the server's stored row and fails closed.
6031        // A primary key that is also a scope column needs no stored-row
6032        // read: the primary key in a sparse payload is by construction the
6033        // row id being patched (§6.1), so its value is proven equal already.
6034        for scope in &schema_table.scope_variables {
6035            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6036                continue;
6037            }
6038            let Some(column) = schema_table
6039                .columns
6040                .iter()
6041                .find(|column| column.name == scope.column)
6042            else {
6043                continue;
6044            };
6045            let stored = self
6046                .conn
6047                .query_row(
6048                    &format!(
6049                        "SELECT {} FROM {} WHERE {}",
6050                        quote_ident(&scope.column),
6051                        visible_table(&schema_table.name),
6052                        row_id_predicate(schema_table)
6053                    ),
6054                    rusqlite::params![row_id],
6055                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6056                )
6057                .optional()
6058                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6059            // Decode the supplied value through the same column-type path the
6060            // rest of the patch encoding uses, so a value that only equals
6061            // the stored one in another number representation still compares
6062            // equal.
6063            let incoming = json_to_column_value(column, partial.get(&scope.column))
6064                .ok()
6065                .map(|value| column_value_to_json(&value));
6066            // A stored blob never equals a patch value, matching the server's
6067            // scope-column comparison (§3.4): bytes are unstorable as scope
6068            // values.
6069            let equal = stored.as_ref().is_some_and(|stored| {
6070                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6071            });
6072            if !equal {
6073                return Err(invalid_authoring(
6074                    "patch cannot write a scope column (§3.4)",
6075                    json!({ "table": table, "column": scope.column }),
6076                ));
6077            }
6078        }
6079        let mut values = partial;
6080        // Presence-set semantics: a proven-equal scope column is dropped, so
6081        // the patch leaves the stored value untouched (§6.2).
6082        for scope in &schema_table.scope_variables {
6083            values.remove(&scope.column);
6084        }
6085        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6086            // §2.4: the rowId of an integer key is its canonical decimal form.
6087            ColumnType::Integer => match row_id.parse::<i64>() {
6088                Ok(v)
6089                    if v.to_string() == row_id
6090                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6091                            .contains(&v) =>
6092                {
6093                    Value::from(v)
6094                }
6095                _ => {
6096                    return Err(format!(
6097                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6098                )
6099                    .into())
6100                }
6101            },
6102            ColumnType::Boolean => match row_id {
6103                "true" => Value::Bool(true),
6104                "false" => Value::Bool(false),
6105                _ => {
6106                    return Err(format!(
6107                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6108                )
6109                    .into())
6110                }
6111            },
6112            _ => Value::from(row_id.to_owned()),
6113        };
6114        values.insert(schema_table.primary_key.clone(), pk_value);
6115        // §6.1: a patch cannot null a non-nullable column.
6116        if let Some(column) = schema_table
6117            .columns
6118            .iter()
6119            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6120        {
6121            return Err(invalid_authoring(
6122                "a non-nullable column is null (§6.1)",
6123                json!({ "table": table, "column": column.name }),
6124            ));
6125        }
6126        // §5.11: structural validation only; an absent key-id selector is
6127        // resolved at the push seam from the stored local row, and a
6128        // key-selection or unknown-key failure becomes a durable
6129        // `client.encrypt_failed` rejection there (§10.3), never an
6130        // author-time error.
6131        self.validate_author_encode(schema_table, row_id, &values)?;
6132        Ok(OutboxOp {
6133            upsert: true,
6134            table: table.to_owned(),
6135            row_id: row_id.to_owned(),
6136            base_version,
6137            values: Some(values),
6138            insert: false,
6139        })
6140    }
6141
6142    /// §5.11 key-id fallback: the stored local row's selector value as a
6143    /// positional row (only the selector slot filled). Returns None when no
6144    /// selector is configured, the row is locally absent, or the stored
6145    /// value is not a usable key id. A failed storage read is an error.
6146    fn stored_key_fallback(
6147        &self,
6148        table: &TableSchema,
6149        row_id: &str,
6150    ) -> Result<Option<Row>, String> {
6151        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6152            return Ok(None);
6153        };
6154        let Some(index) = table
6155            .columns
6156            .iter()
6157            .position(|column| &column.name == selector)
6158        else {
6159            return Ok(None);
6160        };
6161        let sql = format!(
6162            "SELECT {} FROM {} WHERE {}",
6163            quote_ident(selector),
6164            visible_table(&table.name),
6165            row_id_predicate(table)
6166        );
6167        let key = self
6168            .conn
6169            .query_row(&sql, rusqlite::params![row_id], |row| {
6170                row.get::<_, Option<String>>(0)
6171            })
6172            .optional()
6173            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6174            .flatten()
6175            .filter(|key| !key.is_empty());
6176        let Some(key) = key else {
6177            return Ok(None);
6178        };
6179        let mut fallback: Row = vec![None; table.columns.len()];
6180        fallback[index] = Some(ColumnValue::String(key));
6181        Ok(Some(fallback))
6182    }
6183
6184    /// §5.11/§10.3 author-time encode check: a structural failure (a
6185    /// non-nullable NULL, a value the codec rejects) is an author-time
6186    /// error; a key-selection or unknown-key failure is deferred to the push
6187    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6188    fn validate_author_encode(
6189        &self,
6190        table: &TableSchema,
6191        row_id: &str,
6192        values: &Map<String, Value>,
6193    ) -> Result<(), String> {
6194        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6195            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6196            result => result.map(|_| ()),
6197        }
6198    }
6199
6200    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6201    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6202    /// raise one durable rejection per commit. `sync()` never aborts for an
6203    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6204    /// undoing the dropped commit's purely-optimistic rows.
6205    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6206    /// (an envelope or a type an earlier version accepted). `with_encryption`
6207    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6208    /// the startup boundary leaves those to the send seam, where hosts configure
6209    /// keys after open.
6210    /// Recovery runs as one observation. An enclosing observation (the guarded
6211    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6212    /// domains join that change event instead of publishing a second one.
6213    fn drop_unencodable_outbox(
6214        &mut self,
6215        with_encryption: bool,
6216        batch: Option<&mut ChangeAccumulator>,
6217    ) -> Result<bool, String> {
6218        let mut failures = Vec::new();
6219        for commit in self.outbox.iter() {
6220            for op in &commit.ops {
6221                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6222                    continue;
6223                };
6224                // An insert lacking a non-nullable column is left to
6225                // `drop_incompatible_outbox` (§7.4.4).
6226                let Some(values) = op.presented_values(table) else {
6227                    continue;
6228                };
6229                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6230                let failure = match encode_sparse_row_json(
6231                    table,
6232                    &op.row_id,
6233                    &values,
6234                    &self.encryption,
6235                    fallback.as_ref(),
6236                ) {
6237                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6238                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6239                    }
6240                    // §7.1/§7.4.4: a value the current codec refuses was
6241                    // persisted by a version that accepted it; it leaves
6242                    // through the incompatible-outbox path, so the FIFO
6243                    // queue never wedges.
6244                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6245                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6246                        OUTBOX_INCOMPATIBLE_CODE,
6247                    )),
6248                    _ => None,
6249                };
6250                if let Some((error, code)) = failure {
6251                    failures.push((commit.clone(), error, code));
6252                    break;
6253                }
6254            }
6255        }
6256        if failures.is_empty() {
6257            return Ok(false);
6258        }
6259        // §7.4.4: one observation covers the durable outcome writes, the outbox
6260        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6261        // leaves the replica and the client unchanged, a successful recovery
6262        // publishes one batch with the rejection, outcome, and table domains, and
6263        // an ordinary open with nothing to recover publishes nothing.
6264        let rollback = self.capture();
6265        let owns_observation = batch.is_none();
6266        let owns_transaction = self.conn.is_autocommit();
6267        let mut domains = ChangeAccumulator::default();
6268        if owns_observation {
6269            self.begin_observation("syncular_value_recovery")?;
6270        }
6271        let dropped = (|| -> Result<(), String> {
6272            for (commit, message, code) in failures {
6273                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6274                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6275                }
6276                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6277                let rejection = RejectionRecord {
6278                    client_commit_id: commit.client_commit_id.clone(),
6279                    op_index: 0,
6280                    code: code.to_owned(),
6281                    message,
6282                    retryable: false,
6283                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6284                        field_paths: None,
6285                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6286                        required_action: None,
6287                        references: None,
6288                    }),
6289                    operation: representable
6290                        .then(|| commit.ops.first())
6291                        .flatten()
6292                        .map(CommitOperation::from),
6293                };
6294                self.persist_commit_outcome(
6295                    &commit.client_commit_id,
6296                    CommitOutcomeStatus::Rejected,
6297                    &[CommitOperationOutcome::Error {
6298                        rejection: rejection.clone(),
6299                    }],
6300                    representable.then_some(commit.ops.as_slice()),
6301                )?;
6302                self.delete_outbox_persisted(&commit.client_commit_id)?;
6303                self.outbox
6304                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6305                self.rejections.push(rejection);
6306                for operation in &commit.ops {
6307                    domains.table(operation.table.as_str());
6308                }
6309            }
6310            domains.status = true;
6311            domains.rejections = true;
6312            domains.outcomes = true;
6313            self.prune_commit_outcomes()?;
6314            // The visible projection is re-derived from the surviving base + outbox.
6315            self.overlay_dirty.set(true);
6316            self.rebuild_overlay()?;
6317            Ok(())
6318        })();
6319        let result = match batch {
6320            Some(batch) => {
6321                if dropped.is_ok() {
6322                    for table in domains.tables.keys() {
6323                        batch.table(table.as_str());
6324                    }
6325                    batch.status |= domains.status;
6326                    batch.rejections |= domains.rejections;
6327                    batch.outcomes |= domains.outcomes;
6328                }
6329                dropped
6330            }
6331            None => {
6332                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6333            }
6334        };
6335        if let Err(error) = result {
6336            if owns_observation {
6337                if owns_transaction {
6338                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6339                        let mut failure = self.storage_failure.borrow_mut();
6340                        let failure =
6341                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6342                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6343                    }
6344                } else {
6345                    self.rollback_observation("syncular_value_recovery");
6346                }
6347            }
6348            self.restore(rollback);
6349            return Err(error);
6350        }
6351        Ok(true)
6352    }
6353
6354    pub fn pending_commit_ids(&self) -> Vec<String> {
6355        self.outbox
6356            .iter()
6357            .map(|c| c.client_commit_id.clone())
6358            .collect()
6359    }
6360
6361    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6362    /// FIFO — exactly what the next push round would put on the wire. Used by
6363    /// conformance B.21(h) to pin cross-core byte identity.
6364    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6365        let mut payloads = Vec::new();
6366        for commit in self.outbox.iter() {
6367            for op in &commit.ops {
6368                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6369                    continue;
6370                };
6371                let Some(values) = op.presented_values(table) else {
6372                    continue;
6373                };
6374                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6375                if let Ok(payload) = encode_sparse_row_json(
6376                    table,
6377                    &op.row_id,
6378                    &values,
6379                    &self.encryption,
6380                    fallback.as_ref(),
6381                ) {
6382                    payloads.push(payload);
6383                }
6384            }
6385        }
6386        Ok(payloads)
6387    }
6388
6389    pub fn conflicts(&self) -> &[ConflictRecord] {
6390        &self.conflicts
6391    }
6392
6393    pub fn rejections(&self) -> &[RejectionRecord] {
6394        &self.rejections
6395    }
6396
6397    pub fn resolve_commit_outcome(
6398        &mut self,
6399        input: ResolveCommitOutcomeInput,
6400    ) -> Result<CommitOutcome, String> {
6401        let current = self
6402            .commit_outcome(&input.client_commit_id)?
6403            .ok_or_else(|| {
6404                format!(
6405                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6406                    input.client_commit_id
6407                )
6408            })?;
6409        if current.resolution != CommitOutcomeResolution::Active {
6410            return Ok(current);
6411        }
6412        if input.resolution == CommitOutcomeResolution::Active {
6413            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6414        }
6415        match input.resolution {
6416            CommitOutcomeResolution::Superseded => {
6417                let replacement = input
6418                    .replacement_client_commit_id
6419                    .as_deref()
6420                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6421                    .ok_or_else(|| {
6422                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6423                            .to_owned()
6424                    })?;
6425                let _ = replacement;
6426            }
6427            _ if input.replacement_client_commit_id.is_some() => {
6428                return Err(
6429                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6430                        .to_owned(),
6431                );
6432            }
6433            _ => {}
6434        }
6435        let allowed = match current.status {
6436            CommitOutcomeStatus::Conflict => matches!(
6437                input.resolution,
6438                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6439            ),
6440            CommitOutcomeStatus::Rejected => {
6441                matches!(
6442                    input.resolution,
6443                    CommitOutcomeResolution::Superseded
6444                        | CommitOutcomeResolution::ResolvedKeepServer
6445                )
6446            }
6447            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6448                input.resolution == CommitOutcomeResolution::Dismissed
6449            }
6450        };
6451        if !allowed {
6452            return Err(format!(
6453                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6454                input.resolution, current.status
6455            ));
6456        }
6457
6458        let rollback = self.capture();
6459        self.begin_observation("syncular_outcome_resolution")?;
6460        let result = (|| {
6461            self.conn
6462                .execute(
6463                    "UPDATE _syncular_commit_outcomes
6464                        SET resolution = ?1, resolved_at_ms = ?2,
6465                            replacement_client_commit_id = ?3
6466                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6467                    rusqlite::params![
6468                        Self::outcome_resolution_name(input.resolution),
6469                        self.clock_now_ms(),
6470                        input.replacement_client_commit_id,
6471                        input.client_commit_id
6472                    ],
6473                )
6474                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6475            let resolved = self
6476                .commit_outcome(&current.client_commit_id)?
6477                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6478            self.conn
6479                .execute(
6480                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6481                    rusqlite::params![input.client_commit_id],
6482                )
6483                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6484            for (commit, _) in self.failed_commits.iter() {
6485                if commit.client_commit_id == input.client_commit_id {
6486                    for operation in &commit.ops {
6487                        self.overlay_dirty.table(&operation.table);
6488                    }
6489                }
6490            }
6491            self.failed_commits
6492                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6493            self.prune_commit_outcomes()?;
6494            Ok(resolved)
6495        })();
6496        let resolved = match result {
6497            Ok(outcome) => outcome,
6498            Err(error) => {
6499                self.rollback_observation("syncular_outcome_resolution");
6500                self.restore(rollback);
6501                return Err(error);
6502            }
6503        };
6504        self.conflicts
6505            .retain(|record| record.client_commit_id != current.client_commit_id);
6506        self.rejections
6507            .retain(|record| record.client_commit_id != current.client_commit_id);
6508        let mut batch = ChangeAccumulator {
6509            conflicts: current.status == CommitOutcomeStatus::Conflict,
6510            rejections: current.status == CommitOutcomeStatus::Rejected,
6511            outcomes: true,
6512            ..ChangeAccumulator::default()
6513        };
6514        if let Some(operations) = &current.operations {
6515            for operation in operations {
6516                batch.table(&operation.table);
6517            }
6518        }
6519        if let Err(error) = self
6520            .rebuild_overlay_if_dirty()
6521            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6522        {
6523            self.rollback_observation("syncular_outcome_resolution");
6524            self.restore(rollback);
6525            return Err(error);
6526        }
6527        Ok(resolved)
6528    }
6529
6530    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6531        self.schema_floor.as_ref()
6532    }
6533
6534    /// §7.3.5: the client's opaque auth-lease state, if any.
6535    pub fn lease_state(&self) -> Option<&LeaseState> {
6536        self.lease_state.as_ref()
6537    }
6538
6539    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6540    /// the two lease codes set it; other errors leave leaseState untouched.
6541    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6542        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6543            return Ok(());
6544        }
6545        let mut next = self.lease_state.clone().unwrap_or_default();
6546        next.error_code = Some(code.to_owned());
6547        self.set_lease_state(Some(next))
6548    }
6549
6550    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6551        if self.lease_state == next {
6552            return Ok(());
6553        }
6554        self.begin_observation("syncular_lease")?;
6555        let rollback = self.capture();
6556        self.lease_state = next;
6557        let persisted = match &self.lease_state {
6558            Some(lease) => serde_json::to_string(lease)
6559                .map_err(|error| error.to_string())
6560                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6561            None => self.delete_meta(LEASE_STATE_KEY),
6562        };
6563        let batch = ChangeAccumulator {
6564            status: true,
6565            ..ChangeAccumulator::default()
6566        };
6567        if let Err(error) =
6568            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6569        {
6570            self.rollback_observation("syncular_lease");
6571            self.restore(rollback);
6572            return Err(error);
6573        }
6574        Ok(())
6575    }
6576
6577    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6578        if self.schema_floor == next {
6579            return Ok(());
6580        }
6581        self.begin_observation("syncular_schema_floor")?;
6582        let rollback = self.capture();
6583        self.stopped = next.is_some();
6584        self.schema_floor = next;
6585        let persisted = match &self.schema_floor {
6586            Some(floor) => serde_json::to_string(floor)
6587                .map_err(|error| error.to_string())
6588                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6589            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6590        };
6591        let batch = ChangeAccumulator {
6592            status: true,
6593            ..ChangeAccumulator::default()
6594        };
6595        if let Err(error) =
6596            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6597        {
6598            self.rollback_observation("syncular_schema_floor");
6599            self.restore(rollback);
6600            return Err(error);
6601        }
6602        Ok(())
6603    }
6604
6605    fn set_upgrading(&mut self, value: bool) {
6606        if self.upgrading == value {
6607            return;
6608        }
6609        if self.begin_observation("syncular_upgrading").is_err() {
6610            return;
6611        }
6612        self.upgrading = value;
6613        let batch = ChangeAccumulator {
6614            status: true,
6615            ..ChangeAccumulator::default()
6616        };
6617        if self
6618            .finish_observation("syncular_upgrading", batch)
6619            .is_err()
6620        {
6621            self.rollback_observation("syncular_upgrading");
6622        }
6623    }
6624
6625    pub fn sync_needed(&self) -> bool {
6626        self.sync_needed
6627    }
6628
6629    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6630        let schema_table = self
6631            .schema
6632            .table(table)
6633            .ok_or_else(|| format!("unknown table {table:?}"))?;
6634        let sql = format!(
6635            "SELECT * FROM {} ORDER BY {} ASC",
6636            visible_table(table),
6637            quote_ident(&schema_table.primary_key)
6638        );
6639        let mut stmt = self
6640            .conn
6641            .prepare(&sql)
6642            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6643        let mut rows = stmt
6644            .query([])
6645            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6646        let mut out = Vec::new();
6647        while let Some(row) = rows
6648            .next()
6649            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6650        {
6651            let mut values = Map::new();
6652            for (i, column) in schema_table.columns.iter().enumerate() {
6653                let value = row
6654                    .get_ref(i)
6655                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6656                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6657            }
6658            let version: i64 = row
6659                .get(schema_table.columns.len())
6660                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6661            let row_id = match values.get(&schema_table.primary_key) {
6662                Some(Value::String(s)) => s.clone(),
6663                Some(Value::Number(n)) => n.to_string(),
6664                Some(Value::Bool(b)) => b.to_string(),
6665                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6666            };
6667            out.push(RowState {
6668                row_id,
6669                version,
6670                values,
6671            });
6672        }
6673        Ok(out)
6674    }
6675
6676    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6677    //
6678    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6679    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6680    // the op with `yrs`, re-encodes the whole doc state, and records a
6681    // baseVersion-less sparse upsert carrying only the primary key and the
6682    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6683    // the overlay re-materializes the edit immediately (optimistic apply,
6684    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6685    // Byte-compatible with `@syncular/crdt-yjs`.
6686
6687    /// The current stored value of a `crdt` column for one row — the visible
6688    /// (optimistic) bytes, or `None` when the row is absent or the column is
6689    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6690    /// `crdt` column (guards the app against a typo'd column name).
6691    #[cfg(feature = "crdt-yjs")]
6692    fn crdt_column_bytes(
6693        &self,
6694        table: &str,
6695        row_id: &str,
6696        column: &str,
6697    ) -> Result<Option<Vec<u8>>, String> {
6698        let schema_table = self
6699            .schema
6700            .table(table)
6701            .ok_or_else(|| format!("unknown table {table:?}"))?;
6702        let col = schema_table
6703            .columns
6704            .iter()
6705            .find(|c| c.name == column)
6706            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6707        if col.ty != ColumnType::Crdt {
6708            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6709        }
6710        let sql = format!(
6711            "SELECT {} FROM {} WHERE {}",
6712            quote_ident(column),
6713            visible_table(table),
6714            row_id_predicate(schema_table)
6715        );
6716        let bytes: Option<Vec<u8>> = self
6717            .conn
6718            .prepare_cached(&sql)
6719            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6720            .query_row(rusqlite::params![row_id], |row| {
6721                row.get::<_, Option<Vec<u8>>>(0)
6722            })
6723            .map_err(|e| match e {
6724                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6725                other => Self::sqlite_failure(&self.storage_failure, other),
6726            })?;
6727        Ok(bytes)
6728    }
6729
6730    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6731    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
6732    /// An absent row / NULL column is the empty document (empty string).
6733    #[cfg(feature = "crdt-yjs")]
6734    pub fn crdt_text(
6735        &self,
6736        table: &str,
6737        row_id: &str,
6738        column: &str,
6739        name: &str,
6740    ) -> Result<String, String> {
6741        let bytes = self
6742            .crdt_column_bytes(table, row_id, column)?
6743            .unwrap_or_default();
6744        crate::crdt::text(&bytes, name)
6745    }
6746
6747    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
6748    /// record the resulting crdt-only sparse operation. Returns the enqueued
6749    /// `clientCommitId`.
6750    #[cfg(feature = "crdt-yjs")]
6751    pub fn crdt_insert_text(
6752        &mut self,
6753        table: &str,
6754        row_id: &str,
6755        column: &str,
6756        name: &str,
6757        index: u32,
6758        value: &str,
6759    ) -> Result<String, ClientError> {
6760        let prior = self.storage_failure.borrow_mut().take();
6761        let outcome = (|| -> Result<String, ClientError> {
6762            let current = self
6763                .crdt_column_bytes(table, row_id, column)?
6764                .unwrap_or_default();
6765            let update = crate::crdt::insert_text(&current, name, index, value)?;
6766            self.perform_crdt_push_update(table, row_id, column, &update)
6767        })();
6768        self.finish_authoring(prior, outcome)
6769    }
6770
6771    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
6772    /// record the resulting crdt-only sparse operation. Returns the enqueued
6773    /// `clientCommitId`.
6774    #[cfg(feature = "crdt-yjs")]
6775    pub fn crdt_delete_text(
6776        &mut self,
6777        table: &str,
6778        row_id: &str,
6779        column: &str,
6780        name: &str,
6781        index: u32,
6782        len: u32,
6783    ) -> Result<String, ClientError> {
6784        let prior = self.storage_failure.borrow_mut().take();
6785        let outcome = (|| -> Result<String, ClientError> {
6786            let current = self
6787                .crdt_column_bytes(table, row_id, column)?
6788                .unwrap_or_default();
6789            let update = crate::crdt::delete_text(&current, name, index, len)?;
6790            self.perform_crdt_push_update(table, row_id, column, &update)
6791        })();
6792        self.finish_authoring(prior, outcome)
6793    }
6794
6795    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
6796    /// `crdt` column's current state and record the resulting crdt-only sparse
6797    /// operation. The app authored the update with its own `yrs` model.
6798    /// Returns the enqueued `clientCommitId`.
6799    #[cfg(feature = "crdt-yjs")]
6800    pub fn crdt_apply_update(
6801        &mut self,
6802        table: &str,
6803        row_id: &str,
6804        column: &str,
6805        update: &[u8],
6806    ) -> Result<String, ClientError> {
6807        let prior = self.storage_failure.borrow_mut().take();
6808        let outcome = (|| -> Result<String, ClientError> {
6809            let current = self
6810                .crdt_column_bytes(table, row_id, column)?
6811                .unwrap_or_default();
6812            let next = crate::crdt::apply_update(&current, update)?;
6813            self.perform_crdt_push_update(table, row_id, column, &next)
6814        })();
6815        self.finish_authoring(prior, outcome)
6816    }
6817
6818    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
6819    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
6820    /// enqueue it. The server merges the crdt column and leaves every other
6821    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
6822    /// clobber a concurrent edit to another column. A locally absent row
6823    /// records the same partial operation; the server answers per §6.2.
6824    #[cfg(feature = "crdt-yjs")]
6825    fn perform_crdt_push_update(
6826        &mut self,
6827        table: &str,
6828        row_id: &str,
6829        column: &str,
6830        crdt_bytes: &[u8],
6831    ) -> Result<String, ClientError> {
6832        let mut bytes_obj = Map::new();
6833        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
6834        let mut partial = Map::new();
6835        partial.insert(column.to_owned(), Value::Object(bytes_obj));
6836        self.perform_patch(table, row_id, partial, None)
6837    }
6838
6839    /// Run an arbitrary read-only SQL query against the local database and
6840    /// return each row as a `column-name → JSON value` map. This is the seam
6841    /// the React `useSyncQuery` live-query API needs (it takes app-authored
6842    /// SQL over the visible tables/views, not a fixed table read like
6843    /// [`read_rows`]).
6844    ///
6845    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
6846    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
6847    /// same envelope the command surface uses everywhere else. Output BLOB
6848    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
6849    ///
6850    /// The result column typing is dynamic (SQLite's stored affinity), because
6851    /// arbitrary SQL can alias, join, and compute — there is no schema column
6852    /// to consult per output cell, unlike [`read_rows`].
6853    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
6854        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
6855    }
6856
6857    /// Repository benchmark and conformance access, present only with `bench-internals`.
6858    #[cfg(feature = "bench-internals")]
6859    #[doc(hidden)]
6860    pub fn benchmark_connection(&mut self) -> &mut Connection {
6861        &mut self.conn
6862    }
6863
6864    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
6865    #[cfg(feature = "bench-internals")]
6866    #[doc(hidden)]
6867    pub fn benchmark_phases(
6868        &self,
6869        enabled: Option<bool>,
6870        reset: bool,
6871    ) -> Result<Option<Value>, String> {
6872        self.benchmark_phases.configure(enabled, reset)?;
6873        Ok(self.benchmark_phases.snapshot())
6874    }
6875
6876    /// One serialized, read-only authority observation, including durable coverage.
6877    pub fn authority_snapshot(&self) -> Result<Value, String> {
6878        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
6879        if self.limits.authority_reads.is_empty() {
6880            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
6881        }
6882        self.conn
6883            .execute_batch("SAVEPOINT authority_snapshot")
6884            .map_err(|e| e.to_string())?;
6885        let result = (|| {
6886            let revision = self
6887                .get_meta(LOCAL_REVISION_KEY)?
6888                .ok_or("sync.local_corrupt: authority revision is missing")?;
6889            let parsed_revision = revision
6890                .parse::<u64>()
6891                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
6892            if parsed_revision.to_string() != revision {
6893                return Err("sync.local_corrupt: authority revision is invalid".into());
6894            }
6895            let mut tables = Vec::new();
6896            for read in &self.limits.authority_reads {
6897                let table = self
6898                    .schema
6899                    .table(&read.table)
6900                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
6901                let scalar = |column: &str, value: &Value| -> Result<String, String> {
6902                    let ty = table
6903                        .columns
6904                        .iter()
6905                        .find(|c| c.name == column)
6906                        .expect("validated authority column")
6907                        .ty;
6908                    if ty == ColumnType::Boolean {
6909                        if let Some(value) = value.as_bool() {
6910                            return Ok(value.to_string());
6911                        }
6912                        return match value.as_i64() {
6913                            Some(0) => Ok("false".into()),
6914                            Some(1) => Ok("true".into()),
6915                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
6916                        };
6917                    }
6918                    match value {
6919                        Value::String(s) => Ok(s.clone()),
6920                        Value::Number(n) => Ok(n.to_string()),
6921                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
6922                            .get("$bigint")
6923                            .and_then(Value::as_str)
6924                            .map(str::to_owned)
6925                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
6926                        _ => {
6927                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
6928                        }
6929                    }
6930                };
6931                let matches = |row: &QueryRow| -> Result<bool, String> {
6932                    for (variable, values) in &read.scopes {
6933                        let column = table
6934                            .scope_column(variable)
6935                            .expect("validated authority scope");
6936                        let value = row
6937                            .get(column)
6938                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
6939                        if value.is_null() || !values.contains(&scalar(column, value)?) {
6940                            return Ok(false);
6941                        }
6942                    }
6943                    Ok(true)
6944                };
6945                let selected = read
6946                    .columns
6947                    .iter()
6948                    .map(|c| quote_ident(c))
6949                    .collect::<Vec<_>>()
6950                    .join(",");
6951                let mut intent = BTreeSet::new();
6952                let mut relevant = BTreeSet::new();
6953                for row in query_connection(&self.conn,
6954                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
6955                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
6956                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
6957                    intent.insert(id.clone());
6958                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
6959                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
6960                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
6961                    }
6962                }
6963                let mut rows = BTreeMap::new();
6964                for full in [base_table(&read.table), visible_table(&read.table)] {
6965                    for mut values in query_connection(
6966                        &self.conn,
6967                        &format!(
6968                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
6969                        ),
6970                        &[],
6971                    )
6972                    .map_err(|e| e.to_string())?
6973                    {
6974                        if !matches(&values)? {
6975                            continue;
6976                        }
6977                        let id = scalar(
6978                            &table.primary_key,
6979                            values
6980                                .get(&table.primary_key)
6981                                .expect("selected primary key"),
6982                        )?;
6983                        relevant.insert(id.clone());
6984                        if full == base_table(&read.table) {
6985                            let version = values
6986                                .remove("authority_version")
6987                                .expect("selected version");
6988                            let version = version
6989                                .as_i64()
6990                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
6991                            if version >= 0 {
6992                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
6993                            }
6994                        }
6995                    }
6996                }
6997                intent.retain(|id| relevant.contains(id));
6998                let rows = rows.into_values().collect::<Vec<_>>();
6999                // Every unit covering the table, as its (unit, table) pair (§4.4).
7000                let replica = self.stored(replica_cursor(&self.conn))?;
7001                let persisted = self
7002                    .stored(load_units(&self.conn))?
7003                    .into_iter()
7004                    .filter(|unit| covers(table, &unit.scopes))
7005                    .map(|unit| {
7006                        let effective = unit.echo.as_deref().and_then(|echo| {
7007                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7008                        });
7009                        let requested = read
7010                            .scopes
7011                            .keys()
7012                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7013                            .collect::<Map<_, _>>();
7014                        let complete = effective
7015                            .as_ref()
7016                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7017                            && unit.status == UnitStatus::Active
7018                            && !unit.bootstrapping();
7019                        let exposed = effective.map(|scopes| {
7020                            read.scopes
7021                                .keys()
7022                                .map(|v| {
7023                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7024                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7025                                })
7026                                .collect::<Map<_, _>>()
7027                        });
7028                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7029                    })
7030                    .collect::<Vec<_>>();
7031                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7032                    match selectors.split_first() {
7033                        None => !candidates.is_empty(),
7034                        Some(((variable, values), rest)) => values.iter().all(|value| {
7035                            covered(
7036                                rest,
7037                                &candidates
7038                                    .iter()
7039                                    .copied()
7040                                    .filter(|p| {
7041                                        p["effectiveScopes"][variable.as_str()]
7042                                            .as_array()
7043                                            .is_some_and(|held| {
7044                                                held.contains(&Value::String(value.clone()))
7045                                                    || held.contains(&Value::String("*".into()))
7046                                            })
7047                                    })
7048                                    .collect::<Vec<_>>(),
7049                            )
7050                        }),
7051                    }
7052                }
7053                let coverage = if covered(
7054                    &read.scopes.iter().collect::<Vec<_>>(),
7055                    &persisted
7056                        .iter()
7057                        .filter(|p| p["complete"] == true)
7058                        .collect::<Vec<_>>(),
7059                ) {
7060                    "complete"
7061                } else if persisted.iter().any(|p| {
7062                    p["status"] == "active"
7063                        && read.scopes.iter().all(|(v, values)| {
7064                            values.iter().any(|value| {
7065                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7066                                    held.contains(&Value::String(value.clone()))
7067                                })
7068                            })
7069                        })
7070                }) {
7071                    "pending"
7072                } else {
7073                    "missing"
7074                };
7075                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7076            }
7077            Ok(
7078                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7079            )
7080        })();
7081        match result {
7082            Ok(snapshot) => {
7083                self.conn
7084                    .execute_batch("RELEASE authority_snapshot")
7085                    .map_err(|e| e.to_string())?;
7086                Ok(snapshot)
7087            }
7088            Err(error) => {
7089                self.conn
7090                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7091                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7092                Err(error)
7093            }
7094        }
7095    }
7096
7097    /// Rows, coverage, and local revision from one SQLite read snapshot.
7098    /// §7.5 atomic snapshot read. An `owner` records a failure in
7099    /// `diagnostics_snapshot().query_failures` until its next successful read.
7100    pub fn query_snapshot(
7101        &mut self,
7102        sql: &str,
7103        params: &[QueryValue],
7104        coverage: &[WindowCoverage],
7105        owner: Option<QueryOwner<'_>>,
7106    ) -> Result<QuerySnapshot, String> {
7107        let result = snapshot_connection(&self.conn, sql, params, coverage);
7108        if let Some(owner) = owner {
7109            self.record_query_read(owner, result.as_ref().err());
7110        }
7111        result.map_err(|failure| failure.to_string())
7112    }
7113
7114    /// §7.6: record the outcome of one owned snapshot read, including reads a
7115    /// host ran on a separate connection. A success removes the owner's entry;
7116    /// an identical failure leaves it unchanged; a changed failure moves last.
7117    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7118        let position = self
7119            .query_failures
7120            .iter()
7121            .position(|entry| entry.id == owner.id);
7122        let Some(failure) = failure else {
7123            if let Some(index) = position {
7124                self.query_failures.remove(index);
7125            }
7126            return;
7127        };
7128        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7129            .into_iter()
7130            .map(str::to_owned)
7131            .collect();
7132        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7133        if position.is_some_and(|index| {
7134            let entry = &self.query_failures[index];
7135            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7136        }) {
7137            return;
7138        }
7139        if let Some(index) = position {
7140            self.query_failures.remove(index);
7141        }
7142        self.query_failures.push(DiagnosticQueryFailure {
7143            id: owner.id.to_owned(),
7144            tables,
7145            code,
7146            sqlite_code: failure.sqlite_code,
7147            at_ms: self.clock_now_ms(),
7148        });
7149        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7150            self.query_failures.remove(0);
7151        }
7152    }
7153
7154    // -- request building ---------------------------------------------------------
7155
7156    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7157        #[cfg(feature = "bench-internals")]
7158        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7159        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7160            Box::new(SyncOutcome::Failed {
7161                error_code: "storage.failed".into(),
7162                message,
7163                details: None,
7164                recommended_action: None,
7165            })
7166        })?;
7167        let max_commits = self.limits.max_push_commits_per_request;
7168        let max_ops = self
7169            .limits
7170            .max_push_operations_per_request
7171            .unwrap_or(PUSH_OPS_PER_REQUEST);
7172        let byte_cap = self.limits.max_push_request_bytes;
7173        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7174        // bare URL — capability negotiation, decided per transport.
7175        let accept = self.limits.accept.unwrap_or(if url_capable {
7176            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7177        } else {
7178            DEFAULT_ACCEPT
7179        });
7180        let storage_failed = |message: String| {
7181            Box::new(SyncOutcome::Failed {
7182                error_code: "storage.failed".into(),
7183                message,
7184                details: None,
7185                recommended_action: None,
7186            })
7187        };
7188        let header = Frame::ReqHeader {
7189            client_id: self.client_id.clone(),
7190            schema_version: self.schema.version,
7191            log_epoch: log_epoch.clone(),
7192            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7193        };
7194        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7195        // which stays ahead of every push, pull, unit and progress frame; the
7196        // byte-capacity probes below measure this exact sequence.
7197        let mut lead = vec![header];
7198        lead.extend(
7199            self.round_failure
7200                .iter()
7201                .map(|code| Frame::RoundFailure { code: code.clone() }),
7202        );
7203        let registered: Vec<UnitRecord> = self
7204            .stored(load_units(&self.conn))
7205            .map_err(storage_failed)?
7206            .into_iter()
7207            .filter(|unit| unit.status == UnitStatus::Active)
7208            .collect();
7209        let registration_digest = ssp2::registration::registration_digest(
7210            registered
7211                .iter()
7212                .map(|unit| scope_pairs(&unit.scopes))
7213                .collect::<Vec<_>>()
7214                .iter()
7215                .map(Vec::as_slice),
7216        );
7217        let echo_digest = self
7218            .get_meta(ECHO_DIGEST_KEY)
7219            .map_err(storage_failed)?
7220            .filter(|hex| !hex.is_empty())
7221            .map(|hex| {
7222                crate::values::hex_to_bytes(&hex)
7223                    .ok()
7224                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7225                    .ok_or_else(|| {
7226                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7227                    })
7228            })
7229            .transpose()
7230            .map_err(storage_failed)?;
7231        let pull = Frame::PullHeader {
7232            limit_commits: self.limits.limit_commits.unwrap_or(0),
7233            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7234            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7235            accept,
7236            cursor: self
7237                .stored(replica_cursor(&self.conn))
7238                .map_err(storage_failed)?,
7239            registration_digest,
7240            echo_digest,
7241        };
7242        let registration_digest = bytes_to_hex(&registration_digest);
7243        // §4.1: the list travels when it changed since the server stored it,
7244        // after `unregistered` and after a reset (both clear the stored digest).
7245        let travels = self
7246            .get_meta(REGISTERED_DIGEST_KEY)
7247            .map_err(storage_failed)?
7248            .as_deref()
7249            != Some(registration_digest.as_str());
7250        let mut sub_frames: Vec<Frame> = Vec::new();
7251        if travels {
7252            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7253                scopes: scope_pairs(&unit.scopes),
7254            }));
7255        }
7256        // §4.3: one PROGRESS per catching-up unit, in registration order.
7257        sub_frames.extend(registered.iter().filter_map(|unit| {
7258            Some(Frame::Progress {
7259                id: unit.id.clone(),
7260                cursor: unit.cursor?,
7261                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7262            })
7263        }));
7264        // The fixed frames (order-independent size) and the header+pull probe
7265        // both encode canonically, so the byte budget is the exact wire size.
7266        let fixed_bytes = match byte_cap {
7267            Some(_) => {
7268                let mut frames = lead.clone();
7269                frames.push(pull.clone());
7270                frames.extend(sub_frames.iter().cloned());
7271                encode_message(&Message {
7272                    wire_version: WIRE_VERSION,
7273                    msg_kind: MsgKind::Request,
7274                    frames,
7275                })
7276                .len()
7277            }
7278            None => 0,
7279        };
7280        if let Some(cap) = byte_cap {
7281            if fixed_bytes > cap {
7282                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7283            }
7284        }
7285        let base_bytes = match byte_cap {
7286            Some(_) => {
7287                let mut frames = lead.clone();
7288                frames.push(pull.clone());
7289                encode_message(&Message {
7290                    wire_version: WIRE_VERSION,
7291                    msg_kind: MsgKind::Request,
7292                    frames,
7293                })
7294                .len()
7295            }
7296            None => 0,
7297        };
7298        let total_outbox = if log_epoch.is_some() {
7299            self.outbox.len()
7300        } else {
7301            0
7302        };
7303        let mut push_frames: Vec<Frame> = Vec::new();
7304        let mut pushed_ids = Vec::new();
7305        let mut ops_in_request = 0usize;
7306        let mut bytes_in_request = 0usize;
7307        let mut deferred_commits = 0usize;
7308        #[cfg(feature = "bench-internals")]
7309        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7310        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7311            // §6.1/§7.1: the commit-count and operation caps are hard. A
7312            // first commit over the operation cap is a typed capacity error
7313            // and stays queued; a later one defers with the complete suffix.
7314            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7315                deferred_commits = total_outbox - index;
7316                break;
7317            }
7318            if ops_in_request + commit.ops.len() > max_ops {
7319                if push_frames.is_empty() {
7320                    return Err(Self::push_capacity_outcome(
7321                        "operations",
7322                        max_ops,
7323                        commit.ops.len(),
7324                        Some(commit.client_commit_id.as_str()),
7325                    ));
7326                }
7327                deferred_commits = total_outbox - index;
7328                break;
7329            }
7330            // §7.1 chained base: the commit waits for the change of the commit
7331            // it chains on, which supplies its rebased version.
7332            if self.outbox[..index]
7333                .iter()
7334                .flat_map(|earlier| &earlier.ops)
7335                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7336            {
7337                deferred_commits = total_outbox - index;
7338                break;
7339            }
7340            let operations = commit
7341                .ops
7342                .iter()
7343                .map(|op| {
7344                    let payload = match self
7345                        .schema
7346                        .table(&op.table)
7347                        .and_then(|table| Some((table, op.presented_values(table)?)))
7348                    {
7349                        Some((table, values)) => {
7350                            // §0: outbox entries encode at send time with the
7351                            // current codec (validated at mutate()). §6.1: the
7352                            // values map's keys are the sparse presence set; an
7353                            // insert-intent operation presents every column (§7.1).
7354                            // §5.11: encrypted columns are encrypted here, with
7355                            // an absent key-id selector read from the stored row.
7356                            // `sync_inner` runs `drop_unencodable_outbox` first,
7357                            // so an encrypt failure cannot reach this encode; the
7358                            // author seam validated every other failure.
7359                            let fallback =
7360                                self.stored_key_fallback(table, &op.row_id)
7361                                    .map_err(|message| {
7362                                        Box::new(SyncOutcome::Failed {
7363                                            error_code: "storage.failed".into(),
7364                                            message,
7365                                            details: None,
7366                                            recommended_action: None,
7367                                        })
7368                                    })?;
7369                            encode_sparse_row_json(
7370                                table,
7371                                &op.row_id,
7372                                &values,
7373                                &self.encryption,
7374                                fallback.as_ref(),
7375                            )
7376                            .ok()
7377                        }
7378                        None => None,
7379                    };
7380                    Ok(ssp2::model::Operation {
7381                        table: op.table.clone(),
7382                        row_id: op.row_id.clone(),
7383                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7384                        base_version: op.base_version,
7385                        payload,
7386                    })
7387                })
7388                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7389            // §6.3.2, §7.1: the commit's outbox sequence.
7390            let client_seq = self
7391                .conn
7392                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7393                .and_then(|mut statement| {
7394                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7395                })
7396                .map_err(|error| {
7397                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7398                })?;
7399            let frame = Frame::PushCommit {
7400                client_commit_id: commit.client_commit_id.clone(),
7401                client_seq,
7402                operations,
7403            };
7404            let frame_bytes = match byte_cap {
7405                Some(_) => {
7406                    let mut frames = lead.clone();
7407                    frames.push(frame.clone());
7408                    frames.push(pull.clone());
7409                    encode_message(&Message {
7410                        wire_version: WIRE_VERSION,
7411                        msg_kind: MsgKind::Request,
7412                        frames,
7413                    })
7414                    .len()
7415                        - base_bytes
7416                }
7417                None => 0,
7418            };
7419            // The head is always encoded so the caller can report the full
7420            // projected request size; a later commit defers once it no longer
7421            // fits.
7422            if let Some(cap) = byte_cap {
7423                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7424                    deferred_commits = total_outbox - index;
7425                    break;
7426                }
7427            }
7428            push_frames.push(frame);
7429            pushed_ids.push(commit.client_commit_id.clone());
7430            ops_in_request += commit.ops.len();
7431            bytes_in_request += frame_bytes;
7432        }
7433        #[cfg(feature = "bench-internals")]
7434        drop(outbox_phase);
7435        if let Some(cap) = byte_cap {
7436            if fixed_bytes + bytes_in_request > cap {
7437                // The head alone does not fit this budget; the commit stays
7438                // queued and no transport round is attempted.
7439                let blocked = if push_frames.is_empty() {
7440                    None
7441                } else {
7442                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7443                };
7444                return Err(Self::push_capacity_outcome(
7445                    "bytes",
7446                    cap,
7447                    fixed_bytes + bytes_in_request,
7448                    blocked,
7449                ));
7450            }
7451        }
7452        let mut frames = lead;
7453        frames.extend(push_frames);
7454        frames.push(pull);
7455        frames.extend(sub_frames);
7456        let message = Message {
7457            wire_version: WIRE_VERSION,
7458            msg_kind: MsgKind::Request,
7459            frames,
7460        };
7461        Ok((
7462            message,
7463            RequestMeta {
7464                pushed_ids,
7465                registration_digest,
7466                accept,
7467                deferred_commits,
7468            },
7469        ))
7470    }
7471
7472    /// §7.1: a configured push capacity limit cannot carry this request.
7473    fn push_capacity_outcome(
7474        kind: &str,
7475        limit: usize,
7476        size: usize,
7477        client_commit_id: Option<&str>,
7478    ) -> Box<SyncOutcome> {
7479        let mut details = Map::new();
7480        details.insert("kind".to_owned(), Value::from(kind));
7481        details.insert("limit".to_owned(), Value::from(limit as u64));
7482        details.insert("size".to_owned(), Value::from(size as u64));
7483        if let Some(id) = client_commit_id {
7484            details.insert("clientCommitId".to_owned(), Value::from(id));
7485        }
7486        Box::new(SyncOutcome::Failed {
7487            error_code: "client.push_request_too_large".into(),
7488            message: "the push request exceeds a configured capacity limit".into(),
7489            details: Some(Value::Object(details)),
7490            recommended_action: None,
7491        })
7492    }
7493
7494    // -- sync -------------------------------------------------------------------
7495
7496    /// Cloneable observation handle; no client lock is needed to read it.
7497    pub fn progress(&self) -> ProgressObserver {
7498        self.progress.clone()
7499    }
7500
7501    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7502        let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7503            Ok(prepared) => prepared,
7504            Err(outcome) => return *outcome,
7505        };
7506        // The round's network deadline rides `PreparedSyncRound` and is scoped
7507        // to each exchange, so uploads, continuations, the main request, and
7508        // every segment fetch share one monotonic anchor without leaving
7509        // deadline state on a reused transport.
7510        let mut next = prepared;
7511        loop {
7512            match self.apply_sync_round(next.exchange(transport)) {
7513                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7514                crate::AppliedSyncRound::Complete {
7515                    outcome, controls, ..
7516                } => {
7517                    if self.transport_enabled {
7518                        for text in controls {
7519                            let _ = transport.realtime_send(&text);
7520                        }
7521                    } else {
7522                        self.disconnect_realtime(transport);
7523                    }
7524                    return outcome;
7525                }
7526            }
7527        }
7528    }
7529
7530    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7531        if let SyncOutcome::Failed {
7532            error_code,
7533            message,
7534            details,
7535            ..
7536        } = &mut outcome
7537        {
7538            if let Some((failure, code)) = self
7539                .storage_failure
7540                .borrow()
7541                .as_ref()
7542                .and_then(|failure| failure.code.map(|code| (failure, code)))
7543            {
7544                *error_code = code.to_owned();
7545                *message = match code {
7546                    "client.storage_busy" => "local SQLite storage is busy",
7547                    "client.storage_full" => "local SQLite storage is full",
7548                    "client.storage_io" => "local SQLite storage I/O failed",
7549                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7550                    _ => unreachable!("unrecognized classified storage failure"),
7551                }
7552                .to_owned();
7553                *details = failure.details();
7554            }
7555        }
7556        let details = match &outcome {
7557            SyncOutcome::Failed { details, .. } => details.as_ref(),
7558            _ => None,
7559        };
7560        let cause_kind = details
7561            .and_then(|details| details.get("causeKind"))
7562            .and_then(Value::as_str)
7563            .and_then(|cause| match cause {
7564                "network" => Some(crate::TransportCauseKind::Network),
7565                "connect" => Some(crate::TransportCauseKind::Connect),
7566                "tls" => Some(crate::TransportCauseKind::Tls),
7567                "io" => Some(crate::TransportCauseKind::Io),
7568                "request" => Some(crate::TransportCauseKind::Request),
7569                "protocol" => Some(crate::TransportCauseKind::Protocol),
7570                "timeout" => Some(crate::TransportCauseKind::Timeout),
7571                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7572                "aborted" => Some(crate::TransportCauseKind::Aborted),
7573                "body" => Some(crate::TransportCauseKind::Body),
7574                "status" => Some(crate::TransportCauseKind::Status),
7575                "redirect" => Some(crate::TransportCauseKind::Redirect),
7576                "unknown" => Some(crate::TransportCauseKind::Unknown),
7577                _ => None,
7578            });
7579        let http_status = if matches!(
7580            cause_kind,
7581            Some(
7582                crate::TransportCauseKind::Status
7583                    | crate::TransportCauseKind::Body
7584                    | crate::TransportCauseKind::Redirect
7585            )
7586        ) {
7587            details
7588                .and_then(|details| details.get("httpStatus"))
7589                .and_then(Value::as_f64)
7590                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7591                .map(|status| status as u16)
7592        } else {
7593            None
7594        };
7595        let retry_delay_ms = self.round_retry_delay_ms;
7596        self.progress.update(|p| match &outcome {
7597            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7598                if report.failed.is_empty() =>
7599            {
7600                p.state = ProgressState::Complete
7601            }
7602            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7603                p.state = ProgressState::Failed;
7604                p.error_code = Some("sync.scope_revoked".into());
7605            }
7606            SyncOutcome::Failed { error_code, .. } => {
7607                p.state = ProgressState::Failed;
7608                p.error_code = Some(Self::diagnostic_code(error_code));
7609                p.cause_kind = cause_kind;
7610                p.http_status = http_status;
7611                p.retry_delay_ms = retry_delay_ms;
7612            }
7613            SyncOutcome::RealtimeUnavailable { .. } => {
7614                p.state = ProgressState::Failed;
7615                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7616                p.retry_delay_ms = retry_delay_ms;
7617            }
7618        });
7619        let completed_at_ms = self.clock_now_ms();
7620        self.last_round = Some(match &outcome {
7621            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7622                status: "succeeded".to_owned(),
7623                started_at_ms,
7624                completed_at_ms,
7625                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7626                counters: Some(DiagnosticRoundCounters {
7627                    pushed: report.pushed,
7628                    applied: report.applied.len(),
7629                    rejected: report.rejected.len(),
7630                    retryable: report.retryable.len(),
7631                    conflicts: report.conflicts,
7632                    commits_applied: report.commits_applied,
7633                    segment_rows_applied: report.segment_rows_applied,
7634                    bootstrapping: report.bootstrapping.len(),
7635                    resets: report.resets.len(),
7636                    revoked: report.revoked.len(),
7637                    failed: report.failed.len(),
7638                    deferred_commits: report.deferred_commits,
7639                }),
7640                error_code: None,
7641                cause_kind: None,
7642                http_status: None,
7643            },
7644            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7645                status: "failed".to_owned(),
7646                started_at_ms,
7647                completed_at_ms,
7648                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7649                counters: None,
7650                error_code: Some(Self::diagnostic_code(error_code)),
7651                cause_kind,
7652                http_status,
7653            },
7654            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7655                status: "failed".to_owned(),
7656                started_at_ms,
7657                completed_at_ms,
7658                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7659                counters: None,
7660                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7661                cause_kind: None,
7662                http_status: None,
7663            },
7664        });
7665        outcome
7666    }
7667
7668    /// Capture one request on the client owner. Mutations made after this
7669    /// call remain in the outbox for a later round; no network I/O occurs here.
7670    pub fn prepare_sync_round(
7671        &mut self,
7672        url_capable: bool,
7673    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7674        if !self.transport_enabled {
7675            return Err(Box::new(SyncOutcome::Failed {
7676                error_code: "sync.offline".into(),
7677                message: "transport gate is closed".into(),
7678                details: None,
7679                recommended_action: None,
7680            }));
7681        }
7682        if self.active_round.is_some() {
7683            return Err(Box::new(SyncOutcome::Failed {
7684                error_code: "client.round_in_progress".into(),
7685                message: "a sync round is already in flight".into(),
7686                details: None,
7687                recommended_action: None,
7688            }));
7689        }
7690        self.storage_failure.borrow_mut().take();
7691        self.progress.start();
7692        self.round_retry_delay_ms = None;
7693        let started_at_ms = self.clock_now_ms();
7694        let prepared = (|| {
7695            if self.stopped {
7696                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7697                    schema_floor: self.schema_floor.clone(),
7698                    ..SyncReport::default()
7699                })));
7700            }
7701            if self.security_preflight {
7702                return Err(Box::new(SyncOutcome::Failed {
7703                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7704                    message: "security preflight must complete before syncing".into(),
7705                    details: None,
7706                    recommended_action: None,
7707                }));
7708            }
7709            self.set_sync_needed(false, false);
7710            self.drain_pending_evictions()
7711                .and_then(|_| self.drop_incompatible_outbox())
7712                .and_then(|_| self.drop_unencodable_outbox(true, None))
7713                .map_err(|message| {
7714                    Box::new(SyncOutcome::Failed {
7715                        error_code: "storage.failed".into(),
7716                        message,
7717                        details: None,
7718                        recommended_action: None,
7719                    })
7720                })?;
7721            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7722                Box::new(SyncOutcome::Failed {
7723                    error_code: "storage.failed".into(),
7724                    message,
7725                    details: None,
7726                    recommended_action: None,
7727                })
7728            })?;
7729            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
7730                self.pending_blob_uploads().map_err(|e| {
7731                    Box::new(SyncOutcome::Failed {
7732                        error_code: e.code,
7733                        message: e.message,
7734                        details: e.details,
7735                        recommended_action: None,
7736                    })
7737                })?
7738            } else {
7739                Vec::new()
7740            };
7741            if self.realtime_state != RealtimeState::Connected
7742                && self.realtime_policy == RealtimePolicy::Required
7743            {
7744                let retry_delay_ms = self.schedule_background_retry();
7745                self.realtime_retry_delay_ms = Some(retry_delay_ms);
7746                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
7747                    state: self.realtime_state,
7748                    reason_code: self.realtime_reason_code.clone(),
7749                    retry_delay_ms,
7750                }));
7751            }
7752            let (message, meta) = self.build_request(url_capable)?;
7753            let id = uuid::Uuid::new_v4();
7754            self.active_round = Some(id);
7755            let phase = if uploads.is_empty() {
7756                crate::round::RoundPhase::Sync
7757            } else {
7758                crate::round::RoundPhase::UploadGrant
7759            };
7760            Ok(crate::PreparedSyncRound {
7761                id,
7762                started_at_ms,
7763                message,
7764                meta,
7765                uploads: uploads.into(),
7766                phase,
7767                encoded: None,
7768                continuation: None,
7769                completed: None,
7770                realtime: self.realtime_state == RealtimeState::Connected,
7771                // §5.5: the owning unit's scope map, each value a one-element list.
7772                scopes: self
7773                    .stored(load_units(&self.conn))
7774                    .map_err(|message| {
7775                        Box::new(SyncOutcome::Failed {
7776                            error_code: "storage.failed".into(),
7777                            message,
7778                            details: None,
7779                            recommended_action: None,
7780                        })
7781                    })?
7782                    .into_iter()
7783                    .map(|unit| {
7784                        (
7785                            unit.id,
7786                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
7787                        )
7788                    })
7789                    .collect(),
7790                #[cfg(feature = "bench-internals")]
7791                benchmark_phases: self.benchmark_phases.clone(),
7792                progress: self.progress.clone(),
7793                fixed_now: self.now_ms,
7794                round_deadline_at: None,
7795            })
7796        })();
7797        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
7798    }
7799
7800    /// Discard the reply of an old authorization or registration context. Its
7801    /// submitted commits stay durable and can be retried with the same ids.
7802    pub fn cancel_sync_round(&mut self) {
7803        self.active_round = None;
7804    }
7805
7806    /// Apply captured network results on the owner, preserving all normal
7807    /// version, schema, content-address and cursor checks. No network I/O.
7808    pub fn apply_sync_round(
7809        &mut self,
7810        completed: crate::CompletedSyncRound,
7811    ) -> crate::AppliedSyncRound {
7812        let mut prepared = completed.prepared;
7813        if self.active_round != Some(prepared.id) || self.security_preflight {
7814            return crate::AppliedSyncRound::Complete {
7815                outcome: SyncOutcome::Failed {
7816                    error_code: "client.round_cancelled".into(),
7817                    message: "sync round belongs to a released client context".into(),
7818                    details: None,
7819                    recommended_action: None,
7820                },
7821                controls: Vec::new(),
7822                more: false,
7823                bootstrap_advanced: false,
7824            };
7825        }
7826        self.storage_failure.borrow_mut().take();
7827        let (response, mut downloads, transport_failed) = match completed.exchange {
7828            crate::round::ExchangeResult::Upload { id, result } => {
7829                let result = match result {
7830                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
7831                    // is the body's final outcome. The commits naming it take
7832                    // the incompatible-commit path with the server's code and
7833                    // leave this round's request; the round continues.
7834                    Err(error)
7835                        if error
7836                            .details
7837                            .as_ref()
7838                            .and_then(|details| details.get("retryable"))
7839                            == Some(&Value::Bool(false)) =>
7840                    {
7841                        self.conn
7842                            .prepare(
7843                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
7844                            )
7845                            .and_then(|mut statement| {
7846                                statement
7847                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
7848                                    .collect::<Result<HashSet<_>, _>>()
7849                            })
7850                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
7851                            .and_then(|refused| {
7852                                let commits = self
7853                                    .outbox
7854                                    .iter()
7855                                    .filter(|commit| refused.contains(&commit.client_commit_id))
7856                                    .cloned()
7857                                    .collect::<Vec<_>>();
7858                                if !commits.is_empty() {
7859                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
7860                                }
7861                                prepared.message.frames.retain(|frame| {
7862                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
7863                                        if refused.contains(client_commit_id))
7864                                });
7865                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
7866                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
7867                                Ok(())
7868                            })
7869                            .map_err(|message| TransportError::new("client.failed", message))
7870                    }
7871                    result => result,
7872                };
7873                let result = result.and_then(|()| {
7874                    self.conn
7875                        .execute(
7876                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
7877                            rusqlite::params![id],
7878                        )
7879                        .map(|_| ())
7880                        .map_err(|error| {
7881                            TransportError::new(
7882                                "client.failed",
7883                                Self::sqlite_failure(&self.storage_failure, error),
7884                            )
7885                        })
7886                });
7887                match result {
7888                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
7889                    Err(TransportError {
7890                        code,
7891                        message,
7892                        details,
7893                    }) => {
7894                        self.active_round = None;
7895                        if Self::unreachable_transport(&details) {
7896                            // The round's work is still owed; the host's
7897                            // connectivity resume runs it.
7898                            self.set_sync_needed(true, false);
7899                        } else if Self::retryable_failure_code(&code) {
7900                            self.schedule_background_retry();
7901                        }
7902                        let recommended_action = recommended_action_of(&details);
7903                        let outcome = self.finish_sync_round(
7904                            prepared.started_at_ms,
7905                            SyncOutcome::Failed {
7906                                error_code: code,
7907                                message,
7908                                details,
7909                                recommended_action,
7910                            },
7911                        );
7912                        return crate::AppliedSyncRound::Complete {
7913                            outcome,
7914                            controls: Vec::new(),
7915                            more: false,
7916                            bootstrap_advanced: false,
7917                        };
7918                    }
7919                }
7920            }
7921            crate::round::ExchangeResult::Reply {
7922                response,
7923                downloads,
7924                transport_failed,
7925            } => (response, downloads, transport_failed),
7926        };
7927        self.active_round = None;
7928        let before = self.resume_tokens();
7929        let outcome = (|| {
7930            let response = match response {
7931                Ok(response) => {
7932                    // §1.5: a complete, decodable response ends the report, an
7933                    // in-band ERROR included (the server already recorded it).
7934                    self.round_failure = None;
7935                    response
7936                }
7937                Err(TransportError {
7938                    code,
7939                    message,
7940                    details,
7941                }) => {
7942                    // §1.5: the server cannot see a round whose transport
7943                    // failed or whose response never decoded, so its code rides
7944                    // the following requests. A code the frame grammar rejects
7945                    // is not reportable, and the previous code stays.
7946                    if ssp2::model::valid_round_failure_code(&code) {
7947                        self.round_failure = Some(code.clone());
7948                    }
7949                    if transport_failed {
7950                        if prepared.realtime {
7951                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
7952                        }
7953                        if let Err(message) = self.record_lease_error(&code) {
7954                            return SyncOutcome::Failed {
7955                                error_code: "storage.failed".to_owned(),
7956                                message,
7957                                details: None,
7958                                recommended_action: None,
7959                            };
7960                        }
7961                    }
7962                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
7963                    {
7964                        let delay = self.schedule_background_retry();
7965                        if prepared.realtime && transport_failed {
7966                            self.realtime_retry_delay_ms = Some(delay);
7967                        }
7968                    }
7969                    let recommended_action = recommended_action_of(&details);
7970                    return SyncOutcome::Failed {
7971                        error_code: code,
7972                        message,
7973                        details,
7974                        recommended_action,
7975                    };
7976                }
7977            };
7978            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
7979            if let SyncOutcome::Ok(report) = &mut outcome {
7980                report.deferred_commits = prepared.meta.deferred_commits;
7981            }
7982            match &outcome {
7983                SyncOutcome::Ok(_) => self.reset_background_retry(),
7984                SyncOutcome::Failed {
7985                    error_code,
7986                    details,
7987                    ..
7988                } if Self::retryable_failure_code(error_code)
7989                    && !Self::unreachable_transport(details) =>
7990                {
7991                    self.schedule_background_retry();
7992                }
7993                _ => {}
7994            }
7995            if prepared.meta.deferred_commits > 0 {
7996                self.set_sync_needed(true, true);
7997            }
7998            self.previous_version_lifetime_check();
7999            outcome
8000        })();
8001        match &outcome {
8002            // §7.1: a unit still catching up needs another round, also when the
8003            // drive that started it ends early.
8004            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8005                self.set_sync_needed(true, false)
8006            }
8007            // The round's work is still owed; the host's connectivity resume
8008            // runs it.
8009            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8010                self.set_sync_needed(true, false)
8011            }
8012            _ => {}
8013        }
8014        let more = self.transport_enabled
8015            && matches!(&outcome, SyncOutcome::Ok(report)
8016                if report.schema_floor.is_none()
8017                    && (!report.bootstrapping.is_empty()
8018                        || report.commits_applied > 0
8019                        || report.segment_rows_applied > 0
8020                        || !report.resets.is_empty()
8021                        || report.deferred_commits > 0
8022                        || self.sync_needed));
8023        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8024            && before != self.resume_tokens();
8025        crate::AppliedSyncRound::Complete {
8026            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8027            controls: if self.transport_enabled {
8028                downloads.controls
8029            } else {
8030                Vec::new()
8031            },
8032            more,
8033            bootstrap_advanced,
8034        }
8035    }
8036
8037    /// Every unit's resume token: a bootstrap that advanced changes one (§4.7).
8038    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>)>> {
8039        load_units(&self.conn).ok().map(|units| {
8040            units
8041                .into_iter()
8042                .map(|unit| (unit.id, unit.bootstrap_state))
8043                .collect()
8044        })
8045    }
8046
8047    /// §7.8: one idle drive's default total deadline, the TS client's value.
8048    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8049
8050    pub fn sync_until_idle(
8051        &mut self,
8052        transport: &mut dyn Transport,
8053        max_rounds: Option<u32>,
8054    ) -> SyncOutcome {
8055        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8056    }
8057
8058    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8059    /// milliseconds (`None` disables it). The deadline is checked between
8060    /// rounds; one blocking transport call is bounded by the transport's own
8061    /// whole-round deadline (§1.8).
8062    pub fn sync_until_idle_with_deadline(
8063        &mut self,
8064        transport: &mut dyn Transport,
8065        max_rounds: Option<u32>,
8066        timeout_ms: Option<u64>,
8067    ) -> SyncOutcome {
8068        // §7.8: an explicit budget is a strict positive round count. Zero is a
8069        // caller error, not an exhaustion.
8070        if max_rounds == Some(0) {
8071            return SyncOutcome::Failed {
8072                details: None,
8073                error_code: "sync.invalid_request".into(),
8074                message: "maxRounds must be a positive integer".into(),
8075                recommended_action: None,
8076            };
8077        }
8078        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8079            return SyncOutcome::Failed {
8080                details: None,
8081                error_code: "sync.invalid_request".into(),
8082                message: "timeoutMs must be a positive integer".into(),
8083                recommended_action: None,
8084            };
8085        }
8086        let started_at_ms = self.clock_now_ms();
8087        let deadline_ms = timeout_ms.map(|ms| started_at_ms.saturating_add(ms as i64));
8088        self.sync_until_idle_bounded(transport, max_rounds, started_at_ms, deadline_ms)
8089    }
8090
8091    fn sync_until_idle_bounded(
8092        &mut self,
8093        transport: &mut dyn Transport,
8094        max_rounds: Option<u32>,
8095        started_at_ms: i64,
8096        deadline_ms: Option<i64>,
8097    ) -> SyncOutcome {
8098        let rounds = max_rounds.unwrap_or(20);
8099        let mut aggregate = SyncReport::default();
8100        let mut spent = 0;
8101        while spent < rounds {
8102            if let Some(deadline_ms) = deadline_ms {
8103                if self.clock_now_ms() >= deadline_ms {
8104                    return SyncOutcome::Failed {
8105                        error_code: "sync.idle_timeout".into(),
8106                        message: "sync_until_idle exceeded its deadline".into(),
8107                        details: Some(serde_json::json!({
8108                            "timeoutMs": deadline_ms
8109                                .saturating_sub(started_at_ms) as u64,
8110                            "rounds": spent,
8111                        })),
8112                        recommended_action: Some("retryLater".into()),
8113                    };
8114                }
8115            }
8116            spent += 1;
8117            let before = self.resume_tokens();
8118            match self.sync(transport) {
8119                SyncOutcome::Failed {
8120                    error_code,
8121                    message,
8122                    details,
8123                    recommended_action,
8124                } => {
8125                    return SyncOutcome::Failed {
8126                        details,
8127                        error_code,
8128                        message,
8129                        recommended_action,
8130                    };
8131                }
8132                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8133                SyncOutcome::BudgetExhausted(_) => {
8134                    unreachable!("sync() never exhausts a round budget")
8135                }
8136                SyncOutcome::Ok(report) => {
8137                    if max_rounds.is_none()
8138                        && report.segment_rows_applied > 0
8139                        && !report.bootstrapping.is_empty()
8140                        && before != self.resume_tokens()
8141                    {
8142                        spent = 0;
8143                    }
8144                    // TS parity: a disabled transport or a schema floor ends
8145                    // the loop. The latest round still merges first.
8146                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8147                    aggregate.merge(&report);
8148                    if stop {
8149                        return SyncOutcome::Ok(aggregate);
8150                    }
8151                    // §4.5: pull again whenever the response contained
8152                    // commits or segments; resets re-bootstrap; a pending
8153                    // resume token continues paging (§4.7); deferred outbox
8154                    // commits and a raised sync-needed signal push on the next
8155                    // round.
8156                    let more = !report.bootstrapping.is_empty()
8157                        || report.commits_applied > 0
8158                        || report.segment_rows_applied > 0
8159                        || !report.resets.is_empty()
8160                        || report.deferred_commits > 0
8161                        || self.sync_needed;
8162                    if !more {
8163                        return SyncOutcome::Ok(aggregate);
8164                    }
8165                }
8166            }
8167        }
8168        // §7.8: partial success. The aggregate report is retained and the run
8169        // is explicitly not idle.
8170        SyncOutcome::BudgetExhausted(aggregate)
8171    }
8172
8173    fn process_response(
8174        &mut self,
8175        transport: &mut dyn Transport,
8176        response: Message,
8177        meta: &RequestMeta,
8178    ) -> SyncOutcome {
8179        #[cfg(feature = "bench-internals")]
8180        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8181        let mut report = SyncReport {
8182            pushed: meta.pushed_ids.len() as u32,
8183            ..SyncReport::default()
8184        };
8185        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8186            HashMap::new();
8187        let pushed_ids = meta
8188            .pushed_ids
8189            .iter()
8190            .map(String::as_str)
8191            .collect::<HashSet<_>>();
8192        let mut last_final_push_result_id: Option<String> = None;
8193        for frame in &response.frames {
8194            match frame {
8195                Frame::PushResultDetails {
8196                    client_commit_id,
8197                    entries,
8198                } => {
8199                    let details = rejection_details_by_commit
8200                        .entry(client_commit_id.clone())
8201                        .or_default();
8202                    for entry in entries {
8203                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8204                            Ok(value) => value,
8205                            Err(message) => {
8206                                return SyncOutcome::Failed {
8207                                    details: None,
8208                                    error_code: "sync.invalid_request".to_owned(),
8209                                    message,
8210                                    recommended_action: None,
8211                                };
8212                            }
8213                        };
8214                        details.insert(entry.op_index, parsed);
8215                    }
8216                }
8217                Frame::PushResult {
8218                    client_commit_id,
8219                    status,
8220                    results,
8221                    ..
8222                } if pushed_ids.contains(client_commit_id.as_str())
8223                    && Self::push_result_is_final(*status, results) =>
8224                {
8225                    last_final_push_result_id = Some(client_commit_id.clone());
8226                }
8227                _ => {}
8228            }
8229        }
8230        let mut frames = response.frames.into_iter().peekable();
8231        match frames.next() {
8232            Some(Frame::RespHeader {
8233                required_schema_version,
8234                latest_schema_version,
8235                log_epoch,
8236                reset_required,
8237            }) => {
8238                if let Some(required) = required_schema_version {
8239                    // §1.6 schema-floor response: nothing else is processed;
8240                    // stop syncing and surface the upgrade requirement.
8241                    let floor = SchemaFloor {
8242                        required_schema_version: Some(required),
8243                        latest_schema_version,
8244                    };
8245                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8246                        return SyncOutcome::Failed {
8247                            details: None,
8248                            error_code: "storage.failed".to_owned(),
8249                            message,
8250                            recommended_action: None,
8251                        };
8252                    }
8253                    report.schema_floor = Some(floor);
8254                    return SyncOutcome::Ok(report);
8255                }
8256                if reset_required {
8257                    if frames.next().is_some() {
8258                        return SyncOutcome::Failed {
8259                            details: None,
8260                            error_code: "client.invalid_host_response".to_owned(),
8261                            message: "log-epoch reset response contains body frames".to_owned(),
8262                            recommended_action: None,
8263                        };
8264                    }
8265                    match self.run_log_epoch_reset(&log_epoch) {
8266                        Ok(resets) => {
8267                            report.resets = resets;
8268                            return SyncOutcome::Ok(report);
8269                        }
8270                        Err(message) => {
8271                            return SyncOutcome::Failed {
8272                                details: None,
8273                                error_code: "sync.local_corrupt".to_owned(),
8274                                message,
8275                                recommended_action: None,
8276                            };
8277                        }
8278                    }
8279                }
8280                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8281                    Ok(stored) => stored,
8282                    Err(message) => {
8283                        return SyncOutcome::Failed {
8284                            details: None,
8285                            error_code: "storage.failed".to_owned(),
8286                            message,
8287                            recommended_action: None,
8288                        };
8289                    }
8290                };
8291                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8292                    return SyncOutcome::Failed {
8293                        details: None,
8294                        error_code: "client.invalid_host_response".to_owned(),
8295                        message: "server changed logEpoch without requiring a reset".to_owned(),
8296                        recommended_action: None,
8297                    };
8298                }
8299            }
8300            _ => {
8301                return SyncOutcome::Failed {
8302                    details: None,
8303                    error_code: "sync.invalid_request".to_owned(),
8304                    message: "response does not start with RESP_HEADER".to_owned(),
8305                    recommended_action: None,
8306                };
8307            }
8308        }
8309
8310        if let Err(message) = self.begin_coalesced_apply() {
8311            return SyncOutcome::Failed {
8312                details: None,
8313                error_code: "storage.failed".into(),
8314                message,
8315                recommended_action: None,
8316            };
8317        }
8318        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8319        // §4.5 client application: the units whose persisted echo a live
8320        // change must match, fixed at LIVE_START after every SUB_END persisted.
8321        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8322        let mut live_ended = false;
8323        while let Some(frame) = frames.next() {
8324            match frame {
8325                frame @ Frame::PushResult { .. } => {
8326                    let successful = matches!(
8327                        &frame,
8328                        Frame::PushResult {
8329                            status: PushStatus::Applied | PushStatus::Cached,
8330                            ..
8331                        }
8332                    );
8333                    let mut results = vec![frame];
8334                    if successful {
8335                        while matches!(
8336                            frames.peek(),
8337                            Some(Frame::PushResult {
8338                                status: PushStatus::Applied | PushStatus::Cached,
8339                                ..
8340                            })
8341                        ) {
8342                            if let Some(next) = frames.next() {
8343                                results.push(next);
8344                            }
8345                        }
8346                    }
8347                    if self
8348                        .handle_push_results(
8349                            &results,
8350                            &pushed_ids,
8351                            &rejection_details_by_commit,
8352                            &mut report,
8353                            last_final_push_result_id.as_deref(),
8354                        )
8355                        .is_err()
8356                    {
8357                        failure = Some((
8358                            "client.outcome_persistence_failed".to_owned(),
8359                            "local commit outcome could not be persisted".to_owned(),
8360                            None,
8361                            None,
8362                        ));
8363                        break;
8364                    }
8365                }
8366                Frame::PushResultDetails { .. } => {}
8367                Frame::SubStart {
8368                    id,
8369                    status,
8370                    reason_code,
8371                    bootstrap,
8372                    fenced,
8373                } => {
8374                    // §1.4 rule 5: an ERROR closes the section; its body still
8375                    // applied, its SUB_END values are never persisted.
8376                    let mut body = Vec::new();
8377                    let mut sub_end: Option<(i64, Option<String>)> = None;
8378                    let mut error: Option<Frame> = None;
8379                    for inner in frames.by_ref() {
8380                        match inner {
8381                            Frame::SubEnd {
8382                                next_cursor,
8383                                bootstrap_state,
8384                            } => {
8385                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8386                                break;
8387                            }
8388                            inner @ Frame::Error { .. } => {
8389                                error = Some(inner);
8390                                break;
8391                            }
8392                            Frame::Unknown { .. } => {}
8393                            other => body.push(other),
8394                        }
8395                    }
8396                    let ended = sub_end.is_some();
8397                    let section = SectionStart {
8398                        id,
8399                        status,
8400                        reason_code,
8401                        bootstrap,
8402                        fenced,
8403                    };
8404                    match self.process_section(
8405                        transport,
8406                        &section,
8407                        body,
8408                        sub_end,
8409                        meta,
8410                        &mut report,
8411                    ) {
8412                        Err(SectionError::Abort(code, message)) => {
8413                            failure = Some((code, message, None, None));
8414                            break;
8415                        }
8416                        Err(SectionError::Transport(error)) => {
8417                            failure = Some((error.code, error.message, None, error.details));
8418                            break;
8419                        }
8420                        _ => {}
8421                    }
8422                    if ended {
8423                        // The counters stay: `complete` and `failed` report them.
8424                        self.progress.update(|p| {
8425                            p.unit_id = None;
8426                            p.table = None;
8427                            p.segment_id = None;
8428                        });
8429                    }
8430                    match error {
8431                        Some(Frame::Error {
8432                            code,
8433                            message,
8434                            recommended_action,
8435                            details,
8436                            ..
8437                        }) => {
8438                            failure = Some(Self::error_frame_failure(
8439                                code,
8440                                message,
8441                                recommended_action,
8442                                details,
8443                            ));
8444                            break;
8445                        }
8446                        Some(_) => unreachable!("only an ERROR closes a section early"),
8447                        None if !ended => {
8448                            failure = Some((
8449                                "sync.invalid_request".to_owned(),
8450                                "unit section without SUB_END".to_owned(),
8451                                None,
8452                                None,
8453                            ));
8454                            break;
8455                        }
8456                        None => {}
8457                    }
8458                }
8459                Frame::LiveStart {
8460                    status,
8461                    reason_code,
8462                } => {
8463                    let receivers = match self.stored(load_units(&self.conn)) {
8464                        Ok(units) => units
8465                            .into_iter()
8466                            .filter(|unit| {
8467                                unit.status == UnitStatus::Active
8468                                    && unit.echo.is_some()
8469                                    && !unit.bootstrapping()
8470                            })
8471                            .collect(),
8472                        Err(message) => {
8473                            failure = Some(("storage.failed".to_owned(), message, None, None));
8474                            break;
8475                        }
8476                    };
8477                    live = Some((status, reason_code, receivers));
8478                }
8479                Frame::Commit {
8480                    commit_seq,
8481                    tables,
8482                    changes,
8483                    ..
8484                } => {
8485                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8486                        continue;
8487                    };
8488                    let receivers = receivers.clone();
8489                    match self.apply_live_commit(
8490                        &receivers,
8491                        &tables,
8492                        changes,
8493                        commit_seq,
8494                        AppliedServerCommitSource::Pull,
8495                    ) {
8496                        Ok(applied) => report.commits_applied += u32::from(applied),
8497                        Err(error) => {
8498                            failure = Some((error.code, error.message, None, error.details));
8499                            break;
8500                        }
8501                    }
8502                }
8503                Frame::LiveEnd {
8504                    through_seq,
8505                    echo_digest,
8506                } => {
8507                    let Some((status, reason_code, _)) = live.take() else {
8508                        continue;
8509                    };
8510                    live_ended = true;
8511                    live = Some((status, String::new(), Vec::new()));
8512                    if let Err(message) = self.finish_live_section(
8513                        status,
8514                        &reason_code,
8515                        through_seq,
8516                        echo_digest,
8517                        &meta.registration_digest,
8518                        &mut report,
8519                    ) {
8520                        failure = Some(("storage.failed".to_owned(), message, None, None));
8521                        break;
8522                    }
8523                }
8524                Frame::Lease {
8525                    lease_id,
8526                    expires_at_ms,
8527                } => {
8528                    // §7.3.5: persist the opaque lease; a fresh lease clears
8529                    // any prior lease error (the outage/revocation is over).
8530                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8531                        lease_id: Some(lease_id),
8532                        expires_at_ms: Some(expires_at_ms),
8533                        error_code: None,
8534                    })) {
8535                        failure = Some(("storage.failed".to_owned(), message, None, None));
8536                        break;
8537                    }
8538                }
8539                Frame::Error {
8540                    code,
8541                    message,
8542                    recommended_action,
8543                    details,
8544                    ..
8545                } => {
8546                    // §1.6: the whole request failed; already-completed
8547                    // units keep their applied data and cursors.
8548                    failure = Some(Self::error_frame_failure(
8549                        code,
8550                        message,
8551                        recommended_action,
8552                        details,
8553                    ));
8554                    break;
8555                }
8556                Frame::Unknown { .. } => {}
8557                _ => {
8558                    failure = Some((
8559                        "sync.invalid_request".to_owned(),
8560                        "unexpected frame in response".to_owned(),
8561                        None,
8562                        None,
8563                    ));
8564                    break;
8565                }
8566            }
8567        }
8568
8569        let applied = match failure {
8570            Some(failure) => Err(failure),
8571            // Local apply boundaries already reconciled their affected overlay.
8572            // Refuse round success if any remaining dirty work fails to
8573            // reconcile. §4.8 E1: the push half may have drained commits that
8574            // pinned rows of a shrunk window unit, so retry deferred evictions.
8575            None => self
8576                .rebuild_overlay_if_dirty()
8577                .and_then(|()| self.drain_pending_evictions())
8578                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8579        };
8580        // §1.4 rule 4: frames completed before a failure stay committed; a
8581        // failed COMMIT of the coalesced transaction fails the round.
8582        if let Err(message) = self.finish_coalesced_apply() {
8583            return SyncOutcome::Failed {
8584                details: None,
8585                error_code: "storage.failed".into(),
8586                message,
8587                recommended_action: None,
8588            };
8589        }
8590        if let Err((error_code, message, recommended_action, details)) = applied {
8591            return SyncOutcome::Failed {
8592                details,
8593                error_code,
8594                message,
8595                recommended_action,
8596            };
8597        }
8598        let catching_up = match self.stored(load_units(&self.conn)) {
8599            Ok(units) => units
8600                .into_iter()
8601                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8602                .map(|unit| unit.id)
8603                .collect::<Vec<_>>(),
8604            Err(message) => {
8605                return SyncOutcome::Failed {
8606                    details: None,
8607                    error_code: "storage.failed".into(),
8608                    message,
8609                    recommended_action: None,
8610                }
8611            }
8612        };
8613        // §7.1: a response without LIVE_END, one answered `unregistered` or
8614        // `reset`, or a unit still catching up leaves sync-needed raised;
8615        // after `unregistered` the next round starts at once.
8616        let status = live.map(|(status, _, _)| status);
8617        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8618            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8619        }
8620        self.ack_after_pull(transport);
8621        // §7.4.5: the reset is over once the first post-reset pull round
8622        // leaves no unit catching up: the tables are rebuilt and current.
8623        if self.upgrading && catching_up.is_empty() {
8624            self.set_upgrading(false);
8625        }
8626        report.bootstrapping = catching_up;
8627        SyncOutcome::Ok(report)
8628    }
8629
8630    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8631    fn error_frame_failure(
8632        code: String,
8633        message: String,
8634        recommended_action: String,
8635        details: Option<RawJson>,
8636    ) -> (String, String, Option<String>, Option<Value>) {
8637        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8638        let details = details
8639            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8640            .filter(Value::is_object);
8641        (code, message, recommended_action, details)
8642    }
8643
8644    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8645    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8646    /// replica cursor, the echo digest and the digest of the list the server
8647    /// now stores.
8648    fn finish_live_section(
8649        &mut self,
8650        status: LiveStatus,
8651        reason_code: &str,
8652        through_seq: i64,
8653        echo_digest: Option<[u8; 32]>,
8654        registration_digest: &str,
8655        report: &mut SyncReport,
8656    ) -> Result<(), String> {
8657        let rollback = self.capture();
8658        let resets = report.resets.len();
8659        self.begin_observation("syncular_live_end")?;
8660        let finished = (|| {
8661            let mut batch = ChangeAccumulator::default();
8662            let before = self.stored(replica_cursor(&self.conn))?;
8663            let mut cursor = if status == LiveStatus::Unregistered {
8664                before
8665            } else {
8666                -1
8667            };
8668            let joined = before.max(through_seq);
8669            for unit in self.stored(load_units(&self.conn))? {
8670                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8671                    continue;
8672                }
8673                match status {
8674                    // §4.6 replica reset: every live unit bootstraps again;
8675                    // post-bootstrap units keep their cursors.
8676                    LiveStatus::Reset if unit.cursor.is_none() => {
8677                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8678                            batch.window(&base_key, &value);
8679                        }
8680                        report.resets.push(unit.id.clone());
8681                        self.stored(save_unit(
8682                            &self.conn,
8683                            &UnitRecord {
8684                                reason_code: Some(reason_code.to_owned()),
8685                                cursor: Some(-1),
8686                                seeded: Vec::new(),
8687                                ..unit
8688                            },
8689                        ))?;
8690                    }
8691                    // §4.5 join rule J.
8692                    LiveStatus::Active => {
8693                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8694                        if next >= joined {
8695                            cursor = joined;
8696                        }
8697                        if unit.cursor.is_some() {
8698                            self.stored(save_unit(
8699                                &self.conn,
8700                                &UnitRecord {
8701                                    cursor: (next < joined).then_some(next),
8702                                    ..unit
8703                                },
8704                            ))?;
8705                        }
8706                    }
8707                    _ => {}
8708                }
8709            }
8710            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
8711            self.set_meta(
8712                ECHO_DIGEST_KEY,
8713                &echo_digest
8714                    .map(|digest| bytes_to_hex(&digest))
8715                    .unwrap_or_default(),
8716            )?;
8717            if status == LiveStatus::Unregistered {
8718                self.delete_meta(REGISTERED_DIGEST_KEY)?;
8719            } else {
8720                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
8721            }
8722            self.finish_observation("syncular_live_end", batch)
8723        })();
8724        if finished.is_err() {
8725            self.rollback_observation("syncular_live_end");
8726            self.restore(rollback);
8727            report.resets.truncate(resets);
8728        }
8729        finished
8730    }
8731
8732    // -- push results (§6.3, §7.2) ------------------------------------------------
8733
8734    fn handle_push_results(
8735        &mut self,
8736        frames: &[Frame],
8737        pushed_ids: &HashSet<&str>,
8738        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
8739        report: &mut SyncReport,
8740        last_final_push_result_id: Option<&str>,
8741    ) -> Result<(), String> {
8742        self.begin_observation("syncular_push_results")?;
8743        let rollback = self.capture();
8744        let conflict_count = self.conflicts.len();
8745        let applied_count = report.applied.len();
8746        let rejected_count = report.rejected.len();
8747        let retryable_count = report.retryable.len();
8748        let mut drained = Vec::new();
8749        let mut batch = ChangeAccumulator::default();
8750        let persisted = (|| {
8751            for frame in frames {
8752                let Frame::PushResult {
8753                    client_commit_id,
8754                    status,
8755                    results,
8756                    commit_seq,
8757                } = frame
8758                else {
8759                    return Err("unexpected acknowledgement frame".to_owned());
8760                };
8761                if !pushed_ids.contains(client_commit_id.as_str()) {
8762                    continue;
8763                }
8764                let Some(index) = self
8765                    .outbox
8766                    .iter()
8767                    .position(|commit| commit.client_commit_id == *client_commit_id)
8768                else {
8769                    continue;
8770                };
8771                let status = *status;
8772                let rejection_details = rejection_details_by_commit.get(client_commit_id);
8773                let operations = self.outbox[index].ops.clone();
8774                match status {
8775                    PushStatus::Applied | PushStatus::Cached => {
8776                        // §7.2: a lost ack replays as `cached` — proceed as if the
8777                        // ack had arrived.
8778                        let sequence =
8779                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
8780                        let affected: HashSet<_> = operations
8781                            .iter()
8782                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
8783                            .collect();
8784                        let prior = self.acknowledged_operations(Some(&affected), None)?;
8785                        for (op_index, operation) in operations.iter().enumerate() {
8786                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
8787                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
8788                            if delivered {
8789                                continue;
8790                            }
8791                            let table = self
8792                                .schema
8793                                .table(&operation.table)
8794                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
8795                            let mut intended: Map<String, Value> = self
8796                                .stored_row(table, &operation.row_id, true)?
8797                                .map(|(row, _)| {
8798                                    table
8799                                        .columns
8800                                        .iter()
8801                                        .zip(&row)
8802                                        .map(|(column, value)| {
8803                                            (column.name.clone(), column_value_to_json(value))
8804                                        })
8805                                        .collect()
8806                                })
8807                                .unwrap_or_default();
8808                            let mut scope_image = intended.clone();
8809                            for op in prior
8810                                .iter()
8811                                .chain(
8812                                    self.failed_commits
8813                                        .iter()
8814                                        .flat_map(|(commit, _)| &commit.ops),
8815                                )
8816                                .chain(
8817                                    self.outbox
8818                                        .iter()
8819                                        .take(index)
8820                                        .flat_map(|commit| &commit.ops),
8821                                )
8822                                .chain(operations.iter().take(op_index + 1))
8823                            {
8824                                if op.table != operation.table || op.row_id != operation.row_id {
8825                                    continue;
8826                                }
8827                                if let Some(values) = &op.values {
8828                                    intended.extend(values.clone());
8829                                } else {
8830                                    scope_image = intended.clone();
8831                                    intended.clear();
8832                                }
8833                            }
8834                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
8835                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
8836                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
8837                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
8838                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
8839                        }
8840                        let journal_results = results
8841                            .iter()
8842                            .map(|result| {
8843                                let op_index = match result {
8844                                    OpResult::Applied { op_index }
8845                                    | OpResult::Conflict { op_index, .. }
8846                                    | OpResult::Error { op_index, .. } => *op_index,
8847                                };
8848                                CommitOperationOutcome::Applied { op_index }
8849                            })
8850                            .collect::<Vec<_>>();
8851                        let outcome_status = if status == PushStatus::Applied {
8852                            CommitOutcomeStatus::Applied
8853                        } else {
8854                            CommitOutcomeStatus::Cached
8855                        };
8856                        let persisted = self
8857                            .persist_commit_outcome(
8858                                client_commit_id,
8859                                outcome_status,
8860                                &journal_results,
8861                                None,
8862                            )
8863                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
8864                        persisted?;
8865                        self.sync_needed = true;
8866                        batch.status = true;
8867                        batch.outcomes = true;
8868                    }
8869                    PushStatus::Rejected => {
8870                        if results.iter().any(|result| {
8871                            matches!(
8872                                result,
8873                                OpResult::Error {
8874                                    code,
8875                                    retryable: true,
8876                                    ..
8877                                } if code == "sync.idempotency_cache_miss"
8878                            )
8879                        }) {
8880                            // §6.3/§7.2: a serving failure, not an outcome — keep the
8881                            // exact commit queued for an identical retry.
8882                            report.retryable.push(client_commit_id.to_owned());
8883                            continue;
8884                        }
8885
8886                        let mut journal_results = Vec::with_capacity(results.len());
8887                        let mut conflicts = Vec::new();
8888                        let mut rejections = Vec::new();
8889                        for result in results {
8890                            match result {
8891                                OpResult::Applied { op_index } => {
8892                                    journal_results.push(CommitOperationOutcome::Applied {
8893                                        op_index: *op_index,
8894                                    });
8895                                }
8896                                OpResult::Conflict {
8897                                    op_index,
8898                                    code,
8899                                    message,
8900                                    server_version,
8901                                    server_row,
8902                                    conflict_columns: conflict_columns_bitmap,
8903                                } => {
8904                                    let operation = operations
8905                                        .get(*op_index as usize)
8906                                        .map(CommitOperation::from);
8907                                    let (table, row_id) = operation
8908                                        .as_ref()
8909                                        .map(|op| (op.table.clone(), op.row_id.clone()))
8910                                        .unwrap_or_default();
8911                                    let server_row_json = self
8912                                        .schema
8913                                        .table(&table)
8914                                        .and_then(|t| {
8915                                            decode_row_bytes(t, server_row, &self.encryption)
8916                                                .ok()
8917                                                .map(|row| (t, row))
8918                                        })
8919                                        .map(|(t, row)| {
8920                                            let mut map = Map::new();
8921                                            for (i, column) in t.columns.iter().enumerate() {
8922                                                map.insert(
8923                                                    column.name.clone(),
8924                                                    column_value_to_json(
8925                                                        row.get(i).unwrap_or(&None),
8926                                                    ),
8927                                                );
8928                                            }
8929                                            map
8930                                        })
8931                                        .unwrap_or_default();
8932                                    // §6.3: decode the presence-layout bitmap
8933                                    // into the marked column names.
8934                                    let conflict_columns = self
8935                                        .schema
8936                                        .table(&table)
8937                                        .map(|t| {
8938                                            t.columns
8939                                                .iter()
8940                                                .enumerate()
8941                                                .filter(|(index, _)| {
8942                                                    conflict_columns_bitmap
8943                                                        .get(index / 8)
8944                                                        .is_some_and(|byte| {
8945                                                            byte & (1 << (index % 8)) != 0
8946                                                        })
8947                                                })
8948                                                .map(|(_, column)| column.name.clone())
8949                                                .collect::<Vec<_>>()
8950                                        })
8951                                        .unwrap_or_default();
8952                                    let conflict = ConflictRecord {
8953                                        client_commit_id: client_commit_id.to_owned(),
8954                                        op_index: *op_index,
8955                                        table,
8956                                        row_id,
8957                                        code: code.clone(),
8958                                        message: message.clone(),
8959                                        server_version: *server_version,
8960                                        server_row: server_row_json,
8961                                        conflict_columns,
8962                                        operation,
8963                                    };
8964                                    journal_results.push(CommitOperationOutcome::Conflict {
8965                                        conflict: conflict.clone(),
8966                                    });
8967                                    conflicts.push(conflict);
8968                                }
8969                                OpResult::Error {
8970                                    op_index,
8971                                    code,
8972                                    message,
8973                                    retryable,
8974                                } => {
8975                                    let rejection = RejectionRecord {
8976                                        client_commit_id: client_commit_id.to_owned(),
8977                                        op_index: *op_index,
8978                                        code: code.clone(),
8979                                        message: message.clone(),
8980                                        retryable: *retryable,
8981                                        details: rejection_details
8982                                            .and_then(|details| details.get(op_index))
8983                                            .cloned(),
8984                                        operation: operations
8985                                            .get(*op_index as usize)
8986                                            .map(CommitOperation::from),
8987                                    };
8988                                    journal_results.push(CommitOperationOutcome::Error {
8989                                        rejection: rejection.clone(),
8990                                    });
8991                                    rejections.push(rejection);
8992                                }
8993                            }
8994                        }
8995                        if self.retain_failed_commits {
8996                            self.retain_failed_commit(client_commit_id, &operations)?;
8997                        }
8998                        let outcome_status = if conflicts.is_empty() {
8999                            CommitOutcomeStatus::Rejected
9000                        } else {
9001                            CommitOutcomeStatus::Conflict
9002                        };
9003                        let persisted = self
9004                            .persist_commit_outcome(
9005                                client_commit_id,
9006                                outcome_status,
9007                                &journal_results,
9008                                Some(&operations),
9009                            )
9010                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9011                        persisted?;
9012                        batch.conflicts = !conflicts.is_empty();
9013                        batch.rejections = !rejections.is_empty();
9014                        batch.status = true;
9015                        batch.outcomes = true;
9016                        self.conflicts.extend(conflicts);
9017                        self.rejections.extend(rejections);
9018                    }
9019                }
9020                drained.push(self.outbox.remove(index));
9021                if batch.status {
9022                    for operation in &operations {
9023                        if !self.record_row_scopes(
9024                            &mut batch,
9025                            &operation.table,
9026                            &operation.row_id,
9027                            false,
9028                        ) {
9029                            batch.table(&operation.table);
9030                        }
9031                    }
9032                }
9033                match status {
9034                    PushStatus::Applied | PushStatus::Cached => {
9035                        report.applied.push(client_commit_id.to_owned())
9036                    }
9037                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9038                }
9039            }
9040            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9041                self.prune_commit_outcomes()?;
9042            }
9043            self.reconcile_rows(&drained)?;
9044            self.rebuild_overlay_if_dirty()?;
9045            self.finish_observation("syncular_push_results", batch)
9046        })();
9047        if let Err(error) = persisted {
9048            self.rollback_observation("syncular_push_results");
9049            self.restore(rollback);
9050            report.applied.truncate(applied_count);
9051            report.rejected.truncate(rejected_count);
9052            report.retryable.truncate(retryable_count);
9053            return Err(error);
9054        }
9055        if !rollback.sync_needed && self.sync_needed {
9056            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9057        }
9058        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9059        Ok(())
9060    }
9061
9062    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9063        status != PushStatus::Rejected
9064            || !results.iter().any(|result| {
9065                matches!(
9066                    result,
9067                    OpResult::Error {
9068                        code,
9069                        retryable: true,
9070                        ..
9071                    } if code == "sync.idempotency_cache_miss"
9072                )
9073            })
9074    }
9075
9076    // -- unit sections and the live section ------------------------------------------
9077
9078    /// One unit section (§4.4): apply its segments, then persist its
9079    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9080    /// resume token). A section of a unit the app released while the round was
9081    /// in flight is skipped; a fail-closed first page fails the unit and skips
9082    /// the rest of its section (§5.6).
9083    fn process_section(
9084        &mut self,
9085        transport: &mut dyn Transport,
9086        start: &SectionStart,
9087        body: Vec<Frame>,
9088        sub_end: Option<(i64, Option<String>)>,
9089        meta: &RequestMeta,
9090        report: &mut SyncReport,
9091    ) -> Result<(), SectionError> {
9092        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9093        let unit = self
9094            .stored(get_unit(&self.conn, &start.id))
9095            .map_err(storage)?
9096            .filter(|unit| unit.status == UnitStatus::Active);
9097        // §7.7: a section boundary ends the previous payload.
9098        self.progress.update(|p| {
9099            p.phase = ProgressPhase::Import;
9100            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9101            p.table = None;
9102            p.segment_id = None;
9103            p.bytes_received = 0;
9104            p.bytes_total = None;
9105            p.rows_processed = 0;
9106            p.rows_total = None;
9107            p.import_write_ms = None;
9108            p.import_commit_ms = None;
9109        });
9110        let Some(unit) = unit else {
9111            return Ok(());
9112        };
9113        let echo: Vec<EchoRecord> = start
9114            .fenced
9115            .iter()
9116            .map(|record| EchoRecord {
9117                table: record.table.clone(),
9118                status: record.status.name().to_owned(),
9119                scopes: record.scopes.iter().cloned().collect(),
9120            })
9121            .collect();
9122        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9123        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9124            BTreeSet::new()
9125        } else {
9126            unit.seeded.iter().cloned().collect()
9127        };
9128        let applied = (|| {
9129            for frame in body {
9130                match frame {
9131                    Frame::SegmentInline { payload } => {
9132                        let segment = decode_rows_segment(&payload).map_err(|e| {
9133                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9134                        })?;
9135                        self.progress.update(|p| {
9136                            p.phase = ProgressPhase::Import;
9137                            p.unit_id = Some(unit.id.clone());
9138                            p.table = Some(segment.table.clone());
9139                            p.segment_id = None;
9140                            p.bytes_received = payload.len() as u64;
9141                            p.bytes_total = Some(payload.len() as u64);
9142                            p.rows_processed = 0;
9143                            p.rows_total =
9144                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9145                            p.import_write_ms = None;
9146                            p.import_commit_ms = None;
9147                        });
9148                        let (pair, clear, effective) =
9149                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9150                        report.segment_rows_applied +=
9151                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9152                        seeded.insert(segment.table);
9153                    }
9154                    Frame::SegmentRef {
9155                        segment_id,
9156                        byte_length,
9157                        media_type,
9158                        table,
9159                        row_count,
9160                        as_of_commit_seq,
9161                        scope_digest,
9162                        row_cursor,
9163                        next_row_cursor,
9164                        url,
9165                        url_expires_at_ms,
9166                        ..
9167                    } => {
9168                        // §4.2: reject a descriptor whose mediaType was not
9169                        // advertised — never skip or guess.
9170                        let advertised = match media_type {
9171                            MediaType::Rows => {
9172                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9173                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9174                            }
9175                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9176                        };
9177                        if !advertised {
9178                            return Err(SectionError::Abort(
9179                                "sync.invalid_request".to_owned(),
9180                                format!(
9181                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9182                                    media_type.name()
9183                                ),
9184                            ));
9185                        }
9186                        self.progress.update(|p| {
9187                            p.phase = ProgressPhase::Download;
9188                            p.unit_id = Some(unit.id.clone());
9189                            p.table = Some(table.clone());
9190                            p.segment_id = Some(segment_id.clone());
9191                            p.bytes_received = 0;
9192                            p.bytes_total = u64::try_from(byte_length).ok();
9193                            p.rows_processed = 0;
9194                            p.rows_total = u64::try_from(row_count).ok();
9195                            p.import_write_ms = None;
9196                            p.import_commit_ms = None;
9197                        });
9198                        let observer = self.progress.clone();
9199                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9200                        let bytes = if let Some(url) = url {
9201                            // §5.4: a url-carrying descriptor MUST be fetched
9202                            // from that URL; failure invalidates the whole
9203                            // descriptor (no fall-through to §5.5 — re-pull
9204                            // recovers, §1.4 rule 5).
9205                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9206                                return Err(SectionError::Abort(
9207                                    "sync.invalid_request".to_owned(),
9208                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9209                                        .to_owned(),
9210                                ));
9211                            }
9212                            // §5.4: MUST NOT start a fetch at/past expiry.
9213                            if url_expires_at_ms.is_some_and(|exp| {
9214                                exp <= transport
9215                                    .segment_fetch_started_at(&url)
9216                                    .unwrap_or_else(|| self.clock_now_ms())
9217                            }) {
9218                                return Err(SectionError::Abort(
9219                                    "sync.segment_expired".to_owned(),
9220                                    format!(
9221                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9222                                    ),
9223                                ));
9224                            }
9225                            transport
9226                                .fetch_url(
9227                                    &url,
9228                                    u64::try_from(byte_length).unwrap_or(0),
9229                                    &mut on_progress,
9230                                )
9231                                .map_err(|e| self.segment_fetch_failed(e))?
9232                        } else {
9233                            // §5.5: the owning unit's scope map, each value a one-element list.
9234                            let requested_scopes_json =
9235                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9236                            transport
9237                                .download_segment(
9238                                    &SegmentRequest {
9239                                        segment_id: segment_id.clone(),
9240                                        table: table.clone(),
9241                                        requested_scopes_json,
9242                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9243                                    },
9244                                    &mut on_progress,
9245                                )
9246                                .map_err(|e| {
9247                                    // §5.5: access changed since the response;
9248                                    // the next round's SUB_START settles the unit.
9249                                    if e.code == "sync.forbidden" {
9250                                        SectionError::Refused
9251                                    } else {
9252                                        self.segment_fetch_failed(e)
9253                                    }
9254                                })?
9255                        };
9256                        // §5.4: the descriptor's byteLength caps the body before
9257                        // §5.1 verifies the content address; the client discards
9258                        // the segment and re-pulls on either failure.
9259                        if bytes.len() as i64 > byte_length {
9260                            self.schedule_background_retry();
9261                            return Err(SectionError::Abort(
9262                                "sync.invalid_request".to_owned(),
9263                                "segment exceeds its byteLength".to_owned(),
9264                            ));
9265                        }
9266                        on_progress(bytes.len() as u64);
9267                        let digest = Sha256::digest(&bytes);
9268                        let expected = segment_id
9269                            .strip_prefix("sha256:")
9270                            .unwrap_or(segment_id.as_str());
9271                        if bytes_to_hex(&digest) != expected {
9272                            // §5.1: the client discards the segment and re-pulls.
9273                            self.schedule_background_retry();
9274                            return Err(SectionError::Abort(
9275                                "sync.invalid_request".to_owned(),
9276                                "segment bytes do not match the content address (§5.1)".to_owned(),
9277                            ));
9278                        }
9279                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9280                        if media_type == MediaType::Sqlite {
9281                            // §5.3: images are whole-table — a paged sqlite
9282                            // descriptor is invalid.
9283                            if row_cursor.is_some() || next_row_cursor.is_some() {
9284                                return Err(SectionError::Abort(
9285                                    "sync.invalid_request".to_owned(),
9286                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9287                                        .to_owned(),
9288                                ));
9289                            }
9290                            let (pair, clear, effective) =
9291                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9292                            report.segment_rows_applied += self.apply_sqlite_segment(
9293                                &pair,
9294                                &bytes,
9295                                clear,
9296                                &effective,
9297                                &unit.id,
9298                                row_count,
9299                                as_of_commit_seq,
9300                                &scope_digest,
9301                            )?;
9302                            seeded.insert(table);
9303                        } else {
9304                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9305                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9306                            })?;
9307                            let (pair, clear, effective) =
9308                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9309                            report.segment_rows_applied +=
9310                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9311                            seeded.insert(segment.table);
9312                        }
9313                    }
9314                    _ => {
9315                        return Err(SectionError::Abort(
9316                            "sync.invalid_request".to_owned(),
9317                            "unexpected frame inside a unit section".to_owned(),
9318                        ));
9319                    }
9320                }
9321            }
9322            Ok(())
9323        })();
9324        match applied {
9325            Ok(()) => {}
9326            // The unit keeps catching up with its cursor unpersisted.
9327            Err(SectionError::Refused) => return Ok(()),
9328            Err(SectionError::FailClosed) => {
9329                // §5.6: the failure is unit-local; the rest of the response
9330                // still applies and the failed unit leaves the list.
9331                self.begin_observation("syncular_section_failure")
9332                    .map_err(storage)?;
9333                let failed = (|| {
9334                    let mut batch = ChangeAccumulator::default();
9335                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9336                        batch.window(&base_key, &value);
9337                    }
9338                    batch.status = true;
9339                    self.stored(save_unit(
9340                        &self.conn,
9341                        &UnitRecord {
9342                            status: UnitStatus::Failed,
9343                            reason_code: Some("sync.scope_revoked".to_owned()),
9344                            ..unit.clone()
9345                        },
9346                    ))?;
9347                    self.finish_observation("syncular_section_failure", batch)
9348                })();
9349                if let Err(message) = failed {
9350                    self.rollback_observation("syncular_section_failure");
9351                    return Err(storage(message));
9352                }
9353                report.failed.push(unit.id);
9354                return Ok(());
9355            }
9356            Err(error) => return Err(error),
9357        }
9358        let Some((next_cursor, bootstrap_state)) = sub_end else {
9359            return Ok(());
9360        };
9361        let rollback = self.capture();
9362        let counts = (
9363            report.resets.len(),
9364            report.revoked.len(),
9365            report.failed.len(),
9366        );
9367        self.begin_observation("syncular_sub_end")
9368            .map_err(storage)?;
9369        let finished = self
9370            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9371            .and_then(|batch| {
9372                self.rebuild_overlay_if_dirty()?;
9373                self.finish_observation("syncular_sub_end", batch)
9374            });
9375        if let Err(message) = finished {
9376            self.rollback_observation("syncular_sub_end");
9377            self.restore(rollback);
9378            report.resets.truncate(counts.0);
9379            report.revoked.truncate(counts.1);
9380            report.failed.truncate(counts.2);
9381            return Err(storage(message));
9382        }
9383        Ok(())
9384    }
9385
9386    /// §5.6: the table, first-page flag and effective scopes of one segment
9387    /// of `unit`'s section.
9388    #[allow(clippy::type_complexity)]
9389    fn segment_pair(
9390        &self,
9391        unit: &UnitRecord,
9392        echo: &[EchoRecord],
9393        seeded: &BTreeSet<String>,
9394        name: &str,
9395    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9396        let table = self.schema.table(name).cloned().ok_or_else(|| {
9397            SectionError::Abort(
9398                "sync.schema_mismatch".to_owned(),
9399                format!("segment table {name:?} is not in the client schema"),
9400            )
9401        })?;
9402        let effective = covers(&table, &unit.scopes)
9403            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9404            .flatten()
9405            .ok_or_else(|| {
9406                SectionError::Abort(
9407                    "sync.invalid_request".to_owned(),
9408                    "a segment names no active pair of its unit".to_owned(),
9409                )
9410            })?;
9411        Ok((table, !seeded.contains(name), effective))
9412    }
9413
9414    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9415    /// caller's observation: a reset discards cursor, resume token and
9416    /// first-page record and keeps the echo; a revoked pair purges by its last
9417    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9418    /// fenced record that turns active again re-bootstraps the unit, and a
9419    /// bootstrap section persists its pin, resume token and seeded tables.
9420    fn finish_section(
9421        &mut self,
9422        start: &SectionStart,
9423        echo: Vec<EchoRecord>,
9424        next_cursor: i64,
9425        bootstrap_state: Option<String>,
9426        seeded: BTreeSet<String>,
9427        report: &mut SyncReport,
9428    ) -> Result<ChangeAccumulator, String> {
9429        #[cfg(feature = "bench-internals")]
9430        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9431        let mut batch = ChangeAccumulator::default();
9432        // §4.4: SUB_END writes only into a unit that is still registered.
9433        let Some(unit) = self
9434            .stored(get_unit(&self.conn, &start.id))?
9435            .filter(|unit| unit.status == UnitStatus::Active)
9436        else {
9437            return Ok(batch);
9438        };
9439        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9440            batch.window(&base_key, &value);
9441        }
9442        if start.status == SubStatus::Reset {
9443            // §4.6: staleness, never a purge; the rows stay until the
9444            // bootstrap replaces them.
9445            report.resets.push(unit.id.clone());
9446            self.stored(save_unit(
9447                &self.conn,
9448                &UnitRecord {
9449                    reason_code: Some(start.reason_code.clone()),
9450                    cursor: Some(-1),
9451                    bootstrap_state: None,
9452                    seeded: Vec::new(),
9453                    ..unit
9454                },
9455            ))?;
9456            return Ok(batch);
9457        }
9458        let revoked = start.status == SubStatus::Revoked;
9459        let mut units = None;
9460        let mut failed = false;
9461        let mut rearmed = false;
9462        for table in self.schema.tables.clone() {
9463            if !covers(&table, &unit.scopes) {
9464                continue;
9465            }
9466            let last = unit
9467                .echo
9468                .as_deref()
9469                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9470            let next = (!revoked)
9471                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9472                .flatten();
9473            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9474            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9475            // §3.3: purge a pair the response revokes, keyed on the echo the
9476            // client persisted while the pair was active.
9477            if let (Some(last), None) = (&last, &next) {
9478                let units = match units {
9479                    Some(ref units) => units,
9480                    None => units.insert(self.stored(load_units(&self.conn))?),
9481                };
9482                let owners = unit_owners(units, &table, &unit.id);
9483                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9484            }
9485        }
9486        if revoked || failed {
9487            if revoked {
9488                report.revoked.push(unit.id.clone());
9489            }
9490            if failed {
9491                report.failed.push(unit.id.clone());
9492            }
9493            batch.status = true;
9494            self.stored(save_unit(
9495                &self.conn,
9496                &UnitRecord {
9497                    status: if failed {
9498                        UnitStatus::Failed
9499                    } else {
9500                        UnitStatus::Revoked
9501                    },
9502                    reason_code: Some(if failed {
9503                        "sync.scope_revoked".to_owned()
9504                    } else {
9505                        start.reason_code.clone()
9506                    }),
9507                    cursor: Some(-1),
9508                    bootstrap_state: None,
9509                    seeded: Vec::new(),
9510                    echo: None,
9511                    ..unit
9512                },
9513            ))?;
9514            return Ok(batch);
9515        }
9516        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9517        let (cursor, bootstrap_state, seeded) = if rearmed {
9518            (Some(-1), None, Vec::new())
9519        } else if start.bootstrap {
9520            let seeded = if bootstrap_state.is_none() {
9521                Vec::new()
9522            } else {
9523                seeded.into_iter().collect()
9524            };
9525            (Some(next_cursor), bootstrap_state, seeded)
9526        } else {
9527            (
9528                unit.cursor,
9529                unit.bootstrap_state.clone(),
9530                unit.seeded.clone(),
9531            )
9532        };
9533        self.stored(save_unit(
9534            &self.conn,
9535            &UnitRecord {
9536                reason_code: None,
9537                echo: Some(echo.clone()),
9538                cursor,
9539                bootstrap_state,
9540                seeded,
9541                ..unit.clone()
9542            },
9543        ))?;
9544        if !completed {
9545            return Ok(batch);
9546        }
9547        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9548        // at or below its pin.
9549        for table in self.schema.tables.clone() {
9550            let Some(effective) = covers(&table, &unit.scopes)
9551                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9552                .flatten()
9553            else {
9554                continue;
9555            };
9556            let mut statement = self
9557                .conn
9558                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9559                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9560            let rows = statement
9561                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9562                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9563                })
9564                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9565            let mut covered = HashSet::new();
9566            for row in rows {
9567                let (id, intent) =
9568                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9569                let values: Map<String, Value> =
9570                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9571                if owns_local_row(&table, &effective, &[], &values) {
9572                    covered.insert(id);
9573                }
9574            }
9575            drop(statement);
9576            for id in covered {
9577                self.conn
9578                    .execute(
9579                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9580                        rusqlite::params![table.name, id, next_cursor],
9581                    )
9582                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9583                self.overlay_dirty.table(&table.name);
9584                batch.table(&table.name);
9585            }
9586        }
9587        Ok(batch)
9588    }
9589
9590    /// §3.3: purge one revoked pair inside the caller's observation: delete
9591    /// the rows matching its last effective scopes that no owner holds, drop
9592    /// the retained intent and every pending commit that provably writes into
9593    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9594    /// table has no local scope-column mapping (fail closed: nothing deleted).
9595    fn purge_pair(
9596        &mut self,
9597        batch: &mut ChangeAccumulator,
9598        table: &TableSchema,
9599        last: &[(String, Vec<String>)],
9600        owners: &[Vec<(String, Vec<String>)>],
9601    ) -> Result<bool, String> {
9602        if local_scope_predicate(table, last, owners).is_err() {
9603            return Ok(false);
9604        }
9605        for commit in self.drop_failed_commits(|name, values| {
9606            name == table.name && owns_local_row(table, last, owners, values)
9607        })? {
9608            batch.conflicts = true;
9609            batch.rejections = true;
9610            batch.outcomes = true;
9611            for operation in commit.ops {
9612                batch.table(&operation.table);
9613            }
9614        }
9615        if !self.purge_scope_rows(&table.name, last, owners)? {
9616            return Err("storage.failed: the revoked scope purge failed".to_owned());
9617        }
9618        self.record_scope_map(batch, &table.name, last);
9619        if self.drop_doomed_outbox(table, last, owners)? {
9620            batch.status = true;
9621            batch.rejections = true;
9622            batch.outcomes = true;
9623        }
9624        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9625        self.delete_unreferenced_cached_blobs()?;
9626        Ok(true)
9627    }
9628
9629    /// §4.5 client application: apply one live-section or delta `COMMIT`
9630    /// with only the changes that match the persisted echo of a receiving
9631    /// unit; every applied change then goes through the version guard.
9632    /// Returns whether any change of the frame applied.
9633    fn apply_live_commit(
9634        &mut self,
9635        receivers: &[UnitRecord],
9636        tables: &[String],
9637        changes: Vec<ssp2::model::Change>,
9638        commit_seq: i64,
9639        source: AppliedServerCommitSource,
9640    ) -> Result<bool, TransportError> {
9641        let changes: Vec<ssp2::model::Change> = changes
9642            .into_iter()
9643            .filter(|change| {
9644                // An unknown table fails loud in the apply below.
9645                let Some(table) = tables
9646                    .get(change.table_index as usize)
9647                    .and_then(|name| self.schema.table(name))
9648                else {
9649                    return true;
9650                };
9651                receivers.iter().any(|unit| {
9652                    covers(table, &unit.scopes)
9653                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9654                            |scopes| {
9655                                scopes.iter().all(|(variable, values)| {
9656                                    change
9657                                        .scopes
9658                                        .iter()
9659                                        .find(|(key, _)| key == variable)
9660                                        .is_some_and(|(_, value)| values.contains(value))
9661                                })
9662                            },
9663                        )
9664                })
9665            })
9666            .collect();
9667        if changes.is_empty() {
9668            return Ok(false);
9669        }
9670        let affected_tables = changes
9671            .iter()
9672            .filter_map(|change| tables.get(change.table_index as usize))
9673            .cloned()
9674            .collect::<BTreeSet<_>>();
9675        let previous_batches = self.change_queue.len();
9676        self.apply_commit_frame(tables, &changes, commit_seq)?;
9677        if self.change_queue.len() > previous_batches {
9678            if let Some(batch) = self.change_queue.back_mut() {
9679                batch
9680                    .applied_commits
9681                    .extend(
9682                        affected_tables
9683                            .into_iter()
9684                            .map(|table| AppliedServerCommit {
9685                                table,
9686                                commit_seq,
9687                                source,
9688                            }),
9689                    );
9690            }
9691        }
9692        Ok(true)
9693    }
9694
9695    // Both pull and realtime frames use the same durable observation boundary.
9696    fn apply_commit_frame(
9697        &mut self,
9698        tables: &[String],
9699        changes: &[ssp2::model::Change],
9700        sequence: i64,
9701    ) -> Result<(), TransportError> {
9702        #[cfg(feature = "bench-internals")]
9703        let _phase = self.benchmark_phases.start(Phase::CommitApply);
9704        let rollback = self.capture();
9705        self.coalesce_unit(changes.len())
9706            .and_then(|()| self.begin_observation("syncular_commit"))
9707            .map_err(|message| TransportError::new("storage.failed", message))?;
9708        let applied = (|| {
9709            let mut batch = ChangeAccumulator::default();
9710            self.record_commit_changes(&mut batch, tables, changes);
9711            // §7.1 chained base: a change names the version an acknowledged own
9712            // commit wrote; pending upserts chained on it move to that version.
9713            for change in changes {
9714                let Some(table) = tables.get(change.table_index as usize) else {
9715                    continue;
9716                };
9717                let acknowledged = self
9718                    .conn
9719                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
9720                    .and_then(|mut statement| {
9721                        statement
9722                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
9723                            .collect::<Result<Vec<_>, _>>()
9724                    })
9725                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
9726                    .iter()
9727                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
9728                    .collect::<Result<Vec<_>, _>>()
9729                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
9730                for index in 0..self.outbox.len() {
9731                    if !self.outbox[index]
9732                        .ops
9733                        .iter()
9734                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
9735                    {
9736                        continue;
9737                    }
9738                    let commit = Arc::make_mut(&mut self.outbox[index]);
9739                    for op in &mut commit.ops {
9740                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
9741                            op.base_version = Some(change.row_version);
9742                        }
9743                    }
9744                    let ops = serde_json::to_string(
9745                        &commit
9746                            .ops
9747                            .iter()
9748                            .map(CommitOperation::from)
9749                            .collect::<Vec<_>>(),
9750                    )
9751                    .map_err(|_| {
9752                        TransportError::new(
9753                            "storage.failed",
9754                            "sync.local_corrupt: unencodable outbox".to_owned(),
9755                        )
9756                    })?;
9757                    let id = commit.client_commit_id.clone();
9758                    self.conn
9759                        .execute(
9760                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
9761                            rusqlite::params![ops, id],
9762                        )
9763                        .map_err(|error| {
9764                            TransportError::new(
9765                                "storage.failed",
9766                                Self::sqlite_failure(&self.storage_failure, error),
9767                            )
9768                        })?;
9769                }
9770            }
9771            {
9772                let storage_failed = |error| {
9773                    TransportError::new(
9774                        "storage.failed",
9775                        Self::sqlite_failure(&self.storage_failure, error),
9776                    )
9777                };
9778                let pinned: HashSet<(&str, &str)> = self
9779                    .outbox
9780                    .iter()
9781                    .flat_map(|commit| &commit.ops)
9782                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
9783                    .collect();
9784                for change in changes {
9785                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
9786                        TransportError::new(
9787                            "sync.invalid_request",
9788                            "change tableIndex out of range".to_owned(),
9789                        )
9790                    })?;
9791                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
9792                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
9793                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
9794                            .map_err(storage_failed)?;
9795                    }
9796                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
9797                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
9798                        .map_err(storage_failed)?;
9799                }
9800            }
9801            self.apply_commit_changes(tables, changes)?;
9802            self.rebuild_overlay_if_dirty()
9803                .map_err(|message| TransportError::new("storage.failed", message))?;
9804            self.finish_observation("syncular_commit", batch)
9805                .map_err(|message| TransportError::new("storage.failed", message))
9806        })();
9807        if applied.is_err() {
9808            self.rollback_observation("syncular_commit");
9809            self.restore(rollback);
9810        }
9811        applied
9812    }
9813
9814    fn apply_commit_changes(
9815        &mut self,
9816        tables: &[String],
9817        changes: &[ssp2::model::Change],
9818    ) -> Result<(), TransportError> {
9819        // A clean overlay can reconcile changed primary keys independently
9820        // when their tables have no secondary unique constraints. Such a
9821        // constraint can change whether another row's write succeeds, so a
9822        // table with one and any local intent takes the table rebuild.
9823        let changed_tables: BTreeSet<&str> = changes
9824            .iter()
9825            .filter_map(|change| tables.get(change.table_index as usize))
9826            .map(String::as_str)
9827            .collect();
9828        let mut mirror_visible = !self.overlay_dirty.get();
9829        for name in &changed_tables {
9830            if mirror_visible
9831                && self
9832                    .schema
9833                    .table(name)
9834                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
9835            {
9836                let (pending, protected) = self
9837                    .table_intent(name)
9838                    .map_err(|message| TransportError::new("storage.failed", message))?;
9839                mirror_visible = !pending && !protected;
9840            }
9841        }
9842        for change in changes {
9843            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
9844                TransportError::new(
9845                    "sync.invalid_request",
9846                    "change tableIndex out of range".to_owned(),
9847                )
9848            })?;
9849            let table = self.schema.table(table_name).ok_or_else(|| {
9850                TransportError::new(
9851                    "sync.schema_mismatch",
9852                    format!("change targets unknown table {table_name:?}"),
9853                )
9854            })?;
9855            let landed = match change.op {
9856                Op::Upsert => {
9857                    let payload = change.row.as_ref().ok_or_else(|| {
9858                        TransportError::new(
9859                            "sync.invalid_request",
9860                            "upsert change without row payload".to_owned(),
9861                        )
9862                    })?;
9863                    // §5.11: decrypt encrypted columns on apply.
9864                    let row = {
9865                        #[cfg(feature = "bench-internals")]
9866                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
9867                        // §5.11: a decrypt failure keeps its own identity.
9868                        decode_row_bytes(table, payload, &self.encryption)?
9869                    };
9870                    let table_name = table.name.clone();
9871                    // §4.5 version guard: only a newer upsert lands.
9872                    let landed = self
9873                        .write_base_row(&table_name, &row, change.row_version)
9874                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
9875                    if mirror_visible && landed {
9876                        self.write_row(
9877                            &visible_table(&table_name),
9878                            &table_name,
9879                            &row,
9880                            change.row_version,
9881                        )
9882                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
9883                    }
9884                    landed
9885                }
9886                // §4.5 version guard: a delete at or below the local version
9887                // is skipped, a delete of an absent row is a no-op.
9888                Op::Delete => self
9889                    .delete_base_row(table_name, &change.row_id, change.row_version)
9890                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
9891            };
9892            // The change dropped the row's acknowledged intent either way
9893            // (§7.1), so the overlay of a skipped change rebuilds from the
9894            // unchanged base.
9895            self.overlay_dirty.table(table_name);
9896            if mirror_visible && !(landed && change.op == Op::Upsert) {
9897                let predicate = row_id_predicate(table);
9898                let visible = visible_table(table_name);
9899                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
9900                if !landed {
9901                    statements.push(format!(
9902                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
9903                        base_table(table_name)
9904                    ));
9905                }
9906                for statement in &statements {
9907                    self.conn
9908                        .prepare_cached(statement)
9909                        .and_then(|mut statement| {
9910                            statement.execute(rusqlite::params![change.row_id])
9911                        })
9912                        .map_err(|error| {
9913                            TransportError::new("sync.invalid_request", error.to_string())
9914                        })?;
9915                }
9916            }
9917        }
9918        if mirror_visible {
9919            let changed_rows: HashSet<(&str, &str)> = changes
9920                .iter()
9921                .map(|change| {
9922                    (
9923                        tables[change.table_index as usize].as_str(),
9924                        change.row_id.as_str(),
9925                    )
9926                })
9927                .collect();
9928            self.replay_overlay(&changed_tables, Some(&changed_rows))
9929                .map_err(|message| TransportError::new("storage.failed", message))?;
9930            self.overlay_dirty.set(false);
9931        }
9932        Ok(())
9933    }
9934
9935    /// §5.6 segment application: validate against the generated schema,
9936    /// clear the pair's scope on the table's first page (fail closed
9937    /// without a mapping), then upsert each row with its segment-carried
9938    /// server version under the §4.5 version guard (§5.2).
9939    fn apply_segment(
9940        &mut self,
9941        table: &TableSchema,
9942        segment: &RowsSegment,
9943        first_page: bool,
9944        effective: &[(String, Vec<String>)],
9945        unit_id: &str,
9946    ) -> Result<u32, SectionError> {
9947        // §5.2: the column table validates against the generated schema —
9948        // order, names, types, nullability; mismatch is fatal. §5.11: the
9949        // server sends the WIRE types (bytes for an encrypted column), so
9950        // validate against wire_columns.
9951        let column = if segment.columns.len() != table.wire_columns.len() {
9952            Some(Value::from("count"))
9953        } else {
9954            segment
9955                .columns
9956                .iter()
9957                .zip(table.wire_columns.iter())
9958                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
9959                .map(Value::from)
9960        };
9961        if segment.table != table.name
9962            || segment.schema_version != self.schema.version
9963            || column.is_some()
9964        {
9965            let mut details = Map::new();
9966            details.insert("table".to_owned(), Value::from(table.name.as_str()));
9967            if let Some(column) = column {
9968                details.insert("column".to_owned(), column);
9969            }
9970            return Err(SectionError::Transport(TransportError {
9971                code: "sync.schema_mismatch".to_owned(),
9972                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
9973                details: Some(Value::Object(details)),
9974            }));
9975        }
9976        let mut applied = 0u32;
9977        let empty = Vec::new();
9978        let blocks = if segment.blocks.is_empty() {
9979            std::slice::from_ref(&empty)
9980        } else {
9981            &segment.blocks
9982        };
9983        for (index, block) in blocks.iter().enumerate() {
9984            let clear = first_page && index == 0;
9985            if !clear && block.is_empty() {
9986                continue;
9987            }
9988            let rollback = self.capture();
9989            let (pending, protected) = self
9990                .table_intent(&table.name)
9991                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9992            let reconcile_pending =
9993                !protected && pending && table.indexes.iter().any(|index| index.unique);
9994            let mirror_visible = !protected && !reconcile_pending;
9995            self.coalesce_unit(block.len())
9996                .and_then(|()| self.begin_observation("syncular_segment_block"))
9997                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9998            let outcome = (|| {
9999                self.rebuild_overlay_if_dirty()
10000                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10001                let mut changed_keys = Vec::new();
10002                let mut batch = ChangeAccumulator::default();
10003                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10004                    batch.table(&table.name);
10005                }
10006                for (row_index, row) in block.iter().enumerate() {
10007                    let decrypted;
10008                    let values = if table.has_encrypted_columns() {
10009                        let mut values = row.values.clone();
10010                        crate::values::decrypt_segment_row(table, &mut values, &self.encryption)
10011                            .map_err(SectionError::Transport)?;
10012                        decrypted = values;
10013                        &decrypted
10014                    } else {
10015                        &row.values
10016                    };
10017                    if !self
10018                        .write_base_row(&table.name, values, row.server_version)
10019                        .map_err(|message| {
10020                            SectionError::Abort("sync.invalid_request".into(), message)
10021                        })?
10022                    {
10023                        continue;
10024                    }
10025                    if reconcile_pending {
10026                        changed_keys.push(
10027                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10028                                |message| SectionError::Abort("storage.failed".into(), message),
10029                            )?,
10030                        );
10031                    }
10032                    if mirror_visible {
10033                        self.write_row(
10034                            &visible_table(&table.name),
10035                            &table.name,
10036                            values,
10037                            row.server_version,
10038                        )
10039                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10040                    }
10041                    let processed = u64::from(applied) + row_index as u64 + 1;
10042                    if processed.is_multiple_of(1024) {
10043                        self.progress.update(|p| p.rows_processed = processed);
10044                    }
10045                }
10046                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10047                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10048                self.finish_observation("syncular_segment_block", batch)
10049                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10050            })();
10051            if let Err(error) = outcome {
10052                self.rollback_observation("syncular_segment_block");
10053                self.restore(rollback);
10054                return Err(error);
10055            }
10056            applied += block.len() as u32;
10057            self.progress
10058                .update(|p| p.rows_processed = u64::from(applied));
10059            std::thread::yield_now();
10060        }
10061        self.progress
10062            .update(|p| p.rows_processed = u64::from(applied));
10063        Ok(applied)
10064    }
10065
10066    /// §5.3 sqlite-image application: validate the in-file metadata
10067    /// against the descriptor, validate column names/order against the
10068    /// generated schema, run the §5.6 first-page clear when fresh, then
10069    /// replace-or-upsert every image row with its `_syncular_version`.
10070    /// Mechanics: the image lands in a temp file read through a second
10071    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10072    /// Each image chunk commits its writes and revision before readers continue.
10073    #[allow(clippy::too_many_arguments)]
10074    fn apply_sqlite_segment(
10075        &mut self,
10076        table: &TableSchema,
10077        bytes: &[u8],
10078        first_page: bool,
10079        effective: &[(String, Vec<String>)],
10080        unit_id: &str,
10081        row_count: i64,
10082        as_of_commit_seq: i64,
10083        scope_digest: &str,
10084    ) -> Result<u32, SectionError> {
10085        let invalid = |detail: &str| {
10086            SectionError::Abort(
10087                "sync.invalid_request".to_owned(),
10088                format!("sqlite segment rejected: {detail} (§5.3)"),
10089            )
10090        };
10091        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10092        // one for a table with encrypted columns.
10093        if table.has_encrypted_columns() {
10094            return Err(invalid("table has encrypted columns"));
10095        }
10096
10097        // The image lands beside the replica so it shares the replica's
10098        // directory permissions and volume; an in-memory replica has no
10099        // directory and uses the system temp dir. The file is owner-only.
10100        let directory = self
10101            .previous_version_replica_path()
10102            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10103            .unwrap_or_else(std::env::temp_dir);
10104        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10105        let outcome = (|| {
10106            let mut options = std::fs::OpenOptions::new();
10107            options.write(true).create_new(true);
10108            #[cfg(unix)]
10109            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10110            std::io::Write::write_all(
10111                &mut options
10112                    .open(&path)
10113                    .map_err(|_| invalid("image temp file write failed"))?,
10114                bytes,
10115            )
10116            .map_err(|_| invalid("image temp file write failed"))?;
10117            let img = rusqlite::Connection::open_with_flags(
10118                &path,
10119                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10120            )
10121            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10122            self.apply_sqlite_image(
10123                &img,
10124                table,
10125                first_page,
10126                effective,
10127                unit_id,
10128                row_count,
10129                as_of_commit_seq,
10130                scope_digest,
10131            )
10132        })();
10133        let _ = std::fs::remove_file(&path);
10134        outcome
10135    }
10136
10137    #[allow(clippy::too_many_arguments)]
10138    fn apply_sqlite_image(
10139        &mut self,
10140        img: &rusqlite::Connection,
10141        table: &crate::schema::TableSchema,
10142        first_fresh_page: bool,
10143        effective: &[(String, Vec<String>)],
10144        unit_id: &str,
10145        row_count: i64,
10146        as_of_commit_seq: i64,
10147        scope_digest: &str,
10148    ) -> Result<u32, SectionError> {
10149        let invalid = |detail: String| {
10150            SectionError::Abort(
10151                "sync.invalid_request".to_owned(),
10152                format!("sqlite segment rejected: {detail} (§5.3)"),
10153            )
10154        };
10155
10156        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10157        //    one row).
10158        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10159        let meta: MetaRow = img
10160            .query_row(
10161                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10162                        \"scopeDigest\", \"rowCount\",
10163                        (SELECT count(*) FROM _syncular_segment)
10164                 FROM _syncular_segment",
10165                [],
10166                |row| {
10167                    Ok((
10168                        row.get(0)?,
10169                        row.get(1)?,
10170                        row.get(2)?,
10171                        row.get(3)?,
10172                        row.get(4)?,
10173                        row.get(5)?,
10174                        row.get(6)?,
10175                    ))
10176                },
10177            )
10178            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10179        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10180        if meta_count != 1 {
10181            return Err(invalid(format!(
10182                "_syncular_segment must contain exactly one row, found {meta_count}"
10183            )));
10184        }
10185        if format != 1 {
10186            return Err(invalid(format!("format {format}")));
10187        }
10188        if meta_table != table.name {
10189            return Err(invalid(format!("image table {meta_table:?}")));
10190        }
10191        if schema_version != i64::from(self.schema.version) {
10192            return Err(invalid(format!("schemaVersion {schema_version}")));
10193        }
10194        if pin != as_of_commit_seq {
10195            return Err(invalid(format!("asOfCommitSeq {pin}")));
10196        }
10197        if digest != scope_digest {
10198            return Err(invalid("scopeDigest mismatch".to_owned()));
10199        }
10200        if meta_rows != row_count {
10201            return Err(invalid(format!("rowCount {meta_rows}")));
10202        }
10203
10204        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10205        let mut names: Vec<String> = Vec::new();
10206        let mut primary_keys: Vec<String> = Vec::new();
10207        {
10208            let mut stmt = img
10209                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10210                .map_err(|_| invalid("image data table missing".to_owned()))?;
10211            let mut rows = stmt
10212                .query([])
10213                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10214            while let Some(row) = rows
10215                .next()
10216                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10217            {
10218                if row
10219                    .get::<_, i64>(5)
10220                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10221                    > 0
10222                {
10223                    primary_keys.push(row.get(1).map_err(|error| {
10224                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10225                    })?);
10226                }
10227                names.push(
10228                    row.get::<_, String>(1)
10229                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10230                );
10231            }
10232        }
10233        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10234        expected.push("_syncular_version");
10235        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10236            return Err(SectionError::Transport(TransportError {
10237                code: "sync.schema_mismatch".to_owned(),
10238                message: "sqlite segment columns do not match the generated schema (§5.3)"
10239                    .to_owned(),
10240                details: Some(serde_json::json!({ "table": table.name })),
10241            }));
10242        }
10243
10244        if primary_keys != vec![table.primary_key.clone()] {
10245            return Err(invalid(
10246                "image primary key does not match generated schema".into(),
10247            ));
10248        }
10249        let actual_count: i64 = img
10250            .query_row(
10251                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10252                [],
10253                |row| row.get(0),
10254            )
10255            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10256        if actual_count != row_count {
10257            return Err(invalid("image row count does not match descriptor".into()));
10258        }
10259        let insert = self.insert_row_sql(&base_table(&table.name), table);
10260        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10261        let column_list = names
10262            .iter()
10263            .map(|n| quote_ident(n))
10264            .collect::<Vec<_>>()
10265            .join(", ");
10266        let mut stmt = img
10267            .prepare(&format!(
10268                "SELECT {column_list} FROM {} ORDER BY {}",
10269                quote_ident(&table.name),
10270                quote_ident(&table.primary_key)
10271            ))
10272            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10273        let mut rows = stmt
10274            .query([])
10275            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10276        let mut applied = 0u32;
10277        let mut limit = 1024u32;
10278        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10279        loop {
10280            let clear = applied == 0 && first_fresh_page;
10281            let rollback = self.capture();
10282            let (pending, protected) = self
10283                .table_intent(&table.name)
10284                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10285            let reconcile_pending =
10286                !protected && pending && table.indexes.iter().any(|index| index.unique);
10287            let mirror_visible = !protected && !reconcile_pending;
10288            // §5.3: the first chunk holds 1,024 rows. A later chunk holds at
10289            // most twice the rows of the one before, at most 65,536, and at
10290            // most the rows the one before applied per 100 ms from its start
10291            // to its COMMIT; a chunk that replays local writes holds at most
10292            // 1,024.
10293            let chunk = if pending || protected {
10294                limit.min(1024)
10295            } else {
10296                limit
10297            };
10298            self.coalesce_unit(chunk as usize)
10299                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10300                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10301            let started = std::time::Instant::now();
10302            let result = (|| {
10303                self.rebuild_overlay_if_dirty()
10304                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10305                let mut changed_keys = Vec::new();
10306                let mut batch = ChangeAccumulator::default();
10307                if clear && self.clear_scope(table, effective, unit_id)? {
10308                    batch.table(&table.name);
10309                }
10310                let mut processed = 0u32;
10311                {
10312                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10313                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10314                    })?;
10315                    let mut mirror = if mirror_visible {
10316                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10317                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10318                        })?)
10319                    } else {
10320                        None
10321                    };
10322                    for _ in 0..chunk {
10323                        let Some(row) = rows.next().map_err(|error| {
10324                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10325                        })?
10326                        else {
10327                            break;
10328                        };
10329                        for (i, column) in table.columns.iter().enumerate() {
10330                            let cell = row.get_ref(i).map_err(|error| {
10331                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10332                            })?;
10333                            let param = image_cell_param(column, cell).map_err(invalid)?;
10334                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10335                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10336                            })?;
10337                            if let Some(mirror) = &mut mirror {
10338                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10339                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10340                                })?;
10341                            }
10342                        }
10343                        let version_index = table.columns.len();
10344                        let version: i64 = row.get(version_index).map_err(|error| {
10345                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10346                        })?;
10347                        if version < 1 {
10348                            return Err(invalid("image row version must be positive".into()));
10349                        }
10350                        ins.raw_bind_parameter(version_index + 1, version)
10351                            .map_err(|error| {
10352                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10353                            })?;
10354                        // §4.5 version guard: the base upsert decides.
10355                        let landed = ins.raw_execute().map_err(|error| {
10356                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10357                        })? > 0;
10358                        processed += 1;
10359                        if processed.is_multiple_of(1024) && processed < chunk {
10360                            self.progress
10361                                .update(|p| p.rows_processed = u64::from(applied + processed));
10362                        }
10363                        if !landed {
10364                            continue;
10365                        }
10366                        if let Some(mirror) = &mut mirror {
10367                            mirror
10368                                .raw_bind_parameter(version_index + 1, version)
10369                                .map_err(|error| {
10370                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10371                                })?;
10372                            mirror.raw_execute().map_err(|error| {
10373                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10374                            })?;
10375                        }
10376                        if reconcile_pending {
10377                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10378                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10379                            )?);
10380                        }
10381                    }
10382                }
10383                if processed > 0 {
10384                    batch.table(&table.name);
10385                    self.overlay_dirty.table(&table.name);
10386                }
10387                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10388                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10389                let written = std::time::Instant::now();
10390                self.finish_observation("syncular_image_chunk", batch)
10391                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10392                Ok((processed, written))
10393            })();
10394            let (processed, written) = match result {
10395                Ok(done) => done,
10396                Err(error) => {
10397                    self.rollback_observation("syncular_image_chunk");
10398                    self.restore(rollback);
10399                    return Err(error);
10400                }
10401            };
10402            // The chunk commits before the yield, also inside a coalesced apply.
10403            let coalesced = self.coalesced.is_some();
10404            if coalesced {
10405                self.commit_coalesced_apply()
10406                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10407            }
10408            write += written - started;
10409            commit += written.elapsed();
10410            applied += processed;
10411            let last = processed < chunk || i64::from(applied) == actual_count;
10412            // A file replica's WAL keeps its high-water size until truncated.
10413            if last && self.conn.is_autocommit() {
10414                self.conn
10415                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10416                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10417            }
10418            if coalesced {
10419                self.begin_coalesced_apply()
10420                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10421            }
10422            self.progress.update(|p| {
10423                p.rows_processed = u64::from(applied);
10424                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10425                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10426            });
10427            let duration = started.elapsed().as_secs_f64() * 1000.0;
10428            #[cfg(test)]
10429            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10430            std::thread::yield_now();
10431            if last {
10432                break;
10433            }
10434            // A zero duration saturates the cast and leaves the doubling.
10435            limit = (processed * 2)
10436                .min(65_536)
10437                .min(((f64::from(processed) * 100.0 / duration) as u32).max(1));
10438        }
10439        if i64::from(applied) != actual_count {
10440            return Err(invalid(format!(
10441                "image yielded {applied} rows, expected {actual_count}"
10442            )));
10443        }
10444        Ok(applied)
10445    }
10446
10447    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10448    /// them from the base and replay their remaining intent. A table with a
10449    /// secondary unique index restores and replays every row with intent,
10450    /// since one row's value can admit or defer another row's write.
10451    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10452        let names: BTreeSet<&str> = drained
10453            .iter()
10454            .flat_map(|commit| &commit.ops)
10455            .map(|op| op.table.as_str())
10456            .collect();
10457        for name in names {
10458            let Some(table) = self.schema.table(name) else {
10459                continue;
10460            };
10461            let ops = drained.iter().flat_map(|commit| &commit.ops);
10462            let tables = BTreeSet::from([name]);
10463            let rows: HashSet<_> = ops
10464                .clone()
10465                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10466                .collect();
10467            let unique = table.indexes.iter().any(|index| index.unique);
10468            let acknowledged = if unique {
10469                self.acknowledged_operations(None, Some(&tables))?
10470            } else {
10471                Vec::new()
10472            };
10473            let peers = acknowledged
10474                .iter()
10475                .chain(
10476                    self.failed_commits
10477                        .iter()
10478                        .flat_map(|(commit, _)| &commit.ops),
10479                )
10480                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10481                .filter(|_| unique);
10482            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10483            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10484        }
10485        Ok(())
10486    }
10487
10488    /// Whether `table` has pending outbox intent, and whether it has protected
10489    /// (acknowledged or retained failed) intent.
10490    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10491        let touches = |op: &OutboxOp| op.table == table;
10492        Ok((
10493            self.outbox
10494                .iter()
10495                .flat_map(|commit| &commit.ops)
10496                .any(touches),
10497            self.failed_commits
10498                .iter()
10499                .flat_map(|(commit, _)| &commit.ops)
10500                .any(touches)
10501                || self
10502                    .conn
10503                    .query_row(
10504                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10505                        [table],
10506                        |row| row.get(0),
10507                    )
10508                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10509        ))
10510    }
10511
10512    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10513    /// only pending intent, so a table with protected intent rebuilds; a
10514    /// `restore` restores the imported `keys` and every pending row first.
10515    fn reconcile_import(
10516        &mut self,
10517        table: &TableSchema,
10518        keys: &mut Vec<SqlValue>,
10519        protected: bool,
10520        restore: bool,
10521    ) -> Result<(), String> {
10522        if protected {
10523            return self.rebuild_overlay_if_dirty();
10524        }
10525        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10526        if restore {
10527            self.restore_base_rows(table, keys, pending.clone())?;
10528        }
10529        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10530        self.overlay_dirty.set(false);
10531        Ok(())
10532    }
10533
10534    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10535    /// `table` from the base before the caller replays their intent. The
10536    /// caller owns the transaction and publishes its revision after replay.
10537    fn restore_base_rows<'a>(
10538        &self,
10539        table: &TableSchema,
10540        keys: &mut Vec<SqlValue>,
10541        ops: impl IntoIterator<Item = &'a OutboxOp>,
10542    ) -> Result<(), String> {
10543        let visible = visible_table(&table.name);
10544        let base = base_table(&table.name);
10545        let pk = quote_ident(&table.primary_key);
10546        let predicate = row_id_predicate(table);
10547        let mut lookup = self.conn.prepare_cached(&format!(
10548            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10549        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10550        let mut deleted = HashSet::new();
10551        for op in ops.into_iter().filter(|op| op.table == table.name) {
10552            if op.upsert {
10553                if let Some(values) = &op.values {
10554                    let key = json_to_column_value(
10555                        &table.columns[table.pk_index],
10556                        values.get(&table.primary_key),
10557                    )?;
10558                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10559                }
10560            } else if deleted.insert(&op.row_id) {
10561                keys.extend(
10562                    lookup
10563                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10564                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10565                        .collect::<Result<Vec<_>, _>>()
10566                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10567                );
10568            }
10569        }
10570        // Delete the complete affected set before restoring any base value, so
10571        // old optimistic unique values cannot collide with the new server rows.
10572        for chunk in keys.chunks(400) {
10573            let holes = vec!["?"; chunk.len()].join(",");
10574            self.conn
10575                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10576                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10577                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10578        }
10579        let updates = table
10580            .columns
10581            .iter()
10582            .map(|column| column.name.as_str())
10583            .chain(std::iter::once("_syncular_version"))
10584            .filter(|name| *name != table.primary_key)
10585            .map(|name| {
10586                let name = quote_ident(name);
10587                format!("{name}=excluded.{name}")
10588            })
10589            .collect::<Vec<_>>()
10590            .join(",");
10591        for chunk in keys.chunks(400) {
10592            let holes = vec!["?"; chunk.len()].join(",");
10593            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10594                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10595                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10596        }
10597        Ok(())
10598    }
10599
10600    // -- application-authorized local purge ---------------------------------------
10601
10602    fn compile_local_data_purge(
10603        &self,
10604        input: &LocalDataPurgeInput,
10605    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10606        let invalid = |message: String| format!("sync.invalid_request: {message}");
10607        if input.purge_id.is_empty()
10608            || input.purge_id.len() > 128
10609            || !is_local_operation_code_like(&input.purge_id)
10610        {
10611            return Err(invalid(
10612                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10613            ));
10614        }
10615        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10616            return Err(invalid(format!(
10617                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10618            )));
10619        }
10620
10621        let mut deduplicated: BTreeMap<
10622            String,
10623            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10624        > = BTreeMap::new();
10625        for target in &input.targets {
10626            let table = self.schema.table(&target.table).ok_or_else(|| {
10627                invalid(format!(
10628                    "local purge names unknown table {:?}",
10629                    target.table
10630                ))
10631            })?;
10632            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10633                return Err(invalid(format!(
10634                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10635                    target.table
10636                )));
10637            }
10638            let mut selectors = Vec::with_capacity(target.selectors.len());
10639            let mut canonical_selectors = BTreeMap::new();
10640            for (column_name, raw_values) in &target.selectors {
10641                let Some((column_index, column)) = table
10642                    .columns
10643                    .iter()
10644                    .enumerate()
10645                    .find(|(_, column)| column.name == *column_name)
10646                else {
10647                    return Err(invalid(format!(
10648                        "local purge target {:?} names unknown column {:?}",
10649                        target.table, column_name
10650                    )));
10651                };
10652                let encrypted = table
10653                    .encrypted_columns
10654                    .iter()
10655                    .any(|candidate| candidate.index == column_index);
10656                if column.ty != ColumnType::String || encrypted {
10657                    return Err(invalid(format!(
10658                        "local purge selector {:?}.{:?} must be a plaintext string column",
10659                        target.table, column_name
10660                    )));
10661                }
10662                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
10663                    return Err(invalid(format!(
10664                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
10665                        target.table, column_name
10666                    )));
10667                }
10668                let mut values = raw_values.clone();
10669                values.sort();
10670                values.dedup();
10671                if values.iter().any(|value| {
10672                    value.is_empty()
10673                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
10674                        || !is_local_operation_code_like(value)
10675                }) {
10676                    return Err(invalid(format!(
10677                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
10678                    )));
10679                }
10680                selectors.push((column_name.clone(), values.clone()));
10681                canonical_selectors.insert(column_name.clone(), values);
10682            }
10683            let canonical = LocalDataPurgeTarget {
10684                table: target.table.clone(),
10685                selectors: canonical_selectors,
10686            };
10687            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10688            deduplicated.insert(
10689                key,
10690                (
10691                    CompiledLocalDataPurgeTarget {
10692                        table: target.table.clone(),
10693                        selectors,
10694                    },
10695                    canonical,
10696                ),
10697            );
10698        }
10699        let targets = deduplicated
10700            .values()
10701            .map(|(compiled, _)| compiled.clone())
10702            .collect::<Vec<_>>();
10703        let canonical = deduplicated
10704            .values()
10705            .map(|(_, target)| target.clone())
10706            .collect::<Vec<_>>();
10707        let canonical_plan =
10708            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10709        Ok((targets, canonical_plan))
10710    }
10711
10712    fn local_purge_base_row_ids(
10713        &self,
10714        targets: &[CompiledLocalDataPurgeTarget],
10715    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
10716        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
10717        for target in targets {
10718            let table = self
10719                .schema
10720                .table(&target.table)
10721                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
10722            let mut clauses = Vec::with_capacity(target.selectors.len());
10723            let mut params = Vec::new();
10724            for (column, values) in &target.selectors {
10725                clauses.push(format!(
10726                    "{} IN ({})",
10727                    quote_ident(column),
10728                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
10729                ));
10730                params.extend(values.iter().cloned().map(SqlValue::Text));
10731            }
10732            let sql = format!(
10733                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
10734                quote_ident(&table.primary_key),
10735                base_table(&target.table),
10736                clauses.join(" AND ")
10737            );
10738            let mut statement = self
10739                .conn
10740                .prepare(&sql)
10741                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10742            let rows = statement
10743                .query_map(rusqlite::params_from_iter(params), |row| {
10744                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
10745                    render_row_id_json(Some(&value))
10746                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
10747                })
10748                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10749            let ids = by_table.entry(target.table.clone()).or_default();
10750            for row in rows {
10751                ids.insert(
10752                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10753                );
10754            }
10755        }
10756        Ok(by_table)
10757    }
10758
10759    fn local_purge_values_match(
10760        target: &CompiledLocalDataPurgeTarget,
10761        values: &Map<String, Value>,
10762    ) -> bool {
10763        target.selectors.iter().all(|(column, allowed)| {
10764            values
10765                .get(column)
10766                .and_then(Value::as_str)
10767                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
10768        })
10769    }
10770
10771    /// Apply one host-authorized local security purge. The host MUST gate the
10772    /// corresponding subscriptions first; this primitive owns local SQLite
10773    /// cleanup and never grants/revokes server authority by itself.
10774    pub fn purge_local_data(
10775        &mut self,
10776        input: &LocalDataPurgeInput,
10777    ) -> Result<LocalDataPurgeResult, String> {
10778        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
10779        let meta_key = format!("localPurge:{}", input.purge_id);
10780        let applied_plan = self
10781            .conn
10782            .query_row(
10783                "SELECT value FROM _syncular_meta WHERE key = ?1",
10784                rusqlite::params![meta_key],
10785                |row| row.get::<_, String>(0),
10786            )
10787            .optional()
10788            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10789        if let Some(applied_plan) = applied_plan {
10790            if applied_plan != canonical_plan {
10791                return Err(format!(
10792                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
10793                    input.purge_id
10794                ));
10795            }
10796            // A retry after a failed rewrite finishes it.
10797            self.erase_purged_pages()?;
10798            return Ok(LocalDataPurgeResult {
10799                already_applied: true,
10800                purged_rows: 0,
10801                dropped_commits: 0,
10802            });
10803        }
10804
10805        // RFC 0005 D7: the container file cannot be a purge target (its name is
10806        // not in the running schema), so purge drops it as a fixed step, before
10807        // any mirror row is touched and unconditionally.
10808        let _ = self.drop_previous_version();
10809
10810        let rollback = self.capture();
10811        self.begin_observation("syncular_local_purge")?;
10812        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
10813            let row_ids = self.local_purge_base_row_ids(&targets)?;
10814            let retained_dropped = self.drop_failed_commits(|table, values| {
10815                targets.iter().any(|target| {
10816                    target.table == table && Self::local_purge_values_match(target, values)
10817                })
10818            })?;
10819            let doomed = self
10820                .outbox
10821                .iter()
10822                .filter(|commit| {
10823                    commit.ops.iter().any(|operation| {
10824                        let matching_targets = targets
10825                            .iter()
10826                            .filter(|target| target.table == operation.table)
10827                            .collect::<Vec<_>>();
10828                        if matching_targets.is_empty() {
10829                            return false;
10830                        }
10831                        if row_ids
10832                            .get(&operation.table)
10833                            .is_some_and(|ids| ids.contains(&operation.row_id))
10834                        {
10835                            return true;
10836                        }
10837                        operation.values.as_ref().is_some_and(|values| {
10838                            matching_targets
10839                                .iter()
10840                                .any(|target| Self::local_purge_values_match(target, values))
10841                        })
10842                    })
10843                })
10844                .cloned()
10845                .collect::<Vec<_>>();
10846            let doomed_ids = doomed
10847                .iter()
10848                .map(|commit| commit.client_commit_id.clone())
10849                .collect::<BTreeSet<_>>();
10850            let mut batch = ChangeAccumulator::default();
10851            for commit in &retained_dropped {
10852                batch.conflicts = true;
10853                batch.rejections = true;
10854                batch.outcomes = true;
10855                for operation in &commit.ops {
10856                    batch.table(&operation.table);
10857                }
10858            }
10859            let mut rejections = Vec::new();
10860            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
10861            // codes, never its operations, so no purged value survives in it.
10862            for commit in &doomed {
10863                for operation in &commit.ops {
10864                    batch.table(&operation.table);
10865                    self.overlay_dirty.table(&operation.table);
10866                }
10867                let results = (0..commit.ops.len())
10868                    .map(|op_index| {
10869                        let rejection = RejectionRecord {
10870                            client_commit_id: commit.client_commit_id.clone(),
10871                            op_index: op_index as i32,
10872                            code: "client.local_data_purged".to_owned(),
10873                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
10874                            retryable: false,
10875                            details: None,
10876                            operation: None,
10877                        };
10878                        rejections.push(rejection.clone());
10879                        CommitOperationOutcome::Error { rejection }
10880                    })
10881                    .collect::<Vec<_>>();
10882                self.persist_commit_outcome(
10883                    &commit.client_commit_id,
10884                    CommitOutcomeStatus::Rejected,
10885                    &results,
10886                    None,
10887                )?;
10888                self.delete_outbox_persisted(&commit.client_commit_id)?;
10889            }
10890            if !doomed.is_empty() {
10891                self.outbox
10892                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
10893                self.rejections.extend(rejections);
10894                self.prune_commit_outcomes()?;
10895                batch.status = true;
10896                batch.rejections = true;
10897                batch.outcomes = true;
10898            }
10899
10900            let mut purged_rows = 0usize;
10901            for (table, ids) in &row_ids {
10902                if ids.is_empty() {
10903                    continue;
10904                }
10905                batch.table(table);
10906                self.overlay_dirty.table(table);
10907                for row_id in ids {
10908                    self.delete_base_row(table, row_id, i64::MAX)?;
10909                    purged_rows += 1;
10910                }
10911            }
10912            // §5.11 purge erasure: a journaled outcome with an operation or
10913            // conflict row the purge reaches keeps its identity, status, ids
10914            // and codes only.
10915            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
10916                targets.iter().any(|target| {
10917                    target.table == table
10918                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
10919                            || values.is_some_and(|values| {
10920                                Self::local_purge_values_match(target, values)
10921                            }))
10922                })
10923            };
10924            let entries = self
10925                .conn
10926                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
10927                .and_then(|mut statement| {
10928                    statement
10929                        .query_map([], |row| {
10930                            Ok((
10931                                row.get::<_, String>(0)?,
10932                                row.get::<_, String>(1)?,
10933                                row.get::<_, Option<String>>(2)?,
10934                            ))
10935                        })?
10936                        .collect::<Result<Vec<_>, _>>()
10937                })
10938                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10939            for (client_commit_id, results_json, operations_json) in entries {
10940                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
10941                let mut results: Vec<CommitOperationOutcome> =
10942                    serde_json::from_str(&results_json).map_err(corrupt)?;
10943                let operations: Vec<CommitOperation> = operations_json
10944                    .map(|json| serde_json::from_str(&json))
10945                    .transpose()
10946                    .map_err(corrupt)?
10947                    .unwrap_or_default();
10948                let named = operations
10949                    .iter()
10950                    .chain(results.iter().filter_map(|result| match result {
10951                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
10952                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
10953                        CommitOperationOutcome::Applied { .. } => None,
10954                    }))
10955                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
10956                    || results.iter().any(|result| {
10957                        matches!(result, CommitOperationOutcome::Conflict { conflict }
10958                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
10959                    });
10960                if !named {
10961                    continue;
10962                }
10963                for result in &mut results {
10964                    match result {
10965                        CommitOperationOutcome::Conflict { conflict } => {
10966                            conflict.server_row = Map::new();
10967                            conflict.conflict_columns.clear();
10968                            conflict.operation = None;
10969                        }
10970                        CommitOperationOutcome::Error { rejection } => {
10971                            rejection.details = None;
10972                            rejection.operation = None;
10973                        }
10974                        CommitOperationOutcome::Applied { .. } => {}
10975                    }
10976                }
10977                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
10978                self.conn
10979                    .execute(
10980                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
10981                        rusqlite::params![results_json, client_commit_id],
10982                    )
10983                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10984                batch.outcomes = true;
10985            }
10986            self.rebuild_overlay_if_dirty()?;
10987            self.delete_unreferenced_cached_blobs()?;
10988            self.conn
10989                .execute(
10990                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
10991                    rusqlite::params![meta_key, canonical_plan],
10992                )
10993                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10994            Ok((
10995                batch,
10996                LocalDataPurgeResult {
10997                    already_applied: false,
10998                    purged_rows,
10999                    dropped_commits: doomed.len(),
11000                },
11001            ))
11002        })()
11003        .and_then(|(batch, result)| {
11004            self.finish_observation("syncular_local_purge", batch)
11005                .map(|()| result)
11006        });
11007        let result = match applied {
11008            Ok(result) => result,
11009            Err(error) => {
11010                self.rollback_observation("syncular_local_purge");
11011                self.restore(rollback);
11012                return Err(error);
11013            }
11014        };
11015        self.cancel_sync_round();
11016        self.erase_purged_pages()?;
11017        Ok(result)
11018    }
11019
11020    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11021    /// database file, and the WAL keeps the frames written before the purge,
11022    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11023    /// rows, and the checkpoint truncates the WAL.
11024    fn erase_purged_pages(&self) -> Result<(), String> {
11025        let erased = self
11026            .conn
11027            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11028            .and_then(|()| {
11029                self.conn
11030                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11031            });
11032        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11033        erased
11034            .and(reset)
11035            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11036    }
11037
11038    /// Application-authorized recovery of the replicated projection. Unlike a
11039    /// local security purge, this retains the entire outbox, device identity,
11040    /// lease, outcomes, unit registrations, and protected bookkeeping.
11041    /// The projection reset, unit rewind, optimistic replay, and
11042    /// idempotency marker share one savepoint for interruption safety.
11043    pub fn rebootstrap_local_data(
11044        &mut self,
11045        input: &LocalDataRebootstrapInput,
11046    ) -> Result<LocalDataRebootstrapResult, String> {
11047        if input.rebootstrap_id.is_empty()
11048            || input.rebootstrap_id.len() > 128
11049            || !is_local_operation_code_like(&input.rebootstrap_id)
11050        {
11051            return Err(
11052                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11053                    .to_owned(),
11054            );
11055        }
11056        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11057        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11058            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11059        })? {
11060            let (retained_commits, reset_subscriptions) =
11061                decode_local_rebootstrap_receipt(&persisted)?;
11062            return Ok(LocalDataRebootstrapResult {
11063                already_applied: true,
11064                retained_commits,
11065                reset_subscriptions,
11066            });
11067        }
11068        if self.stopped || self.schema_floor.is_some() {
11069            return Err(
11070                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11071                    .to_owned(),
11072            );
11073        }
11074
11075        let retained_commits = self.outbox.len();
11076        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11077        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11078        let rollback = self.capture();
11079        self.begin_observation("syncular_local_rebootstrap")?;
11080        let mut batch = ChangeAccumulator::default();
11081        let applied = (|| -> Result<(), String> {
11082            self.run_schema_reset_observed(&mut batch, false)?;
11083            self.conn
11084                .execute(
11085                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11086                    rusqlite::params![meta_key, receipt],
11087                )
11088                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11089            self.sync_needed = true;
11090            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11091            batch.status = true;
11092            Ok(())
11093        })()
11094        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11095        if let Err(error) = applied {
11096            self.rollback_observation("syncular_local_rebootstrap");
11097            self.restore(rollback);
11098            return Err(error);
11099        }
11100
11101        Ok(LocalDataRebootstrapResult {
11102            already_applied: false,
11103            retained_commits,
11104            reset_subscriptions,
11105        })
11106    }
11107
11108    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11109
11110    /// Delete base and visible rows owned by the effective scopes
11111    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11112    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11113    fn purge_scope_rows(
11114        &mut self,
11115        table_name: &str,
11116        effective: &[(String, Vec<String>)],
11117        owners: &[Vec<(String, Vec<String>)>],
11118    ) -> Result<bool, String> {
11119        let Some(table) = self.schema.table(table_name) else {
11120            return Ok(false);
11121        };
11122        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11123            return Ok(false);
11124        };
11125        let Some((predicate, params)) = predicate else {
11126            return Ok(true);
11127        };
11128        self.overlay_dirty.table(table_name);
11129        for source in [base_table(table_name), visible_table(table_name)] {
11130            self.conn
11131                .execute(
11132                    &format!("DELETE FROM {source} WHERE {predicate}"),
11133                    rusqlite::params_from_iter(&params),
11134                )
11135                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11136        }
11137        Ok(true)
11138    }
11139
11140    /// §3.3: drop pending commits whose upserts provably land in the
11141    /// revoked effective scopes and no other owner covers (row ownership)
11142    /// — whole-commit, never per-operation.
11143    fn drop_doomed_outbox(
11144        &mut self,
11145        table: &TableSchema,
11146        effective: &[(String, Vec<String>)],
11147        owners: &[Vec<(String, Vec<String>)>],
11148    ) -> Result<bool, String> {
11149        let doomed: Vec<_> =
11150            self.outbox
11151                .iter()
11152                .filter(|commit| {
11153                    commit.ops.iter().any(|op| {
11154                        op.upsert
11155                            && op.table == table.name
11156                            && op.values.as_ref().is_some_and(|values| {
11157                                owns_local_row(table, effective, owners, values)
11158                            })
11159                    })
11160                })
11161                .cloned()
11162                .collect();
11163        if doomed.is_empty() {
11164            return Ok(false);
11165        }
11166        let mut rejections = Vec::new();
11167        for commit in &doomed {
11168            let results = commit
11169                .ops
11170                .iter()
11171                .enumerate()
11172                .map(|(op_index, operation)| {
11173                    self.overlay_dirty.table(&operation.table);
11174                    let rejection = RejectionRecord {
11175                        client_commit_id: commit.client_commit_id.clone(),
11176                        op_index: op_index as i32,
11177                        code: "sync.scope_revoked".to_owned(),
11178                        message: "the commit was dropped because its effective scope was revoked"
11179                            .to_owned(),
11180                        retryable: false,
11181                        details: None,
11182                        operation: Some(CommitOperation::from(operation)),
11183                    };
11184                    rejections.push(rejection.clone());
11185                    CommitOperationOutcome::Error { rejection }
11186                })
11187                .collect::<Vec<_>>();
11188            self.persist_commit_outcome(
11189                &commit.client_commit_id,
11190                CommitOutcomeStatus::Rejected,
11191                &results,
11192                Some(&commit.ops),
11193            )?;
11194            self.delete_outbox_persisted(&commit.client_commit_id)?;
11195        }
11196        self.prune_commit_outcomes()?;
11197        let doomed_ids = doomed
11198            .iter()
11199            .map(|commit| commit.client_commit_id.as_str())
11200            .collect::<BTreeSet<_>>();
11201        self.outbox
11202            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11203        self.rejections.extend(rejections);
11204        Ok(true)
11205    }
11206
11207    // -- blobs (§5.9) ----------------------------------------------------------------
11208
11209    /// §5.9.7: hash bytes into the content address, cache them, queue the
11210    /// upload (flushed before the next push, B4). Returns the canonical
11211    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11212    /// column value.
11213    pub fn upload_blob(
11214        &mut self,
11215        bytes: &[u8],
11216        media_type: Option<String>,
11217        name: Option<String>,
11218    ) -> Result<Value, String> {
11219        let blob_id = blob_id_for(bytes);
11220        let now = self.clock_now_ms();
11221        let tx = self
11222            .conn
11223            .transaction()
11224            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11225        tx.execute(
11226            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11227             ON CONFLICT(blob_id) DO NOTHING",
11228            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11229        )
11230        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11231        tx.execute(
11232            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11233            rusqlite::params![blob_id, media_type, now],
11234        )
11235        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11236        tx.commit()
11237            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11238        self.enforce_blob_cache_cap();
11239        let mut obj = Map::new();
11240        obj.insert("blobId".to_owned(), Value::from(blob_id));
11241        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11242        if let Some(mt) = media_type {
11243            obj.insert("mediaType".to_owned(), Value::from(mt));
11244        }
11245        if let Some(n) = name {
11246            obj.insert("name".to_owned(), Value::from(n));
11247        }
11248        Ok(Value::Object(obj))
11249    }
11250
11251    /// §5.9.7: resolve an owned blob body without driver encoding. A
11252    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11253    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11254    pub fn fetch_blob_bytes(
11255        &mut self,
11256        transport: &mut dyn Transport,
11257        blob_id_or_ref: &str,
11258    ) -> Result<FetchedBlob, (String, String)> {
11259        let simple = |m: String| ("client.failed".to_owned(), m);
11260        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11261        // blobId declares none.
11262        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11263            (blob_id_or_ref.to_owned(), None)
11264        } else {
11265            let value: Value = serde_json::from_str(blob_id_or_ref)
11266                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11267            (
11268                value
11269                    .get("blobId")
11270                    .and_then(Value::as_str)
11271                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11272                    .to_owned(),
11273                value.get("byteLength").and_then(Value::as_u64),
11274            )
11275        };
11276        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11277            return Ok(cached);
11278        }
11279        if !self.transport_enabled {
11280            return Err(("sync.offline".into(), "transport gate is closed".into()));
11281        }
11282        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11283        // blob.not_found) verbatim so the harness can assert on it. The
11284        // authorized endpoint serves bytes inline OR (always-issue, presign
11285        // configured) a signed url the client fetches directly — no host auth,
11286        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11287        #[cfg(feature = "bench-internals")]
11288        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11289        let bytes = match transport
11290            .blob_download(&blob_id, byte_length)
11291            .map_err(|e| (e.code, e.message))?
11292        {
11293            BlobDownload::Bytes(bytes) => bytes,
11294            BlobDownload::Url {
11295                url,
11296                url_expires_at_ms,
11297            } => {
11298                // §5.9.5: MUST NOT start a fetch at/past expiry.
11299                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11300                    return Err((
11301                        "sync.segment_expired".to_owned(),
11302                        format!(
11303                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11304                        ),
11305                    ));
11306                }
11307                transport
11308                    .fetch_blob_url(&url, byte_length)
11309                    .map_err(|e| (e.code, e.message))?
11310            }
11311        };
11312        #[cfg(feature = "bench-internals")]
11313        drop(download_phase);
11314        #[cfg(feature = "bench-internals")]
11315        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11316        // §5.9.5 inherits §5.1: the declared size caps the body before the
11317        // content address is verified.
11318        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11319            return Err((
11320                "sync.invalid_request".to_owned(),
11321                "blob exceeds its byteLength".to_owned(),
11322            ));
11323        }
11324        if blob_id_for(&bytes) != blob_id {
11325            return Err(simple(format!(
11326                "blob content address mismatch for {blob_id}"
11327            )));
11328        }
11329        #[cfg(feature = "bench-internals")]
11330        drop(validate_phase);
11331        #[cfg(feature = "bench-internals")]
11332        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11333        let now = self.clock_now_ms();
11334        self.conn
11335            .execute(
11336                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11337                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11338            )
11339            .map_err(|e| simple(e.to_string()))?;
11340        self.conn
11341            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11342            .map_err(|e| simple(e.to_string()))?;
11343        self.enforce_blob_cache_cap();
11344        #[cfg(feature = "bench-internals")]
11345        drop(insert_phase);
11346        self.get_cached_blob_bytes(&blob_id)
11347            .map_err(simple)?
11348            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11349    }
11350
11351    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11352        #[cfg(feature = "bench-internals")]
11353        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11354        let mut stmt = self
11355            .conn
11356            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11357            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11358        let mut rows = stmt
11359            .query(rusqlite::params![blob_id])
11360            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11361        if let Some(row) = rows
11362            .next()
11363            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11364        {
11365            let bytes: Vec<u8> = row
11366                .get(0)
11367                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11368            let byte_length: i64 = row
11369                .get(1)
11370                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11371            let media_type: Option<String> = row
11372                .get(2)
11373                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11374            return Ok(Some(FetchedBlob {
11375                blob_id: blob_id.to_owned(),
11376                byte_length,
11377                bytes,
11378                media_type,
11379            }));
11380        }
11381        Ok(None)
11382    }
11383
11384    /// §5.9.7 B4: upload every queued blob before push.
11385    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11386        let pending: Vec<(String, Option<String>)> = {
11387            let mut stmt = self
11388                .conn
11389                .prepare(
11390                    "SELECT blob_id, media_type FROM (
11391                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11392                      UNION ALL
11393                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11394                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11395                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11396                    ) ORDER BY created_at_ms, blob_id",
11397                )
11398                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11399            let rows = stmt
11400                .query_map([], |row| {
11401                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11402                })
11403                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11404            rows.collect::<Result<_, _>>()
11405                .map_err(|error| match error {
11406                    rusqlite::Error::InvalidColumnType(..)
11407                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11408                        "sync.local_corrupt",
11409                        "Pending blob upload metadata is invalid",
11410                    ),
11411                    _ => TransportError::new("client.failed", error.to_string()),
11412                })?
11413        };
11414        let mut uploads = Vec::new();
11415        for (blob_id, media_type) in pending {
11416            let bytes = (|| {
11417                let stored: Option<(SqlValue, SqlValue)> = self
11418                    .conn
11419                    .query_row(
11420                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11421                        rusqlite::params![blob_id],
11422                        |row| Ok((row.get(0)?, row.get(1)?)),
11423                    )
11424                    .optional()
11425                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11426                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11427                    return Err(TransportError::new(
11428                        "sync.local_corrupt",
11429                        "Pending blob upload body is missing or corrupt",
11430                    ));
11431                };
11432                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11433                    return Err(TransportError::new(
11434                        "sync.local_corrupt",
11435                        "Pending blob upload body is missing or corrupt",
11436                    ));
11437                }
11438                Ok(bytes)
11439            })();
11440            let failed = bytes.is_err();
11441            uploads.push(crate::round::PendingUpload {
11442                id: blob_id,
11443                bytes,
11444                media_type,
11445            });
11446            if failed {
11447                break;
11448            }
11449        }
11450        Ok(uploads)
11451    }
11452
11453    fn visible_blob_ids_sql(&self) -> String {
11454        let mut selects = Vec::new();
11455        for table in &self.schema.tables {
11456            for column in table
11457                .columns
11458                .iter()
11459                .filter(|column| column.ty == ColumnType::BlobRef)
11460            {
11461                let identifier = quote_ident(&column.name);
11462                selects.push(format!(
11463                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11464                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11465                     AND json_type({identifier}, '$.blobId') = 'text'",
11466                    quote_ident(&table.name),
11467                ));
11468            }
11469        }
11470        if selects.is_empty() {
11471            "SELECT NULL AS blob_id WHERE 0".to_owned()
11472        } else {
11473            selects.join(" UNION ")
11474        }
11475    }
11476
11477    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11478    /// staged uploads pin bodies directly; no derived refcount is stored.
11479    fn enforce_blob_cache_cap(&self) {
11480        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11481            return;
11482        };
11483        let mut total: i64 = self
11484            .conn
11485            .query_row(
11486                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11487                [],
11488                |row| row.get(0),
11489            )
11490            .unwrap_or(0);
11491        if total <= max_bytes {
11492            return;
11493        }
11494        let query = format!(
11495            "SELECT blob_id, byte_length FROM _syncular_blobs
11496             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11497               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11498               AND blob_id NOT IN ({})
11499             ORDER BY created_at_ms ASC, blob_id ASC",
11500            self.visible_blob_ids_sql(),
11501        );
11502        let candidates: Vec<(String, i64)> = {
11503            let Ok(mut stmt) = self.conn.prepare(&query) else {
11504                return;
11505            };
11506            let Ok(rows) = stmt.query_map([], |row| {
11507                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11508            }) else {
11509                return;
11510            };
11511            rows.filter_map(Result::ok).collect()
11512        };
11513        for (blob_id, byte_length) in candidates {
11514            if total <= max_bytes {
11515                break;
11516            }
11517            let _ = self.conn.execute(
11518                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11519                rusqlite::params![blob_id],
11520            );
11521            total -= byte_length;
11522        }
11523    }
11524
11525    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11526        #[cfg(feature = "bench-internals")]
11527        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11528        if !self.schema_has_blobs() {
11529            return Ok(());
11530        }
11531        self.rebuild_overlay_if_dirty()?;
11532        self.conn
11533            .execute(
11534                &format!(
11535                    "DELETE FROM _syncular_blobs
11536                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11537                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11538                   AND blob_id NOT IN ({})",
11539                    self.visible_blob_ids_sql(),
11540                ),
11541                [],
11542            )
11543            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11544        Ok(())
11545    }
11546
11547    // -- local row storage ----------------------------------------------------------
11548
11549    /// The cached per-table primary-key upsert SQL for `full_table` (see
11550    /// the `insert_sql` field: built once, reused per row).
11551    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11552        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11553            return sql.clone();
11554        }
11555        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11556        columns.push(quote_ident("_syncular_version"));
11557        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11558        let primary_key = quote_ident(&table.primary_key);
11559        let updates = columns
11560            .iter()
11561            .filter(|column| **column != primary_key)
11562            .map(|column| format!("{column}=excluded.{column}"))
11563            .collect::<Vec<_>>()
11564            .join(", ");
11565        // §4.5 version guard: server state lands in the base table only above
11566        // the stored version; the visible overlay takes every write.
11567        let guard = if full_table == base_table(&table.name) {
11568            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11569        } else {
11570            String::new()
11571        };
11572        let sql = format!(
11573            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11574            columns.join(", "),
11575            placeholders.join(", ")
11576        );
11577        self.insert_sql
11578            .borrow_mut()
11579            .insert(full_table.to_owned(), sql.clone());
11580        sql
11581    }
11582
11583    /// §4.5: write a server row under the version guard; returns whether it
11584    /// landed.
11585    fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<bool, String> {
11586        let landed = self.write_row(&base_table(table_name), table_name, row, version)?;
11587        if landed {
11588            self.overlay_dirty.table(table_name);
11589        }
11590        Ok(landed)
11591    }
11592
11593    fn write_row(
11594        &self,
11595        full_table: &str,
11596        table_name: &str,
11597        row: &Row,
11598        version: i64,
11599    ) -> Result<bool, String> {
11600        #[cfg(feature = "bench-internals")]
11601        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11602        let table = self
11603            .schema
11604            .table(table_name)
11605            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11606        let sql = self.insert_row_sql(full_table, table);
11607        let mut stmt = self
11608            .conn
11609            .prepare_cached(&sql)
11610            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11611        let params = row
11612            .iter()
11613            .map(RowParam::Cell)
11614            .chain(std::iter::once(RowParam::Version(version)));
11615        stmt.execute(rusqlite::params_from_iter(params))
11616            .map(|changed| changed > 0)
11617            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11618    }
11619
11620    /// §4.5: delete a server row whose stored version lies below `version`;
11621    /// returns whether a row left.
11622    fn delete_base_row(
11623        &self,
11624        table_name: &str,
11625        row_id: &str,
11626        version: i64,
11627    ) -> Result<bool, String> {
11628        let table = self
11629            .schema
11630            .table(table_name)
11631            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11632        let sql = format!(
11633            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
11634            base_table(table_name),
11635            row_id_predicate(table)
11636        );
11637        let mut stmt = self
11638            .conn
11639            .prepare_cached(&sql)
11640            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11641        let landed = stmt
11642            .execute(rusqlite::params![row_id, version])
11643            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11644            > 0;
11645        if landed {
11646            self.overlay_dirty.table(table_name);
11647        }
11648        Ok(landed)
11649    }
11650
11651    fn has_acknowledged_rows(&self) -> Result<bool, String> {
11652        self.conn
11653            .query_row(
11654                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
11655                [],
11656                |row| row.get(0),
11657            )
11658            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11659    }
11660
11661    fn acknowledged_operations(
11662        &self,
11663        rows: Option<&HashSet<(&str, &str)>>,
11664        tables: Option<&BTreeSet<&str>>,
11665    ) -> Result<Vec<OutboxOp>, String> {
11666        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
11667        let mut statement = self
11668            .conn
11669            .prepare_cached(if keys.is_some() {
11670                ACKED_ROWS_BY_ROW
11671            } else {
11672                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
11673            })
11674            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11675        let rows = statement
11676            .query_map(
11677                rusqlite::params![
11678                    keys,
11679                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
11680                ],
11681                |row| {
11682                    #[cfg(test)]
11683                    self.acknowledged_replay_count
11684                        .set(self.acknowledged_replay_count.get() + 1);
11685                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11686                },
11687            )
11688            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11689        rows.map(|row| {
11690            let (op, intent) =
11691                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11692            let op: CommitOperation =
11693                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
11694            let table = self
11695                .schema
11696                .table(&op.table)
11697                .ok_or_else(|| "sync.unknown_table".to_owned())?;
11698            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
11699                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
11700            } else {
11701                op.values.clone()
11702            };
11703            Ok(OutboxOp {
11704                values,
11705                ..op.into()
11706            })
11707        })
11708        .collect()
11709    }
11710
11711    fn drop_acknowledged_rows(
11712        &self,
11713        matches: impl Fn(&str, &Map<String, Value>) -> bool,
11714    ) -> Result<Vec<OutboxCommit>, String> {
11715        let mut statement = self
11716            .conn
11717            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
11718            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11719        let rows = statement
11720            .query_map([], |row| {
11721                Ok((
11722                    row.get::<_, String>(0)?,
11723                    row.get::<_, String>(1)?,
11724                    row.get::<_, String>(2)?,
11725                    row.get::<_, String>(3)?,
11726                ))
11727            })
11728            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11729        let mut doomed = HashSet::new();
11730        for row in rows {
11731            let (id, table, intent, row_id) =
11732                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11733            let values =
11734                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
11735            let base_matches = self
11736                .schema
11737                .table(&table)
11738                .map(|schema| self.stored_row(schema, &row_id, true))
11739                .transpose()?
11740                .flatten()
11741                .is_some_and(|(base, _)| {
11742                    let schema = self.schema.table(&table).expect("acknowledged table");
11743                    let values = schema
11744                        .columns
11745                        .iter()
11746                        .zip(base.iter())
11747                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
11748                        .collect();
11749                    matches(&table, &values)
11750                });
11751            if matches(&table, &values) || base_matches {
11752                doomed.insert(id);
11753            }
11754        }
11755        let mut dropped = Vec::new();
11756        for id in doomed {
11757            let mut statement = self
11758                .conn
11759                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
11760                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11761            let operations = statement
11762                .query_map([&id], |row| row.get::<_, String>(0))
11763                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11764            let mut ops = Vec::new();
11765            for operation in operations {
11766                let json = operation
11767                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11768                let op: CommitOperation =
11769                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
11770                self.overlay_dirty.table(&op.table);
11771                ops.push(OutboxOp {
11772                    table: op.table,
11773                    row_id: op.row_id,
11774                    upsert: op.op == "upsert",
11775                    base_version: op.base_version,
11776                    values: op.values,
11777                    insert: op.insert,
11778                });
11779            }
11780            dropped.push(OutboxCommit {
11781                client_commit_id: id.clone(),
11782                ops,
11783            });
11784            self.conn
11785                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
11786                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11787        }
11788        Ok(dropped)
11789    }
11790
11791    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
11792    /// the outbox changed since the last rebuild (the no-op sync round).
11793    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
11794        if self.overlay_dirty.get() {
11795            self.rebuild_overlay()?;
11796        }
11797        Ok(())
11798    }
11799
11800    /// §7.1: local reads see outbox state applied optimistically — rebuild
11801    /// every visible table as (base server state) + (pending outbox replay
11802    /// on top). Optimistic rows carry version `-1`.
11803    fn rebuild_overlay(&mut self) -> Result<(), String> {
11804        #[cfg(feature = "bench-internals")]
11805        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
11806        #[cfg(test)]
11807        self.overlay_rebuild_count
11808            .set(self.overlay_rebuild_count.get() + 1);
11809        let dirty = self.overlay_dirty.snapshot();
11810        let tables: Vec<_> = self
11811            .schema
11812            .tables
11813            .iter()
11814            .filter(|table| {
11815                dirty
11816                    .as_ref()
11817                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
11818            })
11819            .cloned()
11820            .collect();
11821        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
11822        self.begin_observation("syncular_overlay")?;
11823        let rebuilt = (|| {
11824            for table in &tables {
11825                for index in &table.fts_indexes {
11826                    self.drop_fts_triggers(index)?;
11827                }
11828                let visible = visible_table(&table.name);
11829                let base = base_table(&table.name);
11830                self.conn
11831                    .execute_batch(&format!(
11832                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
11833                    ))
11834                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11835            }
11836            self.replay_overlay(&names, None)?;
11837            for table in &tables {
11838                for index in &table.fts_indexes {
11839                    self.rebuild_fts_projection(table, index)?;
11840                    self.create_fts_triggers(table, index)?;
11841                }
11842            }
11843            self.conn
11844                .execute_batch("RELEASE syncular_overlay")
11845                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11846            Ok(())
11847        })();
11848        if rebuilt.is_err() {
11849            self.conn
11850                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
11851                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11852        } else {
11853            self.overlay_dirty.set(false);
11854        }
11855        rebuilt
11856    }
11857
11858    /// §7.1 replay of acknowledged, retained failed and pending intent, in
11859    /// that order, for the operations of `tables` on `rows` (every row for
11860    /// `None`) over their current visible state.
11861    fn replay_overlay(
11862        &self,
11863        tables: &BTreeSet<&str>,
11864        rows: Option<&HashSet<(&str, &str)>>,
11865    ) -> Result<(), String> {
11866        let replays = |op: &&OutboxOp| {
11867            tables.contains(op.table.as_str())
11868                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
11869        };
11870        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
11871        for (commit, initial) in self.failed_commits.iter() {
11872            for operation in commit.ops.iter().filter(replays) {
11873                let Some(table) = self.schema.table(&operation.table) else {
11874                    continue;
11875                };
11876                if operation.upsert
11877                    && self.visible_row(table, &operation.row_id)?.is_none()
11878                    && operation.presented_values(table).is_some_and(|values| {
11879                        table
11880                            .columns
11881                            .iter()
11882                            .all(|column| values.contains_key(&column.name))
11883                    })
11884                {
11885                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
11886                        .expect("row key serialization");
11887                    if let Some(Value::Object(values)) = initial.get(&key) {
11888                        self.apply_outbox_ops(&[OutboxOp {
11889                            values: Some(values.clone()),
11890                            ..operation.clone()
11891                        }])?;
11892                    }
11893                }
11894            }
11895            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
11896        }
11897        self.apply_outbox_ops(
11898            self.outbox
11899                .iter()
11900                .flat_map(|commit| &commit.ops)
11901                .filter(replays),
11902        )
11903    }
11904
11905    /// §7.1 replay: every operation is applied over the current visible
11906    /// state. An operation that no longer applies (a server row now holds its
11907    /// unique value) stays pending and invisible until a later replay admits
11908    /// it or the server answers its push.
11909    fn apply_outbox_ops<'a>(
11910        &self,
11911        ops: impl IntoIterator<Item = &'a OutboxOp>,
11912    ) -> Result<(), String> {
11913        #[cfg(feature = "bench-internals")]
11914        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
11915        for op in ops {
11916            #[cfg(feature = "bench-internals")]
11917            if let Some(phase) = &mut phase {
11918                phase.unit();
11919            }
11920            if op.upsert {
11921                if let Some((table, values)) = self
11922                    .schema
11923                    .table(&op.table)
11924                    .filter(|table| table.indexes.iter().any(|index| index.unique))
11925                    .and_then(|table| Some((table, op.presented_values(table)?)))
11926                {
11927                    let mut intended: Map<String, Value> = self
11928                        .visible_row(table, &op.row_id)?
11929                        .map(|(row, _)| {
11930                            table
11931                                .columns
11932                                .iter()
11933                                .zip(&row)
11934                                .map(|(column, value)| {
11935                                    (column.name.clone(), column_value_to_json(value))
11936                                })
11937                                .collect()
11938                        })
11939                        .unwrap_or_default();
11940                    intended.extend(values.into_owned());
11941                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
11942                        continue;
11943                    }
11944                }
11945            }
11946            self.apply_outbox_op(op)?;
11947        }
11948        Ok(())
11949    }
11950
11951    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
11952        let Some(table) = self.schema.table(&op.table) else {
11953            return Ok(());
11954        };
11955        let visible = visible_table(&table.name);
11956        if !op.upsert {
11957            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
11958            self.conn
11959                .prepare_cached(&sql)
11960                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
11961                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11962            return Ok(());
11963        }
11964        // §7.1: an insert-intent operation that lacks a non-nullable column
11965        // cannot encode; it stays invisible until the send-time prepass drops
11966        // it as sync.outbox_incompatible (§7.4.4).
11967        let Some(values) = op.presented_values(table) else {
11968            return Ok(());
11969        };
11970        // §7.1: the overlay applies the operation's PRESENT columns
11971        // over the current local row. A partial operation over an
11972        // absent local row leaves it absent — the server answers it
11973        // with sync.row_deleted or sync.row_missing (§6.2). An
11974        // insert-intent operation presents every column.
11975        let full = table
11976            .columns
11977            .iter()
11978            .all(|column| values.contains_key(&column.name));
11979        let local = self.visible_row(table, &op.row_id)?;
11980        if local.is_none() && !full {
11981            return Ok(());
11982        }
11983        let mut row: Row = Vec::with_capacity(table.columns.len());
11984        for (index, column) in table.columns.iter().enumerate() {
11985            let incoming = match values.get(&column.name) {
11986                Some(raw) => json_to_column_value(
11987                    if raw.get("$bytes").is_some()
11988                        && table
11989                            .encrypted_columns
11990                            .iter()
11991                            .any(|encrypted| encrypted.index == index)
11992                    {
11993                        &table.wire_columns[index]
11994                    } else {
11995                        column
11996                    },
11997                    Some(raw),
11998                ),
11999                None => Ok(local
12000                    .as_ref()
12001                    .and_then(|(values, _)| values.get(index).cloned())
12002                    .flatten()),
12003            };
12004            row.push(incoming.map_err(|_| {
12005                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12006            })?);
12007        }
12008        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12009        self.write_row(&visible, &table.name, &row, version)
12010            .map(|_| ())
12011            .map_err(|error| {
12012                if error.starts_with("UNIQUE constraint failed:") {
12013                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12014                } else {
12015                    error
12016                }
12017            })
12018    }
12019
12020    /// Whether the row is present in the overlay. Existence reads only the
12021    /// key: locked encrypted values stay opaque, and decoding the complete row
12022    /// would reject a valid plain write.
12023    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12024        self.conn
12025            .query_row(
12026                &format!(
12027                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12028                    visible_table(&table.name),
12029                    row_id_predicate(table)
12030                ),
12031                rusqlite::params![row_id],
12032                |row| row.get(0),
12033            )
12034            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12035    }
12036
12037    /// The visible (optimistic) row's values plus version, or `None` when the
12038    /// row is absent from the overlay.
12039    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12040        self.stored_row(table, row_id, false)
12041    }
12042
12043    fn stored_row(
12044        &self,
12045        table: &TableSchema,
12046        row_id: &str,
12047        base: bool,
12048    ) -> Result<Option<(Row, i64)>, String> {
12049        let sql = format!(
12050            "SELECT * FROM {} WHERE {}",
12051            if base {
12052                base_table(&table.name)
12053            } else {
12054                visible_table(&table.name)
12055            },
12056            row_id_predicate(table)
12057        );
12058        let mut stmt = self
12059            .conn
12060            .prepare_cached(&sql)
12061            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12062        let mut rows = stmt
12063            .query(rusqlite::params![row_id])
12064            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12065        let Some(row) = rows
12066            .next()
12067            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12068        else {
12069            return Ok(None);
12070        };
12071        let mut values = Vec::with_capacity(table.columns.len());
12072        for (index, column) in table.columns.iter().enumerate() {
12073            let raw = row
12074                .get_ref(index)
12075                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12076            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12077                if table
12078                    .encrypted_columns
12079                    .iter()
12080                    .any(|encrypted| encrypted.index == index)
12081                {
12082                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12083                    continue;
12084                }
12085            }
12086            let json = sql_ref_to_json(column, raw);
12087            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12088                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12089            })?);
12090        }
12091        let version: i64 = row
12092            .get(table.columns.len())
12093            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12094        Ok(Some((values, version)))
12095    }
12096
12097    // -- realtime (§8) ---------------------------------------------------------------
12098
12099    /// §8.8: the configured realtime policy.
12100    #[must_use]
12101    pub fn realtime_policy(&self) -> RealtimePolicy {
12102        self.realtime_policy
12103    }
12104
12105    /// §8.8: the explicit realtime availability state.
12106    #[must_use]
12107    pub fn realtime_state(&self) -> RealtimeState {
12108        self.realtime_state
12109    }
12110
12111    /// §8.8: set the realtime policy. Refused while the socket is connected:
12112    /// a policy change never silently re-routes an ownership state.
12113    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12114        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12115            return Err(
12116                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12117                    .to_owned(),
12118            );
12119        }
12120        self.realtime_policy = policy;
12121        match policy {
12122            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12123            RealtimePolicy::Required | RealtimePolicy::Optional
12124                if self.realtime_state == RealtimeState::Disabled =>
12125            {
12126                self.set_realtime_state(RealtimeState::Disconnected, None);
12127            }
12128            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12129        }
12130        Ok(())
12131    }
12132
12133    /// Record an availability transition, keeping the reason code bounded and
12134    /// code-like (§7.6) and clearing it where it cannot apply.
12135    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12136        self.realtime_state = state;
12137        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12138        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12139            self.realtime_reason_code = None;
12140            self.realtime_retry_delay_ms = None;
12141        }
12142    }
12143
12144    #[must_use]
12145    fn realtime_connected(&self) -> bool {
12146        self.realtime_state == RealtimeState::Connected
12147    }
12148
12149    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12150        if !self.transport_enabled {
12151            return Err("sync.offline: transport gate is closed".into());
12152        }
12153        if self.realtime_policy == RealtimePolicy::Off {
12154            return Err(
12155                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12156            );
12157        }
12158        if self.realtime_connected() {
12159            return Ok(());
12160        }
12161        self.realtime_state = RealtimeState::Connecting;
12162        match transport.realtime_connect_for_client(&self.client_id) {
12163            Ok(()) => {
12164                self.set_realtime_state(RealtimeState::Connected, None);
12165                Ok(())
12166            }
12167            Err(error) => {
12168                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12169                Err(format!("{}: {}", error.code, error.message))
12170            }
12171        }
12172    }
12173
12174    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12175        self.cancel_sync_round();
12176        // A deliberately disconnected or policy-disabled client has nothing
12177        // to release; a `lost` socket still does (the host owns the handle).
12178        if self.realtime_state == RealtimeState::Disconnected
12179            || self.realtime_state == RealtimeState::Disabled
12180        {
12181            return;
12182        }
12183        let _ = transport.realtime_close();
12184        self.set_realtime_state(
12185            if self.realtime_policy == RealtimePolicy::Off {
12186                RealtimeState::Disabled
12187            } else {
12188                RealtimeState::Disconnected
12189            },
12190            None,
12191        );
12192        self.presence.clear(); // §8.6.1: presence is per-connection
12193    }
12194
12195    /// §8.8: the host observed the socket end without disconnecting it (a
12196    /// peer close, an I/O error, a failed send; the native transport reports
12197    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12198    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12199    /// one background retry intent so the host reconnects and catches up.
12200    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12201        if self.realtime_state != RealtimeState::Connected {
12202            return;
12203        }
12204        self.cancel_sync_round();
12205        let _ = transport.realtime_close();
12206        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12207        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12208        self.presence.clear();
12209    }
12210
12211    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12212    /// document for `scope_key`. Requires a live socket; the document is
12213    /// ephemeral (lost on disconnect). Authorization is the connection's
12214    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12215    /// server with `presence.forbidden`.
12216    pub fn set_presence(
12217        &mut self,
12218        transport: &mut dyn Transport,
12219        scope_key: &str,
12220        doc: Option<&Value>,
12221    ) -> Result<(), String> {
12222        if !self.transport_enabled {
12223            return Err("sync.offline: transport gate is closed".into());
12224        }
12225        if !self.realtime_connected() {
12226            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12227        }
12228        let text = encode_presence_publish(scope_key, doc);
12229        transport
12230            .realtime_send(&text)
12231            .map_err(|e| format!("{}: {}", e.code, e.message))
12232    }
12233
12234    /// §8.6: the peers currently present on a scope key (ephemeral).
12235    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12236        self.presence
12237            .get(scope_key)
12238            .map(|peers| peers.values().cloned().collect())
12239            .unwrap_or_default()
12240    }
12241
12242    /// §8.6 apply an inbound presence fanout to the local map.
12243    fn apply_presence(
12244        &mut self,
12245        scope_key: String,
12246        kind: Option<PresenceKind>,
12247        actor_id: Option<String>,
12248        client_id: Option<String>,
12249        doc: Option<Value>,
12250        error: Option<String>,
12251    ) {
12252        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12253        // it as a `presence` event carrying the server's `error` code.
12254        if error.is_some() {
12255            return;
12256        }
12257        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12258            return;
12259        };
12260        let peer_key = format!("{actor_id} {client_id}");
12261        match kind {
12262            PresenceKind::Leave => {
12263                if let Some(peers) = self.presence.get_mut(&scope_key) {
12264                    peers.remove(&peer_key);
12265                    if peers.is_empty() {
12266                        self.presence.remove(&scope_key);
12267                    }
12268                }
12269            }
12270            _ => {
12271                let doc = match doc {
12272                    Some(Value::Object(_)) => doc.unwrap(),
12273                    _ => return,
12274                };
12275                self.presence.entry(scope_key).or_default().insert(
12276                    peer_key,
12277                    PresencePeer {
12278                        actor_id,
12279                        client_id,
12280                        doc,
12281                    },
12282                );
12283            }
12284        }
12285    }
12286
12287    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12288    pub fn on_realtime_text(&mut self, text: &str) {
12289        if self.stopped || !self.transport_enabled || self.security_preflight {
12290            return;
12291        }
12292        match parse_control(text) {
12293            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12294                if requires_sync {
12295                    // §8.1: pull before trusting the socket for continuity.
12296                    self.set_sync_needed(true, true);
12297                }
12298            }
12299            Ok(ControlMessage::Presence {
12300                scope_key,
12301                kind,
12302                actor_id,
12303                client_id,
12304                doc,
12305                error,
12306                ..
12307            }) => {
12308                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12309            }
12310            Ok(ControlMessage::Wake { .. }) => {
12311                // §8.3: any wake-up means "run a pull soon", never data.
12312                self.set_sync_needed(true, true);
12313            }
12314            _ => {}
12315        }
12316    }
12317
12318    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12319    /// a pull response per section; an unapplied delta is a wake-up.
12320    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12321    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12322        let mut transport = crate::round::DownloadResults::default();
12323        self.on_realtime_binary(&mut transport, bytes);
12324        transport.controls
12325    }
12326
12327    /// §8.2: a delta applies exactly like a live section and sets the replica
12328    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12329    /// duplicate. A delta the client does not apply (no live unit, a status
12330    /// other than `active`, a decode or apply failure) is a drop and raises
12331    /// sync-needed.
12332    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12333        if self.stopped || !self.transport_enabled || self.security_preflight {
12334            return;
12335        }
12336        #[cfg(feature = "bench-internals")]
12337        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12338        let message = match decode_message(bytes) {
12339            Ok(m) if m.msg_kind == MsgKind::Response => m,
12340            _ => {
12341                self.set_sync_needed(true, true);
12342                return;
12343            }
12344        };
12345        #[cfg(feature = "bench-internals")]
12346        drop(decode_phase);
12347        // §8.2: a delta's header means what a pull response's header means. A
12348        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12349        if let Some(Frame::RespHeader {
12350            required_schema_version: Some(required),
12351            latest_schema_version,
12352            ..
12353        }) = message.frames.first()
12354        {
12355            let floor = SchemaFloor {
12356                required_schema_version: Some(*required),
12357                latest_schema_version: *latest_schema_version,
12358            };
12359            if self.set_schema_floor(Some(floor)).is_err() {
12360                self.set_sync_needed(true, true);
12361            }
12362            return;
12363        }
12364        let through = message.frames.iter().find_map(|frame| match frame {
12365            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12366            _ => None,
12367        });
12368        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12369        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12370                reset_required: false,
12371                log_epoch,
12372                ..
12373            }) if Some(log_epoch) == epoch.as_ref());
12374        let mut applied: Option<i64> = None;
12375        dropped |= self.begin_coalesced_apply().is_err();
12376        let mut receivers: Option<Vec<UnitRecord>> = None;
12377        for frame in message.frames {
12378            if dropped {
12379                break;
12380            }
12381            match frame {
12382                Frame::LiveStart { status, .. } => {
12383                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12384                        dropped = true;
12385                        break;
12386                    };
12387                    if status != LiveStatus::Active || cursor < 0 {
12388                        dropped = true;
12389                    } else if !through.is_some_and(|through| through <= cursor) {
12390                        // A catching-up unit receives no delta (§8.2).
12391                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12392                            units
12393                                .into_iter()
12394                                .filter(|unit| {
12395                                    unit.status == UnitStatus::Active
12396                                        && unit.echo.is_some()
12397                                        && unit.cursor.is_none()
12398                                })
12399                                .collect()
12400                        });
12401                        dropped = receivers.is_none();
12402                    }
12403                }
12404                Frame::Commit {
12405                    commit_seq,
12406                    tables,
12407                    changes,
12408                    ..
12409                } => {
12410                    let Some(receivers) = receivers.clone() else {
12411                        continue;
12412                    };
12413                    dropped = self
12414                        .apply_live_commit(
12415                            &receivers,
12416                            &tables,
12417                            changes,
12418                            commit_seq,
12419                            AppliedServerCommitSource::Delta,
12420                        )
12421                        .is_err();
12422                }
12423                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12424                    dropped = self
12425                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12426                        .is_err();
12427                    applied = Some(through_seq);
12428                }
12429                Frame::Error { .. } => dropped = true,
12430                _ => {}
12431            }
12432        }
12433        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12434        // the replica cursor, so the delta counts as a drop.
12435        if self.finish_coalesced_apply().is_err() || dropped {
12436            applied = None;
12437            dropped = true;
12438        }
12439        if let Some(cursor) = applied {
12440            // §8.2 ack point: the delta's throughSeq.
12441            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12442            let _ = transport.realtime_send(&ack);
12443        }
12444        if dropped {
12445            self.set_sync_needed(true, true);
12446        }
12447    }
12448
12449    /// §8.2 ack point after a pull round on a live connection: the replica
12450    /// cursor while it is at least `0`.
12451    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12452        if !self.realtime_connected() {
12453            return;
12454        }
12455        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12456            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12457            let _ = transport.realtime_send(&ack);
12458        }
12459    }
12460}
12461
12462#[cfg(test)]
12463mod previous_version_wiring_tests;
12464
12465#[cfg(test)]
12466mod identity_tests;
12467
12468#[cfg(test)]
12469mod insert_intent_tests;
12470
12471#[cfg(test)]
12472mod robustness_tests;