Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127    version: u8,
128    retained_commits: u64,
129    reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137    retained_commits: usize,
138    reset_subscriptions: usize,
139) -> Result<String, String> {
140    let retained_commits =
141        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    let reset_subscriptions =
143        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145        return Err(invalid_local_rebootstrap_receipt());
146    }
147    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149        retained_commits,
150        reset_subscriptions,
151    })
152    .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156    // Pre-0.15.36 markers proved application but did not retain the receipt.
157    if value == "v1" {
158        return Ok((0, 0));
159    }
160    let receipt: PersistedLocalDataRebootstrapReceipt =
161        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165    {
166        return Err(invalid_local_rebootstrap_receipt());
167    }
168    Ok((
169        usize::try_from(receipt.retained_commits)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171        usize::try_from(receipt.reset_subscriptions)
172            .map_err(|_| invalid_local_rebootstrap_receipt())?,
173    ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179/// Internal state for hosts that perform network I/O outside the client owner.
180#[doc(hidden)]
181pub struct CurrentBasePreparation {
182    client_commit_id: String,
183    sequence: i64,
184    units: Vec<UnitRecord>,
185    transport_epoch: u64,
186    registration_epoch: u64,
187    security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195    upsert: bool,
196    table: String,
197    row_id: String,
198    base_version: Option<i64>,
199    /// Schema-agnostic local form (§0): driver JSON values, encoded with
200    /// the current codec at send time. A key's presence IS the §6.1 sparse
201    /// presence set — both the overlay and the wire payload derive from it.
202    values: Option<Map<String, Value>>,
203    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
204    /// absent from the visible replica. Replay and send-time encoding present
205    /// every current-schema column; a column `values` lacks is NULL.
206    insert: bool,
207}
208
209impl OutboxOp {
210    /// The values this upsert presents under the current `table` (§7.1). An
211    /// insert-intent operation presents every column, a missing one as NULL;
212    /// `None` when a missing column is non-nullable, which leaves the commit
213    /// unencodable (§7.4.4), or when the operation carries no values.
214    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215        let values = self.values.as_ref()?;
216        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217            return Some(Cow::Borrowed(values));
218        }
219        full_row_values(table, values.clone()).ok().map(Cow::Owned)
220    }
221}
222
223impl From<CommitOperation> for OutboxOp {
224    fn from(operation: CommitOperation) -> Self {
225        Self {
226            upsert: operation.op == "upsert",
227            table: operation.table,
228            row_id: operation.row_id,
229            base_version: operation.base_version,
230            values: operation.values,
231            insert: operation.insert,
232        }
233    }
234}
235
236/// §7.5 static authoring refusal; the table and column travel in details.
237fn invalid_authoring(message: &str, details: Value) -> ClientError {
238    ClientError {
239        code: "sync.invalid_request".to_owned(),
240        message: message.to_owned(),
241        details: Some(details),
242        retryable: false,
243    }
244}
245
246/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
247/// row and presents only columns `earlier` presents.
248fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249    // Only an upsert carries values, and it always carries its primary key.
250    earlier.row_id == later.row_id
251        && earlier.table == later.table
252        && later.base_version.is_some_and(|base| base > 0)
253        && earlier.base_version == later.base_version
254        && later.values.as_ref().is_some_and(|values| {
255            values.keys().all(|column| {
256                earlier
257                    .values
258                    .as_ref()
259                    .is_some_and(|earlier| earlier.contains_key(column))
260            })
261        })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265    fn from(operation: &OutboxOp) -> Self {
266        Self {
267            table: operation.table.clone(),
268            row_id: operation.row_id.clone(),
269            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270            base_version: operation.base_version,
271            values: operation.values.clone(),
272            insert: operation.insert,
273        }
274    }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279    client_commit_id: String,
280    ops: Vec<OutboxOp>,
281}
282
283/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
284/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
285/// can never disagree about which commits cannot re-encode.
286fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287    commit
288        .ops
289        .iter()
290        .filter(|operation| operation.upsert)
291        .map(|operation| {
292            (
293                operation.table.as_str(),
294                operation
295                    .values
296                    .as_ref()
297                    .map(|values| values.keys().map(String::as_str).collect())
298                    .unwrap_or_default(),
299                operation.insert,
300            )
301        })
302        .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309    table: String,
310    selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314    sequence: i64,
315    client_commit_id: String,
316    status: String,
317    recorded_at_ms: i64,
318    results_json: String,
319    operations_json: Option<String>,
320    resolution: String,
321    resolved_at_ms: Option<i64>,
322    replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327        Ok(Self {
328            sequence: row.get(0)?,
329            client_commit_id: row.get(1)?,
330            status: row.get(2)?,
331            recorded_at_ms: row.get(3)?,
332            results_json: row.get(4)?,
333            operations_json: row.get(5)?,
334            resolution: row.get(6)?,
335            resolved_at_ms: row.get(7)?,
336            replacement_client_commit_id: row.get(8)?,
337        })
338    }
339}
340
341/// Section outcome distinguishing the §5.6 unit-local fail-closed
342/// path from a round-aborting failure (§1.4 rule 5).
343/// §4.4 a unit section's `SUB_START`.
344struct SectionStart {
345    id: String,
346    status: SubStatus,
347    reason_code: String,
348    bootstrap: bool,
349    fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353    FailClosed,
354    /// §5.5: the server refused a segment with `sync.forbidden`.
355    Refused,
356    /// §5.3 client memory: the round deferred this image's body to a later
357    /// pull of the same resume token.
358    Deferred,
359    Abort(String, String),
360    Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365    pushed_ids: Vec<String>,
366    /// Hex of the registration digest this pull carried; the server stores
367    /// that list once the round completes (§4.1).
368    registration_digest: String,
369    pub(crate) accept: u8,
370    /// §6.1 splitBatch: outbox commits held back from THIS request because
371    /// the running operation count reached the push cap — the next round
372    /// pushes them (`sync_needed` stays set while any remain).
373    deferred_commits: usize,
374    /// §5.6 applied-segment rule: per unit presenting its recorded cursor
375    /// and token, the segment ids neither downloaded nor applied again.
376    pub(crate) applied: BTreeMap<String, BTreeSet<String>>,
377    /// §5.3 client memory: (unit, segment id) of the first image whose body
378    /// the round did not fetch because the batch already holds two images.
379    /// That unit's section stops there without persisting `SUB_END`.
380    pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385    /// None means table-wide; Some is the exact scoped domain.
386    tables: BTreeMap<String, Option<BTreeSet<String>>>,
387    windows: BTreeMap<String, BTreeSet<String>>,
388    status: bool,
389    conflicts: bool,
390    rejections: bool,
391    outcomes: bool,
392}
393
394impl ChangeAccumulator {
395    fn table(&mut self, table: &str) {
396        self.tables.insert(table.to_owned(), None);
397    }
398
399    fn scope(&mut self, table: &str, key: String) {
400        match self.tables.get_mut(table) {
401            Some(None) => {}
402            Some(Some(keys)) => {
403                keys.insert(key);
404            }
405            None => {
406                self.tables
407                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
408            }
409        }
410    }
411
412    fn window(&mut self, base_key: &str, value: &str) {
413        self.windows
414            .entry(base_key.to_owned())
415            .or_default()
416            .insert(value.to_owned());
417    }
418
419    fn touched(&self) -> bool {
420        !self.tables.is_empty()
421            || !self.windows.is_empty()
422            || self.status
423            || self.conflicts
424            || self.rejections
425            || self.outcomes
426    }
427}
428
429/// §6.1: the server caps total operations per request (reference default
430/// 500) and rejects the whole batch with `sync.too_many_operations`; the
431/// client "splits and retries". Splitting happens at build time: commits are
432/// included IN ORDER until the operation budget is spent, the rest wait for
433/// the next round.
434const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441/// Dirty visible tables; `None` means a whole-replica recovery/reset.
442struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445    fn clean() -> Self {
446        Self(RefCell::new(Some(BTreeSet::new())))
447    }
448    fn get(&self) -> bool {
449        self.0
450            .borrow()
451            .as_ref()
452            .is_none_or(|tables| !tables.is_empty())
453    }
454    fn set(&self, dirty: bool) {
455        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456    }
457    fn table(&self, table: &str) {
458        if let Some(tables) = self.0.borrow_mut().as_mut() {
459            tables.insert(table.to_owned());
460        }
461    }
462    fn snapshot(&self) -> Option<BTreeSet<String>> {
463        self.0.borrow().clone()
464    }
465    fn restore(&self, tables: Option<BTreeSet<String>>) {
466        *self.0.borrow_mut() = tables;
467    }
468}
469
470/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
471/// write after a capture copies it once.
472#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476    type Target = T;
477    fn deref(&self) -> &T {
478        &self.0
479    }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483    fn deref_mut(&mut self) -> &mut T {
484        Arc::make_mut(&mut self.0)
485    }
486}
487
488impl<T> From<T> for Shared<T> {
489    fn from(value: T) -> Self {
490        Self(Arc::new(value))
491    }
492}
493
494/// The in-memory mirrors of durable state, captured before a local
495/// transaction and restored when it rolls back. Queues only grow inside a
496/// transaction, so their lengths suffice.
497#[derive(Debug, Clone)]
498struct Rollback {
499    outbox: Shared<Vec<Arc<OutboxCommit>>>,
500    failed_commits: Shared<Vec<FailedCommit>>,
501    conflicts: Shared<Vec<ConflictRecord>>,
502    rejections: Shared<Vec<RejectionRecord>>,
503    schema_floor: Option<SchemaFloor>,
504    lease_state: Option<LeaseState>,
505    stopped: bool,
506    upgrading: bool,
507    sync_needed: bool,
508    active_round: Option<uuid::Uuid>,
509    overlay_dirty: Option<BTreeSet<String>>,
510    change_queue: usize,
511    sync_intent_queue: usize,
512    last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516    #[cfg(feature = "bench-internals")]
517    benchmark_phases: Recorder,
518    conn: Connection,
519    schema: ClientSchema,
520    client_id: String,
521    limits: ClientLimits,
522    outbox: Shared<Vec<Arc<OutboxCommit>>>,
523    failed_commits: Shared<Vec<FailedCommit>>,
524    retain_failed_commits: bool,
525    conflicts: Shared<Vec<ConflictRecord>>,
526    rejections: Shared<Vec<RejectionRecord>>,
527    schema_floor: Option<SchemaFloor>,
528    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
529    lease_state: Option<LeaseState>,
530    /// §1.6: the schema-floor response stops syncing until an upgrade.
531    stopped: bool,
532    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
533    upgrading: bool,
534    /// §8.4 coalesced sync-needed signal.
535    sync_needed: bool,
536    /// §8.8: how sync rounds treat the realtime binding, and its exact
537    /// availability state with the evidence behind a lost or refused one.
538    realtime_policy: RealtimePolicy,
539    realtime_state: RealtimeState,
540    realtime_reason_code: Option<String>,
541    realtime_retry_delay_ms: Option<u64>,
542    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
543    presence: HashMap<String, HashMap<String, PresencePeer>>,
544    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check and the
545    /// §7.8 drive deadline; the host may pin it and move the pinned value
546    /// (conformance runs on a virtual clock).
547    now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
548    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
549    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
550    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
551    encryption: crate::values::EncryptionConfig,
552    /// Fail-closed host bootstrap gate. While set, command hosts permit only
553    /// status/lifecycle inspection and an exact authorized local purge.
554    security_preflight: bool,
555    security_epoch: u64,
556    transport_enabled: bool,
557    transport_epoch: u64,
558    /// Raised by every registration change; a current-base proof spanning
559    /// one reports missing coverage.
560    registration_epoch: u64,
561    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
562    /// means the feature is off; the descriptor write and the container orphan
563    /// sweep still happen.
564    previous_version: Option<PreviousVersionContextConfig>,
565    /// Per-table primary-key upsert SQL, built once per (full table name) —
566    /// the row write path runs per row during bootstrap (§5.6), so the SQL
567    /// string (and, via `prepare_cached`, its compiled statement) is reused
568    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
569    /// schema reset (the column lists may have changed).
570    insert_sql: RefCell<HashMap<String, String>>,
571    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
572    /// reset and restart explicitly invalidate every table.
573    overlay_dirty: OverlayDirty,
574    // First SQLite failure of the current operation, before savepoint cleanup.
575    storage_failure: RefCell<Option<QueryReadFailure>>,
576    /// Test-only structural performance signal: response processing must not
577    /// turn a batch of acknowledgements into one full overlay rebuild each.
578    #[cfg(test)]
579    overlay_rebuild_count: Cell<usize>,
580    #[cfg(test)]
581    acknowledged_replay_count: Cell<usize>,
582    /// Test-only structural performance signal: outcome retention is enforced
583    /// once per response rather than once per acknowledgement.
584    #[cfg(test)]
585    outcome_prune_count: Cell<usize>,
586    /// Test-only pinned duration of every §5.3 image chunk, in ms.
587    #[cfg(test)]
588    image_chunk_ms: Cell<Option<f64>>,
589    /// Exact observer-transaction output drained by command/FFI hosts.
590    progress: ProgressObserver,
591    change_queue: VecDeque<ClientChangeBatch>,
592    sync_intent_queue: VecDeque<SyncIntent>,
593    /// Explicit exponential retry policy for transient transport failures.
594    retry_delay_ms: u64,
595    /// §7.7: the background retry the current round scheduled, if any.
596    round_retry_delay_ms: Option<u64>,
597    /// §5.6 applied-segment rule: per unit, the cursor and resume token it
598    /// presented and the `SEGMENT_REF`s applied completely in sections whose
599    /// `SUB_END` did not persist.
600    #[allow(clippy::type_complexity)]
601    applied_segments: HashMap<String, (Option<i64>, Option<String>, BTreeSet<String>)>,
602    /// §5.3: a table's next image chunk size, carried from one range image
603    /// of the table to the next.
604    image_chunk_rows: HashMap<String, u32>,
605    /// A round whose response never completed (§1.5): the client keeps its
606    /// code and reports it on every following request until a round receives a
607    /// complete, decodable response. The server already recorded the round, so
608    /// nothing else can tell it the round failed on this side.
609    round_failure: Option<String>,
610    active_round: Option<uuid::Uuid>,
611    last_round: Option<DiagnosticLastRound>,
612    last_change: Option<DiagnosticLastChange>,
613    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
614    query_failures: Vec<DiagnosticQueryFailure>,
615    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
616    coalesced: Option<CoalescedApply>,
617    /// A coalesced apply transaction of the current response failed; every
618    /// later observation of that response fails until it ends.
619    coalesced_failure: Option<String>,
620}
621
622/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
623/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
624/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
625struct CoalescedApply {
626    rows: usize,
627    rollback: Rollback,
628}
629
630const COALESCED_APPLY_ROWS: usize = 1024;
631
632pub(crate) fn quote_ident(name: &str) -> String {
633    format!("\"{}\"", name.replace('"', "\"\""))
634}
635/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
636/// previous-version module reads and writes the SAME records as the client
637/// instead of growing a parallel meta layer.
638pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
639    conn.query_row(
640        "SELECT value FROM _syncular_meta WHERE key = ?1",
641        rusqlite::params![key],
642        |row| row.get::<_, String>(0),
643    )
644    .optional()
645}
646
647pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
648    conn.execute(
649        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
650        rusqlite::params![key, value],
651    )
652    .map(|_| ())
653}
654
655pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
656    conn.execute(
657        "DELETE FROM _syncular_meta WHERE key = ?1",
658        rusqlite::params![key],
659    )
660    .map(|_| ())
661}
662
663/// Seek by the existing key while retaining the exact legacy text match.
664fn row_id_predicate(table: &TableSchema) -> String {
665    let key = quote_ident(&table.primary_key);
666    let text_match = format!("CAST({key} AS TEXT) = ?1");
667    match table.columns[table.pk_index].ty {
668        ColumnType::String | ColumnType::Json => {
669            format!("{key} = ?1 AND {text_match}")
670        }
671        // Native columns have no declared SQLite affinity. Match the stored
672        // integer bind type explicitly. Unary + removes expression affinity so
673        // SQLite can seek the typeless key; the text check preserves exact IDs.
674        ColumnType::Integer | ColumnType::Boolean => {
675            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
676        }
677        // REAL-to-text conversion can round distinct values to the same text.
678        // Narrowing those matches by numeric equality would change behavior.
679        _ => text_match,
680    }
681}
682
683/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
684/// `effective` and that no entry of `owners` matches. `owners` holds the
685/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
686/// Scope values bind by the column's declared type, as `row_id_predicate`
687/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
688/// is a missing local scope-column mapping (the fail-closed case).
689fn local_scope_predicate(
690    table: &TableSchema,
691    effective: &[(String, Vec<String>)],
692    owners: &[Vec<(String, Vec<String>)>],
693) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
694    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
695        return Ok(None);
696    }
697    let column = |variable: &str| {
698        let name = table.scope_column(variable).ok_or(())?;
699        let column = table
700            .columns
701            .iter()
702            .find(|column| column.name == name)
703            .ok_or(())?;
704        Ok((quote_ident(name), column.ty == ColumnType::Integer))
705    };
706    let mut clauses = Vec::new();
707    let mut params = Vec::new();
708    for (variable, values) in effective {
709        let (key, integer) = column(variable)?;
710        let holes = vec!["?"; values.len()].join(", ");
711        clauses.push(if integer {
712            params.extend(values.iter().cloned().map(SqlValue::Text));
713            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
714            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
715        } else {
716            format!("{key} IN ({holes})")
717        });
718        params.extend(values.iter().cloned().map(SqlValue::Text));
719    }
720    // Owners bind as one JSON parameter per variable set, so the statement
721    // keeps its size at any owner count; owners of one variable bind the
722    // union of their values. An owner holding other values of a variable
723    // `effective` names owns none of its rows and drops out. Values compare
724    // as text, the exact semantics of the effective clauses.
725    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
726    for owner in owners {
727        if owner.is_empty()
728            || owner.iter().any(|(variable, values)| {
729                values.is_empty()
730                    || effective.iter().any(|(name, wanted)| {
731                        name == variable && wanted.iter().all(|value| !values.contains(value))
732                    })
733            })
734        {
735            continue;
736        }
737        let mut owner: Vec<_> = owner.iter().collect();
738        owner.sort_by(|a, b| a.0.cmp(&b.0));
739        groups
740            .entry(
741                owner
742                    .iter()
743                    .map(|(variable, _)| variable.as_str())
744                    .collect(),
745            )
746            .or_default()
747            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
748    }
749    let within = |variable: &str, list: &str| {
750        let (key, integer) = column(variable)?;
751        let key = if integer {
752            format!("CAST({key} AS TEXT)")
753        } else {
754            key
755        };
756        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
757    };
758    for (variables, tuples) in groups {
759        if let [variable] = variables[..] {
760            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
761            params.push(SqlValue::Text(
762                Value::from_iter(values.into_iter().cloned()).to_string(),
763            ));
764            clauses.push(format!("NOT {}", within(variable, "?")?));
765        } else {
766            params.push(SqlValue::Text(
767                serde_json::to_string(&tuples).expect("owner tuples serialize"),
768            ));
769            let matches = variables
770                .iter()
771                .enumerate()
772                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
773                .collect::<Result<Vec<_>, ()>>()?;
774            clauses.push(format!(
775                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
776                matches.join(" AND ")
777            ));
778        }
779    }
780    Ok(Some((clauses.join(" AND "), params)))
781}
782
783/// [`local_scope_predicate`] over decoded row values (retained rows).
784fn owns_local_row(
785    table: &TableSchema,
786    effective: &[(String, Vec<String>)],
787    owners: &[Vec<(String, Vec<String>)>],
788    values: &Map<String, Value>,
789) -> bool {
790    let matches = |scopes: &[(String, Vec<String>)]| {
791        !scopes.is_empty()
792            && scopes.iter().all(|(variable, allowed)| {
793                table
794                    .scope_column(variable)
795                    .and_then(|column| values.get(column))
796                    .filter(|value| !value.is_null())
797                    .is_some_and(|value| {
798                        allowed.contains(
799                            &value
800                                .as_str()
801                                .map_or_else(|| value.to_string(), str::to_owned),
802                        )
803                    })
804            })
805    };
806    matches(effective) && !owners.iter().any(|owner| matches(owner))
807}
808
809fn is_local_operation_code_like(value: &str) -> bool {
810    let bytes = value.as_bytes();
811    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
812        && bytes
813            .iter()
814            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
815}
816
817fn base_table(name: &str) -> String {
818    quote_ident(&format!("_syncular_base_{name}"))
819}
820
821/// §4.8 deferred eviction record: (unit id, table, effective scope map).
822type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
823
824/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
825fn window_base_key(base: &WindowBase) -> String {
826    format!(
827        "{}\0{}",
828        base.variable,
829        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
830    )
831}
832
833/// The unit of one window value: the fixed scopes plus the value.
834fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
835    let mut scopes = base.fixed_scopes.clone();
836    scopes.insert(base.variable.clone(), value.to_owned());
837    scopes
838}
839
840/// A unit's scope map as the codec and the digests take it.
841fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
842    let mut pairs: Vec<(String, String)> = scopes
843        .iter()
844        .map(|(variable, value)| (variable.clone(), value.clone()))
845        .collect();
846    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
847    pairs
848}
849
850/// §4.1: the unit covers every table declaring all of its variables.
851fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
852    scopes
853        .keys()
854        .all(|variable| table.scope_column(variable).is_some())
855}
856
857/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
858/// scope map as one-element lists plus the fenced lists. `None` when the
859/// fence revoked the pair.
860fn pair_scopes(
861    scopes: &UnitScopes,
862    table: &str,
863    echo: Option<&[EchoRecord]>,
864) -> Option<Vec<(String, Vec<String>)>> {
865    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
866    if fenced.is_some_and(|record| record.status == "revoked") {
867        return None;
868    }
869    let mut effective: BTreeMap<String, Vec<String>> = fenced
870        .map(|record| record.scopes.clone())
871        .unwrap_or_default();
872    for (variable, value) in scopes {
873        effective.insert(variable.clone(), vec![value.clone()]);
874    }
875    Some(effective.into_iter().collect())
876}
877
878/// §3.3 row ownership: the pair scopes of every owner of `table` except
879/// `except_id`. An owner is active with a persisted echo and live or
880/// post-bootstrap, or bootstrapping with a segment of the table in its
881/// current bootstrap.
882fn unit_owners(
883    units: &[UnitRecord],
884    table: &TableSchema,
885    except_id: &str,
886) -> Vec<Vec<(String, Vec<String>)>> {
887    units
888        .iter()
889        .filter(|unit| {
890            unit.id != except_id
891                && unit.status == UnitStatus::Active
892                && unit.echo.is_some()
893                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
894                && covers(table, &unit.scopes)
895        })
896        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
897        .collect()
898}
899
900/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
901fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
902    scopes
903        .iter()
904        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
905        .collect()
906}
907
908/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
909/// It joins through the (tbl, id) index: a scan per acknowledgement would
910/// make draining a batch quadratic.
911const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
912
913const UNIT_COLUMNS: &str =
914    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
915
916/// A unit row; a field that does not parse is a conversion failure, which
917/// both read paths classify as `sync.local_corrupt`.
918fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
919    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
920        serde_json::from_str(raw).map_err(|error| {
921            rusqlite::Error::FromSqlConversionFailure(
922                index,
923                rusqlite::types::Type::Text,
924                Box::new(error),
925            )
926        })
927    }
928    Ok(UnitRecord {
929        id: row.get(0)?,
930        scopes: json(1, &row.get::<_, String>(1)?)?,
931        explicit: row.get::<_, i64>(2)? == 1,
932        status: json(
933            3,
934            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
935        )?,
936        reason_code: row.get(4)?,
937        cursor: row.get(5)?,
938        bootstrap_state: row.get(6)?,
939        seeded: json(7, &row.get::<_, String>(7)?)?,
940        echo: row
941            .get::<_, Option<String>>(8)?
942            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
943            .transpose()?
944            .map(|echo| {
945                if echo
946                    .iter()
947                    .all(|record| record.status == "active" || record.status == "revoked")
948                {
949                    Ok(echo)
950                } else {
951                    Err(rusqlite::Error::FromSqlConversionFailure(
952                        8,
953                        rusqlite::types::Type::Text,
954                        "echo status is neither active nor revoked".into(),
955                    ))
956                }
957            })
958            .transpose()?,
959    })
960}
961
962/// Every unit in registration order (§4.1).
963fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
964    conn.prepare_cached(&format!(
965        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
966    ))?
967    .query_map([], unit_from_row)?
968    .collect()
969}
970
971fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
972    conn.query_row(
973        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
974        [id],
975        unit_from_row,
976    )
977    .optional()
978}
979
980/// Insert or update in place: an update keeps the registration order.
981fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
982    conn.prepare_cached(&format!(
983        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
984         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
985           reason_code=excluded.reason_code, cursor=excluded.cursor,
986           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo"
987    ))?
988    .execute(rusqlite::params![
989        unit.id,
990        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
991        i64::from(unit.explicit),
992        unit.status.name(),
993        unit.reason_code,
994        unit.cursor,
995        unit.bootstrap_state,
996        serde_json::to_string(&unit.seeded).expect("table list serializes"),
997        unit.echo
998            .as_ref()
999            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
1000    ])
1001    .map(|_| ())
1002}
1003
1004/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1005fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1006    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1007        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1008        .collect()
1009}
1010
1011/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1012/// registrations and discard the synced state of every unit outside `keep`,
1013/// so the next round sends the list and fresh-bootstraps those units. The
1014/// replica cursor survives only while a kept unit is live. The caller owns
1015/// the transaction.
1016fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1017    conn.execute(
1018        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1019           bootstrap_state=NULL, seeded='[]', echo=NULL
1020         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1021        [serde_json::to_string(keep).expect("unit ids serialize")],
1022    )?;
1023    conn.execute_batch(
1024        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1025         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1026           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1027    )
1028}
1029
1030/// §4.5: the replica cursor, `-1` while no unit is live.
1031fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1032    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1033        raw.parse().map_err(|error| {
1034            rusqlite::Error::FromSqlConversionFailure(
1035                0,
1036                rusqlite::types::Type::Text,
1037                Box::new(error),
1038            )
1039        })
1040    })
1041}
1042
1043fn visible_table(name: &str) -> String {
1044    quote_ident(name)
1045}
1046
1047const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1048
1049/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1050/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1051fn is_synced_table_name(name: &str) -> bool {
1052    if name.starts_with("sqlite_") {
1053        return false;
1054    }
1055    if name.starts_with("_syncular_base_") {
1056        return true; // the base half of a synced table pair
1057    }
1058    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1059    !name.starts_with("_syncular_")
1060}
1061
1062/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1063fn blob_id_for(bytes: &[u8]) -> String {
1064    let digest = Sha256::digest(bytes);
1065    format!("sha256:{}", bytes_to_hex(&digest))
1066}
1067
1068/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1069/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1070/// per row on the §5.6 bootstrap path), scalars bind owned.
1071enum RowParam<'a> {
1072    Cell(&'a Option<ColumnValue>),
1073    Version(i64),
1074    Text(Option<&'a str>),
1075}
1076
1077impl rusqlite::ToSql for RowParam<'_> {
1078    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1079        Ok(match self {
1080            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1081            RowParam::Text(text) => match text {
1082                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1083                None => ToSqlOutput::Owned(SqlValue::Null),
1084            },
1085            RowParam::Cell(cell) => match cell {
1086                None => ToSqlOutput::Owned(SqlValue::Null),
1087                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1088                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1089                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1090                Some(ColumnValue::Boolean(b)) => {
1091                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1092                }
1093                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1094                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1095                }
1096                // §5.10: crdt bytes store as BLOB, like bytes.
1097                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1098                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1099                }
1100            },
1101        })
1102    }
1103}
1104
1105// Retain a typed primary key across the current import chunk without copying row payloads.
1106fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1107    match param.to_sql().map_err(|error| error.to_string())? {
1108        ToSqlOutput::Owned(value) => Ok(value),
1109        ToSqlOutput::Borrowed(value) => {
1110            SqlValue::try_from(value).map_err(|error| error.to_string())
1111        }
1112        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1113    }
1114}
1115
1116/// §5.3 image cell → bind parameter, strict per the declared column type
1117/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1118/// nullable). Mismatches are image-producer violations. Returns the cell
1119/// borrowed when the stored representation already matches what the row
1120/// codec would write, or the normalized scalar (boolean → 0/1, float from
1121/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1122/// old convert-then-insert path without allocating per cell.
1123fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1124    use ssp2::segment::ColumnType;
1125    let mismatch = || {
1126        Err(format!(
1127            "image column {:?} holds a value of the wrong type",
1128            column.name
1129        ))
1130    };
1131    match value {
1132        ValueRef::Null => {
1133            if !column.nullable {
1134                return Err(format!(
1135                    "image column {:?} is NULL but not nullable",
1136                    column.name
1137                ));
1138            }
1139            Ok(ToSqlOutput::Owned(SqlValue::Null))
1140        }
1141        ValueRef::Integer(i) => match column.ty {
1142            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1143            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1144            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1145            _ => mismatch(),
1146        },
1147        ValueRef::Real(_) => match column.ty {
1148            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1149            _ => mismatch(),
1150        },
1151        ValueRef::Text(t) => {
1152            std::str::from_utf8(t)
1153                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1154            match column.ty {
1155                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1156                    Ok(ToSqlOutput::Borrowed(value))
1157                }
1158                _ => mismatch(),
1159            }
1160        }
1161        ValueRef::Blob(_) => match column.ty {
1162            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1163            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1164            _ => mismatch(),
1165        },
1166    }
1167}
1168
1169pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1170    use rusqlite::types::ValueRef;
1171    match value {
1172        ValueRef::Null => Value::Null,
1173        ValueRef::Integer(i) => match column.ty {
1174            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1175            ssp2::segment::ColumnType::Float => {
1176                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1177            }
1178            _ => Value::from(i),
1179        },
1180        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1181        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1182        ValueRef::Blob(b) => {
1183            let mut map = Map::new();
1184            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1185            Value::Object(map)
1186        }
1187    }
1188}
1189
1190/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1191/// Objects are accepted in lossless `{"$bytes": hex}` and
1192/// `{"$bigint": decimal}` envelope forms.
1193fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1194    Ok(match value {
1195        Value::Null => SqlValue::Null,
1196        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1197        Value::Number(n) => {
1198            if let Some(i) = n.as_i64() {
1199                SqlValue::Integer(i)
1200            } else if let Some(f) = n.as_f64() {
1201                SqlValue::Real(f)
1202            } else {
1203                return Err(format!("query param number {n} is out of range"));
1204            }
1205        }
1206        Value::String(s) => SqlValue::Text(s.clone()),
1207        Value::Object(_) => {
1208            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1209                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1210            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1211                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1212                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1213                })?)
1214            } else {
1215                return Err(
1216                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1217                        .to_owned(),
1218                );
1219            }
1220        }
1221        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1222    })
1223}
1224
1225/// Map a rusqlite value with no schema column to consult (arbitrary query
1226/// output): integers/reals/text pass through by stored affinity, blobs ride
1227/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1228/// schema column type to recover booleans/floats/json.
1229fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1230    use rusqlite::types::ValueRef;
1231    match value {
1232        ValueRef::Null => Value::Null,
1233        ValueRef::Integer(i) => {
1234            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1235            // ordinary UI-sized integers ergonomic and envelope only values
1236            // beyond JavaScript's safe range.
1237            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1238            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1239                Value::from(i)
1240            } else {
1241                let mut map = Map::new();
1242                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1243                Value::Object(map)
1244            }
1245        }
1246        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1247        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1248        ValueRef::Blob(b) => {
1249            let mut map = Map::new();
1250            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1251            Value::Object(map)
1252        }
1253    }
1254}
1255
1256/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1257fn query_connection(
1258    conn: &Connection,
1259    sql: &str,
1260    params: &[QueryValue],
1261) -> Result<Vec<QueryRow>, QueryReadFailure> {
1262    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1263    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1264    let bound: Vec<SqlValue> = params
1265        .iter()
1266        .map(json_param_to_sql)
1267        .collect::<Result<_, _>>()?;
1268    // §2.4: an unaliased physical version column stays engine-internal, so a
1269    // `SELECT *` row feeds straight back into mutation values. An explicit
1270    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1271    // §5.11 sealed columns follow the same rule.
1272    let columns: Vec<(usize, String)> = stmt
1273        .column_names()
1274        .enumerate()
1275        .filter(|(_, name)| {
1276            !matches!(
1277                *name,
1278                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1279            )
1280        })
1281        .map(|(i, name)| (i, name.to_owned()))
1282        .collect();
1283    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1284        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1285    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1286    let mut out = Vec::new();
1287    while let Some(row) = sql_rows.next()? {
1288        let mut record = Map::new();
1289        for (i, name) in &columns {
1290            let value = row.get_ref(*i)?;
1291            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1292        }
1293        out.push(record);
1294    }
1295    Ok(out)
1296}
1297
1298fn persisted_window_state(
1299    conn: &Connection,
1300    base: &WindowBase,
1301) -> Result<WindowState, QueryReadFailure> {
1302    let mut stmt = conn.prepare(
1303        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1304           FROM _syncular_windows AS windows
1305           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1306          WHERE windows.base = ?1
1307          ORDER BY windows.unit ASC",
1308    )?;
1309    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1310        Ok((
1311            row.get::<_, String>(0)?,
1312            row.get::<_, Option<String>>(1)?,
1313            row.get::<_, Option<i64>>(2)?,
1314            row.get::<_, Option<String>>(3)?,
1315        ))
1316    })?;
1317    let mut units = Vec::new();
1318    let mut pending = Vec::new();
1319    for row in rows {
1320        let (unit, status, cursor, bootstrap_state) = row?;
1321        // §4.8: complete iff registered, active and not bootstrapping.
1322        if status.as_deref() != Some("active")
1323            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1324        {
1325            pending.push(unit.clone());
1326        }
1327        units.push(unit);
1328    }
1329    Ok(WindowState { units, pending })
1330}
1331
1332/// One read-only statement in a §7.5 snapshot read.
1333#[derive(Debug, Clone, Copy)]
1334pub struct SnapshotStatement<'a> {
1335    pub sql: &'a str,
1336    pub params: &'a [Value],
1337}
1338
1339/// A §7.5 atomic read request: read-only statements, window coverage, unit
1340/// catch-up states, and commit deliveries, all resolved from one SQLite read
1341/// transaction and one revision.
1342#[derive(Debug, Clone, Default)]
1343pub struct SnapshotReadRequest<'a> {
1344    pub statements: Vec<SnapshotStatement<'a>>,
1345    pub coverage: &'a [WindowCoverage],
1346    pub units: Vec<String>,
1347    pub commit_ids: Vec<String>,
1348}
1349
1350#[cfg(test)]
1351thread_local! {
1352    /// Test-only rendezvous that runs between the statements of one snapshot
1353    /// read. A test uses it to commit on another connection mid-read and prove
1354    /// the read transaction's snapshot and revision stay stable.
1355    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1356        std::cell::RefCell::new(None);
1357}
1358
1359#[cfg(test)]
1360fn run_snapshot_read_hook() {
1361    SNAPSHOT_READ_HOOK.with(|slot| {
1362        if let Some(hook) = slot.borrow_mut().take() {
1363            hook();
1364        }
1365    });
1366}
1367
1368/// §7.5: resolve a request over one SQLite read transaction. The revision,
1369/// every statement's rows, coverage, catch-up states, and deliveries are
1370/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1371/// read back and releases the connection.
1372fn snapshot_read_connection(
1373    conn: &Connection,
1374    request: &SnapshotReadRequest<'_>,
1375) -> Result<SnapshotRead, QueryReadFailure> {
1376    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1377    let corrupt = |message: &str| QueryReadFailure {
1378        code: Some("sync.local_corrupt"),
1379        sqlite_code: None,
1380        sqlite_message: None,
1381        rollback_failure: None,
1382        message: message.to_owned(),
1383    };
1384    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1385        rusqlite::Error::Utf8Error(..)
1386        | rusqlite::Error::InvalidColumnType(..)
1387        | rusqlite::Error::FromSqlConversionFailure(..)
1388        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1389        _ => QueryReadFailure::from(error),
1390    };
1391    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1392        // The durable revision as a canonical decimal `u64`; a missing marker
1393        // is the legacy zero, and any other non-canonical value is corrupt.
1394        let revision: Option<String> = conn
1395            .query_row(
1396                "SELECT value FROM _syncular_meta WHERE key = ?1",
1397                rusqlite::params![LOCAL_REVISION_KEY],
1398                |row| row.get(0),
1399            )
1400            .optional()
1401            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1402        let revision = match revision {
1403            None => "0".to_owned(),
1404            Some(value)
1405                if value == "0"
1406                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1407                        && value.bytes().all(|b| b.is_ascii_digit())
1408                        && value.parse::<u64>().is_ok()) =>
1409            {
1410                value
1411            }
1412            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1413        };
1414        let mut queries = Vec::with_capacity(request.statements.len());
1415        for statement in &request.statements {
1416            queries.push(query_connection(conn, statement.sql, statement.params)?);
1417            #[cfg(test)]
1418            run_snapshot_read_hook();
1419        }
1420        let mut pending = Vec::new();
1421        let mut missing = Vec::new();
1422        for requested in request.coverage {
1423            let base_key = window_base_key(&requested.base);
1424            let state = persisted_window_state(conn, &requested.base)?;
1425            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1426                let reference = WindowUnitRef {
1427                    base_key: base_key.clone(),
1428                    unit: unit.clone(),
1429                };
1430                if !state.units.iter().any(|held| held == &unit) {
1431                    missing.push(reference);
1432                } else if state.pending.iter().any(|held| held == &unit) {
1433                    pending.push(reference);
1434                }
1435            }
1436        }
1437        let mut units = Vec::with_capacity(request.units.len());
1438        for id in &request.units {
1439            let exists: bool = conn.query_row(
1440                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1441                [id],
1442                |row| row.get(0),
1443            )?;
1444            if !exists {
1445                units.push(UnitCatchup::Unknown { id: id.clone() });
1446                continue;
1447            }
1448            // Every persisted field is required to be well typed; a corrupt
1449            // record fails typed instead of inventing a state.
1450            let unit = get_unit(conn, id)
1451                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1452                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1453            let cursor = match unit.cursor {
1454                Some(cursor) => cursor,
1455                None => replica_cursor(conn)
1456                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1457            };
1458            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1459                return Err(corrupt("persisted unit state is invalid"));
1460            }
1461            let active = unit.status == UnitStatus::Active;
1462            let pending = unit.bootstrapping();
1463            units.push(UnitCatchup::Known {
1464                id: id.clone(),
1465                status: unit.status,
1466                live: unit.cursor.is_none(),
1467                cursor,
1468                has_resume_token: unit.bootstrap_state.is_some(),
1469                bootstrap_complete: active && !pending,
1470                known_pending_pages: active && pending,
1471                reason_code: unit.reason_code,
1472            });
1473        }
1474        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1475        for id in &request.commit_ids {
1476            let pending: bool = conn.query_row(
1477                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1478                rusqlite::params![id],
1479                |row| row.get(0),
1480            )?;
1481            if pending {
1482                deliveries.push(CommitDelivery::Pending {
1483                    client_commit_id: id.clone(),
1484                });
1485                continue;
1486            }
1487            let row = conn
1488                .query_row(
1489                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1490                            resolution, resolved_at_ms, replacement_client_commit_id
1491                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1492                    rusqlite::params![id],
1493                    StoredCommitOutcomeRow::from_row,
1494                )
1495                .optional()
1496                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1497            deliveries.push(match row {
1498                Some(row) => CommitDelivery::Known {
1499                    client_commit_id: id.clone(),
1500                    outcome: SyncClient::outcome_from_row(row)
1501                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1502                },
1503                None => CommitDelivery::Unknown {
1504                    client_commit_id: id.clone(),
1505                },
1506            });
1507        }
1508        Ok(SnapshotRead {
1509            revision,
1510            queries,
1511            coverage: CoverageSnapshot {
1512                complete: pending.is_empty() && missing.is_empty(),
1513                pending,
1514                missing,
1515            },
1516            units,
1517            deliveries,
1518        })
1519    })();
1520    match result {
1521        Ok(read) => {
1522            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1523            Ok(read)
1524        }
1525        Err(mut error) => {
1526            if let Err(rollback) = conn
1527                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1528            {
1529                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1530            }
1531            Err(error)
1532        }
1533    }
1534}
1535
1536fn snapshot_connection(
1537    conn: &Connection,
1538    sql: &str,
1539    params: &[Value],
1540    coverage: &[WindowCoverage],
1541) -> Result<QuerySnapshot, QueryReadFailure> {
1542    let read = snapshot_read_connection(
1543        conn,
1544        &SnapshotReadRequest {
1545            statements: vec![SnapshotStatement { sql, params }],
1546            coverage,
1547            units: Vec::new(),
1548            commit_ids: Vec::new(),
1549        },
1550    )?;
1551    Ok(QuerySnapshot {
1552        revision: read.revision,
1553        rows: read
1554            .queries
1555            .into_iter()
1556            .next()
1557            .expect("single-query snapshot returns one result"),
1558        coverage: read.coverage,
1559    })
1560}
1561
1562/// A long-lived read-only SQLite sidecar for latency-critical native views.
1563/// Network rounds stay serialized on the mutable core owner, while atomic
1564/// query snapshots use this independent connection and therefore never queue
1565/// behind HTTP/WebSocket latency.
1566pub struct FileQuerySnapshotReader {
1567    path: String,
1568    conn: Option<Connection>,
1569}
1570
1571impl FileQuerySnapshotReader {
1572    #[must_use]
1573    pub fn new(path: impl Into<String>) -> Self {
1574        Self {
1575            path: path.into(),
1576            conn: None,
1577        }
1578    }
1579
1580    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1581        if self.conn.is_none() {
1582            let conn = Connection::open_with_flags(
1583                &self.path,
1584                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1585            )
1586            .map_err(QueryReadFailure::from)?;
1587            conn.busy_timeout(std::time::Duration::from_millis(250))
1588                .map_err(QueryReadFailure::from)?;
1589            self.conn = Some(conn);
1590        }
1591        self.conn
1592            .as_ref()
1593            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1594    }
1595
1596    /// One atomic read over the shared primitive: every statement, coverage,
1597    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1598    pub fn snapshot_read(
1599        &mut self,
1600        request: &SnapshotReadRequest<'_>,
1601    ) -> Result<SnapshotRead, QueryReadFailure> {
1602        snapshot_read_connection(self.connection()?, request)
1603    }
1604
1605    /// The current durable local revision. A host can skip a read when it is
1606    /// unchanged; a corrupt or unreadable marker fails typed.
1607    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1608        Ok(self
1609            .snapshot_read(&SnapshotReadRequest::default())?
1610            .revision)
1611    }
1612
1613    /// One unit's persisted catch-up state (see the §7.5 contract).
1614    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1615        let read = self.snapshot_read(&SnapshotReadRequest {
1616            units: vec![id.to_owned()],
1617            ..Default::default()
1618        })?;
1619        Ok(read
1620            .units
1621            .into_iter()
1622            .next()
1623            .expect("one requested catch-up"))
1624    }
1625
1626    /// One client commit's delivery status: pending, its retained outcome, or
1627    /// an explicit unknown (never held, or pruned by retention).
1628    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1629        let read = self.snapshot_read(&SnapshotReadRequest {
1630            commit_ids: vec![id.to_owned()],
1631            ..Default::default()
1632        })?;
1633        Ok(read
1634            .deliveries
1635            .into_iter()
1636            .next()
1637            .expect("one requested delivery"))
1638    }
1639
1640    /// Hosts report an owned failure, and the first success after it, to the
1641    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1642    pub fn query_snapshot(
1643        &mut self,
1644        sql: &str,
1645        params: &[Value],
1646        coverage: &[WindowCoverage],
1647    ) -> Result<QuerySnapshot, QueryReadFailure> {
1648        snapshot_connection(self.connection()?, sql, params, coverage)
1649    }
1650}
1651
1652fn validate_authority_reads(
1653    schema: &ClientSchema,
1654    declarations: &[crate::api::AuthorityReadDeclaration],
1655) -> Result<(), String> {
1656    let mut seen = BTreeSet::new();
1657    for read in declarations {
1658        let table = schema
1659            .table(&read.table)
1660            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1661        let plain = |name: &str| {
1662            table.columns.iter().enumerate().any(|(i, c)| {
1663                c.name == name
1664                    && name != "authority_version"
1665                    && !name.starts_with("_sync")
1666                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1667                    && matches!(
1668                        c.ty,
1669                        ssp2::segment::ColumnType::String
1670                            | ssp2::segment::ColumnType::Integer
1671                            | ssp2::segment::ColumnType::Float
1672                            | ssp2::segment::ColumnType::Boolean
1673                            | ssp2::segment::ColumnType::Json
1674                    )
1675            })
1676        };
1677        if !seen.insert(&read.table)
1678            || read.columns.is_empty()
1679            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1680            || !read.columns.contains(&table.primary_key)
1681            || !read.columns.iter().all(|c| plain(c))
1682            || read.scopes.is_empty()
1683            || !read.scopes.iter().all(|(v, values)| {
1684                table
1685                    .scope_column(v)
1686                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1687                    && !values.is_empty()
1688                    && values.iter().all(|v| !v.is_empty() && v != "*")
1689            })
1690        {
1691            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1692        }
1693    }
1694    Ok(())
1695}
1696
1697// §7.4.2: every open and recreation checks before any persistent writes.
1698/// A marker read that hit a recognized SQLite storage failure keeps that
1699/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1700/// failure is the invalid-marker `sync.local_corrupt`.
1701fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1702    let failure = QueryReadFailure::from(error);
1703    if failure.code.is_some() {
1704        failure.to_string()
1705    } else {
1706        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1707    }
1708}
1709
1710/// §7.1: reject an invalid configured client limit before any storage is
1711/// opened or written. Zero would wedge the push queue and an out-of-range
1712/// value would differ per target; the TS core enforces the same bounds.
1713fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1714    if limits.outcome_retention_max_entries == Some(0) {
1715        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1716    }
1717    if [
1718        limits.max_push_commits_per_request,
1719        limits.max_push_operations_per_request,
1720        limits.max_push_request_bytes,
1721    ]
1722    .into_iter()
1723    .flatten()
1724    .any(|value| value == 0 || value > u32::MAX as usize)
1725    {
1726        return Err(
1727            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1728        );
1729    }
1730    Ok(())
1731}
1732
1733fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1734    let corrupt =
1735        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1736    let table_type: Option<String> = conn
1737        .query_row(
1738            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1739            [],
1740            |row| row.get(0),
1741        )
1742        .optional()
1743        .map_err(schema_marker_read_failure)?;
1744    let Some(table_type) = table_type else {
1745        return Ok(None);
1746    };
1747    if table_type != "table" {
1748        return Err(corrupt());
1749    }
1750    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1751    // whichever row SQLite returns first, and an older one would drive a
1752    // destructive reset over a newer replica.
1753    let mut statement = conn
1754        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1755        .map_err(schema_marker_read_failure)?;
1756    let mut rows = statement
1757        .query([LOCAL_SCHEMA_VERSION_KEY])
1758        .map_err(schema_marker_read_failure)?;
1759    let value: Option<String> = rows
1760        .next()
1761        .map_err(schema_marker_read_failure)?
1762        .map(|row| row.get(0))
1763        .transpose()
1764        .map_err(schema_marker_read_failure)?;
1765    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1766        return Err(corrupt());
1767    }
1768    let Some(value) = value else {
1769        let has_descriptor = conn
1770            .query_row(
1771                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1772                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1773                |_| Ok(()),
1774            )
1775            .optional()
1776            .map_err(schema_marker_read_failure)?
1777            .is_some();
1778        if has_descriptor {
1779            return Err(corrupt());
1780        }
1781        return Ok(None);
1782    };
1783    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1784        || !value.bytes().all(|b| b.is_ascii_digit())
1785    {
1786        return Err(corrupt());
1787    }
1788    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1789    if version > requested {
1790        return Err(
1791            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1792                .to_owned(),
1793        );
1794    }
1795    Ok(Some(version))
1796}
1797
1798/// A file replica on a browser worker must recover a hot journal before any
1799/// statement reads it. `wasm_vfs` replaces the installed OPFS VFS's reserved
1800/// lock callback, so install it before the connection opens its file.
1801#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1802fn prepare_wasm_replica(path: &str) -> Result<(), String> {
1803    if path.is_empty() || path == ":memory:" {
1804        return Ok(());
1805    }
1806    crate::wasm_vfs::disable_foreign_lock_check()
1807}
1808
1809#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
1810fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
1811    Ok(())
1812}
1813
1814impl SyncClient {
1815    pub fn new_with_identity(
1816        client_id: Option<String>,
1817        schema_json: &Value,
1818        limits: ClientLimits,
1819    ) -> Result<Self, String> {
1820        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1821        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1822        Self::with_connection(resolved, schema_json, limits, conn)
1823    }
1824
1825    pub fn new(
1826        client_id: String,
1827        schema_json: &Value,
1828        limits: ClientLimits,
1829    ) -> Result<Self, String> {
1830        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1831        Self::with_connection(client_id, schema_json, limits, conn)
1832    }
1833
1834    /// Build a client backed by an on-disk SQLite database at `path` — the
1835    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1836    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1837    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1838    /// of the command router's dependency set (the router only holds a path).
1839    pub fn open_path(
1840        client_id: String,
1841        schema_json: &Value,
1842        limits: ClientLimits,
1843        path: &str,
1844    ) -> Result<Self, String> {
1845        validate_client_limits(&limits)?;
1846        parse_schema_json(schema_json)?;
1847        prepare_wasm_replica(path)?;
1848        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1849        Self::with_connection(client_id, schema_json, limits, conn)
1850    }
1851
1852    /// Open a file replica and resolve its client id. Native targets require
1853    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1854    /// accepts the sqlite-wasm worker journal instead.
1855    pub fn open_path_with_identity(
1856        client_id: Option<String>,
1857        schema_json: &Value,
1858        limits: ClientLimits,
1859        path: &str,
1860    ) -> Result<Self, String> {
1861        Self::open_path_with_identity_journal(
1862            client_id,
1863            schema_json,
1864            limits,
1865            path,
1866            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1867        )
1868    }
1869
1870    fn open_path_with_identity_journal(
1871        client_id: Option<String>,
1872        schema_json: &Value,
1873        limits: ClientLimits,
1874        path: &str,
1875        require_wal: bool,
1876    ) -> Result<Self, String> {
1877        validate_client_limits(&limits)?;
1878        // §7.4.2: an invalid requested version is refused before the replica is
1879        // even created, so a rejected constructor leaves no file behind.
1880        parse_schema_json(schema_json)?;
1881        prepare_wasm_replica(path)?;
1882        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1883        // A file-backed native client uses an independent read connection for
1884        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1885        // concurrency mode: a view read never holds a rollback-journal lock
1886        // that delays the client's next commit. The busy timeout must precede
1887        // this replica's own startup transaction; WAL is applied only after
1888        // startup succeeded, because a refused open must leave the replica's
1889        // journal mode and contents untouched (§7.4.2).
1890        conn.busy_timeout(std::time::Duration::from_millis(250))
1891            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1892        // Durability is the caller's contract here, not the VFS default: a
1893        // rollback-journal commit needs FULL before it writes the database
1894        // pages, or a killed worker can leave a half-written image behind.
1895        // WAL below inherits the same setting.
1896        conn.pragma_update(None, "synchronous", "FULL")
1897            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1898        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1899        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1900        // host has no independent snapshot reader. The WAL requirement exists
1901        // for native file replicas, where it is the reader/writer boundary.
1902        if require_wal {
1903            let journal_mode: String = client
1904                .conn
1905                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1906                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1907            if journal_mode != "wal" {
1908                return Err(
1909                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1910                );
1911            }
1912        }
1913        Ok(client)
1914    }
1915
1916    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1917    /// `-shm` sidecars, through the VFS that owns it. Use this after
1918    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1919    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1920    /// pool's slot accounting.
1921    ///
1922    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1923    /// a VFS cannot delete a file another handle still owns. A file that does
1924    /// not exist counts as deleted, so a cleanly closed replica without
1925    /// sidecars resets. The next open
1926    /// creates a fresh replica (new client id) and re-bootstraps from the
1927    /// server. A memory replica has nothing to delete and fails with
1928    /// `sync.invalid_request`.
1929    pub fn reset_replica(path: &str) -> Result<(), String> {
1930        if path.is_empty() || path == ":memory:" {
1931            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1932        }
1933        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1934        // the same one `Connection::open(path)` above selects. `xDelete` is
1935        // that VFS's own function pointer, and `c_name` outlives the call.
1936        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1937        if vfs.is_null() {
1938            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1939        }
1940        let delete = unsafe { (*vfs).xDelete }
1941            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1942        for suffix in ["-wal", "-shm", "-journal", ""] {
1943            let file_name = format!("{path}{suffix}");
1944            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1945                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1946            })?;
1947            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1948            // SQLite library; `c_name` is a valid NUL-terminated path.
1949            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1950            // A cleanly closed WAL replica has no sidecars; a missing file is
1951            // already deleted.
1952            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1953            {
1954                return Err(format!(
1955                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1956                ));
1957            }
1958        }
1959        Ok(())
1960    }
1961
1962    #[must_use]
1963    pub fn client_id(&self) -> &str {
1964        &self.client_id
1965    }
1966
1967    /// Rotate the caller's client id (§1.5) and return the new one. Use it
1968    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
1969    /// resetClientId`): the next round carries the new id. A live realtime
1970    /// socket is closed so the next connection registers with the new id.
1971    /// The replica and its outbox stay in place; wipe them yourself when the
1972    /// stored data belongs to the account that signed out.
1973    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
1974        self.disconnect_realtime(transport);
1975        let client_id = uuid::Uuid::new_v4().to_string();
1976        self.set_meta(CLIENT_ID_KEY, &client_id)?;
1977        self.client_id.clone_from(&client_id);
1978        self.round_failure = None;
1979        Ok(client_id)
1980    }
1981
1982    /// Build a client over a caller-supplied rusqlite connection — the seam a
1983    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
1984    /// with an on-disk database (`Connection::open(path)`) rather than the
1985    /// default `:memory:`. The connection MUST be fresh (no pre-existing
1986    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
1987    /// the same file across process restarts reuses the persisted rows.
1988    pub fn with_connection(
1989        client_id: String,
1990        schema_json: &Value,
1991        limits: ClientLimits,
1992        conn: Connection,
1993    ) -> Result<Self, String> {
1994        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
1995    }
1996
1997    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
1998    /// means a durable replica's persisted `clientId` wins over a fresh one.
1999    fn with_connection_identity(
2000        client_id: Option<String>,
2001        schema_json: &Value,
2002        limits: ClientLimits,
2003        conn: Connection,
2004    ) -> Result<Self, String> {
2005        validate_client_limits(&limits)?;
2006        let schema = parse_schema_json(schema_json)?;
2007        validate_authority_reads(&schema, &limits.authority_reads)?;
2008        // RFC 0005 D8: resolve the previous-version config before the opening
2009        // reset runs — the capture happens inside it. Bad bounds fail loud.
2010        let previous_version = match limits.previous_version_context {
2011            Some(config) => {
2012                config.validate()?;
2013                Some(config)
2014            }
2015            None => None,
2016        };
2017        let mut client = SyncClient {
2018            #[cfg(feature = "bench-internals")]
2019            benchmark_phases: Recorder::default(),
2020            conn,
2021            schema,
2022            client_id: client_id
2023                .clone()
2024                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2025            limits,
2026            outbox: Shared::default(),
2027            failed_commits: Shared::default(),
2028            retain_failed_commits: false,
2029            conflicts: Shared::default(),
2030            rejections: Shared::default(),
2031            schema_floor: None,
2032            lease_state: None,
2033            stopped: false,
2034            upgrading: false,
2035            sync_needed: false,
2036            realtime_policy: RealtimePolicy::Optional,
2037            realtime_state: RealtimeState::Disconnected,
2038            realtime_reason_code: None,
2039            realtime_retry_delay_ms: None,
2040            presence: HashMap::new(),
2041            now_ms: None,
2042            encryption: crate::values::EncryptionConfig::default(),
2043            security_preflight: false,
2044            security_epoch: 0,
2045            transport_enabled: true,
2046            transport_epoch: 0,
2047            registration_epoch: 0,
2048            previous_version,
2049            insert_sql: RefCell::new(HashMap::new()),
2050            overlay_dirty: OverlayDirty::clean(),
2051            storage_failure: RefCell::new(None),
2052            #[cfg(test)]
2053            overlay_rebuild_count: Cell::new(0),
2054            #[cfg(test)]
2055            acknowledged_replay_count: Cell::new(0),
2056            #[cfg(test)]
2057            outcome_prune_count: Cell::new(0),
2058            #[cfg(test)]
2059            image_chunk_ms: Cell::new(None),
2060            progress: ProgressObserver::default(),
2061            change_queue: VecDeque::new(),
2062            sync_intent_queue: VecDeque::new(),
2063            retry_delay_ms: 250,
2064            round_retry_delay_ms: None,
2065            applied_segments: HashMap::new(),
2066            image_chunk_rows: HashMap::new(),
2067            round_failure: None,
2068            active_round: None,
2069            last_round: None,
2070            last_change: None,
2071            query_failures: Vec::new(),
2072            coalesced: None,
2073            coalesced_failure: None,
2074        };
2075        // The row write path leans on the prepared-statement cache (two
2076        // insert statements per synced table, plus the bookkeeping
2077        // statements); size it so a multi-table schema never thrashes.
2078        client
2079            .conn
2080            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2081        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2082        // writer) covers the marker read, the identity check, and every
2083        // bookkeeping/schema write, so a concurrent open that upgraded the
2084        // replica cannot interleave between the guard and the writes, and a
2085        // refusal leaves the replica unchanged.
2086        client
2087            .conn
2088            .execute_batch("BEGIN IMMEDIATE")
2089            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2090        let startup = (|| -> Result<(), String> {
2091            // The marker guard runs inside the write transaction, so the version
2092            // it validates is the version every write below commits under.
2093            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2094            // New app indexes may reference columns that only exist after a
2095            // version-bump reset.
2096            client.create_bookkeeping_tables()?;
2097            match client.get_meta(CLIENT_ID_KEY)? {
2098                Some(existing) => match client_id {
2099                    Some(requested) if existing != requested => {
2100                        return Err(format!(
2101                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2102                        ))
2103                    }
2104                    _ => client.client_id = existing,
2105                },
2106                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2107            }
2108            client.restore_persisted_state()?;
2109            client.failed_commits = client.load_failed_commits()?.into();
2110            match marker {
2111                None => {
2112                    client.create_synced_tables()?;
2113                    client.persist_schema_version()?;
2114                    client.save_unit_scope_schema()?;
2115                }
2116                Some(version) if version == client.schema.version => {
2117                    client.create_synced_tables()?;
2118                    // RFC 0005 D1: the same-version open backfills the descriptor
2119                    // for a database first opened by an unaware binary, so the
2120                    // NEXT bump can capture without needing a schema bump first.
2121                    set_local_schema_descriptor(&client.conn, &client.schema)
2122                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2123                    client.save_unit_scope_schema()?;
2124                }
2125                Some(_) => client.run_schema_reset()?,
2126            }
2127            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2128            // committed in its OWN file while the replica's savepoint never
2129            // released — a crash at that seam leaves a container beside a matching
2130            // OLD marker, and the next open at that version runs no reset. Discard
2131            // it before anything can read it. No-op when no container exists.
2132            client.reconcile_previous_version_at_boot()?;
2133            client.clear_satisfied_persisted_schema_floor()?;
2134            client.prune_unknown_units(false)?;
2135            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2136            // the full startup/reset reconciliation boundary, inside this open
2137            // transaction and after the schema and identity guards, so a commit
2138            // an earlier version accepted is dropped with durable rejection
2139            // evidence before any replay. A table the current schema removed
2140            // keeps the §7.4.4 send-time classification, and an unresolved
2141            // encryption key stays with the send seam because hosts configure
2142            // keys after open.
2143            client.drop_unencodable_outbox(false, None)?;
2144            if marker == Some(client.schema.version)
2145                && (!client.outbox.is_empty()
2146                    || !client.failed_commits.is_empty()
2147                    || client.has_acknowledged_rows()?)
2148            {
2149                // Reconstruct the visible optimistic overlay from the durable
2150                // base plus outbox instead of trusting a process-interrupted
2151                // mirror.
2152                client.overlay_dirty.set(true);
2153                client.rebuild_overlay()?;
2154            }
2155            // Every persisted active unit needs one catch-up pull on
2156            // open: realtime only covers changes after connection, while an
2157            // idempotent setWindow correctly creates no fresh command effect.
2158            // Pending outbox work has the same restart requirement. The core
2159            // owns this intent so native hosts never poll or require an
2160            // application-issued sync().
2161            client.enqueue_startup_sync_if_needed()?;
2162            Ok(())
2163        })();
2164        match startup {
2165            Ok(()) => match client.conn.execute_batch("COMMIT") {
2166                // §4.8: an eviction a crash interrupted finishes before any
2167                // read, with or without a transport; a replica reopened in
2168                // preflight finishes it at activation.
2169                Ok(()) => {
2170                    if !client.security_preflight {
2171                        client.drain_pending_evictions()?;
2172                    }
2173                    Ok(client)
2174                }
2175                Err(error) => {
2176                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2177                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2178                        // Keep a failed rollback's typed storage classification.
2179                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2180                    }
2181                    Err(failure)
2182                }
2183            },
2184            Err(error) => {
2185                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2186                    // Keep a failed rollback's typed storage classification.
2187                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2188                }
2189                Err(error)
2190            }
2191        }
2192    }
2193
2194    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2195    /// against this instead of system time (conformance virtual clock).
2196    pub fn set_now_ms(&mut self, now_ms: i64) {
2197        self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2198    }
2199
2200    /// The pinned clock, shared: a host that stores into it moves the client
2201    /// clock while a call runs (a virtual clock that advances per request).
2202    pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2203        self.now_ms.clone()
2204    }
2205
2206    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2207    /// The command router parses these from the `create` command's
2208    /// `encryption` config (keys as `{$bytes: hex}`).
2209    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2210        self.encryption = encryption;
2211    }
2212
2213    #[must_use]
2214    pub fn security_lifecycle(&self) -> &'static str {
2215        if self.security_preflight {
2216            "preflight"
2217        } else {
2218            "active"
2219        }
2220    }
2221
2222    #[must_use]
2223    pub fn security_preflight(&self) -> bool {
2224        self.security_preflight
2225    }
2226
2227    /// Host-owned, non-persisted network gate. Local operations remain available.
2228    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2229        if self.transport_enabled != enabled {
2230            self.transport_enabled = enabled;
2231            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2232            self.sync_intent_queue.clear();
2233            self.sync_intent_queue
2234                .push_back(if enabled && !self.security_preflight {
2235                    SyncIntent::Interactive
2236                } else {
2237                    SyncIntent::None
2238                });
2239        }
2240        // A captured reply keeps its normal apply/revocation semantics.
2241        if !enabled && self.active_round.is_none() {
2242            self.disconnect_realtime(transport);
2243        }
2244    }
2245
2246    pub fn transport_enabled(&self) -> bool {
2247        self.transport_enabled
2248    }
2249
2250    /// Quarantine this replica: enter the fail-closed gate, release all
2251    /// core-owned key material, and record the gate in the database so
2252    /// reopening re-enters preflight until `activate_security` clears it.
2253    ///
2254    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2255    /// which must NOT leave that durable mark.
2256    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2257        let entering = !self.security_preflight;
2258        self.seal_security_on_teardown();
2259        // Persist the quarantine so it survives handle teardown and restart:
2260        // reopening this replica re-enters preflight until activation clears it.
2261        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2262        if entering {
2263            self.emit_status_batch()?;
2264        }
2265        Ok(())
2266    }
2267
2268    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2269    fn emit_status_batch(&mut self) -> Result<(), String> {
2270        self.begin_observation("syncular_status")?;
2271        let batch = ChangeAccumulator {
2272            status: true,
2273            ..ChangeAccumulator::default()
2274        };
2275        self.finish_observation("syncular_status", batch)
2276            .inspect_err(|_| self.rollback_observation("syncular_status"))
2277    }
2278
2279    /// Release core-owned key material as a host tears the client down,
2280    /// WITHOUT recording a quarantine.
2281    ///
2282    /// Shutting an activated client down is not a quarantine event. Persisting
2283    /// the gate here marks every cleanly closed replica as pending, and the
2284    /// next plain `create` is then refused permanently: the reopen path
2285    /// restores the flag from the marker and the create guard rejects it. The
2286    /// in-memory flag still closes the gate for anything still holding this
2287    /// instance, which is all a teardown barrier needs.
2288    pub fn seal_security_on_teardown(&mut self) {
2289        self.cancel_sync_round();
2290        self.security_preflight = true;
2291        self.security_epoch = self.security_epoch.wrapping_add(1);
2292        self.encryption = crate::values::EncryptionConfig::default();
2293        self.sync_intent_queue.clear();
2294    }
2295
2296    /// Install the post-authentication keyring and release the host loop.
2297    pub fn activate_security(
2298        &mut self,
2299        encryption: crate::values::EncryptionConfig,
2300    ) -> Result<(), String> {
2301        if !self.security_preflight {
2302            return Err(
2303                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2304            );
2305        }
2306        // §4.8: an interrupted eviction finishes before protected reads open.
2307        self.drain_pending_evictions()?;
2308        self.enqueue_startup_sync_if_needed()?;
2309        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2310        self.encryption = encryption;
2311        self.security_preflight = false;
2312        // §5.11: rows sealed under a key this keyring now holds open at once.
2313        self.reveal_sealed_rows()
2314            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2315        self.emit_status_batch()
2316    }
2317
2318    fn clock_now_ms(&self) -> i64 {
2319        self.now_ms
2320            .as_ref()
2321            .map_or_else(crate::wall_clock_ms, |clock| {
2322                clock.load(std::sync::atomic::Ordering::Relaxed)
2323            })
2324    }
2325
2326    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2327        let exists = |name: &str| {
2328            self.conn
2329                .query_row(
2330                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2331                    [name],
2332                    |row| row.get::<_, bool>(0),
2333                )
2334                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2335        };
2336        let outbox_existed = exists("_syncular_outbox")?;
2337        // Durable client bookkeeping (outbox + units + meta).
2338        self.conn
2339            .execute_batch(
2340                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2341                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2342                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2343                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2344                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2345                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2346                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2347                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2348                   PRIMARY KEY(commit_id, idx));
2349                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2350                   ON _syncular_acked_rows(tbl, id);
2351                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2352                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2353                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2354                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2355                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2356                   client_commit_id TEXT NOT NULL UNIQUE,
2357                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2358                   recorded_at_ms INTEGER NOT NULL,
2359                   results_json TEXT NOT NULL,
2360                   operations_json TEXT,
2361                   resolution TEXT NOT NULL DEFAULT 'active'
2362                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2363                   resolved_at_ms INTEGER,
2364                   replacement_client_commit_id TEXT);
2365                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2366                   ON _syncular_commit_outcomes(resolution, seq);
2367                 CREATE TABLE IF NOT EXISTS _syncular_units (
2368                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2369                   explicit INTEGER NOT NULL DEFAULT 0,
2370                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2371                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2372                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT);
2373                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2374                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2375                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2376                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2377                   PRIMARY KEY (base, unit));
2378                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2379                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2380                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2381                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2382                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2383                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2384                   PRIMARY KEY(commit_id, blob_id));
2385                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2386                   ON _syncular_blob_commit_refs(blob_id);",
2387            )
2388            .map_err(|e| e.to_string())?;
2389        // Migrate an outcome journal created before failed aggregate
2390        // envelopes were retained. Historical rows intentionally stay NULL.
2391        let _ = self
2392            .conn
2393            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2394        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2395            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2396        }
2397        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2398        // a new sequence, so it replaces the id in the same transaction.
2399        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2400            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2401        }
2402        // §9: a replica written under wire version 3 keeps its outbox, journal,
2403        // rows and pending evictions and registers its units fresh. Its
2404        // subscriptions, cursors, echoes, resume tokens and window entries go;
2405        // units left from an earlier version 4 life restart.
2406        if exists("_syncular_window_pending_evict")? {
2407            let invalid: bool = self
2408                .conn
2409                .query_row(
2410                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2411                       WHERE NOT json_valid(effective_scopes)
2412                          OR json_type(effective_scopes) <> 'object'
2413                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2414                               WHERE type <> 'array'
2415                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2416                                       WHERE v.type <> 'text')))",
2417                    [],
2418                    |row| row.get(0),
2419                )
2420                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2421            if invalid {
2422                return Err(
2423                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2424                        .to_owned(),
2425                );
2426            }
2427            self.conn
2428                .execute_batch(
2429                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2430                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2431                     DROP TABLE _syncular_window_pending_evict;
2432                     DROP TABLE IF EXISTS _syncular_subscriptions;
2433                     DELETE FROM _syncular_windows;",
2434                )
2435                .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2436                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2437        }
2438        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2439        // completing a large upload does not rewrite its BLOB record.
2440        if self.schema_has_blobs() {
2441            if exists("_syncular_blobs")? {
2442                let mut statement = self
2443                    .conn
2444                    .prepare("PRAGMA table_info(_syncular_blobs)")
2445                    .map_err(|e| e.to_string())?;
2446                let mut columns = statement
2447                    .query_map([], |row| row.get::<_, String>(1))
2448                    .map_err(|e| e.to_string())?
2449                    .collect::<Result<Vec<_>, _>>()
2450                    .map_err(|e| e.to_string())?;
2451                columns.sort();
2452                let mut expected = vec![
2453                    "blob_id",
2454                    "byte_length",
2455                    "bytes",
2456                    "created_at_ms",
2457                    "media_type",
2458                ];
2459                expected.sort_unstable();
2460                if columns != expected {
2461                    return Err(
2462                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2463                    );
2464                }
2465            } else {
2466                self.conn
2467                    .execute_batch(
2468                        "CREATE TABLE _syncular_blobs(
2469                           blob_id TEXT PRIMARY KEY,
2470                           bytes BLOB NOT NULL,
2471                           byte_length INTEGER NOT NULL,
2472                           media_type TEXT,
2473                           created_at_ms INTEGER NOT NULL);",
2474                    )
2475                    .map_err(|e| e.to_string())?;
2476            }
2477            self.conn
2478                .execute_batch(
2479                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2480                       blob_id TEXT PRIMARY KEY,
2481                       media_type TEXT,
2482                       created_at_ms INTEGER NOT NULL);",
2483                )
2484                .map_err(|e| e.to_string())?;
2485        }
2486        Ok(())
2487    }
2488
2489    /// True iff any synced table declares a `blob_ref` column (§5.9).
2490    fn schema_has_blobs(&self) -> bool {
2491        self.schema
2492            .tables
2493            .iter()
2494            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2495    }
2496
2497    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2498
2499    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2500        meta_get(&self.conn, key)
2501            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2502    }
2503
2504    // The caller holds the startup/reset transaction. Either both schema
2505    // records persist or the enclosing operation fails and rolls back.
2506    fn persist_schema_version(&self) -> Result<(), String> {
2507        self.conn
2508            .execute(
2509                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2510                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2511            )
2512            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2513        set_local_schema_descriptor(&self.conn, &self.schema)
2514            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2515    }
2516
2517    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2518        meta_set(&self.conn, key, value)
2519            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2520    }
2521
2522    fn delete_meta(&self, key: &str) -> Result<(), String> {
2523        meta_delete(&self.conn, key)
2524            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2525    }
2526
2527    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2528    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2529        let mut stmt = self
2530            .conn
2531            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2532            .map_err(|error| error.to_string())?;
2533        let rows = stmt
2534            .query_map([], |row| {
2535                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2536            })
2537            .map_err(|error| error.to_string())?;
2538        let mut commits = Vec::new();
2539        for row in rows {
2540            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2541            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2542                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2543            })?;
2544            let mut ops = Vec::with_capacity(entries.len());
2545            for entry in entries {
2546                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2547                ops.push(OutboxOp {
2548                    upsert: op == "upsert",
2549                    table: entry
2550                        .get("table")
2551                        .and_then(Value::as_str)
2552                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2553                        .to_owned(),
2554                    row_id: entry
2555                        .get("rowId")
2556                        .and_then(Value::as_str)
2557                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2558                        .to_owned(),
2559                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2560                    // A legacy entry's `changedFields` key is ignored:
2561                    // the values map is the presence set (§6.1).
2562                    values: entry.get("values").and_then(Value::as_object).cloned(),
2563                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2564                });
2565            }
2566            commits.push(Arc::new(OutboxCommit {
2567                client_commit_id,
2568                ops,
2569            }));
2570        }
2571        Ok(commits)
2572    }
2573
2574    fn restore_persisted_state(&mut self) -> Result<(), String> {
2575        self.outbox = self.load_outbox()?.into();
2576
2577        self.prune_commit_outcomes()?;
2578        let active = self.commit_outcomes(CommitOutcomeQuery {
2579            active_only: true,
2580            ..CommitOutcomeQuery::default()
2581        })?;
2582        self.conflicts = active
2583            .iter()
2584            .flat_map(|outcome| outcome.results.iter())
2585            .filter_map(|result| match result {
2586                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2587                _ => None,
2588            })
2589            .collect::<Vec<_>>()
2590            .into();
2591        self.rejections = active
2592            .iter()
2593            .flat_map(|outcome| outcome.results.iter())
2594            .filter_map(|result| match result {
2595                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2596                _ => None,
2597            })
2598            .collect::<Vec<_>>()
2599            .into();
2600
2601        self.lease_state = self
2602            .get_meta(LEASE_STATE_KEY)?
2603            .map(|raw| serde_json::from_str(&raw))
2604            .transpose()
2605            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2606        self.schema_floor = self
2607            .get_meta(SCHEMA_FLOOR_KEY)?
2608            .map(|raw| serde_json::from_str(&raw))
2609            .transpose()
2610            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2611        self.stopped = self.schema_floor.is_some();
2612        // A replica left in unactivated preflight reopens gated: the quarantine
2613        // decision persists with the data across handle teardown and restart.
2614        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2615            self.security_preflight = true;
2616        }
2617        Ok(())
2618    }
2619
2620    fn save_unit_scope_schema(&self) -> Result<(), String> {
2621        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2622            .schema
2623            .tables
2624            .iter()
2625            .map(|table| {
2626                (
2627                    table.name.clone(),
2628                    table
2629                        .scope_variables
2630                        .iter()
2631                        .map(|scope| {
2632                            (
2633                                scope.variable.clone(),
2634                                [scope.prefix.clone(), scope.column.clone()],
2635                            )
2636                        })
2637                        .collect(),
2638                )
2639            })
2640            .collect();
2641        self.set_meta(
2642            "subscriptionScopeSchema",
2643            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2644        )?;
2645        let generations: BTreeMap<_, _> = self
2646            .schema
2647            .tables
2648            .iter()
2649            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2650            .collect();
2651        self.set_meta(
2652            TABLE_GENERATIONS_KEY,
2653            &serde_json::to_string(&generations).expect("generations serialize"),
2654        )
2655    }
2656
2657    /// Drop units that cover no table of the running schema, with their window
2658    /// registry rows and pending evictions; never translate scopes. At a bump
2659    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2660    /// on every table it covered before.
2661    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2662        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2663            self.get_meta("subscriptionScopeSchema")?
2664                .map(|raw| serde_json::from_str(&raw))
2665                .transpose()
2666                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2667        } else {
2668            None
2669        };
2670        // A table keeps the variables' meaning when each has the same prefix
2671        // and column as before. An old binary stored no declaration evidence;
2672        // matching names cannot prove unchanged meaning, so the app registers
2673        // again.
2674        let unchanged = |name: &str, variables: &[&String]| {
2675            let (Some(old), Some(table)) = (
2676                previous.as_ref().and_then(|previous| previous.get(name)),
2677                self.schema.table(name),
2678            ) else {
2679                return false;
2680            };
2681            variables.iter().all(|variable| {
2682                table
2683                    .scope_variables
2684                    .iter()
2685                    .find(|scope| &scope.variable == *variable)
2686                    .is_some_and(|scope| {
2687                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2688                    })
2689            })
2690        };
2691        let compatible = |scopes: &UnitScopes| {
2692            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2693                return false;
2694            }
2695            if !bump {
2696                return true;
2697            }
2698            let Some(previous) = &previous else {
2699                return false;
2700            };
2701            previous.iter().all(|(name, old)| {
2702                !scopes.keys().all(|variable| old.contains_key(variable))
2703                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2704            })
2705        };
2706        let mut stale: BTreeSet<String> = self
2707            .stored(load_units(&self.conn))?
2708            .into_iter()
2709            .filter(|unit| !compatible(&unit.scopes))
2710            .map(|unit| unit.id)
2711            .collect();
2712        // A shrunk window's deferred eviction outlives its unit row and keeps
2713        // its per-table scopes. A table the schema no longer has cannot be
2714        // evicted, and at a bump a table whose scope declarations changed must
2715        // not be evicted by the outdated rule.
2716        for (id, table, effective) in self.load_pending_evictions()? {
2717            if self.schema.table(&table).is_none()
2718                || (bump
2719                    && !unchanged(
2720                        &table,
2721                        &effective
2722                            .iter()
2723                            .map(|(variable, _)| variable)
2724                            .collect::<Vec<_>>(),
2725                    ))
2726            {
2727                stale.insert(id);
2728            }
2729        }
2730        for id in &stale {
2731            for sql in [
2732                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2733                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2734                "DELETE FROM _syncular_units WHERE id = ?1",
2735            ] {
2736                self.conn
2737                    .execute(sql, rusqlite::params![id])
2738                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2739            }
2740        }
2741        Ok(())
2742    }
2743
2744    pub fn local_revision(&self) -> Result<u64, String> {
2745        Ok(self
2746            .get_meta(LOCAL_REVISION_KEY)?
2747            .and_then(|value| value.parse().ok())
2748            .unwrap_or(0))
2749    }
2750
2751    /// §7.5: resolve a snapshot read on the mutable owner connection through
2752    /// the same primitive the read-only sidecar uses, so the command surface
2753    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2754    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2755    /// `owner` records its failure (and its next success) in diagnostics.
2756    pub fn snapshot_read(
2757        &mut self,
2758        request: &SnapshotReadRequest<'_>,
2759        owner: Option<QueryOwner<'_>>,
2760    ) -> Result<SnapshotRead, QueryReadFailure> {
2761        let result = snapshot_read_connection(&self.conn, request);
2762        if let Some(owner) = owner {
2763            self.record_query_read(owner, result.as_ref().err());
2764        }
2765        result
2766    }
2767
2768    #[must_use]
2769    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2770        SyncStatusSnapshot {
2771            current_schema_version: self.schema.version,
2772            outbox: self.outbox.len(),
2773            upgrading: self.upgrading,
2774            lease_state: self.lease_state.clone(),
2775            schema_floor: self.schema_floor.clone(),
2776            sync_needed: self.sync_needed,
2777            previous_version_context: self.previous_version_status(),
2778            security_lifecycle: self.security_lifecycle(),
2779        }
2780    }
2781
2782    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2783    /// rollback precondition), read without creating the file.
2784    fn previous_version_status(&self) -> PreviousVersionStatus {
2785        let Some(replica_path) = self.previous_version_replica_path() else {
2786            return PreviousVersionStatus {
2787                present: false,
2788                created_at_ms: None,
2789            };
2790        };
2791        match crate::previous_version::read_previous_version_record(&replica_path) {
2792            Ok(Some(record)) => PreviousVersionStatus {
2793                present: true,
2794                created_at_ms: Some(record.created_at_ms),
2795            },
2796            _ => PreviousVersionStatus {
2797                present: std::path::Path::new(
2798                    &crate::previous_version::previous_version_container_path(&replica_path),
2799                )
2800                .exists(),
2801                created_at_ms: None,
2802            },
2803        }
2804    }
2805
2806    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2807    /// is always `previousVersion`; the container never makes
2808    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2809    /// revocation, coverage completion and the TTL DISCARD the container and
2810    /// close the read with their own reason.
2811    pub fn previous_version_snapshot(
2812        &self,
2813        spec: &PreviousVersionReadSpec,
2814    ) -> Result<PreviousVersionSnapshot, String> {
2815        let now_ms = self.clock_now_ms();
2816        crate::previous_version::previous_version_snapshot(
2817            &self.conn,
2818            self.previous_version_replica_path().as_deref(),
2819            self.previous_version.as_ref(),
2820            PreviousVersionLifecycle {
2821                lease_inactive: self.previous_version_lease_inactive(now_ms),
2822                scope_revoked: self
2823                    .stored(load_units(&self.conn))?
2824                    .iter()
2825                    .any(|unit| unit.status == UnitStatus::Revoked),
2826                coverage_complete: self.previous_version_coverage_complete()?,
2827            },
2828            now_ms,
2829            self.schema.version,
2830            spec,
2831        )
2832    }
2833
2834    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2835    pub fn previous_version_audit(
2836        &self,
2837    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2838        crate::previous_version::stored_previous_version_audit(&self.conn)
2839            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2840    }
2841
2842    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2843    /// and both metadata records, and report whether anything was present.
2844    /// Idempotent: absence is a successful no-op.
2845    pub fn previous_version_discard(
2846        &self,
2847    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2848        self.drop_previous_version()
2849    }
2850
2851    fn drop_previous_version(
2852        &self,
2853    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2854        let Some(replica_path) = self.previous_version_replica_path() else {
2855            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2856                present: false,
2857                discarded: false,
2858            });
2859        };
2860        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2861    }
2862
2863    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2864    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2865        let Some(lease) = &self.lease_state else {
2866            return false;
2867        };
2868        if lease.error_code.is_some() {
2869            return true;
2870        }
2871        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2872    }
2873
2874    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2875    /// bootstrapping. An empty active set is not completion (mirrors the TS
2876    /// predicate exactly).
2877    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2878        let units = self.stored(load_units(&self.conn))?;
2879        let mut active = units
2880            .iter()
2881            .filter(|unit| unit.status == UnitStatus::Active)
2882            .peekable();
2883        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2884    }
2885
2886    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2887    /// apply. A container that no longer has a reason to exist is discarded so
2888    /// the rows do not outlive the window they were kept for.
2889    fn previous_version_lifetime_check(&self) {
2890        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2891            return;
2892        };
2893        let now_ms = self.clock_now_ms();
2894        if self.previous_version_lease_inactive(now_ms) {
2895            let _ = self.drop_previous_version();
2896            return;
2897        }
2898        let Some(replica_path) = self.previous_version_replica_path() else {
2899            return;
2900        };
2901        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2902        else {
2903            return;
2904        };
2905        let Ok(units) = load_units(&self.conn) else {
2906            return;
2907        };
2908        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2909            let _ = self.drop_previous_version();
2910            return;
2911        }
2912        if now_ms - record.created_at_ms > config.max_age_ms {
2913            let _ = self.drop_previous_version();
2914            return;
2915        }
2916        if self.previous_version_coverage_complete() == Ok(true) {
2917            let _ = self.drop_previous_version();
2918        }
2919    }
2920
2921    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2922        let registered = self.stored(load_units(&self.conn))?;
2923        // Both meta values in one statement: the FFI host recomputes
2924        // diagnostics after every command.
2925        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2926            self.conn
2927                .prepare_cached(
2928                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2929                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2930                )
2931                .and_then(|mut statement| {
2932                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2933                        Ok((row.get(0)?, row.get(1)?))
2934                    })
2935                }),
2936        )?;
2937        let mut units = DiagnosticUnits::default();
2938        for unit in &registered {
2939            *match unit.status {
2940                UnitStatus::Revoked => &mut units.revoked,
2941                UnitStatus::Failed => &mut units.failed,
2942                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2943                UnitStatus::Active
2944                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2945                {
2946                    &mut units.reset
2947                }
2948                UnitStatus::Active => &mut units.bootstrapping,
2949            } += 1;
2950        }
2951        units.reason_codes = registered
2952            .iter()
2953            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
2954            .collect::<BTreeSet<_>>()
2955            .into_iter()
2956            .collect();
2957        let captured_at_ms = self.clock_now_ms();
2958        let lease = if let Some(error_code) = self
2959            .lease_state
2960            .as_ref()
2961            .and_then(|state| state.error_code.clone())
2962        {
2963            ClientDiagnosticsLease {
2964                state: "stopped".to_owned(),
2965                expires_at_ms: self
2966                    .lease_state
2967                    .as_ref()
2968                    .and_then(|state| state.expires_at_ms),
2969                error_code: Some(Self::diagnostic_code(&error_code)),
2970            }
2971        } else if let Some(expires_at_ms) = self
2972            .lease_state
2973            .as_ref()
2974            .and_then(|state| state.expires_at_ms)
2975        {
2976            ClientDiagnosticsLease {
2977                state: if expires_at_ms <= captured_at_ms {
2978                    "expired".to_owned()
2979                } else {
2980                    "active".to_owned()
2981                },
2982                expires_at_ms: Some(expires_at_ms),
2983                error_code: None,
2984            }
2985        } else {
2986            ClientDiagnosticsLease {
2987                state: "none".to_owned(),
2988                expires_at_ms: None,
2989                error_code: None,
2990            }
2991        };
2992        let connectivity = match self.last_round.as_ref() {
2993            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
2994            Some(round) if round.status == "succeeded" => "online",
2995            Some(round)
2996                if round.status == "failed"
2997                    && matches!(
2998                        round.cause_kind,
2999                        Some(
3000                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3001                        )
3002                    ) =>
3003            {
3004                "offline"
3005            }
3006            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3007            _ => "unknown",
3008        };
3009        Ok(ClientDiagnosticsSnapshot {
3010            version: CLIENT_DIAGNOSTICS_VERSION,
3011            captured_at_ms,
3012            host: ClientDiagnosticsHost {
3013                kind: "direct".to_owned(),
3014                role: "single".to_owned(),
3015                connectivity: connectivity.to_owned(),
3016                realtime: self.realtime_state,
3017                realtime_policy: self.realtime_policy,
3018                realtime_reason_code: self.realtime_reason_code.clone(),
3019                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3020            },
3021            security_lifecycle: self.security_lifecycle().to_owned(),
3022            schema: ClientDiagnosticsSchema {
3023                current_version: self.schema.version,
3024                upgrading: self.upgrading,
3025                required_version: self
3026                    .schema_floor
3027                    .as_ref()
3028                    .and_then(|floor| floor.required_schema_version),
3029                latest_version: self
3030                    .schema_floor
3031                    .as_ref()
3032                    .and_then(|floor| floor.latest_schema_version),
3033            },
3034            replica: ClientDiagnosticsReplica {
3035                local_revision: local_revision
3036                    .and_then(|value| value.parse::<u64>().ok())
3037                    .unwrap_or(0)
3038                    .to_string(),
3039                sync_needed: self.sync_needed,
3040                pending_outbox: self.outbox.len(),
3041            },
3042            lease,
3043            active_units: registered
3044                .iter()
3045                .filter(|unit| unit.status == UnitStatus::Active)
3046                .count(),
3047            units,
3048            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3049                raw.parse()
3050                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3051            })?,
3052            last_round: self.last_round.clone(),
3053            last_change: self.last_change.clone(),
3054            storage: self.diagnostics_storage(),
3055            query_failures: self.query_failures.clone(),
3056        })
3057    }
3058
3059    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3060        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3061            let page_count: i64 = self
3062                .conn
3063                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3064            let page_size: i64 = self
3065                .conn
3066                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3067            // One cached aggregate, since the FFI host recomputes diagnostics
3068            // after every command; every invocation still reads current storage.
3069            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3070                .conn
3071                .prepare_cached(
3072                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3073                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3074                       FROM _syncular_commit_outcomes",
3075                )?
3076                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3077            let blob_bytes = if self.schema_has_blobs() {
3078                self.conn
3079                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3080                    .query_row([], |row| row.get(0))?
3081            } else {
3082                0
3083            };
3084            let pressure = self
3085                .limits
3086                .blob_cache_max_bytes
3087                .is_some_and(|limit| blob_bytes > limit);
3088            Ok(ClientDiagnosticsStorage {
3089                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3090                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3091                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3092                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3093                retained_outcome_entries: Some(outcome_entries.max(0)),
3094                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3095                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3096            })
3097        };
3098        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3099            status: "unreadable".to_owned(),
3100            database_bytes_approx: None,
3101            pending_outbox_bytes_approx: None,
3102            retained_outcome_bytes_approx: None,
3103            retained_outcome_entries: None,
3104            blob_cache_bytes_approx: None,
3105            pressure_reason_code: None,
3106        })
3107    }
3108
3109    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3110        self.change_queue.drain(..).collect()
3111    }
3112
3113    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3114        let intents = self.sync_intent_queue.drain(..).collect();
3115        if self.transport_enabled {
3116            intents
3117        } else {
3118            vec![SyncIntent::None]
3119        }
3120    }
3121
3122    fn schedule_background_retry(&mut self) -> u64 {
3123        let delay_ms = self.retry_delay_ms;
3124        self.sync_intent_queue
3125            .push_back(SyncIntent::Background { delay_ms });
3126        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3127        self.round_retry_delay_ms = Some(delay_ms);
3128        delay_ms
3129    }
3130
3131    fn reset_background_retry(&mut self) {
3132        self.retry_delay_ms = 250;
3133    }
3134
3135    fn retryable_transport_code(code: &str) -> bool {
3136        code == "transport.failed"
3137            || code == "transport.timeout"
3138            || code == "transport.unavailable"
3139            || code == "sync.transport_failed"
3140    }
3141
3142    /// §1.8: the transport reported a refused or unreachable endpoint. The
3143    /// native transport says `connect` (DNS failure, connection refused,
3144    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3145    /// the request on a backoff repairs neither, so no retry is scheduled.
3146    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3147        matches!(
3148            details
3149                .as_ref()
3150                .and_then(|details| details.get("causeKind"))
3151                .and_then(serde_json::Value::as_str),
3152            Some("connect" | "network")
3153        )
3154    }
3155
3156    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3157    /// invisible to the server (the round response arrived whole), so its code
3158    /// rides the following requests like a socket-round failure. A code the
3159    /// frame grammar rejects is not reportable and the previous code stays.
3160    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3161        if ssp2::model::valid_round_failure_code(&error.code) {
3162            self.round_failure = Some(error.code.clone());
3163        }
3164        SectionError::Transport(error)
3165    }
3166
3167    /// A failed round worth a background retry: a transport failure, or a
3168    /// §10.2 code the catalog marks retryable (clients may hardcode the
3169    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3170    /// on the server's `retryable` flag.
3171    fn retryable_failure_code(code: &str) -> bool {
3172        Self::retryable_transport_code(code)
3173            || matches!(
3174                code,
3175                "sync.auth_required"
3176                    | "sync.auth_lease_required"
3177                    | "sync.auth_lease_revoked"
3178                    | "sync.internal_error"
3179                    | "sync.idempotency_cache_miss"
3180                    | "sync.schema_not_ready"
3181                    | "sync.segment_expired"
3182                    | "sync.rate_limited"
3183                    | "sync.websocket_connection_limit"
3184                    | "sync.idle_timeout"
3185            )
3186    }
3187
3188    fn diagnostic_code(code: &str) -> String {
3189        let valid = !code.is_empty()
3190            && code.len() <= 96
3191            && code.contains('.')
3192            && code
3193                .bytes()
3194                .next()
3195                .is_some_and(|byte| byte.is_ascii_lowercase())
3196            && code.bytes().all(|byte| {
3197                byte.is_ascii_lowercase()
3198                    || byte.is_ascii_digit()
3199                    || matches!(byte, b'.' | b'_' | b'-')
3200            });
3201        if valid {
3202            code.to_owned()
3203        } else {
3204            "client.unknown_failure".to_owned()
3205        }
3206    }
3207
3208    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3209        if self.sync_needed != value {
3210            if self.begin_observation("syncular_status").is_ok() {
3211                self.sync_needed = value;
3212                let batch = ChangeAccumulator {
3213                    status: true,
3214                    ..ChangeAccumulator::default()
3215                };
3216                if self.finish_observation("syncular_status", batch).is_err() {
3217                    self.rollback_observation("syncular_status");
3218                }
3219            } else {
3220                self.sync_needed = value;
3221            }
3222        }
3223        if value && interactive {
3224            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3225        }
3226    }
3227
3228    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3229    /// every other SQLite error keeps its storage classification.
3230    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3231        result.map_err(|error| match error {
3232            rusqlite::Error::FromSqlConversionFailure(..) => {
3233                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3234            }
3235            error => Self::sqlite_failure(&self.storage_failure, error),
3236        })
3237    }
3238
3239    fn sqlite_failure(
3240        storage_failure: &RefCell<Option<QueryReadFailure>>,
3241        error: rusqlite::Error,
3242    ) -> String {
3243        let message = error.to_string();
3244        let failure = QueryReadFailure::from(error);
3245        let code = failure.code;
3246        if code.is_some() && storage_failure.borrow().is_none() {
3247            *storage_failure.borrow_mut() = Some(failure);
3248        }
3249        match code {
3250            Some(code) => format!("{code}: {message}"),
3251            None => message,
3252        }
3253    }
3254
3255    fn begin_observation(&self, name: &str) -> Result<(), String> {
3256        if let Some(failure) = &self.coalesced_failure {
3257            return Err(failure.clone());
3258        }
3259        // SQLite rolls a whole transaction back on some I/O and capacity
3260        // errors; later frames must not commit on their own after that.
3261        if self.coalesced.is_some() && self.conn.is_autocommit() {
3262            return Err("the coalesced apply transaction was rolled back".to_owned());
3263        }
3264        self.conn
3265            .execute_batch(&format!("SAVEPOINT {name}"))
3266            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3267    }
3268
3269    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3270    /// already owns one.
3271    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3272        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3273            return Ok(());
3274        }
3275        self.conn
3276            .execute_batch("BEGIN")
3277            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3278        self.coalesced = Some(CoalescedApply {
3279            rows: 0,
3280            rollback: self.capture(),
3281        });
3282        Ok(())
3283    }
3284
3285    fn capture(&self) -> Rollback {
3286        Rollback {
3287            outbox: self.outbox.clone(),
3288            failed_commits: self.failed_commits.clone(),
3289            conflicts: self.conflicts.clone(),
3290            rejections: self.rejections.clone(),
3291            schema_floor: self.schema_floor.clone(),
3292            lease_state: self.lease_state.clone(),
3293            stopped: self.stopped,
3294            upgrading: self.upgrading,
3295            sync_needed: self.sync_needed,
3296            active_round: self.active_round,
3297            overlay_dirty: self.overlay_dirty.snapshot(),
3298            change_queue: self.change_queue.len(),
3299            sync_intent_queue: self.sync_intent_queue.len(),
3300            last_change: self.last_change.clone(),
3301        }
3302    }
3303
3304    fn restore(&mut self, rollback: Rollback) {
3305        self.outbox = rollback.outbox;
3306        self.failed_commits = rollback.failed_commits;
3307        self.conflicts = rollback.conflicts;
3308        self.rejections = rollback.rejections;
3309        self.schema_floor = rollback.schema_floor;
3310        self.lease_state = rollback.lease_state;
3311        self.stopped = rollback.stopped;
3312        self.upgrading = rollback.upgrading;
3313        self.sync_needed = rollback.sync_needed;
3314        self.active_round = rollback.active_round;
3315        self.overlay_dirty.restore(rollback.overlay_dirty);
3316        self.change_queue.truncate(rollback.change_queue);
3317        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3318        self.last_change = rollback.last_change;
3319    }
3320
3321    /// Account the rows of the next frame or block. A unit that would take the
3322    /// coalesced transaction past the bound commits the earlier units first.
3323    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3324        let Some(open) = &self.coalesced else {
3325            return Ok(());
3326        };
3327        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3328            self.commit_coalesced_apply()?;
3329            self.begin_coalesced_apply()?;
3330        }
3331        if let Some(open) = &mut self.coalesced {
3332            open.rows += rows;
3333        }
3334        Ok(())
3335    }
3336
3337    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3338    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3339    /// change batches and sync intents queued since, and fails the rest of the
3340    /// response.
3341    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3342        let Some(open) = self.coalesced.take() else {
3343            return Ok(());
3344        };
3345        let committed = if self.conn.is_autocommit() {
3346            Err("the coalesced apply transaction was rolled back".to_owned())
3347        } else {
3348            self.conn
3349                .execute_batch("COMMIT")
3350                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3351        };
3352        let Err(error) = committed else {
3353            return Ok(());
3354        };
3355        if !self.conn.is_autocommit() {
3356            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3357                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3358            }
3359        }
3360        self.restore(open.rollback);
3361        self.coalesced_failure = Some(error.clone());
3362        Err(error)
3363    }
3364
3365    /// End the response's coalescing: COMMIT what is open and clear a failure.
3366    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3367        let committed = self.commit_coalesced_apply();
3368        match self.coalesced_failure.take() {
3369            Some(failure) => Err(failure),
3370            None => committed,
3371        }
3372    }
3373
3374    fn rollback_observation(&self, name: &str) {
3375        if let Err(error) = self
3376            .conn
3377            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3378        {
3379            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3380                if failure.rollback_failure.is_none() {
3381                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3382                }
3383            }
3384        }
3385    }
3386
3387    fn finish_observation(
3388        &mut self,
3389        name: &str,
3390        mut batch: ChangeAccumulator,
3391    ) -> Result<(), String> {
3392        #[cfg(feature = "bench-internals")]
3393        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3394        if self.failed_commits.iter().any(|(commit, _)| {
3395            commit
3396                .ops
3397                .iter()
3398                .any(|operation| batch.tables.contains_key(&operation.table))
3399        }) {
3400            batch.outcomes = true;
3401        }
3402        if !batch.touched() {
3403            self.conn
3404                .execute_batch(&format!("RELEASE {name}"))
3405                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3406            return Ok(());
3407        }
3408        let revision = self
3409            .local_revision()?
3410            .checked_add(1)
3411            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3412        self.conn
3413            .execute(
3414                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3415                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3416            )
3417            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3418        let status = batch.status.then(|| self.status_snapshot());
3419        let event = ClientChangeBatch {
3420            revision: revision.to_string(),
3421            tables: batch
3422                .tables
3423                .into_iter()
3424                .map(|(table, scope_keys)| TableChange {
3425                    table,
3426                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3427                })
3428                .collect(),
3429            windows: batch
3430                .windows
3431                .into_iter()
3432                .map(|(base_key, units)| WindowChange {
3433                    base_key,
3434                    units: units.into_iter().collect(),
3435                })
3436                .collect(),
3437            status,
3438            conflicts_changed: batch.conflicts,
3439            rejections_changed: batch.rejections,
3440            outcomes_changed: batch.outcomes,
3441            applied_commits: Vec::new(),
3442        };
3443        self.conn
3444            .execute_batch(&format!("RELEASE {name}"))
3445            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3446        let mut diagnostic_tables = event
3447            .tables
3448            .iter()
3449            .map(|entry| entry.table.clone())
3450            .collect::<BTreeSet<_>>()
3451            .into_iter()
3452            .collect::<Vec<_>>();
3453        // §7.6: the selected variables of the changed window bases.
3454        let mut diagnostic_windows = event
3455            .windows
3456            .iter()
3457            .map(|entry| {
3458                entry
3459                    .base_key
3460                    .split('\0')
3461                    .next()
3462                    .unwrap_or_default()
3463                    .to_owned()
3464            })
3465            .collect::<BTreeSet<_>>()
3466            .into_iter()
3467            .collect::<Vec<_>>();
3468        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3469            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3470        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3471        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3472        self.last_change = Some(DiagnosticLastChange {
3473            revision: event.revision.clone(),
3474            recorded_at_ms: self.clock_now_ms(),
3475            tables: diagnostic_tables,
3476            windows: diagnostic_windows,
3477            domains_truncated,
3478            status_changed: event.status.is_some(),
3479            conflicts_changed: event.conflicts_changed,
3480            rejections_changed: event.rejections_changed,
3481            outcomes_changed: event.outcomes_changed,
3482        });
3483        self.change_queue.push_back(event);
3484        Ok(())
3485    }
3486
3487    fn record_scope_map(
3488        &self,
3489        batch: &mut ChangeAccumulator,
3490        table_name: &str,
3491        scopes: &[(String, Vec<String>)],
3492    ) {
3493        let Some(table) = self.schema.table(table_name) else {
3494            return;
3495        };
3496        // A bulk scoped delete whose map leaves a declared variable unnamed
3497        // cannot list the deleted rows' keys for it: the change is table-wide.
3498        if table.scope_variables.iter().any(|scope| {
3499            !scopes
3500                .iter()
3501                .any(|(variable, _)| *variable == scope.variable)
3502        }) {
3503            batch.table(table_name);
3504            return;
3505        }
3506        for (variable, values) in scopes {
3507            let Some(scope) = table
3508                .scope_variables
3509                .iter()
3510                .find(|scope| &scope.variable == variable)
3511            else {
3512                continue;
3513            };
3514            for value in values {
3515                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3516            }
3517        }
3518    }
3519
3520    /// Record a row's current scope keys from the base or visible table.
3521    fn record_row_scopes(
3522        &self,
3523        batch: &mut ChangeAccumulator,
3524        table_name: &str,
3525        row_id: &str,
3526        base: bool,
3527    ) -> bool {
3528        let Some(table) = self.schema.table(table_name) else {
3529            return false;
3530        };
3531        if table.scope_variables.is_empty() {
3532            return false;
3533        }
3534        let columns = table
3535            .scope_variables
3536            .iter()
3537            .map(|scope| quote_ident(&scope.column))
3538            .collect::<Vec<_>>()
3539            .join(", ");
3540        let full_table = if base {
3541            base_table(table_name)
3542        } else {
3543            visible_table(table_name)
3544        };
3545        let sql = format!(
3546            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3547            row_id_predicate(table)
3548        );
3549        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3550            return false;
3551        };
3552        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3553            let mut values = Vec::with_capacity(table.scope_variables.len());
3554            for index in 0..table.scope_variables.len() {
3555                values.push(row.get::<_, Option<String>>(index)?);
3556            }
3557            Ok(values)
3558        });
3559        let Ok(values) = values else {
3560            return false;
3561        };
3562        let mut recorded = false;
3563        for (scope, value) in table.scope_variables.iter().zip(values) {
3564            if let Some(value) = value {
3565                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3566                recorded = true;
3567            }
3568        }
3569        recorded
3570    }
3571
3572    fn record_commit_changes(
3573        &self,
3574        batch: &mut ChangeAccumulator,
3575        tables: &[String],
3576        changes: &[ssp2::model::Change],
3577    ) {
3578        #[cfg(feature = "bench-internals")]
3579        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3580        for change in changes {
3581            let Some(table_name) = tables.get(change.table_index as usize) else {
3582                continue;
3583            };
3584            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3585            if let Some(table) = self.schema.table(table_name) {
3586                for (variable, value) in &change.scopes {
3587                    if let Some(scope) = table
3588                        .scope_variables
3589                        .iter()
3590                        .find(|scope| &scope.variable == variable)
3591                    {
3592                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3593                        precise = true;
3594                    }
3595                }
3596            }
3597            if !precise {
3598                batch.table(table_name);
3599            }
3600        }
3601    }
3602
3603    /// Whether a base or visible row matches the scope predicate; `false`
3604    /// without a local scope-column mapping. A SQLite failure is an error.
3605    fn scoped_rows_exist(
3606        &self,
3607        table_name: &str,
3608        effective: &[(String, Vec<String>)],
3609        owners: &[Vec<(String, Vec<String>)>],
3610    ) -> Result<bool, String> {
3611        let Some(table) = self.schema.table(table_name) else {
3612            return Ok(false);
3613        };
3614        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3615            return Ok(false);
3616        };
3617        for source in [base_table(table_name), visible_table(table_name)] {
3618            if self
3619                .conn
3620                .query_row(
3621                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3622                    rusqlite::params_from_iter(&params),
3623                    |row| row.get::<_, bool>(0),
3624                )
3625                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3626            {
3627                return Ok(true);
3628            }
3629        }
3630        Ok(false)
3631    }
3632
3633    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3634    /// those another owner covers (§3.3), failing closed without a local
3635    /// scope-column mapping. Owners are read only when a row matches, inside
3636    /// the caller's transaction. Returns whether rows matched.
3637    fn clear_scope(
3638        &mut self,
3639        table: &TableSchema,
3640        effective: &[(String, Vec<String>)],
3641        unit_id: &str,
3642    ) -> Result<bool, SectionError> {
3643        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3644        let matched = self
3645            .scoped_rows_exist(&table.name, effective, &[])
3646            .map_err(storage)?;
3647        let owners = if matched {
3648            unit_owners(
3649                &self.stored(load_units(&self.conn)).map_err(storage)?,
3650                table,
3651                unit_id,
3652            )
3653        } else {
3654            Vec::new()
3655        };
3656        let changed = matched
3657            && (owners.is_empty()
3658                || self
3659                    .scoped_rows_exist(&table.name, effective, &owners)
3660                    .map_err(storage)?);
3661        if !self
3662            .purge_scope_rows(&table.name, effective, &owners)
3663            .map_err(storage)?
3664        {
3665            return Err(SectionError::FailClosed);
3666        }
3667        Ok(changed)
3668    }
3669
3670    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3671    pub fn upgrading(&self) -> bool {
3672        self.upgrading
3673    }
3674
3675    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3676    /// keeping this client's local database (identity, outbox, tables). The
3677    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3678    /// version increases. Recreation runs the same marker guard as opening
3679    /// a durable replica, holds the guard and every write it authorizes under
3680    /// one write transaction, and preserves the current client (compiled
3681    /// schema, outbox, tables) on refusal.
3682    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3683        let new_schema = parse_schema_json(schema_json)?;
3684        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3685        self.conn
3686            .execute_batch("BEGIN IMMEDIATE")
3687            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3688        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3689        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3690        // state (dropped outbox commits, rejections, upgrade and floor flags,
3691        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3692        // fails after the reset succeeded rolls the database back, so every
3693        // one of those fields is captured here and restored with it.
3694        let rollback = self.capture();
3695        let guarded = (|| -> Result<(), String> {
3696            // §7.4.2: the marker is re-read under the write lock the caller
3697            // holds, so a replica another process upgraded meanwhile refuses
3698            // this recreation instead of having its marker and tables
3699            // downgraded.
3700            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3701            self.create_bookkeeping_tables()?;
3702            match marker {
3703                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3704                None => {
3705                    self.create_synced_tables()?;
3706                    self.persist_schema_version()?;
3707                }
3708                _ => {}
3709            }
3710            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3711            // durable outbox the same-version path is about to replay. Only the
3712            // outbox is reloaded: the recovery below reads it, and the caller's
3713            // rollback snapshot already covers this field.
3714            self.outbox = self.load_outbox()?.into();
3715            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3716            // this guarded boundary once the schema and marker agree. A bump went
3717            // through `run_schema_reset`, which performs the same recovery after
3718            // it creates the new physical tables and before its final replay.
3719            self.drop_unencodable_outbox(false, None)?;
3720            if marker == Some(self.schema.version)
3721                && (!self.outbox.is_empty()
3722                    || !self.failed_commits.is_empty()
3723                    || self.has_acknowledged_rows()?)
3724            {
3725                self.overlay_dirty.set(true);
3726                self.rebuild_overlay()?;
3727            }
3728            self.prune_unknown_units(false)?;
3729            self.save_unit_scope_schema()?;
3730            // The conformance recreate is the in-memory equivalent of reopening
3731            // a durable client. Apply the same startup catch-up contract even
3732            // when the schema itself did not change.
3733            self.enqueue_startup_sync_if_needed()?;
3734            Ok(())
3735        })();
3736        let outcome = match guarded {
3737            Ok(()) => self
3738                .conn
3739                .execute_batch("COMMIT")
3740                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3741            Err(error) => Err(error),
3742        };
3743        if let Err(error) = outcome {
3744            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3745                // Keep a failed rollback's typed storage classification.
3746                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3747            }
3748            self.schema = prior_schema;
3749            self.restore(rollback);
3750            // Prepared insert statements were derived from the reverted schema.
3751            self.insert_sql.borrow_mut().clear();
3752            return Err(error);
3753        }
3754        if !self.security_preflight {
3755            self.drain_pending_evictions()?;
3756        }
3757        Ok(())
3758    }
3759
3760    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3761        let startup_work = !self.stopped
3762            && (!self.outbox.is_empty()
3763                || self.has_acknowledged_rows()?
3764                || !self.load_pending_evictions()?.is_empty()
3765                || self
3766                    .stored(load_units(&self.conn))?
3767                    .iter()
3768                    .any(|unit| unit.status == UnitStatus::Active));
3769        if startup_work {
3770            self.sync_needed = true;
3771            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3772        }
3773        Ok(())
3774    }
3775
3776    /// A native client persists schema-floor stops across process restarts.
3777    /// Once the running app already satisfies that floor, the persisted stop
3778    /// is only stale evidence from an older server round (for example, the
3779    /// server was restarted after catching up to an app that was ahead).
3780    /// Clear it and let the normal startup pull re-negotiate. If the server is
3781    /// still incompatible it will return the floor again in that first round.
3782    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3783        let satisfied = self
3784            .schema_floor
3785            .as_ref()
3786            .and_then(|floor| floor.required_schema_version)
3787            .is_some_and(|required| self.schema.version >= required);
3788        if !satisfied {
3789            return Ok(());
3790        }
3791        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3792        self.schema_floor = None;
3793        self.stopped = false;
3794        Ok(())
3795    }
3796
3797    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3798    /// path already swept it; this covers a same-version or fresh-install open,
3799    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3800    /// an aware binary can apply.
3801    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3802        let Some(replica_path) = self.previous_version_replica_path() else {
3803            return Ok(());
3804        };
3805        let max_age_ms = self
3806            .previous_version
3807            .filter(|config| config.enabled)
3808            .map(|config| config.max_age_ms);
3809        reconcile_previous_version_at_boot(
3810            &self.conn,
3811            &replica_path,
3812            self.schema.version,
3813            max_age_ms,
3814            self.clock_now_ms(),
3815        )
3816    }
3817
3818    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3819    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3820    /// has no plain sibling path — so the feature resolves as off there.
3821    fn previous_version_replica_path(&self) -> Option<String> {
3822        let path = self.conn.path()?;
3823        if path.is_empty() || path.starts_with("file:") {
3824            return None;
3825        }
3826        Some(path.to_owned())
3827    }
3828
3829    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3830    /// and leaseState. Drops/recreates every synced table from the new
3831    /// schema, resets unit sync state (keeping registrations), clears
3832    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3833    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3834    fn run_schema_reset(&mut self) -> Result<(), String> {
3835        self.begin_observation("syncular_schema_reset")?;
3836        let rollback = self.capture();
3837        let mut batch = ChangeAccumulator::default();
3838        let result = self
3839            .run_schema_reset_observed(&mut batch, true)
3840            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3841        if let Err(error) = result {
3842            self.rollback_observation("syncular_schema_reset");
3843            self.restore(rollback);
3844            return Err(error);
3845        }
3846        Ok(())
3847    }
3848
3849    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3850        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3851            self.begin_observation("syncular_epoch_acquisition")?;
3852            let rollback = self.capture();
3853            if let Err(error) = self.conn.execute(
3854                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3855                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3856            ) {
3857                self.rollback_observation("syncular_epoch_acquisition");
3858                return Err(Self::sqlite_failure(&self.storage_failure, error));
3859            }
3860            self.sync_needed = true;
3861            let batch = ChangeAccumulator {
3862                status: true,
3863                ..ChangeAccumulator::default()
3864            };
3865            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3866                self.rollback_observation("syncular_epoch_acquisition");
3867                self.restore(rollback);
3868                return Err(error);
3869            }
3870            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3871            return Ok(Vec::new());
3872        }
3873        let resets = self
3874            .stored(load_units(&self.conn))?
3875            .into_iter()
3876            .map(|unit| unit.id)
3877            .collect();
3878        self.begin_observation("syncular_log_epoch_reset")?;
3879        let rollback = self.capture();
3880        let mut batch = ChangeAccumulator::default();
3881        let result = self
3882            .run_schema_reset_observed(&mut batch, false)
3883            .and_then(|()| {
3884                self.conn
3885                    .execute(
3886                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3887                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3888                    )
3889                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3890                self.sync_needed = true;
3891                batch.status = true;
3892                self.finish_observation("syncular_log_epoch_reset", batch)
3893            });
3894        if let Err(error) = result {
3895            self.rollback_observation("syncular_log_epoch_reset");
3896            self.restore(rollback);
3897            return Err(error);
3898        }
3899        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3900        Ok(resets)
3901    }
3902
3903    fn run_schema_reset_observed(
3904        &mut self,
3905        batch: &mut ChangeAccumulator,
3906        capture: bool,
3907    ) -> Result<(), String> {
3908        // §7.4.2: the reset is the only path that wipes local tables for a
3909        // version bump, and it rewrites the marker last. Re-read the marker
3910        // under this reset's transaction: a replica another process upgraded
3911        // meanwhile refuses here instead of being reset to this client's
3912        // older schema.
3913        read_local_schema_version(&self.conn, self.schema.version)?;
3914        // §7.4.3: a schema bump keeps every table whose generation equals the
3915        // persisted one; the log-epoch reset keeps nothing. The previous
3916        // declaration evidence decides which units cover only kept tables.
3917        let mut kept = BTreeSet::new();
3918        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3919        if capture {
3920            let persisted: BTreeMap<String, String> = self
3921                .get_meta(TABLE_GENERATIONS_KEY)?
3922                .map(|raw| serde_json::from_str(&raw))
3923                .transpose()
3924                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3925                .unwrap_or_default();
3926            kept = self
3927                .schema
3928                .tables
3929                .iter()
3930                .filter(|table| {
3931                    table.generation.is_some()
3932                        && table.generation.as_ref() == persisted.get(&table.name)
3933                })
3934                .map(|table| table.name.clone())
3935                .collect();
3936            previous_declarations = self
3937                .get_meta("subscriptionScopeSchema")?
3938                .map(|raw| serde_json::from_str(&raw))
3939                .transpose()
3940                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
3941        }
3942        // Kept tables keep both halves, their FTS projections with the FTS5
3943        // shadow tables and id mapping, and their row-level bookkeeping.
3944        let mut kept_objects = BTreeSet::new();
3945        for table in self
3946            .schema
3947            .tables
3948            .iter()
3949            .filter(|table| kept.contains(&table.name))
3950        {
3951            kept_objects.insert(table.name.clone());
3952            kept_objects.insert(format!("_syncular_base_{}", table.name));
3953            for index in &table.fts_indexes {
3954                kept_objects.insert(index.name.clone());
3955                for shadow in ["data", "idx", "content", "docsize", "config"] {
3956                    kept_objects.insert(format!("{}_{shadow}", index.name));
3957                }
3958            }
3959        }
3960        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
3961        for sql in [
3962            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
3963            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
3964        ] {
3965            self.conn
3966                .execute(sql, [&kept_json])
3967                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3968        }
3969        self.cancel_sync_round();
3970        self.upgrading = true;
3971        batch.status = true;
3972        // RFC 0005 D6: classify the pending outbox against the NEW compiled
3973        // schema BEFORE the wipe, and record the advisory audit. This pass
3974        // drops nothing, so the audit sees the original pending intent; the
3975        // §7.4.4 send-time drop and the intrinsic value recovery later in this
3976        // reset are the paths that remove a commit. The classification is
3977        // shared with the send-time drop so the two cannot disagree about what
3978        // is incompatible.
3979        if self.previous_version.is_some_and(|config| config.enabled) {
3980            let previous_version = self
3981                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
3982                .and_then(|value| value.parse::<i32>().ok())
3983                .unwrap_or(self.schema.version);
3984            let pending = self
3985                .outbox
3986                .iter()
3987                .map(|commit| PendingCommitAudit {
3988                    commit_id: &commit.client_commit_id,
3989                    operations: commit_audit_operations(commit),
3990                })
3991                .collect::<Vec<_>>();
3992            crate::previous_version::write_previous_version_audit(
3993                &self.conn,
3994                &crate::previous_version::build_previous_version_audit(
3995                    &self.schema,
3996                    &pending,
3997                    previous_version,
3998                    self.schema.version,
3999                    self.clock_now_ms(),
4000                ),
4001            )?;
4002        }
4003        // RFC 0005 D5: the container lives in its own FILE, so the reset can
4004        // never reach it. Sweep first (unconditional — also on the log-epoch
4005        // reset, which captures nothing), then capture BEFORE the wipe drops
4006        // any row. A refused capture records its reason and leaves no file.
4007        if let Some(replica_path) = self.previous_version_replica_path() {
4008            sweep_previous_version_container(&self.conn, &replica_path)?;
4009            if capture {
4010                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4011                    let previous_version = self
4012                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4013                        .and_then(|value| value.parse::<i32>().ok())
4014                        .unwrap_or(self.schema.version);
4015                    capture_previous_version_from_replica(
4016                        &self.conn,
4017                        &replica_path,
4018                        previous_version,
4019                        self.schema.version,
4020                        &config,
4021                        self.clock_now_ms(),
4022                    )?;
4023                }
4024            }
4025        }
4026        for table in self
4027            .schema
4028            .tables
4029            .iter()
4030            .filter(|table| !kept.contains(&table.name))
4031        {
4032            batch.table(&table.name);
4033        }
4034        for (base_key, value) in self.load_window_registry()? {
4035            batch.window(&base_key, &value);
4036        }
4037        if capture {
4038            self.prune_unknown_units(true)?;
4039        }
4040        // The per-table insert SQL is derived from the OLD column lists.
4041        self.insert_sql.borrow_mut().clear();
4042        self.overlay_dirty.set(true);
4043        // Drop every synced table (base + visible) that currently exists —
4044        // discovered from sqlite_master so a bump that adds/removes tables is
4045        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4046        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4047        // so they are matched explicitly, not by the bookkeeping filter.
4048        // Drop virtual tables first so SQLite removes every FTS shadow table
4049        // atomically instead of the generic discovery tearing it apart.
4050        let virtual_tables: Vec<String> = {
4051            let mut stmt = self
4052                .conn
4053                .prepare(
4054                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4055                )
4056                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4057            let rows = stmt
4058                .query_map([], |row| row.get::<_, String>(0))
4059                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4060            rows.filter_map(Result::ok)
4061                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4062                .collect()
4063        };
4064        for name in virtual_tables {
4065            self.conn
4066                .execute_batch(&format!(
4067                    "DROP TABLE IF EXISTS {}",
4068                    quote_ident(&format!("_syncular_fts_{name}"))
4069                ))
4070                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4071            self.conn
4072                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4073                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4074        }
4075        let existing: Vec<String> = {
4076            let mut stmt = self
4077                .conn
4078                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4079                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4080            let rows = stmt
4081                .query_map([], |row| row.get::<_, String>(0))
4082                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4083            rows.filter_map(Result::ok)
4084                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4085                .collect()
4086        };
4087        for name in &existing {
4088            if let Some(table) = name.strip_prefix("_syncular_base_") {
4089                batch.table(table);
4090            } else if !name.starts_with("_syncular_") {
4091                batch.table(name);
4092            }
4093        }
4094        for name in existing {
4095            self.conn
4096                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4097                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4098        }
4099        // Recreate the synced tables from the NEW schema.
4100        self.create_synced_tables()?;
4101        // Reset the sync state of every unit that covers a table the reset
4102        // did not keep, under the previous declarations or the new schema,
4103        // keeping the registration.
4104        let keep_units = match &previous_declarations {
4105            Some(previous) if !kept.is_empty() => self
4106                .stored(load_units(&self.conn))?
4107                .into_iter()
4108                .filter(|unit| {
4109                    previous.iter().all(|(name, old)| {
4110                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4111                    }) && self
4112                        .schema
4113                        .tables
4114                        .iter()
4115                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4116                })
4117                .map(|unit| unit.id)
4118                .collect(),
4119            _ => BTreeSet::new(),
4120        };
4121        reset_units(&self.conn, &keep_units)
4122            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4123        // The stop state is over: this client now ships a servable schema.
4124        self.stopped = false;
4125        self.schema_floor = None;
4126        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4127        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4128        self.persist_schema_version()?;
4129        self.save_unit_scope_schema()?;
4130        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4131        // original pending intent; the synced tables now carry the new schema, so
4132        // recover intrinsically codec-invalid persisted values here, before the
4133        // final replay. A pending upsert that names a column the new schema
4134        // removed stays pending for the send-time `drop_incompatible_outbox`
4135        // prepass. The reset wrapper snapshots and restores the outbox,
4136        // rejections, and change queues, so a failed commit rolls this recovery
4137        // back with the rest of the reset.
4138        // §7.4.4: the enclosing reset observation owns the publication, so the
4139        // recovery contributes its rejection, outcome, and table domains to it.
4140        self.drop_unencodable_outbox(false, Some(batch))?;
4141        // Re-apply the surviving outbox optimistically over the empty tables.
4142        self.rebuild_overlay()?;
4143        Ok(())
4144    }
4145
4146    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4147    /// column the current schema lacks (or a removed table) cannot be encoded
4148    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4149    /// encodable and is validated by the server), nor can an insert-intent
4150    /// upsert that lacks a non-nullable column (§7.1, the audit's
4151    /// `missing-required-column`). Drop the commit, undo
4152    /// its optimistic projection, and raise a client-local
4153    /// `sync.outbox_incompatible` rejection.
4154    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4155        let incompatible = self
4156            .outbox
4157            .iter()
4158            .filter(|commit| {
4159                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4160            })
4161            .cloned()
4162            .collect::<Vec<_>>();
4163        if incompatible.is_empty() {
4164            return Ok(false);
4165        }
4166        self.drop_outbox_commits(
4167            &incompatible,
4168            OUTBOX_INCOMPATIBLE_CODE,
4169            "the persisted commit cannot encode under the current schema",
4170        )?;
4171        Ok(true)
4172    }
4173
4174    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4175    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4176    /// remove them from the outbox, and undo their optimistic rows.
4177    fn drop_outbox_commits(
4178        &mut self,
4179        incompatible: &[Arc<OutboxCommit>],
4180        code: &str,
4181        message: &str,
4182    ) -> Result<(), String> {
4183        // §7.4.4: one observation covers the durable outcome writes, the outbox
4184        // removal, the in-memory drop, and the overlay rebuild, so a storage
4185        // fault leaves the replica and the client unchanged and publishes no
4186        // change; a successful drop publishes exactly one batch carrying the
4187        // rejections, outcomes, status, and the tables whose projection changed.
4188        let rollback = self.capture();
4189        let owns_transaction = self.conn.is_autocommit();
4190        self.begin_observation("syncular_outbox_drop")?;
4191        let mut batch = ChangeAccumulator::default();
4192        let dropped = (|| -> Result<(), String> {
4193            let mut rejections = Vec::new();
4194            for commit in incompatible {
4195                let results = commit
4196                    .ops
4197                    .iter()
4198                    .enumerate()
4199                    .map(|(op_index, operation)| {
4200                        let rejection = RejectionRecord {
4201                            client_commit_id: commit.client_commit_id.clone(),
4202                            op_index: op_index as i32,
4203                            code: code.to_owned(),
4204                            message: message.to_owned(),
4205                            retryable: false,
4206                            details: None,
4207                            operation: Some(CommitOperation::from(operation)),
4208                        };
4209                        rejections.push(rejection.clone());
4210                        CommitOperationOutcome::Error { rejection }
4211                    })
4212                    .collect::<Vec<_>>();
4213                self.persist_commit_outcome(
4214                    &commit.client_commit_id,
4215                    CommitOutcomeStatus::Rejected,
4216                    &results,
4217                    Some(&commit.ops),
4218                )?;
4219                self.delete_outbox_persisted(&commit.client_commit_id)?;
4220                // A removed table has no local projection to invalidate; every
4221                // surviving table the dropped commit touched does.
4222                for operation in &commit.ops {
4223                    if self.schema.table(&operation.table).is_some() {
4224                        batch.table(&operation.table);
4225                    }
4226                }
4227            }
4228            self.prune_commit_outcomes()?;
4229            let incompatible_ids = incompatible
4230                .iter()
4231                .map(|commit| commit.client_commit_id.as_str())
4232                .collect::<BTreeSet<_>>();
4233            self.outbox
4234                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4235            self.rejections.extend(rejections);
4236            // The dropped commits' purely-optimistic rows are undone: the
4237            // visible projection is re-derived from the surviving base + outbox.
4238            self.overlay_dirty.set(true);
4239            self.rebuild_overlay()?;
4240            batch.status = true;
4241            batch.rejections = true;
4242            batch.outcomes = true;
4243            Ok(())
4244        })();
4245        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4246        if let Err(error) = result {
4247            // An outer RELEASE is the COMMIT. Roll back that transaction
4248            // directly: retrying RELEASE after ROLLBACK TO still needs the
4249            // reader's lock. A nested observation only rolls back its savepoint.
4250            if owns_transaction {
4251                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4252                    let mut failure = self.storage_failure.borrow_mut();
4253                    let failure =
4254                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4255                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4256                }
4257            } else {
4258                self.rollback_observation("syncular_outbox_drop");
4259            }
4260            self.restore(rollback);
4261            return Err(error);
4262        }
4263        Ok(())
4264    }
4265
4266    /// §7.4.3: (re)create the base + visible table pair for every synced
4267    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4268    fn create_synced_tables(&self) -> Result<(), String> {
4269        for table in &self.schema.tables {
4270            // The base half + the visible half form the synced-table pair. An
4271            // index name is global in SQLite, so the base half's indexes are
4272            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4273            for (full, index_prefix) in [
4274                (base_table(&table.name), "_syncular_base_"),
4275                (visible_table(&table.name), ""),
4276            ] {
4277                let mut cols: Vec<String> =
4278                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4279                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4280                cols.extend(
4281                    table
4282                        .sealed_columns()
4283                        .iter()
4284                        .map(|name| format!("{} TEXT", quote_ident(name))),
4285                );
4286                let sql = format!(
4287                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4288                    cols.join(", "),
4289                    quote_ident(&table.primary_key)
4290                );
4291                self.conn
4292                    .execute(&sql, [])
4293                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4294                // §5.11: a replica created before sealed apply gains the
4295                // reserved columns in place, on both halves in the same order,
4296                // so data, indexes and the FTS projection stay.
4297                for name in table.sealed_columns() {
4298                    let present: bool = self
4299                        .conn
4300                        .query_row(
4301                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4302                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4303                            |row| row.get(0),
4304                        )
4305                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4306                    if !present {
4307                        self.conn
4308                            .execute(
4309                                &format!(
4310                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4311                                    quote_ident(name)
4312                                ),
4313                                [],
4314                            )
4315                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4316                    }
4317                }
4318                // The reveal at every round start reads the sealed rows of the
4319                // base half through this partial index.
4320                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4321                    self.conn
4322                        .execute(
4323                            &format!(
4324                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4325                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4326                            ),
4327                            [],
4328                        )
4329                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4330                }
4331                // Local secondary indexes (CREATE INDEX subset). Created on
4332                // both halves so mirror reads hit an index on either. Runs on
4333                // both the initial create and the §7.4.3 reset recreate path.
4334                for index in &table.indexes {
4335                    let unique = if index.unique { "UNIQUE " } else { "" };
4336                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4337                    let cols_sql = index
4338                        .columns
4339                        .iter()
4340                        .map(|c| quote_ident(c))
4341                        .collect::<Vec<_>>()
4342                        .join(", ");
4343                    let index_sql = format!(
4344                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4345                    );
4346                    self.conn
4347                        .execute(&index_sql, [])
4348                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4349                }
4350            }
4351        }
4352        // FTS exists only on the visible half. It is a local search
4353        // projection, never a synced/base table or a wire-schema member.
4354        for table in &self.schema.tables {
4355            for index in &table.fts_indexes {
4356                self.create_fts_projection(table, index)?;
4357            }
4358        }
4359        Ok(())
4360    }
4361
4362    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4363        let count: i64 = self
4364            .conn
4365            .query_row(
4366                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4367                rusqlite::params![index.name],
4368                |row| row.get(0),
4369            )
4370            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4371        Ok(count > 0)
4372    }
4373
4374    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4375        for suffix in ["ai", "ad", "au"] {
4376            self.conn
4377                .execute(
4378                    &format!(
4379                        "DROP TRIGGER IF EXISTS {}",
4380                        quote_ident(&format!("{}_{suffix}", index.name))
4381                    ),
4382                    [],
4383                )
4384                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4385        }
4386        Ok(())
4387    }
4388
4389    fn create_fts_triggers(
4390        &self,
4391        table: &TableSchema,
4392        index: &FtsIndexSchema,
4393    ) -> Result<(), String> {
4394        let fts = quote_ident(&index.name);
4395        let source = visible_table(&table.name);
4396        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4397        let pk = quote_ident(&table.primary_key);
4398        let projection_columns = std::iter::once(source_id.clone())
4399            .chain(index.columns.iter().map(|column| quote_ident(column)))
4400            .collect::<Vec<_>>()
4401            .join(", ");
4402        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4403            .chain(
4404                index
4405                    .columns
4406                    .iter()
4407                    .map(|column| format!("new.{}", quote_ident(column))),
4408            )
4409            .collect::<Vec<_>>()
4410            .join(", ");
4411        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4412        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4413        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4414        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4415        let bi = quote_ident(&format!("{}_bi", index.name));
4416        let ai = quote_ident(&format!("{}_ai", index.name));
4417        let ad = quote_ident(&format!("{}_ad", index.name));
4418        let au = quote_ident(&format!("{}_au", index.name));
4419        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4420        let sql = format!(
4421            "DROP TRIGGER IF EXISTS {bi};
4422             DROP TRIGGER IF EXISTS {ai};
4423             DROP TRIGGER IF EXISTS {ad};
4424             DROP TRIGGER IF EXISTS {au};
4425             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4426             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4427             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4428             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4429        );
4430        self.conn
4431            .execute_batch(&sql)
4432            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4433    }
4434
4435    fn rebuild_fts_projection(
4436        &self,
4437        table: &TableSchema,
4438        index: &FtsIndexSchema,
4439    ) -> Result<(), String> {
4440        let fts = quote_ident(&index.name);
4441        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4442        let pk = quote_ident(&table.primary_key);
4443        let indexed_columns = index
4444            .columns
4445            .iter()
4446            .map(|column| quote_ident(column))
4447            .collect::<Vec<_>>();
4448        let projection_columns = std::iter::once(source_id.clone())
4449            .chain(indexed_columns.iter().cloned())
4450            .collect::<Vec<_>>()
4451            .join(", ");
4452        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4453        let sql = format!(
4454            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4455            columns = indexed_columns.join(", "),
4456            source = visible_table(&table.name),
4457        );
4458        self.conn
4459            .execute_batch(&sql)
4460            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4461    }
4462
4463    fn create_fts_projection(
4464        &self,
4465        table: &TableSchema,
4466        index: &FtsIndexSchema,
4467    ) -> Result<(), String> {
4468        self.begin_observation("syncular_fts_schema")?;
4469        let result = (|| {
4470            let existed = self.fts_projection_exists(index)?;
4471            let tokenizer = index.tokenize.replace('\'', "''");
4472            let columns = index
4473                .columns
4474                .iter()
4475                .map(|column| quote_ident(column))
4476                .collect::<Vec<_>>()
4477                .join(", ");
4478            let sql = format!(
4479            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4480            fts = quote_ident(&index.name),
4481            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4482        );
4483            self.conn.execute(&sql, []).map_err(|error| {
4484                format!(
4485                    "cannot create local FTS5 projection {:?}: {error}",
4486                    index.name
4487                )
4488            })?;
4489            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4490            let mapped: bool = self
4491                .conn
4492                .query_row(
4493                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4494                    [format!("_syncular_fts_{}", index.name)],
4495                    |row| row.get(0),
4496                )
4497                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4498            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4499            if !mapped && existed {
4500                self.conn
4501                    .execute_batch(&format!(
4502                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4503                        quote_ident(FTS_SOURCE_ID_COLUMN),
4504                        quote_ident(&index.name)
4505                    ))
4506                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4507            }
4508            self.create_fts_triggers(table, index)?;
4509            if !existed {
4510                self.rebuild_fts_projection(table, index)?;
4511            }
4512            Ok(())
4513        })();
4514        if let Err(error) = result {
4515            self.rollback_observation("syncular_fts_schema");
4516            return Err(error);
4517        }
4518        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4519            self.rollback_observation("syncular_fts_schema");
4520            return Err(error.to_string());
4521        }
4522        Ok(())
4523    }
4524
4525    // -- persistence write-through --------------------------------------------
4526
4527    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4528        let ops: Vec<Value> = commit
4529            .ops
4530            .iter()
4531            .map(|op| {
4532                let mut entry = serde_json::json!({
4533                    "op": if op.upsert { "upsert" } else { "delete" },
4534                    "table": op.table,
4535                    "rowId": op.row_id,
4536                    "baseVersion": op.base_version,
4537                    "values": op.values.clone().map(Value::Object),
4538                });
4539                if op.insert {
4540                    entry["insert"] = Value::Bool(true);
4541                }
4542                entry
4543            })
4544            .collect();
4545        self.conn
4546            .execute(
4547                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4548                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4549            )
4550            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4551        for operation in &commit.ops {
4552            let Some(values) = &operation.values else {
4553                continue;
4554            };
4555            let Some(table) = self.schema.table(&operation.table) else {
4556                continue;
4557            };
4558            for column in table
4559                .columns
4560                .iter()
4561                .filter(|column| column.ty == ColumnType::BlobRef)
4562            {
4563                let Some(Value::String(raw)) = values.get(&column.name) else {
4564                    continue;
4565                };
4566                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4567                    continue;
4568                };
4569                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4570                    continue;
4571                };
4572                self.conn
4573                    .execute(
4574                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4575                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4576                        rusqlite::params![commit.client_commit_id, blob_id],
4577                    )
4578                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4579            }
4580        }
4581        Ok(())
4582    }
4583
4584    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4585        self.conn
4586            .execute(
4587                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4588                rusqlite::params![client_commit_id],
4589            )
4590            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4591        self.conn
4592            .execute(
4593                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4594                rusqlite::params![client_commit_id],
4595            )
4596            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4597        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4598            .map(|_| ())
4599            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4600    }
4601
4602    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4603        match status {
4604            CommitOutcomeStatus::Applied => "applied",
4605            CommitOutcomeStatus::Cached => "cached",
4606            CommitOutcomeStatus::Conflict => "conflict",
4607            CommitOutcomeStatus::Rejected => "rejected",
4608        }
4609    }
4610
4611    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4612        match resolution {
4613            CommitOutcomeResolution::Active => "active",
4614            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4615            CommitOutcomeResolution::Superseded => "superseded",
4616            CommitOutcomeResolution::Dismissed => "dismissed",
4617        }
4618    }
4619
4620    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4621        match value {
4622            "applied" => Ok(CommitOutcomeStatus::Applied),
4623            "cached" => Ok(CommitOutcomeStatus::Cached),
4624            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4625            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4626            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4627        }
4628    }
4629
4630    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4631        match value {
4632            "active" => Ok(CommitOutcomeResolution::Active),
4633            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4634            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4635            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4636            _ => Err(format!(
4637                "invalid persisted commit outcome resolution {value:?}"
4638            )),
4639        }
4640    }
4641
4642    fn persist_commit_outcome(
4643        &self,
4644        client_commit_id: &str,
4645        status: CommitOutcomeStatus,
4646        results: &[CommitOperationOutcome],
4647        operations: Option<&[OutboxOp]>,
4648    ) -> Result<(), String> {
4649        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4650        let operations_json = operations
4651            .map(|items| {
4652                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4653            })
4654            .transpose()
4655            .map_err(|error| error.to_string())?;
4656        self.conn
4657            .execute(
4658                "INSERT INTO _syncular_commit_outcomes (
4659                   client_commit_id, status, recorded_at_ms, results_json,
4660                   operations_json, resolution
4661                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4662                rusqlite::params![
4663                    client_commit_id,
4664                    Self::outcome_status_name(status),
4665                    self.clock_now_ms(),
4666                    results_json,
4667                    operations_json
4668                ],
4669            )
4670            .map(|_| ())
4671            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4672    }
4673
4674    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4675    /// the byte-envelope rule refuses cannot be journaled verbatim.
4676    fn stored_cell_is_representable(cell: &Value) -> bool {
4677        cell.is_null()
4678            || cell.is_string()
4679            || cell.is_boolean()
4680            || cell.is_number()
4681            || is_canonical_bytes_envelope(cell)
4682    }
4683
4684    fn stored_row_is_representable(value: &Value) -> bool {
4685        value
4686            .as_object()
4687            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4688    }
4689
4690    fn stored_operation_is_representable(value: &Value) -> bool {
4691        value.get("table").is_some_and(Value::is_string)
4692            && value.get("rowId").is_some_and(Value::is_string)
4693            && value.get("baseVersion").is_none_or(|v| {
4694                v.as_i64()
4695                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4696            })
4697            && match value.get("op").and_then(Value::as_str) {
4698                Some("upsert") => {
4699                    value
4700                        .get("values")
4701                        .is_some_and(Self::stored_row_is_representable)
4702                        && value
4703                            .get("insert")
4704                            .is_none_or(|insert| *insert == Value::Bool(true))
4705                }
4706                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4707                _ => false,
4708            }
4709    }
4710
4711    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4712    /// only when the journal reader accepts the serialized operation, which is the
4713    /// rule `outcome_from_row` applies to this same row later.
4714    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4715        serde_json::to_value(CommitOperation::from(operation))
4716            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4717    }
4718
4719    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4720        let StoredCommitOutcomeRow {
4721            sequence,
4722            client_commit_id,
4723            status,
4724            recorded_at_ms,
4725            results_json,
4726            operations_json,
4727            resolution,
4728            resolved_at_ms,
4729            replacement_client_commit_id,
4730        } = row;
4731        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4732            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4733            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4734        {
4735            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4736        }
4737        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4738        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4739        for result in &stored {
4740            let nested = match result.get("status").and_then(Value::as_str) {
4741                Some("conflict") => result.get("conflict"),
4742                Some("error") => result.get("rejection"),
4743                _ => None,
4744            };
4745            if nested
4746                .and_then(|record| record.get("operation"))
4747                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4748                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4749                    && nested
4750                        .and_then(|record| record.get("serverRow"))
4751                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4752            {
4753                return Err(invalid());
4754            }
4755        }
4756        let results: Vec<CommitOperationOutcome> =
4757            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4758        let has_conflict = results
4759            .iter()
4760            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4761        let has_error = results
4762            .iter()
4763            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4764        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4765        let redacted = results.is_empty() && operations_json.is_none();
4766        if match status {
4767            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4768            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4769            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4770        } {
4771            return Err(invalid());
4772        }
4773        let operations: Option<Vec<CommitOperation>> = operations_json
4774            .map(|raw| {
4775                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4776                if !stored.iter().all(Self::stored_operation_is_representable) {
4777                    return Err(invalid());
4778                }
4779                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4780            })
4781            .transpose()?;
4782        for result in &results {
4783            match result {
4784                CommitOperationOutcome::Conflict { conflict }
4785                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4786                {
4787                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4788                }
4789                CommitOperationOutcome::Error { rejection } => {
4790                    if let Some(details) = &rejection.details {
4791                        let encoded = serde_json::to_string(details).map_err(|_| {
4792                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4793                        })?;
4794                        RejectionDetails::parse(&encoded).map_err(|_| {
4795                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4796                        })?;
4797                    }
4798                }
4799                _ => {}
4800            }
4801        }
4802        Ok(CommitOutcome {
4803            sequence,
4804            client_commit_id,
4805            status,
4806            recorded_at_ms,
4807            results,
4808            operations,
4809            retained_rows: None,
4810            resolution: Self::parse_outcome_resolution(&resolution)?,
4811            resolved_at_ms,
4812            replacement_client_commit_id,
4813        })
4814    }
4815
4816    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4817    fn unique_conflicts(
4818        &self,
4819        table: &TableSchema,
4820        values: &Map<String, Value>,
4821        base: bool,
4822    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4823        let source = if base {
4824            base_table(&table.name)
4825        } else {
4826            visible_table(&table.name)
4827        };
4828        let mut conflicts = Vec::new();
4829        for index in table.indexes.iter().filter(|index| index.unique) {
4830            let mut params = vec![json_param_to_sql(
4831                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4832            )?];
4833            for name in &index.columns {
4834                let column = table
4835                    .columns
4836                    .iter()
4837                    .find(|column| column.name == *name)
4838                    .ok_or("sync.local_corrupt")?;
4839                let cell = json_to_column_value(column, values.get(name))?;
4840                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4841            }
4842            let sql = format!(
4843                "SELECT * FROM {source} WHERE {} != ? AND {}",
4844                quote_ident(&table.primary_key),
4845                index
4846                    .columns
4847                    .iter()
4848                    .map(|column| format!("{} = ?", quote_ident(column)))
4849                    .collect::<Vec<_>>()
4850                    .join(" AND ")
4851            );
4852            let mut statement = self
4853                .conn
4854                .prepare_cached(&sql)
4855                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4856            let records = statement
4857                .query_map(rusqlite::params_from_iter(params), |record| {
4858                    let values = table
4859                        .columns
4860                        .iter()
4861                        .enumerate()
4862                        .map(|(index, column)| {
4863                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4864                            json_to_column_value(column, Some(&value)).map_err(|message| {
4865                                rusqlite::Error::ToSqlConversionFailure(message.into())
4866                            })
4867                        })
4868                        .collect::<Result<Row, _>>()?;
4869                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4870                })
4871                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4872            for record in records {
4873                let (row, version) =
4874                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4875                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4876                conflicts.push(RetainedUniqueConflict {
4877                    index: index.name.clone(),
4878                    columns: index.columns.clone(),
4879                    row_id,
4880                    server_row: table
4881                        .columns
4882                        .iter()
4883                        .zip(&row)
4884                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4885                        .collect(),
4886                    server_version: version,
4887                });
4888            }
4889        }
4890        Ok(conflicts)
4891    }
4892
4893    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4894        if let Some((commit, initial)) = self
4895            .failed_commits
4896            .iter()
4897            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4898        {
4899            let mut rows = Vec::new();
4900            for operation in &commit.ops {
4901                // A table the current schema removed has no rows to retain.
4902                let Some(table) = self.schema.table(&operation.table) else {
4903                    continue;
4904                };
4905                let key =
4906                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4907                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4908                let base = self.stored_row(table, &operation.row_id, true)?;
4909                rows.push(RetainedCommitRow {
4910                    table: operation.table.clone(),
4911                    row_id: operation.row_id.clone(),
4912                    unique_conflicts: if operation.upsert {
4913                        self.unique_conflicts(
4914                            table,
4915                            initial
4916                                .get(&key)
4917                                .and_then(Value::as_object)
4918                                .or(operation.values.as_ref())
4919                                .ok_or("sync.local_corrupt")?,
4920                            true,
4921                        )?
4922                    } else {
4923                        Vec::new()
4924                    },
4925                    local_row: if operation.upsert {
4926                        initial
4927                            .get(&key)
4928                            .and_then(Value::as_object)
4929                            .cloned()
4930                            .or_else(|| operation.values.clone())
4931                    } else {
4932                        None
4933                    },
4934                    server_row: base.as_ref().map(|(values, _)| {
4935                        table
4936                            .columns
4937                            .iter()
4938                            .zip(values)
4939                            .map(|(column, value)| {
4940                                (column.name.clone(), column_value_to_json(value))
4941                            })
4942                            .collect()
4943                    }),
4944                    server_version: base.map(|(_, version)| version),
4945                });
4946            }
4947            outcome.retained_rows = Some(rows);
4948        }
4949        Ok(outcome)
4950    }
4951
4952    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
4953        let row = self
4954            .conn
4955            .query_row(
4956                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
4957                        resolution, resolved_at_ms, replacement_client_commit_id
4958                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
4959                rusqlite::params![client_commit_id],
4960                |row| {
4961                    Ok(StoredCommitOutcomeRow {
4962                        sequence: row.get(0)?,
4963                        client_commit_id: row.get(1)?,
4964                        status: row.get(2)?,
4965                        recorded_at_ms: row.get(3)?,
4966                        results_json: row.get(4)?,
4967                        operations_json: row.get(5)?,
4968                        resolution: row.get(6)?,
4969                        resolved_at_ms: row.get(7)?,
4970                        replacement_client_commit_id: row.get(8)?,
4971                    })
4972                },
4973            )
4974            .optional()
4975            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4976        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
4977            .transpose()
4978    }
4979
4980    /// Internal host step. The host must run its idle drive before finishing.
4981    #[doc(hidden)]
4982    pub fn prepare_current_base(
4983        &self,
4984        client_commit_id: &str,
4985        sequence: i64,
4986    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
4987        let outcome = self
4988            .commit_outcome(client_commit_id)
4989            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
4990        let valid = outcome.as_ref().is_some_and(|outcome| {
4991            outcome.sequence == sequence
4992                && outcome.resolution == CommitOutcomeResolution::Active
4993                && matches!(
4994                    outcome.status,
4995                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
4996                )
4997                && outcome
4998                    .operations
4999                    .as_ref()
5000                    .zip(outcome.retained_rows.as_ref())
5001                    .is_some_and(|(operations, rows)| {
5002                        !rows.is_empty() && operations.len() == rows.len()
5003                    })
5004        });
5005        if !valid {
5006            return Err(CurrentBaseProof::failure(
5007                CurrentBaseProofReason::OutcomeUnavailable,
5008            ));
5009        }
5010        if !self.transport_enabled {
5011            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5012        }
5013        Ok(CurrentBasePreparation {
5014            client_commit_id: client_commit_id.to_owned(),
5015            sequence,
5016            units: load_units(&self.conn)
5017                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5018            transport_epoch: self.transport_epoch,
5019            registration_epoch: self.registration_epoch,
5020            security_epoch: self.security_epoch,
5021        })
5022    }
5023
5024    /// Internal host step after the completed post-rejection idle drive.
5025    #[doc(hidden)]
5026    pub fn finish_current_base(
5027        &mut self,
5028        prepared: CurrentBasePreparation,
5029        drive: SyncOutcome,
5030    ) -> CurrentBaseProof {
5031        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5032            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5033        }
5034        if self.registration_epoch != prepared.registration_epoch {
5035            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5036        }
5037        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5038            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5039        }
5040        let report = match drive {
5041            SyncOutcome::Ok(report) => report,
5042            SyncOutcome::BudgetExhausted(_) => {
5043                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5044            }
5045            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5046        };
5047        if report.schema_floor.is_some() {
5048            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5049        }
5050        if self
5051            .conn
5052            .execute_batch("SAVEPOINT syncular_base_proof")
5053            .is_err()
5054        {
5055            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5056        }
5057        let proof = (|| {
5058            let snapshot = snapshot_read_connection(
5059                &self.conn,
5060                &SnapshotReadRequest {
5061                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5062                    commit_ids: vec![prepared.client_commit_id.clone()],
5063                    ..Default::default()
5064                },
5065            )
5066            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5067            let current = self
5068                .commit_outcome(&prepared.client_commit_id)
5069                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5070            let unavailable = if report.revoked.is_empty() {
5071                CurrentBaseProofReason::OutcomeUnavailable
5072            } else {
5073                CurrentBaseProofReason::Revoked
5074            };
5075            let Some(outcome) = current else {
5076                return Err(unavailable);
5077            };
5078            if !matches!(
5079                snapshot.deliveries.first(),
5080                Some(CommitDelivery::Known { .. })
5081            ) || outcome.sequence != prepared.sequence
5082                || outcome.resolution != CommitOutcomeResolution::Active
5083                || !matches!(
5084                    outcome.status,
5085                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5086                )
5087            {
5088                return Err(unavailable);
5089            }
5090            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5091                return Err(unavailable);
5092            };
5093            if rows.is_empty() || operations.len() != rows.len() {
5094                return Err(unavailable);
5095            }
5096            let after =
5097                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5098            let complete: HashSet<&str> = snapshot
5099                .units
5100                .iter()
5101                .filter_map(|unit| match unit {
5102                    UnitCatchup::Known {
5103                        id,
5104                        bootstrap_complete: true,
5105                        ..
5106                    } => Some(id.as_str()),
5107                    _ => None,
5108                })
5109                .collect();
5110            // A unit id names one scope map (§4.1), so the same id before and
5111            // after the drive is the same registration.
5112            let candidates: Vec<&UnitRecord> = after
5113                .iter()
5114                .filter(|unit| {
5115                    complete.contains(unit.id.as_str())
5116                        && prepared.units.iter().any(|before| before.id == unit.id)
5117                })
5118                .collect();
5119            let covered = |table_name: &str, row: &Map<String, Value>| {
5120                let Some(table) = self.schema.table(table_name) else {
5121                    return false;
5122                };
5123                candidates.iter().any(|unit| {
5124                    covers(table, &unit.scopes)
5125                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5126                            |scopes| {
5127                                scopes.iter().all(|(variable, allowed)| {
5128                                    table
5129                                        .scope_column(variable)
5130                                        .and_then(|column| match row.get(column) {
5131                                            Some(Value::String(value)) => Some(value.clone()),
5132                                            Some(Value::Bool(value)) => Some(value.to_string()),
5133                                            Some(Value::Number(value)) => {
5134                                                value.as_f64().map(|number| {
5135                                                    ryu_js::Buffer::new()
5136                                                        .format_finite(number)
5137                                                        .to_owned()
5138                                                })
5139                                            }
5140                                            _ => None,
5141                                        })
5142                                        .is_some_and(|value| {
5143                                            !value.is_empty() && allowed.contains(&value)
5144                                        })
5145                                })
5146                            },
5147                        )
5148                })
5149            };
5150            for row in &rows {
5151                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5152                    return Err(if report.revoked.is_empty() {
5153                        CurrentBaseProofReason::MissingCoverage
5154                    } else {
5155                        CurrentBaseProofReason::Revoked
5156                    });
5157                };
5158                if !covered(&row.table, base)
5159                    || row
5160                        .unique_conflicts
5161                        .iter()
5162                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5163                {
5164                    return Err(if report.revoked.is_empty() {
5165                        CurrentBaseProofReason::MissingCoverage
5166                    } else {
5167                        CurrentBaseProofReason::Revoked
5168                    });
5169                }
5170            }
5171            Ok(rows)
5172        })();
5173        if self
5174            .conn
5175            .execute_batch("RELEASE syncular_base_proof")
5176            .is_err()
5177        {
5178            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5179        }
5180        match proof {
5181            Ok(rows) => CurrentBaseProof {
5182                proven: true,
5183                rows: Some(rows),
5184                reason: None,
5185            },
5186            Err(reason) => CurrentBaseProof::failure(reason),
5187        }
5188    }
5189
5190    pub fn prove_current_base(
5191        &mut self,
5192        transport: &mut dyn Transport,
5193        client_commit_id: &str,
5194        sequence: i64,
5195        max_rounds: Option<u32>,
5196    ) -> CurrentBaseProof {
5197        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5198            Ok(prepared) => prepared,
5199            Err(proof) => return proof,
5200        };
5201        let drive = self.sync_until_idle(transport, max_rounds);
5202        self.finish_current_base(prepared, drive)
5203    }
5204
5205    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5206        if query.limit == Some(0) {
5207            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5208        }
5209        let mut sql = String::from(
5210            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5211                    resolution, resolved_at_ms, replacement_client_commit_id
5212               FROM _syncular_commit_outcomes",
5213        );
5214        if query.active_only {
5215            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5216        }
5217        sql.push_str(" ORDER BY seq DESC");
5218        if let Some(limit) = query.limit {
5219            sql.push_str(&format!(" LIMIT {limit}"));
5220        }
5221        let mut stmt = self
5222            .conn
5223            .prepare(&sql)
5224            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5225        let rows = stmt
5226            .query_map([], |row| {
5227                Ok(StoredCommitOutcomeRow {
5228                    sequence: row.get(0)?,
5229                    client_commit_id: row.get(1)?,
5230                    status: row.get(2)?,
5231                    recorded_at_ms: row.get(3)?,
5232                    results_json: row.get(4)?,
5233                    operations_json: row.get(5)?,
5234                    resolution: row.get(6)?,
5235                    resolved_at_ms: row.get(7)?,
5236                    replacement_client_commit_id: row.get(8)?,
5237                })
5238            })
5239            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5240        let mut outcomes = Vec::new();
5241        for row in rows {
5242            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5243                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5244            )?)?);
5245        }
5246        Ok(outcomes)
5247    }
5248
5249    fn prune_commit_outcomes(&self) -> Result<(), String> {
5250        #[cfg(test)]
5251        self.outcome_prune_count
5252            .set(self.outcome_prune_count.get() + 1);
5253        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5254        let count = self
5255            .conn
5256            .query_row(
5257                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5258                [],
5259                |row| row.get::<_, i64>(0),
5260            )
5261            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5262            as usize;
5263        let excess = count.saturating_sub(max_entries);
5264        if excess == 0 {
5265            return Ok(());
5266        }
5267        let mut stmt = self
5268            .conn
5269            .prepare(
5270                "SELECT seq FROM _syncular_commit_outcomes
5271                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5272                  ORDER BY seq ASC LIMIT ?1",
5273            )
5274            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5275        let rows = stmt
5276            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5277            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5278        let sequences = rows
5279            .collect::<Result<Vec<_>, _>>()
5280            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5281        drop(stmt);
5282        for sequence in sequences {
5283            self.conn
5284                .execute(
5285                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5286                    rusqlite::params![sequence],
5287                )
5288                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5289        }
5290        Ok(())
5291    }
5292
5293    // -- units (§4.1) ------------------------------------------------------------
5294
5295    /// Every registered unit in registration order (§4.1).
5296    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5297        self.stored(load_units(&self.conn))
5298    }
5299
5300    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5301        self.stored(get_unit(&self.conn, id))
5302    }
5303
5304    /// Register one unit (§4.1) and return its id.
5305    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5306        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5307    }
5308
5309    /// Register several units in one local transaction and return their ids:
5310    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5311    /// cold start with hundreds of units pays it once). An error rolls back
5312    /// every registration of the call. Registering an active unit again
5313    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5314    /// re-arms a unit raises sync-needed with an interactive intent.
5315    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5316        self.begin_observation("syncular_subscribe")?;
5317        let mut changed = false;
5318        let registered = (|| {
5319            let mut ids = Vec::with_capacity(units.len());
5320            for scopes in units {
5321                let (id, held) = self.hold_unit(scopes, true)?;
5322                changed |= held;
5323                ids.push(id);
5324            }
5325            self.conn
5326                .execute_batch("RELEASE syncular_subscribe")
5327                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5328            Ok(ids)
5329        })();
5330        let ids = match registered {
5331            Ok(ids) => ids,
5332            Err(error) => {
5333                self.rollback_observation("syncular_subscribe");
5334                return Err(error);
5335            }
5336        };
5337        if changed {
5338            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5339            self.set_sync_needed(true, true);
5340        }
5341        Ok(ids)
5342    }
5343
5344    /// Release the explicit reference to a unit. The unit leaves the
5345    /// registration list when no window base holds it (§4.1); its rows stay.
5346    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5347        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5348            return Ok(());
5349        };
5350        if !unit.explicit {
5351            return Ok(());
5352        }
5353        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5354            return self.stored(save_unit(
5355                &self.conn,
5356                &UnitRecord {
5357                    explicit: false,
5358                    ..unit
5359                },
5360            ));
5361        }
5362        self.conn
5363            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5364            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5365        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5366        if unit.status == UnitStatus::Active {
5367            self.set_sync_needed(true, true);
5368        }
5369        Ok(())
5370    }
5371
5372    /// Take one reference on a unit (§4.1): validate the scope map, register a
5373    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5374    /// Returns the id and whether the registration list changed.
5375    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5376        if scopes.is_empty()
5377            || scopes.values().any(|value| value == "*")
5378            || !self
5379                .schema
5380                .tables
5381                .iter()
5382                .any(|table| covers(table, &scopes))
5383        {
5384            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5385        }
5386        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5387        let existing = self.stored(get_unit(&self.conn, &id))?;
5388        if let Some(existing) = existing
5389            .as_ref()
5390            .filter(|unit| unit.status != UnitStatus::Revoked)
5391        {
5392            if explicit && !existing.explicit {
5393                self.stored(save_unit(
5394                    &self.conn,
5395                    &UnitRecord {
5396                        explicit,
5397                        ..existing.clone()
5398                    },
5399                ))?;
5400            }
5401            return Ok((id, false));
5402        }
5403        self.delete_pending_evict(&id)?;
5404        self.stored(save_unit(
5405            &self.conn,
5406            &UnitRecord {
5407                id: id.clone(),
5408                scopes,
5409                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5410                status: UnitStatus::Active,
5411                reason_code: None,
5412                cursor: Some(-1),
5413                bootstrap_state: None,
5414                seeded: Vec::new(),
5415                echo: None,
5416            },
5417        ))?;
5418        Ok((id, true))
5419    }
5420
5421    // -- windows (§4.8) -----------------------------------------------------------
5422
5423    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5424    /// values take a reference on their unit (a new unit bootstraps on the
5425    /// next sync); removed values release theirs, and a unit no reference
5426    /// holds leaves the registration list fused with its first eviction chunk
5427    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5428    pub fn set_window(
5429        &mut self,
5430        base: &WindowBase,
5431        units: &[String],
5432    ) -> Result<CommandEffects, String> {
5433        let base_key = window_base_key(base);
5434        let live = self.load_window_units(&base_key)?;
5435        let mut changed = false;
5436        for value in units {
5437            if live.iter().any(|(held, _)| held == value) {
5438                continue;
5439            }
5440            let rollback = self.capture();
5441            self.begin_observation("syncular_window")?;
5442            let widened = (|| {
5443                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5444                self.sync_needed |= registered;
5445                self.conn
5446                    .execute(
5447                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5448                        rusqlite::params![base_key, value, id],
5449                    )
5450                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5451                let mut batch = ChangeAccumulator::default();
5452                batch.window(&base_key, value);
5453                batch.status = true;
5454                self.finish_observation("syncular_window", batch)
5455            })();
5456            if let Err(error) = widened {
5457                self.rollback_observation("syncular_window");
5458                self.restore(rollback);
5459                return Err(error);
5460            }
5461            changed = true;
5462            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5463        }
5464        for (value, id) in live {
5465            if units.contains(&value) {
5466                continue;
5467            }
5468            self.rebuild_overlay_if_dirty()?;
5469            let rollback = self.capture();
5470            self.begin_observation("syncular_window")?;
5471            let remaining = self.release_window_unit(&base_key, &value, &id);
5472            let remaining = match remaining {
5473                Ok(remaining) => remaining,
5474                Err(error) => {
5475                    self.rollback_observation("syncular_window");
5476                    self.restore(rollback);
5477                    return Err(error);
5478                }
5479            };
5480            changed = true;
5481            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5482            if remaining {
5483                self.drain_pending_evictions()?;
5484            }
5485        }
5486        Ok(if changed && self.transport_enabled {
5487            CommandEffects::interactive()
5488        } else {
5489            CommandEffects::none()
5490        })
5491    }
5492
5493    /// §4.8 completeness oracle: the windowed-in values of a base and those
5494    /// whose unit is not complete. A value is complete once its unit is active
5495    /// and no longer bootstrapping; registration alone is not completeness.
5496    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5497        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5498    }
5499
5500    /// Registry rows for a base, ordered by value: (value, unit id).
5501    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5502        let mut statement = self
5503            .conn
5504            .prepare_cached(
5505                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5506            )
5507            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5508        let rows = statement
5509            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5510            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5511        rows.collect::<Result<_, _>>()
5512            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5513    }
5514
5515    /// Every window registry row: (base key, value).
5516    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5517        let mut statement = self
5518            .conn
5519            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5520            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5521        let rows = statement
5522            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5523            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5524        rows.collect::<Result<_, _>>()
5525            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5526    }
5527
5528    /// §4.8: release a window value's reference inside the caller's
5529    /// observation. When no reference holds the unit, unregister it fused with
5530    /// its first eviction chunk (E3): delete the rows of every covered table
5531    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5532    /// owned by another unit (§3.3), and park the per-table scopes of what
5533    /// remains in the pending-eviction registry. Fails closed without a local
5534    /// scope-column mapping. Returns whether unpinned rows remain.
5535    fn release_window_unit(
5536        &mut self,
5537        base_key: &str,
5538        value: &str,
5539        id: &str,
5540    ) -> Result<bool, String> {
5541        self.conn
5542            .execute(
5543                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5544                rusqlite::params![base_key, value],
5545            )
5546            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5547        let mut batch = ChangeAccumulator::default();
5548        batch.window(base_key, value);
5549        batch.status = true;
5550        let unit = self.stored(get_unit(&self.conn, id))?;
5551        let mut remaining = false;
5552        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5553            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5554                let units = self.stored(load_units(&self.conn))?;
5555                self.conn
5556                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5557                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5558                self.sync_needed = true;
5559                for table in self.schema.tables.clone() {
5560                    let Some(effective) = unit
5561                        .echo
5562                        .as_deref()
5563                        .filter(|_| covers(&table, &unit.scopes))
5564                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5565                    else {
5566                        continue;
5567                    };
5568                    let pinned = self.pinned_row_ids(&table.name);
5569                    let owners = unit_owners(&units, &table, id);
5570                    let (more, deferred) =
5571                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5572                    remaining |= more;
5573                    if deferred {
5574                        self.save_pending_evict(id, &table.name, &effective)?;
5575                    }
5576                    self.record_scope_map(&mut batch, &table.name, &effective);
5577                }
5578            }
5579        }
5580        self.finish_observation("syncular_window", batch)?;
5581        Ok(remaining)
5582    }
5583
5584    /// §4.8 E1: delete base rows owned by the effective scopes
5585    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5586    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5587    /// fails without deleting rows.
5588    fn evict_scope_rows(
5589        &self,
5590        table_name: &str,
5591        effective: &[(String, Vec<String>)],
5592        owners: &[Vec<(String, Vec<String>)>],
5593        pinned: &std::collections::HashSet<String>,
5594    ) -> Result<(bool, bool), String> {
5595        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5596        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5597            .map_err(|()| "sync.scope_revoked".to_owned())?
5598        else {
5599            return Ok((false, false));
5600        };
5601        let scope_params = params.len();
5602        let pk = quote_ident(&table.primary_key);
5603        // Pinned ids are rowId text; the text form matches every key type.
5604        let predicate = if pinned.is_empty() {
5605            scope.clone()
5606        } else {
5607            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5608            let holes = vec!["?"; pinned.len()].join(", ");
5609            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5610        };
5611        let base = base_table(table_name);
5612        let visible = visible_table(table_name);
5613        let mut ids: Vec<SqlValue> = Vec::new();
5614        for source in [&base, &visible] {
5615            let mut stmt = self
5616                .conn
5617                .prepare_cached(&format!(
5618                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5619                ))
5620                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5621            ids = stmt
5622                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5623                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5624                .collect::<Result<_, _>>()
5625                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5626            if !ids.is_empty() {
5627                break;
5628            }
5629        }
5630        if !ids.is_empty() {
5631            for id in &ids {
5632                self.conn
5633                    .execute(
5634                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5635                        rusqlite::params![table_name, id],
5636                    )
5637                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5638            }
5639            let holes = vec!["?"; ids.len()].join(", ");
5640            for source in [&base, &visible] {
5641                self.conn
5642                    .execute(
5643                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5644                        rusqlite::params_from_iter(&ids),
5645                    )
5646                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5647            }
5648        }
5649        let mut remaining = false;
5650        let mut deferred = false;
5651        for source in [&base, &visible] {
5652            remaining |= self
5653                .conn
5654                .query_row(
5655                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5656                    rusqlite::params_from_iter(&params),
5657                    |row| row.get::<_, bool>(0),
5658                )
5659                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5660            deferred |= self
5661                .conn
5662                .query_row(
5663                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5664                    rusqlite::params_from_iter(&params[..scope_params]),
5665                    |row| row.get::<_, bool>(0),
5666                )
5667                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5668        }
5669        Ok((remaining, deferred))
5670    }
5671
5672    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5673    /// chunks, retaining outbox pins and their durable marker. A unit that
5674    /// re-entered the window has no pending entry left.
5675    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5676        let pending = self.load_pending_evictions()?;
5677        if pending.is_empty() {
5678            return Ok(());
5679        }
5680        self.rebuild_overlay_if_dirty()?;
5681        for (unit_id, table_name, effective) in pending {
5682            let Some(table) = self.schema.table(&table_name).cloned() else {
5683                self.delete_pending_evict_table(&unit_id, &table_name)?;
5684                continue;
5685            };
5686            loop {
5687                std::thread::yield_now();
5688                self.begin_observation("syncular_evict")?;
5689                let chunk = (|| {
5690                    let pinned = self.pinned_row_ids(&table_name);
5691                    let owners =
5692                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5693                    let (remaining, deferred) =
5694                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5695                    if !deferred {
5696                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5697                    }
5698                    let mut batch = ChangeAccumulator::default();
5699                    self.record_scope_map(&mut batch, &table_name, &effective);
5700                    self.finish_observation("syncular_evict", batch)?;
5701                    Ok(remaining)
5702                })();
5703                match chunk {
5704                    Ok(true) => {}
5705                    Ok(false) => break,
5706                    Err(error) => {
5707                        self.rollback_observation("syncular_evict");
5708                        return Err(error);
5709                    }
5710                }
5711            }
5712        }
5713        Ok(())
5714    }
5715
5716    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5717    /// commit — rows that MUST NOT be evicted until the commit drains.
5718    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5719        let mut pinned = std::collections::HashSet::new();
5720        for commit in self
5721            .outbox
5722            .iter()
5723            .map(|commit| &**commit)
5724            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5725        {
5726            for op in &commit.ops {
5727                if op.table == table {
5728                    pinned.insert(op.row_id.clone());
5729                }
5730            }
5731        }
5732        pinned
5733    }
5734
5735    fn save_pending_evict(
5736        &self,
5737        unit_id: &str,
5738        table: &str,
5739        effective: &[(String, Vec<String>)],
5740    ) -> Result<(), String> {
5741        self.conn.execute(
5742            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5743            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5744        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5745    }
5746
5747    /// Re-entry cancels every pending eviction of the unit.
5748    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5749        self.conn
5750            .execute(
5751                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5752                [unit_id],
5753            )
5754            .map(|_| ())
5755            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5756    }
5757
5758    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5759        self.conn
5760            .execute(
5761                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5762                [unit_id, table],
5763            )
5764            .map(|_| ())
5765            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5766    }
5767
5768    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5769        let mut stmt = self
5770            .conn
5771            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5772            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5773        let rows = stmt
5774            .query_map([], |row| {
5775                Ok((
5776                    row.get::<_, String>(0)?,
5777                    row.get::<_, String>(1)?,
5778                    row.get::<_, String>(2)?,
5779                ))
5780            })
5781            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5782        rows.map(|row| {
5783            let (unit_id, table, json) =
5784                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5785            let value: Value = serde_json::from_str(&json)
5786                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5787            let effective = json_to_scope_map(&value)
5788                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5789            Ok((unit_id, table, effective))
5790        })
5791        .collect()
5792    }
5793
5794    /// Select failed-intent retention; reload durable retained aggregates.
5795    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5796        self.retain_failed_commits = enabled;
5797        for (commit, _) in self.failed_commits.iter() {
5798            for operation in &commit.ops {
5799                self.overlay_dirty.table(&operation.table);
5800            }
5801        }
5802        self.failed_commits = self.load_failed_commits()?.into();
5803        for (commit, _) in self.failed_commits.iter() {
5804            for operation in &commit.ops {
5805                self.overlay_dirty.table(&operation.table);
5806            }
5807        }
5808        Ok(())
5809    }
5810
5811    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5812        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5813            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5814        let rows = statement
5815            .query_map([], |row| {
5816                Ok((
5817                    row.get::<_, String>(0)?,
5818                    row.get::<_, String>(1)?,
5819                    row.get::<_, String>(2)?,
5820                ))
5821            })
5822            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5823        rows.map(|row| {
5824            let (id, operations, initial) =
5825                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5826            let operations: Vec<CommitOperation> =
5827                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5828            let initial =
5829                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5830            Ok((
5831                OutboxCommit {
5832                    client_commit_id: id,
5833                    ops: operations
5834                        .into_iter()
5835                        .map(|op| OutboxOp {
5836                            upsert: op.op == "upsert",
5837                            table: op.table,
5838                            row_id: op.row_id,
5839                            base_version: op.base_version,
5840                            values: op.values,
5841                            insert: op.insert,
5842                        })
5843                        .collect(),
5844                },
5845                initial,
5846            ))
5847        })
5848        .collect()
5849    }
5850
5851    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5852        let mut initial = Map::new();
5853        for operation in operations {
5854            // A table the current schema removed has no rows to retain.
5855            let Some(table) = self.schema.table(&operation.table) else {
5856                continue;
5857            };
5858            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5859                Some(row) => Some(row),
5860                None => self.visible_row(table, &operation.row_id)?,
5861            };
5862            if let Some((values, _)) = initial_row {
5863                let mut intended: Map<String, Value> = table
5864                    .columns
5865                    .iter()
5866                    .zip(values.iter())
5867                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5868                    .collect();
5869                if let Some(patch) = &operation.values {
5870                    intended.extend(patch.clone());
5871                }
5872                initial.insert(
5873                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5874                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5875                    Value::Object(intended),
5876                );
5877            }
5878        }
5879        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5880            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5881            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5882            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5883        self.failed_commits.push((
5884            OutboxCommit {
5885                client_commit_id: id.to_owned(),
5886                ops: operations.to_vec(),
5887            },
5888            initial,
5889        ));
5890        Ok(())
5891    }
5892
5893    fn drop_failed_commits(
5894        &mut self,
5895        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5896    ) -> Result<Vec<OutboxCommit>, String> {
5897        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5898        for (commit, initial) in self.failed_commits.iter() {
5899            for operation in &commit.ops {
5900                let key =
5901                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5902                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5903                if initial
5904                    .get(&key)
5905                    .and_then(Value::as_object)
5906                    .is_some_and(|values| matches(&operation.table, values))
5907                {
5908                    doomed.push(commit.clone());
5909                    break;
5910                }
5911            }
5912        }
5913        for commit in &doomed {
5914            self.conn
5915                .execute(
5916                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5917                    rusqlite::params![commit.client_commit_id],
5918                )
5919                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5920        }
5921        for commit in &doomed {
5922            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5923        }
5924        self.conflicts.retain(|record| {
5925            !doomed
5926                .iter()
5927                .any(|commit| commit.client_commit_id == record.client_commit_id)
5928        });
5929        self.rejections.retain(|record| {
5930            !doomed
5931                .iter()
5932                .any(|commit| commit.client_commit_id == record.client_commit_id)
5933        });
5934        self.failed_commits.retain(|(commit, _)| {
5935            !doomed
5936                .iter()
5937                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
5938        });
5939        for commit in &doomed {
5940            for operation in &commit.ops {
5941                self.overlay_dirty.table(&operation.table);
5942            }
5943        }
5944        Ok(doomed)
5945    }
5946
5947    /// Record one commit of local mutations. The returned commit ID is the
5948    /// §2.3 idempotency key. Failures are structured so callers read
5949    /// `code`/`message`/`details` instead of parsing a string prefix.
5950    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
5951        self.mutate_with_commit_id(mutations, None)
5952    }
5953
5954    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
5955    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
5956    /// one this replica still holds fails with `client.commit_id_reused`.
5957    pub fn mutate_with_commit_id(
5958        &mut self,
5959        mutations: Vec<Mutation>,
5960        client_commit_id: Option<String>,
5961    ) -> Result<String, ClientError> {
5962        let prior = self.storage_failure.borrow_mut().take();
5963        if let Some(id) = &client_commit_id {
5964            let held = if id.is_empty() {
5965                Err("sync.invalid_request: clientCommitId is empty".to_owned())
5966            } else {
5967                self.conn
5968                    .query_row(
5969                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
5970                        [id],
5971                        |row| row.get(0),
5972                    )
5973                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5974            };
5975            match held {
5976                Ok(false) => {}
5977                Ok(true) => {
5978                    return self.finish_authoring(
5979                        prior,
5980                        Err(ClientError {
5981                            code: "client.commit_id_reused".to_owned(),
5982                            message: "clientCommitId is held".to_owned(),
5983                            details: Some(json!({ "clientCommitId": id })),
5984                            retryable: false,
5985                        }),
5986                    )
5987                }
5988                Err(message) => return self.finish_authoring(prior, Err(message.into())),
5989            }
5990        }
5991        let outcome = (|| -> Result<String, ClientError> {
5992            if mutations.is_empty() {
5993                return Err(
5994                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
5995                        .to_owned()
5996                        .into(),
5997                );
5998            }
5999            let mut ops = Vec::with_capacity(mutations.len());
6000            for mutation in mutations {
6001                match mutation {
6002                    Mutation::Patch {
6003                        table,
6004                        values,
6005                        base_version,
6006                    } => {
6007                        let schema_table = self
6008                            .schema
6009                            .table(&table)
6010                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
6011                        let values = normalize_values_casing(schema_table, values)?;
6012                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6013                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
6014                    }
6015                    Mutation::Upsert {
6016                        table,
6017                        values,
6018                        base_version,
6019                    } => {
6020                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6021                            format!("sync.unknown_table: unknown table {table:?}")
6022                        })?;
6023                        // §5: value keys are accepted in snake_case AND the
6024                        // generated row types' camelCase; normalize to SQL truth
6025                        // before the pk lookup / codec see them.
6026                        let values = normalize_values_casing(schema_table, values)?;
6027                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6028                        // §6.7 `mutate` marks every column present: fill missing
6029                        // nullable columns with NULL before the payload is built.
6030                        let values = full_row_values(schema_table, values)?;
6031                        self.refuse_sealed_row(schema_table, &row_id)?;
6032                        // §5.11: validate the payload encodes with the current
6033                        // codec. A key-selection or unknown-key failure is a
6034                        // durable rejection at the push seam (§10.3), never an
6035                        // author-time error. Full-row `mutate` presents every
6036                        // column, so no stored fallback is needed.
6037                        self.validate_author_encode(schema_table, &row_id, &values)?;
6038                        // §7.1 insert intent: the row is absent from the
6039                        // visible replica at call time, before this batch
6040                        // applies anything.
6041                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6042                        ops.push(OutboxOp {
6043                            upsert: true,
6044                            table,
6045                            row_id,
6046                            base_version,
6047                            values: Some(values),
6048                            insert,
6049                        });
6050                    }
6051                    Mutation::Delete {
6052                        table,
6053                        row_id,
6054                        base_version,
6055                    } => {
6056                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6057                            format!("sync.unknown_table: unknown table {table:?}")
6058                        })?;
6059                        self.refuse_sealed_row(schema_table, &row_id)?;
6060                        // §2.4: the server rejects a rowId holding U+0000.
6061                        if row_id.contains('\0') {
6062                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6063                                .to_owned()
6064                                .into());
6065                        }
6066                        ops.push(OutboxOp {
6067                            upsert: false,
6068                            table,
6069                            row_id,
6070                            base_version,
6071                            values: None,
6072                            insert: false,
6073                        });
6074                    }
6075                }
6076            }
6077            Ok(self.record_outbox_commit(ops, client_commit_id)?)
6078        })();
6079        self.finish_authoring(prior, outcome)
6080    }
6081
6082    /// Scope the retained `storage_failure` diagnostic to this call so an
6083    /// earlier operation's failure cannot classify a new one, and surface the
6084    /// call's own classified failure with its details. A fresh classified
6085    /// failure replaces the retained diagnostic; otherwise the earlier one is
6086    /// kept for the next sync.
6087    fn finish_authoring(
6088        &self,
6089        prior: Option<QueryReadFailure>,
6090        outcome: Result<String, ClientError>,
6091    ) -> Result<String, ClientError> {
6092        let fresh = self.storage_failure.borrow_mut().take();
6093        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6094        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6095    }
6096
6097    fn record_outbox_commit(
6098        &mut self,
6099        ops: Vec<OutboxOp>,
6100        client_commit_id: Option<String>,
6101    ) -> Result<String, String> {
6102        let commit = OutboxCommit {
6103            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6104            ops,
6105        };
6106        self.begin_observation("syncular_mutation")?;
6107        let mut batch = ChangeAccumulator::default();
6108        for op in &commit.ops {
6109            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6110            if let Some(values) = &op.values {
6111                if let Some(table) = self.schema.table(&op.table) {
6112                    for scope in &table.scope_variables {
6113                        if let Some(Value::String(value)) = values.get(&scope.column) {
6114                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6115                            precise = true;
6116                        }
6117                    }
6118                }
6119            }
6120            if !precise {
6121                batch.table(&op.table);
6122            }
6123        }
6124        if let Err(error) = self.persist_outbox_insert(&commit) {
6125            self.rollback_observation("syncular_mutation");
6126            return Err(error);
6127        }
6128        // A clean overlay contains the FIFO fold of every previous commit, so
6129        // the new commit applies over it. §7.1: an operation that does not
6130        // apply (a secondary unique collision) fails the whole commit; the
6131        // savepoint rollback removes the outbox entry and every visible write.
6132        // The append undoes itself: a rollback capture would copy the outbox.
6133        let was_dirty = self.overlay_dirty.snapshot();
6134        let id = commit.client_commit_id.clone();
6135        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6136            commit
6137                .ops
6138                .iter()
6139                .try_for_each(|op| self.apply_outbox_op(op))?;
6140            self.outbox.push(Arc::new(commit));
6141            batch.status = true;
6142            self.finish_observation("syncular_mutation", batch)
6143                .inspect_err(|_| drop(self.outbox.pop()))
6144        });
6145        if let Err(error) = appended {
6146            self.rollback_observation("syncular_mutation");
6147            self.overlay_dirty.restore(was_dirty);
6148            return Err(error);
6149        }
6150        Ok(id)
6151    }
6152
6153    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6154    /// non-scope columns are present; every other column is absent and stays
6155    /// untouched on the server and in the local overlay. An absent local base
6156    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6157    /// Failures are structured so callers read `code`/`message`/`details`.
6158    pub fn patch(
6159        &mut self,
6160        table: &str,
6161        row_id: &str,
6162        partial: Map<String, Value>,
6163        base_version: Option<i64>,
6164    ) -> Result<String, ClientError> {
6165        let prior = self.storage_failure.borrow_mut().take();
6166        let outcome = self.perform_patch(table, row_id, partial, base_version);
6167        self.finish_authoring(prior, outcome)
6168    }
6169
6170    /// The unscoped patch body. Public authoring entry points scope
6171    /// `storage_failure` around the whole operation, including any pre-read.
6172    fn perform_patch(
6173        &mut self,
6174        table: &str,
6175        row_id: &str,
6176        partial: Map<String, Value>,
6177        base_version: Option<i64>,
6178    ) -> Result<String, ClientError> {
6179        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6180        Ok(self.record_outbox_commit(vec![operation], None)?)
6181    }
6182
6183    fn prepare_patch(
6184        &self,
6185        table: &str,
6186        row_id: &str,
6187        partial: Map<String, Value>,
6188        base_version: Option<i64>,
6189    ) -> Result<OutboxOp, ClientError> {
6190        let schema_table = self
6191            .schema
6192            .table(table)
6193            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6194        let partial = normalize_values_casing(schema_table, partial)?;
6195        if let Some(pk) = partial.get(&schema_table.primary_key) {
6196            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6197                return Err(invalid_authoring(
6198                    "patch cannot change the primary key",
6199                    json!({ "table": table }),
6200                ));
6201            }
6202        }
6203        if !self.visible_row_exists(schema_table, row_id)? {
6204            return Err("sync.row_missing: a sparse patch requires a local row"
6205                .to_owned()
6206                .into());
6207        }
6208        self.refuse_sealed_row(schema_table, row_id)?;
6209        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6210        // server accepts a present scope column whose value equals the
6211        // stored row and applies it as a no-op, so a decoded envelope
6212        // round-trips. The local row is the only value the client can prove
6213        // equality against; when the row is absent locally the client cannot
6214        // prove equality against the server's stored row and fails closed.
6215        // A primary key that is also a scope column needs no stored-row
6216        // read: the primary key in a sparse payload is by construction the
6217        // row id being patched (§6.1), so its value is proven equal already.
6218        for scope in &schema_table.scope_variables {
6219            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6220                continue;
6221            }
6222            let Some(column) = schema_table
6223                .columns
6224                .iter()
6225                .find(|column| column.name == scope.column)
6226            else {
6227                continue;
6228            };
6229            let stored = self
6230                .conn
6231                .query_row(
6232                    &format!(
6233                        "SELECT {} FROM {} WHERE {}",
6234                        quote_ident(&scope.column),
6235                        visible_table(&schema_table.name),
6236                        row_id_predicate(schema_table)
6237                    ),
6238                    rusqlite::params![row_id],
6239                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6240                )
6241                .optional()
6242                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6243            // Decode the supplied value through the same column-type path the
6244            // rest of the patch encoding uses, so a value that only equals
6245            // the stored one in another number representation still compares
6246            // equal.
6247            let incoming = json_to_column_value(column, partial.get(&scope.column))
6248                .ok()
6249                .map(|value| column_value_to_json(&value));
6250            // A stored blob never equals a patch value, matching the server's
6251            // scope-column comparison (§3.4): bytes are unstorable as scope
6252            // values.
6253            let equal = stored.as_ref().is_some_and(|stored| {
6254                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6255            });
6256            if !equal {
6257                return Err(invalid_authoring(
6258                    "patch cannot write a scope column (§3.4)",
6259                    json!({ "table": table, "column": scope.column }),
6260                ));
6261            }
6262        }
6263        let mut values = partial;
6264        // Presence-set semantics: a proven-equal scope column is dropped, so
6265        // the patch leaves the stored value untouched (§6.2).
6266        for scope in &schema_table.scope_variables {
6267            values.remove(&scope.column);
6268        }
6269        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6270            // §2.4: the rowId of an integer key is its canonical decimal form.
6271            ColumnType::Integer => match row_id.parse::<i64>() {
6272                Ok(v)
6273                    if v.to_string() == row_id
6274                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6275                            .contains(&v) =>
6276                {
6277                    Value::from(v)
6278                }
6279                _ => {
6280                    return Err(format!(
6281                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6282                )
6283                    .into())
6284                }
6285            },
6286            ColumnType::Boolean => match row_id {
6287                "true" => Value::Bool(true),
6288                "false" => Value::Bool(false),
6289                _ => {
6290                    return Err(format!(
6291                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6292                )
6293                    .into())
6294                }
6295            },
6296            _ => Value::from(row_id.to_owned()),
6297        };
6298        values.insert(schema_table.primary_key.clone(), pk_value);
6299        // §6.1: a patch cannot null a non-nullable column.
6300        if let Some(column) = schema_table
6301            .columns
6302            .iter()
6303            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6304        {
6305            return Err(invalid_authoring(
6306                "a non-nullable column is null (§6.1)",
6307                json!({ "table": table, "column": column.name }),
6308            ));
6309        }
6310        // §5.11: structural validation only; an absent key-id selector is
6311        // resolved at the push seam from the stored local row, and a
6312        // key-selection or unknown-key failure becomes a durable
6313        // `client.encrypt_failed` rejection there (§10.3), never an
6314        // author-time error.
6315        self.validate_author_encode(schema_table, row_id, &values)?;
6316        Ok(OutboxOp {
6317            upsert: true,
6318            table: table.to_owned(),
6319            row_id: row_id.to_owned(),
6320            base_version,
6321            values: Some(values),
6322            insert: false,
6323        })
6324    }
6325
6326    /// §5.11 key-id fallback: the stored local row's selector value as a
6327    /// positional row (only the selector slot filled). Returns None when no
6328    /// selector is configured, the row is locally absent, or the stored
6329    /// value is not a usable key id. A failed storage read is an error.
6330    fn stored_key_fallback(
6331        &self,
6332        table: &TableSchema,
6333        row_id: &str,
6334    ) -> Result<Option<Row>, String> {
6335        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6336            return Ok(None);
6337        };
6338        let Some(index) = table
6339            .columns
6340            .iter()
6341            .position(|column| &column.name == selector)
6342        else {
6343            return Ok(None);
6344        };
6345        let sql = format!(
6346            "SELECT {} FROM {} WHERE {}",
6347            quote_ident(selector),
6348            visible_table(&table.name),
6349            row_id_predicate(table)
6350        );
6351        let key = self
6352            .conn
6353            .query_row(&sql, rusqlite::params![row_id], |row| {
6354                row.get::<_, Option<String>>(0)
6355            })
6356            .optional()
6357            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6358            .flatten()
6359            .filter(|key| !key.is_empty());
6360        let Some(key) = key else {
6361            return Ok(None);
6362        };
6363        let mut fallback: Row = vec![None; table.columns.len()];
6364        fallback[index] = Some(ColumnValue::String(key));
6365        Ok(Some(fallback))
6366    }
6367
6368    /// §5.11/§10.3 author-time encode check: a structural failure (a
6369    /// non-nullable NULL, a value the codec rejects) is an author-time
6370    /// error; a key-selection or unknown-key failure is deferred to the push
6371    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6372    fn validate_author_encode(
6373        &self,
6374        table: &TableSchema,
6375        row_id: &str,
6376        values: &Map<String, Value>,
6377    ) -> Result<(), String> {
6378        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6379            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6380            result => result.map(|_| ()),
6381        }
6382    }
6383
6384    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6385    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6386    /// raise one durable rejection per commit. `sync()` never aborts for an
6387    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6388    /// undoing the dropped commit's purely-optimistic rows.
6389    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6390    /// (an envelope or a type an earlier version accepted). `with_encryption`
6391    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6392    /// the startup boundary leaves those to the send seam, where hosts configure
6393    /// keys after open.
6394    /// Recovery runs as one observation. An enclosing observation (the guarded
6395    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6396    /// domains join that change event instead of publishing a second one.
6397    fn drop_unencodable_outbox(
6398        &mut self,
6399        with_encryption: bool,
6400        batch: Option<&mut ChangeAccumulator>,
6401    ) -> Result<bool, String> {
6402        let mut failures = Vec::new();
6403        for commit in self.outbox.iter() {
6404            for op in &commit.ops {
6405                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6406                    continue;
6407                };
6408                // An insert lacking a non-nullable column is left to
6409                // `drop_incompatible_outbox` (§7.4.4).
6410                let Some(values) = op.presented_values(table) else {
6411                    continue;
6412                };
6413                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6414                let failure = match encode_sparse_row_json(
6415                    table,
6416                    &op.row_id,
6417                    &values,
6418                    &self.encryption,
6419                    fallback.as_ref(),
6420                ) {
6421                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6422                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6423                    }
6424                    // §7.1/§7.4.4: a value the current codec refuses was
6425                    // persisted by a version that accepted it; it leaves
6426                    // through the incompatible-outbox path, so the FIFO
6427                    // queue never wedges.
6428                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6429                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6430                        OUTBOX_INCOMPATIBLE_CODE,
6431                    )),
6432                    _ => None,
6433                };
6434                if let Some((error, code)) = failure {
6435                    failures.push((commit.clone(), error, code));
6436                    break;
6437                }
6438            }
6439        }
6440        if failures.is_empty() {
6441            return Ok(false);
6442        }
6443        // §7.4.4: one observation covers the durable outcome writes, the outbox
6444        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6445        // leaves the replica and the client unchanged, a successful recovery
6446        // publishes one batch with the rejection, outcome, and table domains, and
6447        // an ordinary open with nothing to recover publishes nothing.
6448        let rollback = self.capture();
6449        let owns_observation = batch.is_none();
6450        let owns_transaction = self.conn.is_autocommit();
6451        let mut domains = ChangeAccumulator::default();
6452        if owns_observation {
6453            self.begin_observation("syncular_value_recovery")?;
6454        }
6455        let dropped = (|| -> Result<(), String> {
6456            for (commit, message, code) in failures {
6457                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6458                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6459                }
6460                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6461                let rejection = RejectionRecord {
6462                    client_commit_id: commit.client_commit_id.clone(),
6463                    op_index: 0,
6464                    code: code.to_owned(),
6465                    message,
6466                    retryable: false,
6467                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6468                        field_paths: None,
6469                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6470                        required_action: None,
6471                        references: None,
6472                    }),
6473                    operation: representable
6474                        .then(|| commit.ops.first())
6475                        .flatten()
6476                        .map(CommitOperation::from),
6477                };
6478                self.persist_commit_outcome(
6479                    &commit.client_commit_id,
6480                    CommitOutcomeStatus::Rejected,
6481                    &[CommitOperationOutcome::Error {
6482                        rejection: rejection.clone(),
6483                    }],
6484                    representable.then_some(commit.ops.as_slice()),
6485                )?;
6486                self.delete_outbox_persisted(&commit.client_commit_id)?;
6487                self.outbox
6488                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6489                self.rejections.push(rejection);
6490                for operation in &commit.ops {
6491                    domains.table(operation.table.as_str());
6492                }
6493            }
6494            domains.status = true;
6495            domains.rejections = true;
6496            domains.outcomes = true;
6497            self.prune_commit_outcomes()?;
6498            // The visible projection is re-derived from the surviving base + outbox.
6499            self.overlay_dirty.set(true);
6500            self.rebuild_overlay()?;
6501            Ok(())
6502        })();
6503        let result = match batch {
6504            Some(batch) => {
6505                if dropped.is_ok() {
6506                    for table in domains.tables.keys() {
6507                        batch.table(table.as_str());
6508                    }
6509                    batch.status |= domains.status;
6510                    batch.rejections |= domains.rejections;
6511                    batch.outcomes |= domains.outcomes;
6512                }
6513                dropped
6514            }
6515            None => {
6516                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6517            }
6518        };
6519        if let Err(error) = result {
6520            if owns_observation {
6521                if owns_transaction {
6522                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6523                        let mut failure = self.storage_failure.borrow_mut();
6524                        let failure =
6525                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6526                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6527                    }
6528                } else {
6529                    self.rollback_observation("syncular_value_recovery");
6530                }
6531            }
6532            self.restore(rollback);
6533            return Err(error);
6534        }
6535        Ok(true)
6536    }
6537
6538    pub fn pending_commit_ids(&self) -> Vec<String> {
6539        self.outbox
6540            .iter()
6541            .map(|c| c.client_commit_id.clone())
6542            .collect()
6543    }
6544
6545    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6546    /// FIFO — exactly what the next push round would put on the wire. Used by
6547    /// conformance B.21(h) to pin cross-core byte identity.
6548    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6549        let mut payloads = Vec::new();
6550        for commit in self.outbox.iter() {
6551            for op in &commit.ops {
6552                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6553                    continue;
6554                };
6555                let Some(values) = op.presented_values(table) else {
6556                    continue;
6557                };
6558                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6559                if let Ok(payload) = encode_sparse_row_json(
6560                    table,
6561                    &op.row_id,
6562                    &values,
6563                    &self.encryption,
6564                    fallback.as_ref(),
6565                ) {
6566                    payloads.push(payload);
6567                }
6568            }
6569        }
6570        Ok(payloads)
6571    }
6572
6573    pub fn conflicts(&self) -> &[ConflictRecord] {
6574        &self.conflicts
6575    }
6576
6577    pub fn rejections(&self) -> &[RejectionRecord] {
6578        &self.rejections
6579    }
6580
6581    pub fn resolve_commit_outcome(
6582        &mut self,
6583        input: ResolveCommitOutcomeInput,
6584    ) -> Result<CommitOutcome, String> {
6585        let current = self
6586            .commit_outcome(&input.client_commit_id)?
6587            .ok_or_else(|| {
6588                format!(
6589                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6590                    input.client_commit_id
6591                )
6592            })?;
6593        if current.resolution != CommitOutcomeResolution::Active {
6594            return Ok(current);
6595        }
6596        if input.resolution == CommitOutcomeResolution::Active {
6597            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6598        }
6599        match input.resolution {
6600            CommitOutcomeResolution::Superseded => {
6601                let replacement = input
6602                    .replacement_client_commit_id
6603                    .as_deref()
6604                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6605                    .ok_or_else(|| {
6606                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6607                            .to_owned()
6608                    })?;
6609                let _ = replacement;
6610            }
6611            _ if input.replacement_client_commit_id.is_some() => {
6612                return Err(
6613                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6614                        .to_owned(),
6615                );
6616            }
6617            _ => {}
6618        }
6619        let allowed = match current.status {
6620            CommitOutcomeStatus::Conflict => matches!(
6621                input.resolution,
6622                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6623            ),
6624            CommitOutcomeStatus::Rejected => {
6625                matches!(
6626                    input.resolution,
6627                    CommitOutcomeResolution::Superseded
6628                        | CommitOutcomeResolution::ResolvedKeepServer
6629                )
6630            }
6631            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6632                input.resolution == CommitOutcomeResolution::Dismissed
6633            }
6634        };
6635        if !allowed {
6636            return Err(format!(
6637                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6638                input.resolution, current.status
6639            ));
6640        }
6641
6642        let rollback = self.capture();
6643        self.begin_observation("syncular_outcome_resolution")?;
6644        let result = (|| {
6645            self.conn
6646                .execute(
6647                    "UPDATE _syncular_commit_outcomes
6648                        SET resolution = ?1, resolved_at_ms = ?2,
6649                            replacement_client_commit_id = ?3
6650                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6651                    rusqlite::params![
6652                        Self::outcome_resolution_name(input.resolution),
6653                        self.clock_now_ms(),
6654                        input.replacement_client_commit_id,
6655                        input.client_commit_id
6656                    ],
6657                )
6658                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6659            let resolved = self
6660                .commit_outcome(&current.client_commit_id)?
6661                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6662            self.conn
6663                .execute(
6664                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6665                    rusqlite::params![input.client_commit_id],
6666                )
6667                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6668            for (commit, _) in self.failed_commits.iter() {
6669                if commit.client_commit_id == input.client_commit_id {
6670                    for operation in &commit.ops {
6671                        self.overlay_dirty.table(&operation.table);
6672                    }
6673                }
6674            }
6675            self.failed_commits
6676                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6677            self.prune_commit_outcomes()?;
6678            Ok(resolved)
6679        })();
6680        let resolved = match result {
6681            Ok(outcome) => outcome,
6682            Err(error) => {
6683                self.rollback_observation("syncular_outcome_resolution");
6684                self.restore(rollback);
6685                return Err(error);
6686            }
6687        };
6688        self.conflicts
6689            .retain(|record| record.client_commit_id != current.client_commit_id);
6690        self.rejections
6691            .retain(|record| record.client_commit_id != current.client_commit_id);
6692        let mut batch = ChangeAccumulator {
6693            conflicts: current.status == CommitOutcomeStatus::Conflict,
6694            rejections: current.status == CommitOutcomeStatus::Rejected,
6695            outcomes: true,
6696            ..ChangeAccumulator::default()
6697        };
6698        if let Some(operations) = &current.operations {
6699            for operation in operations {
6700                batch.table(&operation.table);
6701            }
6702        }
6703        if let Err(error) = self
6704            .rebuild_overlay_if_dirty()
6705            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6706        {
6707            self.rollback_observation("syncular_outcome_resolution");
6708            self.restore(rollback);
6709            return Err(error);
6710        }
6711        Ok(resolved)
6712    }
6713
6714    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6715        self.schema_floor.as_ref()
6716    }
6717
6718    /// §7.3.5: the client's opaque auth-lease state, if any.
6719    pub fn lease_state(&self) -> Option<&LeaseState> {
6720        self.lease_state.as_ref()
6721    }
6722
6723    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6724    /// the two lease codes set it; other errors leave leaseState untouched.
6725    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6726        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6727            return Ok(());
6728        }
6729        let mut next = self.lease_state.clone().unwrap_or_default();
6730        next.error_code = Some(code.to_owned());
6731        self.set_lease_state(Some(next))
6732    }
6733
6734    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6735        if self.lease_state == next {
6736            return Ok(());
6737        }
6738        self.begin_observation("syncular_lease")?;
6739        let rollback = self.capture();
6740        self.lease_state = next;
6741        let persisted = match &self.lease_state {
6742            Some(lease) => serde_json::to_string(lease)
6743                .map_err(|error| error.to_string())
6744                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6745            None => self.delete_meta(LEASE_STATE_KEY),
6746        };
6747        let batch = ChangeAccumulator {
6748            status: true,
6749            ..ChangeAccumulator::default()
6750        };
6751        if let Err(error) =
6752            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6753        {
6754            self.rollback_observation("syncular_lease");
6755            self.restore(rollback);
6756            return Err(error);
6757        }
6758        Ok(())
6759    }
6760
6761    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6762        if self.schema_floor == next {
6763            return Ok(());
6764        }
6765        self.begin_observation("syncular_schema_floor")?;
6766        let rollback = self.capture();
6767        self.stopped = next.is_some();
6768        self.schema_floor = next;
6769        let persisted = match &self.schema_floor {
6770            Some(floor) => serde_json::to_string(floor)
6771                .map_err(|error| error.to_string())
6772                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6773            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6774        };
6775        let batch = ChangeAccumulator {
6776            status: true,
6777            ..ChangeAccumulator::default()
6778        };
6779        if let Err(error) =
6780            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6781        {
6782            self.rollback_observation("syncular_schema_floor");
6783            self.restore(rollback);
6784            return Err(error);
6785        }
6786        Ok(())
6787    }
6788
6789    fn set_upgrading(&mut self, value: bool) {
6790        if self.upgrading == value {
6791            return;
6792        }
6793        if self.begin_observation("syncular_upgrading").is_err() {
6794            return;
6795        }
6796        self.upgrading = value;
6797        let batch = ChangeAccumulator {
6798            status: true,
6799            ..ChangeAccumulator::default()
6800        };
6801        if self
6802            .finish_observation("syncular_upgrading", batch)
6803            .is_err()
6804        {
6805            self.rollback_observation("syncular_upgrading");
6806        }
6807    }
6808
6809    pub fn sync_needed(&self) -> bool {
6810        self.sync_needed
6811    }
6812
6813    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6814        let schema_table = self
6815            .schema
6816            .table(table)
6817            .ok_or_else(|| format!("unknown table {table:?}"))?;
6818        let sql = format!(
6819            "SELECT * FROM {} ORDER BY {} ASC",
6820            visible_table(table),
6821            quote_ident(&schema_table.primary_key)
6822        );
6823        let mut stmt = self
6824            .conn
6825            .prepare(&sql)
6826            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6827        let mut rows = stmt
6828            .query([])
6829            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6830        let mut out = Vec::new();
6831        while let Some(row) = rows
6832            .next()
6833            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6834        {
6835            let mut values = Map::new();
6836            for (i, column) in schema_table.columns.iter().enumerate() {
6837                let value = row
6838                    .get_ref(i)
6839                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6840                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6841            }
6842            let version: i64 = row
6843                .get(schema_table.columns.len())
6844                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6845            let row_id = match values.get(&schema_table.primary_key) {
6846                Some(Value::String(s)) => s.clone(),
6847                Some(Value::Number(n)) => n.to_string(),
6848                Some(Value::Bool(b)) => b.to_string(),
6849                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6850            };
6851            let sealed_key = if schema_table.has_encrypted_columns() {
6852                row.get(schema_table.columns.len() + 1)
6853                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6854            } else {
6855                None
6856            };
6857            out.push(RowState {
6858                row_id,
6859                version,
6860                values,
6861                sealed_key,
6862            });
6863        }
6864        Ok(out)
6865    }
6866
6867    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6868    //
6869    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6870    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6871    // the op with `yrs`, re-encodes the whole doc state, and records a
6872    // baseVersion-less sparse upsert carrying only the primary key and the
6873    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6874    // the overlay re-materializes the edit immediately (optimistic apply,
6875    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6876    // Byte-compatible with `@syncular/crdt-yjs`.
6877
6878    /// The current stored value of a `crdt` column for one row — the visible
6879    /// (optimistic) bytes, or `None` when the row is absent or the column is
6880    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6881    /// `crdt` column (guards the app against a typo'd column name).
6882    #[cfg(feature = "crdt-yjs")]
6883    fn crdt_column_bytes(
6884        &self,
6885        table: &str,
6886        row_id: &str,
6887        column: &str,
6888    ) -> Result<Option<Vec<u8>>, String> {
6889        let schema_table = self
6890            .schema
6891            .table(table)
6892            .ok_or_else(|| format!("unknown table {table:?}"))?;
6893        let col = schema_table
6894            .columns
6895            .iter()
6896            .find(|c| c.name == column)
6897            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6898        if col.ty != ColumnType::Crdt {
6899            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6900        }
6901        let sql = format!(
6902            "SELECT {} FROM {} WHERE {}",
6903            quote_ident(column),
6904            visible_table(table),
6905            row_id_predicate(schema_table)
6906        );
6907        let bytes: Option<Vec<u8>> = self
6908            .conn
6909            .prepare_cached(&sql)
6910            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6911            .query_row(rusqlite::params![row_id], |row| {
6912                row.get::<_, Option<Vec<u8>>>(0)
6913            })
6914            .map_err(|e| match e {
6915                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6916                other => Self::sqlite_failure(&self.storage_failure, other),
6917            })?;
6918        Ok(bytes)
6919    }
6920
6921    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6922    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
6923    /// An absent row / NULL column is the empty document (empty string).
6924    #[cfg(feature = "crdt-yjs")]
6925    pub fn crdt_text(
6926        &self,
6927        table: &str,
6928        row_id: &str,
6929        column: &str,
6930        name: &str,
6931    ) -> Result<String, String> {
6932        let bytes = self
6933            .crdt_column_bytes(table, row_id, column)?
6934            .unwrap_or_default();
6935        crate::crdt::text(&bytes, name)
6936    }
6937
6938    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
6939    /// record the resulting crdt-only sparse operation. Returns the enqueued
6940    /// `clientCommitId`.
6941    #[cfg(feature = "crdt-yjs")]
6942    pub fn crdt_insert_text(
6943        &mut self,
6944        table: &str,
6945        row_id: &str,
6946        column: &str,
6947        name: &str,
6948        index: u32,
6949        value: &str,
6950    ) -> Result<String, ClientError> {
6951        let prior = self.storage_failure.borrow_mut().take();
6952        let outcome = (|| -> Result<String, ClientError> {
6953            let current = self
6954                .crdt_column_bytes(table, row_id, column)?
6955                .unwrap_or_default();
6956            let update = crate::crdt::insert_text(&current, name, index, value)?;
6957            self.perform_crdt_push_update(table, row_id, column, &update)
6958        })();
6959        self.finish_authoring(prior, outcome)
6960    }
6961
6962    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
6963    /// record the resulting crdt-only sparse operation. Returns the enqueued
6964    /// `clientCommitId`.
6965    #[cfg(feature = "crdt-yjs")]
6966    pub fn crdt_delete_text(
6967        &mut self,
6968        table: &str,
6969        row_id: &str,
6970        column: &str,
6971        name: &str,
6972        index: u32,
6973        len: u32,
6974    ) -> Result<String, ClientError> {
6975        let prior = self.storage_failure.borrow_mut().take();
6976        let outcome = (|| -> Result<String, ClientError> {
6977            let current = self
6978                .crdt_column_bytes(table, row_id, column)?
6979                .unwrap_or_default();
6980            let update = crate::crdt::delete_text(&current, name, index, len)?;
6981            self.perform_crdt_push_update(table, row_id, column, &update)
6982        })();
6983        self.finish_authoring(prior, outcome)
6984    }
6985
6986    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
6987    /// `crdt` column's current state and record the resulting crdt-only sparse
6988    /// operation. The app authored the update with its own `yrs` model.
6989    /// Returns the enqueued `clientCommitId`.
6990    #[cfg(feature = "crdt-yjs")]
6991    pub fn crdt_apply_update(
6992        &mut self,
6993        table: &str,
6994        row_id: &str,
6995        column: &str,
6996        update: &[u8],
6997    ) -> Result<String, ClientError> {
6998        let prior = self.storage_failure.borrow_mut().take();
6999        let outcome = (|| -> Result<String, ClientError> {
7000            let current = self
7001                .crdt_column_bytes(table, row_id, column)?
7002                .unwrap_or_default();
7003            let next = crate::crdt::apply_update(&current, update)?;
7004            self.perform_crdt_push_update(table, row_id, column, &next)
7005        })();
7006        self.finish_authoring(prior, outcome)
7007    }
7008
7009    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7010    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7011    /// enqueue it. The server merges the crdt column and leaves every other
7012    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7013    /// clobber a concurrent edit to another column. A locally absent row
7014    /// records the same partial operation; the server answers per §6.2.
7015    #[cfg(feature = "crdt-yjs")]
7016    fn perform_crdt_push_update(
7017        &mut self,
7018        table: &str,
7019        row_id: &str,
7020        column: &str,
7021        crdt_bytes: &[u8],
7022    ) -> Result<String, ClientError> {
7023        let mut bytes_obj = Map::new();
7024        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7025        let mut partial = Map::new();
7026        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7027        self.perform_patch(table, row_id, partial, None)
7028    }
7029
7030    /// Run an arbitrary read-only SQL query against the local database and
7031    /// return each row as a `column-name → JSON value` map. This is the seam
7032    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7033    /// SQL over the visible tables/views, not a fixed table read like
7034    /// [`read_rows`]).
7035    ///
7036    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7037    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7038    /// same envelope the command surface uses everywhere else. Output BLOB
7039    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7040    ///
7041    /// The result column typing is dynamic (SQLite's stored affinity), because
7042    /// arbitrary SQL can alias, join, and compute — there is no schema column
7043    /// to consult per output cell, unlike [`read_rows`].
7044    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7045        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7046    }
7047
7048    /// Repository benchmark and conformance access, present only with `bench-internals`.
7049    #[cfg(feature = "bench-internals")]
7050    #[doc(hidden)]
7051    pub fn benchmark_connection(&mut self) -> &mut Connection {
7052        &mut self.conn
7053    }
7054
7055    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7056    #[cfg(feature = "bench-internals")]
7057    #[doc(hidden)]
7058    pub fn benchmark_phases(
7059        &self,
7060        enabled: Option<bool>,
7061        reset: bool,
7062    ) -> Result<Option<Value>, String> {
7063        self.benchmark_phases.configure(enabled, reset)?;
7064        Ok(self.benchmark_phases.snapshot())
7065    }
7066
7067    /// One serialized, read-only authority observation, including durable coverage.
7068    pub fn authority_snapshot(&self) -> Result<Value, String> {
7069        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7070        if self.limits.authority_reads.is_empty() {
7071            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7072        }
7073        self.conn
7074            .execute_batch("SAVEPOINT authority_snapshot")
7075            .map_err(|e| e.to_string())?;
7076        let result = (|| {
7077            let revision = self
7078                .get_meta(LOCAL_REVISION_KEY)?
7079                .ok_or("sync.local_corrupt: authority revision is missing")?;
7080            let parsed_revision = revision
7081                .parse::<u64>()
7082                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7083            if parsed_revision.to_string() != revision {
7084                return Err("sync.local_corrupt: authority revision is invalid".into());
7085            }
7086            let mut tables = Vec::new();
7087            for read in &self.limits.authority_reads {
7088                let table = self
7089                    .schema
7090                    .table(&read.table)
7091                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7092                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7093                    let ty = table
7094                        .columns
7095                        .iter()
7096                        .find(|c| c.name == column)
7097                        .expect("validated authority column")
7098                        .ty;
7099                    if ty == ColumnType::Boolean {
7100                        if let Some(value) = value.as_bool() {
7101                            return Ok(value.to_string());
7102                        }
7103                        return match value.as_i64() {
7104                            Some(0) => Ok("false".into()),
7105                            Some(1) => Ok("true".into()),
7106                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7107                        };
7108                    }
7109                    match value {
7110                        Value::String(s) => Ok(s.clone()),
7111                        Value::Number(n) => Ok(n.to_string()),
7112                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7113                            .get("$bigint")
7114                            .and_then(Value::as_str)
7115                            .map(str::to_owned)
7116                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7117                        _ => {
7118                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7119                        }
7120                    }
7121                };
7122                let matches = |row: &QueryRow| -> Result<bool, String> {
7123                    for (variable, values) in &read.scopes {
7124                        let column = table
7125                            .scope_column(variable)
7126                            .expect("validated authority scope");
7127                        let value = row
7128                            .get(column)
7129                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7130                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7131                            return Ok(false);
7132                        }
7133                    }
7134                    Ok(true)
7135                };
7136                let selected = read
7137                    .columns
7138                    .iter()
7139                    .map(|c| quote_ident(c))
7140                    .collect::<Vec<_>>()
7141                    .join(",");
7142                let mut intent = BTreeSet::new();
7143                let mut relevant = BTreeSet::new();
7144                for row in query_connection(&self.conn,
7145                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7146                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7147                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7148                    intent.insert(id.clone());
7149                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7150                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7151                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7152                    }
7153                }
7154                let mut rows = BTreeMap::new();
7155                for full in [base_table(&read.table), visible_table(&read.table)] {
7156                    for mut values in query_connection(
7157                        &self.conn,
7158                        &format!(
7159                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7160                        ),
7161                        &[],
7162                    )
7163                    .map_err(|e| e.to_string())?
7164                    {
7165                        if !matches(&values)? {
7166                            continue;
7167                        }
7168                        let id = scalar(
7169                            &table.primary_key,
7170                            values
7171                                .get(&table.primary_key)
7172                                .expect("selected primary key"),
7173                        )?;
7174                        relevant.insert(id.clone());
7175                        if full == base_table(&read.table) {
7176                            let version = values
7177                                .remove("authority_version")
7178                                .expect("selected version");
7179                            let version = version
7180                                .as_i64()
7181                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7182                            if version >= 0 {
7183                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7184                            }
7185                        }
7186                    }
7187                }
7188                intent.retain(|id| relevant.contains(id));
7189                let rows = rows.into_values().collect::<Vec<_>>();
7190                // Every unit covering the table, as its (unit, table) pair (§4.4).
7191                let replica = self.stored(replica_cursor(&self.conn))?;
7192                let persisted = self
7193                    .stored(load_units(&self.conn))?
7194                    .into_iter()
7195                    .filter(|unit| covers(table, &unit.scopes))
7196                    .map(|unit| {
7197                        let effective = unit.echo.as_deref().and_then(|echo| {
7198                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7199                        });
7200                        let requested = read
7201                            .scopes
7202                            .keys()
7203                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7204                            .collect::<Map<_, _>>();
7205                        let complete = effective
7206                            .as_ref()
7207                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7208                            && unit.status == UnitStatus::Active
7209                            && !unit.bootstrapping();
7210                        let exposed = effective.map(|scopes| {
7211                            read.scopes
7212                                .keys()
7213                                .map(|v| {
7214                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7215                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7216                                })
7217                                .collect::<Map<_, _>>()
7218                        });
7219                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7220                    })
7221                    .collect::<Vec<_>>();
7222                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7223                    match selectors.split_first() {
7224                        None => !candidates.is_empty(),
7225                        Some(((variable, values), rest)) => values.iter().all(|value| {
7226                            covered(
7227                                rest,
7228                                &candidates
7229                                    .iter()
7230                                    .copied()
7231                                    .filter(|p| {
7232                                        p["effectiveScopes"][variable.as_str()]
7233                                            .as_array()
7234                                            .is_some_and(|held| {
7235                                                held.contains(&Value::String(value.clone()))
7236                                                    || held.contains(&Value::String("*".into()))
7237                                            })
7238                                    })
7239                                    .collect::<Vec<_>>(),
7240                            )
7241                        }),
7242                    }
7243                }
7244                let coverage = if covered(
7245                    &read.scopes.iter().collect::<Vec<_>>(),
7246                    &persisted
7247                        .iter()
7248                        .filter(|p| p["complete"] == true)
7249                        .collect::<Vec<_>>(),
7250                ) {
7251                    "complete"
7252                } else if persisted.iter().any(|p| {
7253                    p["status"] == "active"
7254                        && read.scopes.iter().all(|(v, values)| {
7255                            values.iter().any(|value| {
7256                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7257                                    held.contains(&Value::String(value.clone()))
7258                                })
7259                            })
7260                        })
7261                }) {
7262                    "pending"
7263                } else {
7264                    "missing"
7265                };
7266                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7267            }
7268            Ok(
7269                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7270            )
7271        })();
7272        match result {
7273            Ok(snapshot) => {
7274                self.conn
7275                    .execute_batch("RELEASE authority_snapshot")
7276                    .map_err(|e| e.to_string())?;
7277                Ok(snapshot)
7278            }
7279            Err(error) => {
7280                self.conn
7281                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7282                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7283                Err(error)
7284            }
7285        }
7286    }
7287
7288    /// Rows, coverage, and local revision from one SQLite read snapshot.
7289    /// §7.5 atomic snapshot read. An `owner` records a failure in
7290    /// `diagnostics_snapshot().query_failures` until its next successful read.
7291    pub fn query_snapshot(
7292        &mut self,
7293        sql: &str,
7294        params: &[QueryValue],
7295        coverage: &[WindowCoverage],
7296        owner: Option<QueryOwner<'_>>,
7297    ) -> Result<QuerySnapshot, String> {
7298        let result = snapshot_connection(&self.conn, sql, params, coverage);
7299        if let Some(owner) = owner {
7300            self.record_query_read(owner, result.as_ref().err());
7301        }
7302        result.map_err(|failure| failure.to_string())
7303    }
7304
7305    /// §7.6: record the outcome of one owned snapshot read, including reads a
7306    /// host ran on a separate connection. A success removes the owner's entry;
7307    /// an identical failure leaves it unchanged; a changed failure moves last.
7308    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7309        let position = self
7310            .query_failures
7311            .iter()
7312            .position(|entry| entry.id == owner.id);
7313        let Some(failure) = failure else {
7314            if let Some(index) = position {
7315                self.query_failures.remove(index);
7316            }
7317            return;
7318        };
7319        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7320            .into_iter()
7321            .map(str::to_owned)
7322            .collect();
7323        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7324        if position.is_some_and(|index| {
7325            let entry = &self.query_failures[index];
7326            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7327        }) {
7328            return;
7329        }
7330        if let Some(index) = position {
7331            self.query_failures.remove(index);
7332        }
7333        self.query_failures.push(DiagnosticQueryFailure {
7334            id: owner.id.to_owned(),
7335            tables,
7336            code,
7337            sqlite_code: failure.sqlite_code,
7338            at_ms: self.clock_now_ms(),
7339        });
7340        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7341            self.query_failures.remove(0);
7342        }
7343    }
7344
7345    // -- request building ---------------------------------------------------------
7346
7347    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7348        #[cfg(feature = "bench-internals")]
7349        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7350        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7351            Box::new(SyncOutcome::Failed {
7352                error_code: "storage.failed".into(),
7353                message,
7354                details: None,
7355                recommended_action: None,
7356            })
7357        })?;
7358        let max_commits = self.limits.max_push_commits_per_request;
7359        let max_ops = self
7360            .limits
7361            .max_push_operations_per_request
7362            .unwrap_or(PUSH_OPS_PER_REQUEST);
7363        let byte_cap = self.limits.max_push_request_bytes;
7364        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7365        // bare URL — capability negotiation, decided per transport.
7366        let accept = self.limits.accept.unwrap_or(if url_capable {
7367            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7368        } else {
7369            DEFAULT_ACCEPT
7370        });
7371        let storage_failed = |message: String| {
7372            Box::new(SyncOutcome::Failed {
7373                error_code: "storage.failed".into(),
7374                message,
7375                details: None,
7376                recommended_action: None,
7377            })
7378        };
7379        let header = Frame::ReqHeader {
7380            client_id: self.client_id.clone(),
7381            schema_version: self.schema.version,
7382            log_epoch: log_epoch.clone(),
7383            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7384        };
7385        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7386        // which stays ahead of every push, pull, unit and progress frame; the
7387        // byte-capacity probes below measure this exact sequence.
7388        let mut lead = vec![header];
7389        lead.extend(
7390            self.round_failure
7391                .iter()
7392                .map(|code| Frame::RoundFailure { code: code.clone() }),
7393        );
7394        let registered: Vec<UnitRecord> = self
7395            .stored(load_units(&self.conn))
7396            .map_err(storage_failed)?
7397            .into_iter()
7398            .filter(|unit| unit.status == UnitStatus::Active)
7399            .collect();
7400        let registration_digest = ssp2::registration::registration_digest(
7401            registered
7402                .iter()
7403                .map(|unit| scope_pairs(&unit.scopes))
7404                .collect::<Vec<_>>()
7405                .iter()
7406                .map(Vec::as_slice),
7407        );
7408        let echo_digest = self
7409            .get_meta(ECHO_DIGEST_KEY)
7410            .map_err(storage_failed)?
7411            .filter(|hex| !hex.is_empty())
7412            .map(|hex| {
7413                crate::values::hex_to_bytes(&hex)
7414                    .ok()
7415                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7416                    .ok_or_else(|| {
7417                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7418                    })
7419            })
7420            .transpose()
7421            .map_err(storage_failed)?;
7422        let pull = Frame::PullHeader {
7423            limit_commits: self.limits.limit_commits.unwrap_or(0),
7424            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7425            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7426            accept,
7427            cursor: self
7428                .stored(replica_cursor(&self.conn))
7429                .map_err(storage_failed)?,
7430            registration_digest,
7431            echo_digest,
7432        };
7433        let registration_digest = bytes_to_hex(&registration_digest);
7434        // §4.1: the list travels when it changed since the server stored it,
7435        // after `unregistered` and after a reset (both clear the stored digest).
7436        let travels = self
7437            .get_meta(REGISTERED_DIGEST_KEY)
7438            .map_err(storage_failed)?
7439            .as_deref()
7440            != Some(registration_digest.as_str());
7441        let mut sub_frames: Vec<Frame> = Vec::new();
7442        if travels {
7443            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7444                scopes: scope_pairs(&unit.scopes),
7445            }));
7446        }
7447        // §4.3: one PROGRESS per catching-up unit, in registration order.
7448        sub_frames.extend(registered.iter().filter_map(|unit| {
7449            Some(Frame::Progress {
7450                id: unit.id.clone(),
7451                cursor: unit.cursor?,
7452                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7453            })
7454        }));
7455        // The fixed frames (order-independent size) and the header+pull probe
7456        // both encode canonically, so the byte budget is the exact wire size.
7457        let fixed_bytes = match byte_cap {
7458            Some(_) => {
7459                let mut frames = lead.clone();
7460                frames.push(pull.clone());
7461                frames.extend(sub_frames.iter().cloned());
7462                encode_message(&Message {
7463                    wire_version: WIRE_VERSION,
7464                    msg_kind: MsgKind::Request,
7465                    frames,
7466                })
7467                .len()
7468            }
7469            None => 0,
7470        };
7471        if let Some(cap) = byte_cap {
7472            if fixed_bytes > cap {
7473                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7474            }
7475        }
7476        let base_bytes = match byte_cap {
7477            Some(_) => {
7478                let mut frames = lead.clone();
7479                frames.push(pull.clone());
7480                encode_message(&Message {
7481                    wire_version: WIRE_VERSION,
7482                    msg_kind: MsgKind::Request,
7483                    frames,
7484                })
7485                .len()
7486            }
7487            None => 0,
7488        };
7489        let total_outbox = if log_epoch.is_some() {
7490            self.outbox.len()
7491        } else {
7492            0
7493        };
7494        let mut push_frames: Vec<Frame> = Vec::new();
7495        let mut pushed_ids = Vec::new();
7496        let mut ops_in_request = 0usize;
7497        let mut bytes_in_request = 0usize;
7498        let mut deferred_commits = 0usize;
7499        #[cfg(feature = "bench-internals")]
7500        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7501        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7502            // §6.1/§7.1: the commit-count and operation caps are hard. A
7503            // first commit over the operation cap is a typed capacity error
7504            // and stays queued; a later one defers with the complete suffix.
7505            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7506                deferred_commits = total_outbox - index;
7507                break;
7508            }
7509            if ops_in_request + commit.ops.len() > max_ops {
7510                if push_frames.is_empty() {
7511                    return Err(Self::push_capacity_outcome(
7512                        "operations",
7513                        max_ops,
7514                        commit.ops.len(),
7515                        Some(commit.client_commit_id.as_str()),
7516                    ));
7517                }
7518                deferred_commits = total_outbox - index;
7519                break;
7520            }
7521            // §7.1 chained base: the commit waits for the change of the commit
7522            // it chains on, which supplies its rebased version.
7523            if self.outbox[..index]
7524                .iter()
7525                .flat_map(|earlier| &earlier.ops)
7526                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7527            {
7528                deferred_commits = total_outbox - index;
7529                break;
7530            }
7531            let operations = commit
7532                .ops
7533                .iter()
7534                .map(|op| {
7535                    let payload = match self
7536                        .schema
7537                        .table(&op.table)
7538                        .and_then(|table| Some((table, op.presented_values(table)?)))
7539                    {
7540                        Some((table, values)) => {
7541                            // §0: outbox entries encode at send time with the
7542                            // current codec (validated at mutate()). §6.1: the
7543                            // values map's keys are the sparse presence set; an
7544                            // insert-intent operation presents every column (§7.1).
7545                            // §5.11: encrypted columns are encrypted here, with
7546                            // an absent key-id selector read from the stored row.
7547                            // `sync_inner` runs `drop_unencodable_outbox` first,
7548                            // so an encrypt failure cannot reach this encode; the
7549                            // author seam validated every other failure.
7550                            let fallback =
7551                                self.stored_key_fallback(table, &op.row_id)
7552                                    .map_err(|message| {
7553                                        Box::new(SyncOutcome::Failed {
7554                                            error_code: "storage.failed".into(),
7555                                            message,
7556                                            details: None,
7557                                            recommended_action: None,
7558                                        })
7559                                    })?;
7560                            encode_sparse_row_json(
7561                                table,
7562                                &op.row_id,
7563                                &values,
7564                                &self.encryption,
7565                                fallback.as_ref(),
7566                            )
7567                            .ok()
7568                        }
7569                        None => None,
7570                    };
7571                    Ok(ssp2::model::Operation {
7572                        table: op.table.clone(),
7573                        row_id: op.row_id.clone(),
7574                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7575                        base_version: op.base_version,
7576                        payload,
7577                    })
7578                })
7579                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7580            // §6.3.2, §7.1: the commit's outbox sequence.
7581            let client_seq = self
7582                .conn
7583                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7584                .and_then(|mut statement| {
7585                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7586                })
7587                .map_err(|error| {
7588                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7589                })?;
7590            let frame = Frame::PushCommit {
7591                client_commit_id: commit.client_commit_id.clone(),
7592                client_seq,
7593                operations,
7594            };
7595            let frame_bytes = match byte_cap {
7596                Some(_) => {
7597                    let mut frames = lead.clone();
7598                    frames.push(frame.clone());
7599                    frames.push(pull.clone());
7600                    encode_message(&Message {
7601                        wire_version: WIRE_VERSION,
7602                        msg_kind: MsgKind::Request,
7603                        frames,
7604                    })
7605                    .len()
7606                        - base_bytes
7607                }
7608                None => 0,
7609            };
7610            // The head is always encoded so the caller can report the full
7611            // projected request size; a later commit defers once it no longer
7612            // fits.
7613            if let Some(cap) = byte_cap {
7614                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7615                    deferred_commits = total_outbox - index;
7616                    break;
7617                }
7618            }
7619            push_frames.push(frame);
7620            pushed_ids.push(commit.client_commit_id.clone());
7621            ops_in_request += commit.ops.len();
7622            bytes_in_request += frame_bytes;
7623        }
7624        #[cfg(feature = "bench-internals")]
7625        drop(outbox_phase);
7626        if let Some(cap) = byte_cap {
7627            if fixed_bytes + bytes_in_request > cap {
7628                // The head alone does not fit this budget; the commit stays
7629                // queued and no transport round is attempted.
7630                let blocked = if push_frames.is_empty() {
7631                    None
7632                } else {
7633                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7634                };
7635                return Err(Self::push_capacity_outcome(
7636                    "bytes",
7637                    cap,
7638                    fixed_bytes + bytes_in_request,
7639                    blocked,
7640                ));
7641            }
7642        }
7643        let mut frames = lead;
7644        frames.extend(push_frames);
7645        frames.push(pull);
7646        frames.extend(sub_frames);
7647        let message = Message {
7648            wire_version: WIRE_VERSION,
7649            msg_kind: MsgKind::Request,
7650            frames,
7651        };
7652        Ok((
7653            message,
7654            RequestMeta {
7655                pushed_ids,
7656                registration_digest,
7657                accept,
7658                deferred_commits,
7659                applied: BTreeMap::new(),
7660                deferred: BTreeSet::new(),
7661            },
7662        ))
7663    }
7664
7665    /// §7.1: a configured push capacity limit cannot carry this request.
7666    fn push_capacity_outcome(
7667        kind: &str,
7668        limit: usize,
7669        size: usize,
7670        client_commit_id: Option<&str>,
7671    ) -> Box<SyncOutcome> {
7672        let mut details = Map::new();
7673        details.insert("kind".to_owned(), Value::from(kind));
7674        details.insert("limit".to_owned(), Value::from(limit as u64));
7675        details.insert("size".to_owned(), Value::from(size as u64));
7676        if let Some(id) = client_commit_id {
7677            details.insert("clientCommitId".to_owned(), Value::from(id));
7678        }
7679        Box::new(SyncOutcome::Failed {
7680            error_code: "client.push_request_too_large".into(),
7681            message: "the push request exceeds a configured capacity limit".into(),
7682            details: Some(Value::Object(details)),
7683            recommended_action: None,
7684        })
7685    }
7686
7687    // -- sync -------------------------------------------------------------------
7688
7689    /// Cloneable observation handle; no client lock is needed to read it.
7690    pub fn progress(&self) -> ProgressObserver {
7691        self.progress.clone()
7692    }
7693
7694    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7695        self.sync_bounded(transport, None)
7696    }
7697
7698    /// One round; `idle_bound` is the idle drive's round bound (§7.8).
7699    fn sync_bounded(
7700        &mut self,
7701        transport: &mut dyn Transport,
7702        idle_bound: Option<(u64, u32)>,
7703    ) -> SyncOutcome {
7704        let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7705            Ok(prepared) => prepared,
7706            Err(outcome) => return *outcome,
7707        };
7708        prepared.idle_bound = idle_bound;
7709        // The round's network deadline rides `PreparedSyncRound` and is scoped
7710        // to each exchange, so uploads, continuations, the main request, and
7711        // every segment fetch share one monotonic anchor without leaving
7712        // deadline state on a reused transport.
7713        let mut next = prepared;
7714        loop {
7715            match self.apply_sync_round(next.exchange(transport)) {
7716                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7717                crate::AppliedSyncRound::Complete {
7718                    outcome, controls, ..
7719                } => {
7720                    if self.transport_enabled {
7721                        for text in controls {
7722                            let _ = transport.realtime_send(&text);
7723                        }
7724                    } else {
7725                        self.disconnect_realtime(transport);
7726                    }
7727                    return outcome;
7728                }
7729            }
7730        }
7731    }
7732
7733    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7734        if let SyncOutcome::Failed {
7735            error_code,
7736            message,
7737            details,
7738            ..
7739        } = &mut outcome
7740        {
7741            if let Some((failure, code)) = self
7742                .storage_failure
7743                .borrow()
7744                .as_ref()
7745                .and_then(|failure| failure.code.map(|code| (failure, code)))
7746            {
7747                *error_code = code.to_owned();
7748                *message = match code {
7749                    "client.storage_busy" => "local SQLite storage is busy",
7750                    "client.storage_full" => "local SQLite storage is full",
7751                    "client.storage_io" => "local SQLite storage I/O failed",
7752                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7753                    _ => unreachable!("unrecognized classified storage failure"),
7754                }
7755                .to_owned();
7756                *details = failure.details();
7757            }
7758        }
7759        let details = match &outcome {
7760            SyncOutcome::Failed { details, .. } => details.as_ref(),
7761            _ => None,
7762        };
7763        let cause_kind = details
7764            .and_then(|details| details.get("causeKind"))
7765            .and_then(Value::as_str)
7766            .and_then(|cause| match cause {
7767                "network" => Some(crate::TransportCauseKind::Network),
7768                "connect" => Some(crate::TransportCauseKind::Connect),
7769                "tls" => Some(crate::TransportCauseKind::Tls),
7770                "io" => Some(crate::TransportCauseKind::Io),
7771                "request" => Some(crate::TransportCauseKind::Request),
7772                "protocol" => Some(crate::TransportCauseKind::Protocol),
7773                "timeout" => Some(crate::TransportCauseKind::Timeout),
7774                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7775                "aborted" => Some(crate::TransportCauseKind::Aborted),
7776                "body" => Some(crate::TransportCauseKind::Body),
7777                "status" => Some(crate::TransportCauseKind::Status),
7778                "redirect" => Some(crate::TransportCauseKind::Redirect),
7779                "unknown" => Some(crate::TransportCauseKind::Unknown),
7780                _ => None,
7781            });
7782        let http_status = if matches!(
7783            cause_kind,
7784            Some(
7785                crate::TransportCauseKind::Status
7786                    | crate::TransportCauseKind::Body
7787                    | crate::TransportCauseKind::Redirect
7788            )
7789        ) {
7790            details
7791                .and_then(|details| details.get("httpStatus"))
7792                .and_then(Value::as_f64)
7793                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7794                .map(|status| status as u16)
7795        } else {
7796            None
7797        };
7798        let retry_delay_ms = self.round_retry_delay_ms;
7799        self.progress.update(|p| match &outcome {
7800            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7801                if report.failed.is_empty() =>
7802            {
7803                p.state = ProgressState::Complete
7804            }
7805            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7806                p.state = ProgressState::Failed;
7807                p.error_code = Some("sync.scope_revoked".into());
7808            }
7809            SyncOutcome::Failed { error_code, .. } => {
7810                p.state = ProgressState::Failed;
7811                p.error_code = Some(Self::diagnostic_code(error_code));
7812                p.cause_kind = cause_kind;
7813                p.http_status = http_status;
7814                p.retry_delay_ms = retry_delay_ms;
7815            }
7816            SyncOutcome::RealtimeUnavailable { .. } => {
7817                p.state = ProgressState::Failed;
7818                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7819                p.retry_delay_ms = retry_delay_ms;
7820            }
7821        });
7822        let completed_at_ms = self.clock_now_ms();
7823        self.last_round = Some(match &outcome {
7824            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7825                status: "succeeded".to_owned(),
7826                started_at_ms,
7827                completed_at_ms,
7828                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7829                counters: Some(DiagnosticRoundCounters {
7830                    pushed: report.pushed,
7831                    applied: report.applied.len(),
7832                    rejected: report.rejected.len(),
7833                    retryable: report.retryable.len(),
7834                    conflicts: report.conflicts,
7835                    commits_applied: report.commits_applied,
7836                    segment_rows_applied: report.segment_rows_applied,
7837                    bootstrapping: report.bootstrapping.len(),
7838                    resets: report.resets.len(),
7839                    revoked: report.revoked.len(),
7840                    failed: report.failed.len(),
7841                    deferred_commits: report.deferred_commits,
7842                }),
7843                error_code: None,
7844                cause_kind: None,
7845                http_status: None,
7846            },
7847            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7848                status: "failed".to_owned(),
7849                started_at_ms,
7850                completed_at_ms,
7851                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7852                counters: None,
7853                error_code: Some(Self::diagnostic_code(error_code)),
7854                cause_kind,
7855                http_status,
7856            },
7857            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7858                status: "failed".to_owned(),
7859                started_at_ms,
7860                completed_at_ms,
7861                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7862                counters: None,
7863                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7864                cause_kind: None,
7865                http_status: None,
7866            },
7867        });
7868        outcome
7869    }
7870
7871    /// Capture one request on the client owner. Mutations made after this
7872    /// call remain in the outbox for a later round; no network I/O occurs here.
7873    pub fn prepare_sync_round(
7874        &mut self,
7875        url_capable: bool,
7876    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7877        if !self.transport_enabled {
7878            return Err(Box::new(SyncOutcome::Failed {
7879                error_code: "sync.offline".into(),
7880                message: "transport gate is closed".into(),
7881                details: None,
7882                recommended_action: None,
7883            }));
7884        }
7885        if self.active_round.is_some() {
7886            return Err(Box::new(SyncOutcome::Failed {
7887                error_code: "client.round_in_progress".into(),
7888                message: "a sync round is already in flight".into(),
7889                details: None,
7890                recommended_action: None,
7891            }));
7892        }
7893        self.storage_failure.borrow_mut().take();
7894        self.progress.start();
7895        self.round_retry_delay_ms = None;
7896        let started_at_ms = self.clock_now_ms();
7897        let prepared = (|| {
7898            if self.stopped {
7899                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7900                    schema_floor: self.schema_floor.clone(),
7901                    ..SyncReport::default()
7902                })));
7903            }
7904            if self.security_preflight {
7905                return Err(Box::new(SyncOutcome::Failed {
7906                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7907                    message: "security preflight must complete before syncing".into(),
7908                    details: None,
7909                    recommended_action: None,
7910                }));
7911            }
7912            self.set_sync_needed(false, false);
7913            self.drain_pending_evictions()
7914                .and_then(|_| self.drop_incompatible_outbox())
7915                .and_then(|_| self.drop_unencodable_outbox(true, None))
7916                .map_err(|message| {
7917                    Box::new(SyncOutcome::Failed {
7918                        error_code: "storage.failed".into(),
7919                        message,
7920                        details: None,
7921                        recommended_action: None,
7922                    })
7923                })?;
7924            // §5.11: reveal rows sealed under a key the keyring gained since.
7925            self.reveal_sealed_rows().map_err(|error| {
7926                Box::new(SyncOutcome::Failed {
7927                    error_code: error.code,
7928                    message: error.message,
7929                    details: error.details,
7930                    recommended_action: None,
7931                })
7932            })?;
7933            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7934                Box::new(SyncOutcome::Failed {
7935                    error_code: "storage.failed".into(),
7936                    message,
7937                    details: None,
7938                    recommended_action: None,
7939                })
7940            })?;
7941            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
7942                self.pending_blob_uploads().map_err(|e| {
7943                    Box::new(SyncOutcome::Failed {
7944                        error_code: e.code,
7945                        message: e.message,
7946                        details: e.details,
7947                        recommended_action: None,
7948                    })
7949                })?
7950            } else {
7951                Vec::new()
7952            };
7953            if self.realtime_state != RealtimeState::Connected
7954                && self.realtime_policy == RealtimePolicy::Required
7955            {
7956                let retry_delay_ms = self.schedule_background_retry();
7957                self.realtime_retry_delay_ms = Some(retry_delay_ms);
7958                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
7959                    state: self.realtime_state,
7960                    reason_code: self.realtime_reason_code.clone(),
7961                    retry_delay_ms,
7962                }));
7963            }
7964            let (message, mut meta) = self.build_request(url_capable)?;
7965            let units = self.stored(load_units(&self.conn)).map_err(|message| {
7966                Box::new(SyncOutcome::Failed {
7967                    error_code: "storage.failed".into(),
7968                    message,
7969                    details: None,
7970                    recommended_action: None,
7971                })
7972            })?;
7973            meta.applied = units
7974                .iter()
7975                .filter_map(|unit| {
7976                    let (cursor, token, ids) = self.applied_segments.get(&unit.id)?;
7977                    (*cursor == unit.cursor && *token == unit.bootstrap_state)
7978                        .then(|| (unit.id.clone(), ids.clone()))
7979                })
7980                .collect();
7981            let id = uuid::Uuid::new_v4();
7982            self.active_round = Some(id);
7983            let phase = if uploads.is_empty() {
7984                crate::round::RoundPhase::Sync
7985            } else {
7986                crate::round::RoundPhase::UploadGrant
7987            };
7988            Ok(crate::PreparedSyncRound {
7989                id,
7990                started_at_ms,
7991                message,
7992                meta,
7993                uploads: uploads.into(),
7994                phase,
7995                encoded: None,
7996                continuation: None,
7997                completed: None,
7998                realtime: self.realtime_state == RealtimeState::Connected,
7999                // §5.5: the owning unit's scope map, each value a one-element list.
8000                scopes: units
8001                    .into_iter()
8002                    .map(|unit| {
8003                        (
8004                            unit.id,
8005                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8006                        )
8007                    })
8008                    .collect(),
8009                #[cfg(feature = "bench-internals")]
8010                benchmark_phases: self.benchmark_phases.clone(),
8011                progress: self.progress.clone(),
8012                fixed_now: self
8013                    .now_ms
8014                    .as_ref()
8015                    .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8016                round_deadline_at: None,
8017                idle_bound: None,
8018            })
8019        })();
8020        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
8021    }
8022
8023    /// Discard the reply of an old authorization or registration context. Its
8024    /// submitted commits stay durable and can be retried with the same ids.
8025    pub fn cancel_sync_round(&mut self) {
8026        self.active_round = None;
8027    }
8028
8029    /// Apply captured network results on the owner, preserving all normal
8030    /// version, schema, content-address and cursor checks. No network I/O.
8031    pub fn apply_sync_round(
8032        &mut self,
8033        completed: crate::CompletedSyncRound,
8034    ) -> crate::AppliedSyncRound {
8035        let mut prepared = completed.prepared;
8036        if self.active_round != Some(prepared.id) || self.security_preflight {
8037            return crate::AppliedSyncRound::Complete {
8038                outcome: SyncOutcome::Failed {
8039                    error_code: "client.round_cancelled".into(),
8040                    message: "sync round belongs to a released client context".into(),
8041                    details: None,
8042                    recommended_action: None,
8043                },
8044                controls: Vec::new(),
8045                more: false,
8046                bootstrap_advanced: false,
8047            };
8048        }
8049        self.storage_failure.borrow_mut().take();
8050        let (response, mut downloads, transport_failed) = match completed.exchange {
8051            crate::round::ExchangeResult::Upload { id, result } => {
8052                let result = match result {
8053                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8054                    // is the body's final outcome. The commits naming it take
8055                    // the incompatible-commit path with the server's code and
8056                    // leave this round's request; the round continues.
8057                    Err(error)
8058                        if error
8059                            .details
8060                            .as_ref()
8061                            .and_then(|details| details.get("retryable"))
8062                            == Some(&Value::Bool(false)) =>
8063                    {
8064                        self.conn
8065                            .prepare(
8066                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8067                            )
8068                            .and_then(|mut statement| {
8069                                statement
8070                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8071                                    .collect::<Result<HashSet<_>, _>>()
8072                            })
8073                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8074                            .and_then(|refused| {
8075                                let commits = self
8076                                    .outbox
8077                                    .iter()
8078                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8079                                    .cloned()
8080                                    .collect::<Vec<_>>();
8081                                if !commits.is_empty() {
8082                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8083                                }
8084                                prepared.message.frames.retain(|frame| {
8085                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8086                                        if refused.contains(client_commit_id))
8087                                });
8088                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8089                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8090                                Ok(())
8091                            })
8092                            .map_err(|message| TransportError::new("client.failed", message))
8093                    }
8094                    result => result,
8095                };
8096                let result = result.and_then(|()| {
8097                    self.conn
8098                        .execute(
8099                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8100                            rusqlite::params![id],
8101                        )
8102                        .map(|_| ())
8103                        .map_err(|error| {
8104                            TransportError::new(
8105                                "client.failed",
8106                                Self::sqlite_failure(&self.storage_failure, error),
8107                            )
8108                        })
8109                });
8110                match result {
8111                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8112                    Err(TransportError {
8113                        code,
8114                        message,
8115                        details,
8116                    }) => {
8117                        self.active_round = None;
8118                        if Self::unreachable_transport(&details) {
8119                            // The round's work is still owed; the host's
8120                            // connectivity resume runs it.
8121                            self.set_sync_needed(true, false);
8122                        } else if Self::retryable_failure_code(&code) {
8123                            self.schedule_background_retry();
8124                        }
8125                        let recommended_action = recommended_action_of(&details);
8126                        let outcome = self.finish_sync_round(
8127                            prepared.started_at_ms,
8128                            SyncOutcome::Failed {
8129                                error_code: code,
8130                                message,
8131                                details,
8132                                recommended_action,
8133                            },
8134                        );
8135                        return crate::AppliedSyncRound::Complete {
8136                            outcome,
8137                            controls: Vec::new(),
8138                            more: false,
8139                            bootstrap_advanced: false,
8140                        };
8141                    }
8142                }
8143            }
8144            crate::round::ExchangeResult::Reply {
8145                response,
8146                downloads,
8147                transport_failed,
8148            } => (response, downloads, transport_failed),
8149        };
8150        self.active_round = None;
8151        let before = self.resume_tokens();
8152        let outcome = (|| {
8153            let response = match response {
8154                Ok(response) => {
8155                    // §1.5: a complete, decodable response ends the report, an
8156                    // in-band ERROR included (the server already recorded it).
8157                    self.round_failure = None;
8158                    response
8159                }
8160                Err(TransportError {
8161                    code,
8162                    message,
8163                    details,
8164                }) => {
8165                    // §1.5: the server cannot see a round whose transport
8166                    // failed or whose response never decoded, so its code rides
8167                    // the following requests. A code the frame grammar rejects
8168                    // is not reportable, and the previous code stays; the
8169                    // client-local drive timeout never goes on the wire (§7.8).
8170                    if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8171                        self.round_failure = Some(code.clone());
8172                    }
8173                    if transport_failed {
8174                        if prepared.realtime {
8175                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8176                        }
8177                        if let Err(message) = self.record_lease_error(&code) {
8178                            return SyncOutcome::Failed {
8179                                error_code: "storage.failed".to_owned(),
8180                                message,
8181                                details: None,
8182                                recommended_action: None,
8183                            };
8184                        }
8185                    }
8186                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8187                    {
8188                        let delay = self.schedule_background_retry();
8189                        if prepared.realtime && transport_failed {
8190                            self.realtime_retry_delay_ms = Some(delay);
8191                        }
8192                    }
8193                    let recommended_action = recommended_action_of(&details);
8194                    return SyncOutcome::Failed {
8195                        error_code: code,
8196                        message,
8197                        details,
8198                        recommended_action,
8199                    };
8200                }
8201            };
8202            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8203            if let SyncOutcome::Ok(report) = &mut outcome {
8204                report.deferred_commits = prepared.meta.deferred_commits;
8205            }
8206            match &outcome {
8207                SyncOutcome::Ok(_) => self.reset_background_retry(),
8208                SyncOutcome::Failed {
8209                    error_code,
8210                    details,
8211                    ..
8212                } if Self::retryable_failure_code(error_code)
8213                    && !Self::unreachable_transport(details) =>
8214                {
8215                    self.schedule_background_retry();
8216                }
8217                _ => {}
8218            }
8219            if prepared.meta.deferred_commits > 0 {
8220                self.set_sync_needed(true, true);
8221            }
8222            self.previous_version_lifetime_check();
8223            outcome
8224        })();
8225        match &outcome {
8226            // §7.1: a unit still catching up needs another round, also when the
8227            // drive that started it ends early.
8228            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8229                self.set_sync_needed(true, false)
8230            }
8231            // The round's work is still owed; the host's connectivity resume
8232            // runs it.
8233            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8234                self.set_sync_needed(true, false)
8235            }
8236            _ => {}
8237        }
8238        let more = self.transport_enabled
8239            && matches!(&outcome, SyncOutcome::Ok(report)
8240                if report.schema_floor.is_none()
8241                    && (!report.bootstrapping.is_empty()
8242                        || report.commits_applied > 0
8243                        || report.segment_rows_applied > 0
8244                        || !report.resets.is_empty()
8245                        || report.deferred_commits > 0
8246                        || self.sync_needed));
8247        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8248            && before != self.resume_tokens();
8249        crate::AppliedSyncRound::Complete {
8250            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8251            controls: if self.transport_enabled {
8252                downloads.controls
8253            } else {
8254                Vec::new()
8255            },
8256            more,
8257            bootstrap_advanced,
8258        }
8259    }
8260
8261    /// Every unit's resume token and applied-segment count: a bootstrap that
8262    /// advanced changes one (§4.7), also when it applied images of a section
8263    /// it deferred (§5.3, §5.6).
8264    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, usize)>> {
8265        load_units(&self.conn).ok().map(|units| {
8266            units
8267                .into_iter()
8268                .map(|unit| {
8269                    let applied = self
8270                        .applied_segments
8271                        .get(&unit.id)
8272                        .map_or(0, |(_, _, ids)| ids.len());
8273                    (unit.id, unit.bootstrap_state, applied)
8274                })
8275                .collect()
8276        })
8277    }
8278
8279    /// §7.8: one idle drive's default total deadline, the TS client's value.
8280    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8281
8282    pub fn sync_until_idle(
8283        &mut self,
8284        transport: &mut dyn Transport,
8285        max_rounds: Option<u32>,
8286    ) -> SyncOutcome {
8287        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8288    }
8289
8290    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8291    /// milliseconds (`None` disables it). The deadline is checked against the
8292    /// client clock between rounds: a drive past it after a completed round
8293    /// ends as `BudgetExhausted`. Each round is bounded by `timeout_ms` on its
8294    /// own, or the transport's tighter whole-round deadline (§1.8); a round
8295    /// that reaches the drive bound fails with `sync.idle_timeout`.
8296    pub fn sync_until_idle_with_deadline(
8297        &mut self,
8298        transport: &mut dyn Transport,
8299        max_rounds: Option<u32>,
8300        timeout_ms: Option<u64>,
8301    ) -> SyncOutcome {
8302        // §7.8: an explicit budget is a strict positive round count. Zero is a
8303        // caller error, not an exhaustion.
8304        if max_rounds == Some(0) {
8305            return SyncOutcome::Failed {
8306                details: None,
8307                error_code: "sync.invalid_request".into(),
8308                message: "maxRounds must be a positive integer".into(),
8309                recommended_action: None,
8310            };
8311        }
8312        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8313            return SyncOutcome::Failed {
8314                details: None,
8315                error_code: "sync.invalid_request".into(),
8316                message: "timeoutMs must be a positive integer".into(),
8317                recommended_action: None,
8318            };
8319        }
8320        let started_at_ms = self.clock_now_ms();
8321        let rounds = max_rounds.unwrap_or(20);
8322        let mut aggregate = SyncReport::default();
8323        let mut spent = 0;
8324        let mut completed = false;
8325        while spent < rounds {
8326            // §7.8: a deadline that elapsed across completed rounds ends the
8327            // drive like an exhausted round budget; the next drive resumes.
8328            if completed
8329                && timeout_ms.is_some_and(|ms| {
8330                    self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8331                })
8332            {
8333                return SyncOutcome::BudgetExhausted(aggregate);
8334            }
8335            let idle_bound = timeout_ms.map(|ms| (ms, spent));
8336            spent += 1;
8337            let before = self.resume_tokens();
8338            match self.sync_bounded(transport, idle_bound) {
8339                SyncOutcome::Failed {
8340                    error_code,
8341                    message,
8342                    details,
8343                    recommended_action,
8344                } => {
8345                    return SyncOutcome::Failed {
8346                        details,
8347                        error_code,
8348                        message,
8349                        recommended_action,
8350                    };
8351                }
8352                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8353                SyncOutcome::BudgetExhausted(_) => {
8354                    unreachable!("sync() never exhausts a round budget")
8355                }
8356                SyncOutcome::Ok(report) => {
8357                    completed = true;
8358                    if max_rounds.is_none()
8359                        && report.segment_rows_applied > 0
8360                        && !report.bootstrapping.is_empty()
8361                        && before != self.resume_tokens()
8362                    {
8363                        spent = 0;
8364                    }
8365                    // TS parity: a disabled transport or a schema floor ends
8366                    // the loop. The latest round still merges first.
8367                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8368                    aggregate.merge(&report);
8369                    if stop {
8370                        return SyncOutcome::Ok(aggregate);
8371                    }
8372                    // §4.5: pull again whenever the response contained
8373                    // commits or segments; resets re-bootstrap; a pending
8374                    // resume token continues paging (§4.7); deferred outbox
8375                    // commits and a raised sync-needed signal push on the next
8376                    // round.
8377                    let more = !report.bootstrapping.is_empty()
8378                        || report.commits_applied > 0
8379                        || report.segment_rows_applied > 0
8380                        || !report.resets.is_empty()
8381                        || report.deferred_commits > 0
8382                        || self.sync_needed;
8383                    if !more {
8384                        return SyncOutcome::Ok(aggregate);
8385                    }
8386                }
8387            }
8388        }
8389        // §7.8: partial success. The aggregate report is retained and the run
8390        // is explicitly not idle.
8391        SyncOutcome::BudgetExhausted(aggregate)
8392    }
8393
8394    fn process_response(
8395        &mut self,
8396        transport: &mut dyn Transport,
8397        response: Message,
8398        meta: &RequestMeta,
8399    ) -> SyncOutcome {
8400        #[cfg(feature = "bench-internals")]
8401        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8402        let mut report = SyncReport {
8403            pushed: meta.pushed_ids.len() as u32,
8404            ..SyncReport::default()
8405        };
8406        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8407            HashMap::new();
8408        let pushed_ids = meta
8409            .pushed_ids
8410            .iter()
8411            .map(String::as_str)
8412            .collect::<HashSet<_>>();
8413        let mut last_final_push_result_id: Option<String> = None;
8414        for frame in &response.frames {
8415            match frame {
8416                Frame::PushResultDetails {
8417                    client_commit_id,
8418                    entries,
8419                } => {
8420                    let details = rejection_details_by_commit
8421                        .entry(client_commit_id.clone())
8422                        .or_default();
8423                    for entry in entries {
8424                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8425                            Ok(value) => value,
8426                            Err(message) => {
8427                                return SyncOutcome::Failed {
8428                                    details: None,
8429                                    error_code: "sync.invalid_request".to_owned(),
8430                                    message,
8431                                    recommended_action: None,
8432                                };
8433                            }
8434                        };
8435                        details.insert(entry.op_index, parsed);
8436                    }
8437                }
8438                Frame::PushResult {
8439                    client_commit_id,
8440                    status,
8441                    results,
8442                    ..
8443                } if pushed_ids.contains(client_commit_id.as_str())
8444                    && Self::push_result_is_final(*status, results) =>
8445                {
8446                    last_final_push_result_id = Some(client_commit_id.clone());
8447                }
8448                _ => {}
8449            }
8450        }
8451        let mut frames = response.frames.into_iter().peekable();
8452        match frames.next() {
8453            Some(Frame::RespHeader {
8454                required_schema_version,
8455                latest_schema_version,
8456                log_epoch,
8457                reset_required,
8458            }) => {
8459                if let Some(required) = required_schema_version {
8460                    // §1.6 schema-floor response: nothing else is processed;
8461                    // stop syncing and surface the upgrade requirement.
8462                    let floor = SchemaFloor {
8463                        required_schema_version: Some(required),
8464                        latest_schema_version,
8465                    };
8466                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8467                        return SyncOutcome::Failed {
8468                            details: None,
8469                            error_code: "storage.failed".to_owned(),
8470                            message,
8471                            recommended_action: None,
8472                        };
8473                    }
8474                    report.schema_floor = Some(floor);
8475                    return SyncOutcome::Ok(report);
8476                }
8477                if reset_required {
8478                    if frames.next().is_some() {
8479                        return SyncOutcome::Failed {
8480                            details: None,
8481                            error_code: "client.invalid_host_response".to_owned(),
8482                            message: "log-epoch reset response contains body frames".to_owned(),
8483                            recommended_action: None,
8484                        };
8485                    }
8486                    match self.run_log_epoch_reset(&log_epoch) {
8487                        Ok(resets) => {
8488                            report.resets = resets;
8489                            return SyncOutcome::Ok(report);
8490                        }
8491                        Err(message) => {
8492                            return SyncOutcome::Failed {
8493                                details: None,
8494                                error_code: "sync.local_corrupt".to_owned(),
8495                                message,
8496                                recommended_action: None,
8497                            };
8498                        }
8499                    }
8500                }
8501                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8502                    Ok(stored) => stored,
8503                    Err(message) => {
8504                        return SyncOutcome::Failed {
8505                            details: None,
8506                            error_code: "storage.failed".to_owned(),
8507                            message,
8508                            recommended_action: None,
8509                        };
8510                    }
8511                };
8512                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8513                    return SyncOutcome::Failed {
8514                        details: None,
8515                        error_code: "client.invalid_host_response".to_owned(),
8516                        message: "server changed logEpoch without requiring a reset".to_owned(),
8517                        recommended_action: None,
8518                    };
8519                }
8520            }
8521            _ => {
8522                return SyncOutcome::Failed {
8523                    details: None,
8524                    error_code: "sync.invalid_request".to_owned(),
8525                    message: "response does not start with RESP_HEADER".to_owned(),
8526                    recommended_action: None,
8527                };
8528            }
8529        }
8530
8531        if let Err(message) = self.begin_coalesced_apply() {
8532            return SyncOutcome::Failed {
8533                details: None,
8534                error_code: "storage.failed".into(),
8535                message,
8536                recommended_action: None,
8537            };
8538        }
8539        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8540        // §4.5 client application: the units whose persisted echo a live
8541        // change must match, fixed at LIVE_START after every SUB_END persisted.
8542        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8543        let mut live_ended = false;
8544        while let Some(frame) = frames.next() {
8545            match frame {
8546                frame @ Frame::PushResult { .. } => {
8547                    let successful = matches!(
8548                        &frame,
8549                        Frame::PushResult {
8550                            status: PushStatus::Applied | PushStatus::Cached,
8551                            ..
8552                        }
8553                    );
8554                    let mut results = vec![frame];
8555                    if successful {
8556                        while matches!(
8557                            frames.peek(),
8558                            Some(Frame::PushResult {
8559                                status: PushStatus::Applied | PushStatus::Cached,
8560                                ..
8561                            })
8562                        ) {
8563                            if let Some(next) = frames.next() {
8564                                results.push(next);
8565                            }
8566                        }
8567                    }
8568                    if self
8569                        .handle_push_results(
8570                            &results,
8571                            &pushed_ids,
8572                            &rejection_details_by_commit,
8573                            &mut report,
8574                            last_final_push_result_id.as_deref(),
8575                        )
8576                        .is_err()
8577                    {
8578                        failure = Some((
8579                            "client.outcome_persistence_failed".to_owned(),
8580                            "local commit outcome could not be persisted".to_owned(),
8581                            None,
8582                            None,
8583                        ));
8584                        break;
8585                    }
8586                }
8587                Frame::PushResultDetails { .. } => {}
8588                Frame::SubStart {
8589                    id,
8590                    status,
8591                    reason_code,
8592                    bootstrap,
8593                    fenced,
8594                } => {
8595                    // §1.4 rule 5: an ERROR closes the section; its body still
8596                    // applied, its SUB_END values are never persisted.
8597                    let mut body = Vec::new();
8598                    let mut sub_end: Option<(i64, Option<String>)> = None;
8599                    let mut error: Option<Frame> = None;
8600                    for inner in frames.by_ref() {
8601                        match inner {
8602                            Frame::SubEnd {
8603                                next_cursor,
8604                                bootstrap_state,
8605                            } => {
8606                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8607                                break;
8608                            }
8609                            inner @ Frame::Error { .. } => {
8610                                error = Some(inner);
8611                                break;
8612                            }
8613                            Frame::Unknown { .. } => {}
8614                            other => body.push(other),
8615                        }
8616                    }
8617                    let ended = sub_end.is_some();
8618                    let section = SectionStart {
8619                        id,
8620                        status,
8621                        reason_code,
8622                        bootstrap,
8623                        fenced,
8624                    };
8625                    match self.process_section(
8626                        transport,
8627                        &section,
8628                        body,
8629                        sub_end,
8630                        meta,
8631                        &mut report,
8632                    ) {
8633                        Err(SectionError::Abort(code, message)) => {
8634                            failure = Some((code, message, None, None));
8635                            break;
8636                        }
8637                        Err(SectionError::Transport(error)) => {
8638                            failure = Some((error.code, error.message, None, error.details));
8639                            break;
8640                        }
8641                        _ => {}
8642                    }
8643                    if ended {
8644                        // The counters stay: `complete` and `failed` report them.
8645                        self.progress.update(|p| {
8646                            p.unit_id = None;
8647                            p.table = None;
8648                            p.segment_id = None;
8649                        });
8650                    }
8651                    match error {
8652                        Some(Frame::Error {
8653                            code,
8654                            message,
8655                            recommended_action,
8656                            details,
8657                            ..
8658                        }) => {
8659                            failure = Some(Self::error_frame_failure(
8660                                code,
8661                                message,
8662                                recommended_action,
8663                                details,
8664                            ));
8665                            break;
8666                        }
8667                        Some(_) => unreachable!("only an ERROR closes a section early"),
8668                        None if !ended => {
8669                            failure = Some((
8670                                "sync.invalid_request".to_owned(),
8671                                "unit section without SUB_END".to_owned(),
8672                                None,
8673                                None,
8674                            ));
8675                            break;
8676                        }
8677                        None => {}
8678                    }
8679                }
8680                Frame::LiveStart {
8681                    status,
8682                    reason_code,
8683                } => {
8684                    let receivers = match self.stored(load_units(&self.conn)) {
8685                        Ok(units) => units
8686                            .into_iter()
8687                            .filter(|unit| {
8688                                unit.status == UnitStatus::Active
8689                                    && unit.echo.is_some()
8690                                    && !unit.bootstrapping()
8691                            })
8692                            .collect(),
8693                        Err(message) => {
8694                            failure = Some(("storage.failed".to_owned(), message, None, None));
8695                            break;
8696                        }
8697                    };
8698                    live = Some((status, reason_code, receivers));
8699                }
8700                Frame::Commit {
8701                    commit_seq,
8702                    tables,
8703                    changes,
8704                    ..
8705                } => {
8706                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8707                        continue;
8708                    };
8709                    let receivers = receivers.clone();
8710                    match self.apply_live_commit(
8711                        &receivers,
8712                        &tables,
8713                        changes,
8714                        commit_seq,
8715                        AppliedServerCommitSource::Pull,
8716                    ) {
8717                        Ok(applied) => report.commits_applied += u32::from(applied),
8718                        Err(error) => {
8719                            failure = Some((error.code, error.message, None, error.details));
8720                            break;
8721                        }
8722                    }
8723                }
8724                Frame::LiveEnd {
8725                    through_seq,
8726                    echo_digest,
8727                } => {
8728                    let Some((status, reason_code, _)) = live.take() else {
8729                        continue;
8730                    };
8731                    live_ended = true;
8732                    live = Some((status, String::new(), Vec::new()));
8733                    if let Err(message) = self.finish_live_section(
8734                        status,
8735                        &reason_code,
8736                        through_seq,
8737                        echo_digest,
8738                        &meta.registration_digest,
8739                        &mut report,
8740                    ) {
8741                        failure = Some(("storage.failed".to_owned(), message, None, None));
8742                        break;
8743                    }
8744                }
8745                Frame::Lease {
8746                    lease_id,
8747                    expires_at_ms,
8748                } => {
8749                    // §7.3.5: persist the opaque lease; a fresh lease clears
8750                    // any prior lease error (the outage/revocation is over).
8751                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8752                        lease_id: Some(lease_id),
8753                        expires_at_ms: Some(expires_at_ms),
8754                        error_code: None,
8755                    })) {
8756                        failure = Some(("storage.failed".to_owned(), message, None, None));
8757                        break;
8758                    }
8759                }
8760                Frame::Error {
8761                    code,
8762                    message,
8763                    recommended_action,
8764                    details,
8765                    ..
8766                } => {
8767                    // §1.6: the whole request failed; already-completed
8768                    // units keep their applied data and cursors.
8769                    failure = Some(Self::error_frame_failure(
8770                        code,
8771                        message,
8772                        recommended_action,
8773                        details,
8774                    ));
8775                    break;
8776                }
8777                Frame::Unknown { .. } => {}
8778                _ => {
8779                    failure = Some((
8780                        "sync.invalid_request".to_owned(),
8781                        "unexpected frame in response".to_owned(),
8782                        None,
8783                        None,
8784                    ));
8785                    break;
8786                }
8787            }
8788        }
8789
8790        let applied = match failure {
8791            Some(failure) => Err(failure),
8792            // Local apply boundaries already reconciled their affected overlay.
8793            // Refuse round success if any remaining dirty work fails to
8794            // reconcile. §4.8 E1: the push half may have drained commits that
8795            // pinned rows of a shrunk window unit, so retry deferred evictions.
8796            None => self
8797                .rebuild_overlay_if_dirty()
8798                .and_then(|()| self.drain_pending_evictions())
8799                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8800        };
8801        // §1.4 rule 4: frames completed before a failure stay committed; a
8802        // failed COMMIT of the coalesced transaction fails the round.
8803        if let Err(message) = self.finish_coalesced_apply() {
8804            return SyncOutcome::Failed {
8805                details: None,
8806                error_code: "storage.failed".into(),
8807                message,
8808                recommended_action: None,
8809            };
8810        }
8811        if let Err((error_code, message, recommended_action, details)) = applied {
8812            return SyncOutcome::Failed {
8813                details,
8814                error_code,
8815                message,
8816                recommended_action,
8817            };
8818        }
8819        let catching_up = match self.stored(load_units(&self.conn)) {
8820            Ok(units) => units
8821                .into_iter()
8822                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8823                .map(|unit| unit.id)
8824                .collect::<Vec<_>>(),
8825            Err(message) => {
8826                return SyncOutcome::Failed {
8827                    details: None,
8828                    error_code: "storage.failed".into(),
8829                    message,
8830                    recommended_action: None,
8831                }
8832            }
8833        };
8834        // §7.1: a response without LIVE_END, one answered `unregistered` or
8835        // `reset`, or a unit still catching up leaves sync-needed raised;
8836        // after `unregistered` the next round starts at once.
8837        let status = live.map(|(status, _, _)| status);
8838        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8839            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8840        }
8841        self.ack_after_pull(transport);
8842        // §7.4.5: the reset is over once the first post-reset pull round
8843        // leaves no unit catching up: the tables are rebuilt and current.
8844        if self.upgrading && catching_up.is_empty() {
8845            self.set_upgrading(false);
8846        }
8847        report.bootstrapping = catching_up;
8848        SyncOutcome::Ok(report)
8849    }
8850
8851    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8852    fn error_frame_failure(
8853        code: String,
8854        message: String,
8855        recommended_action: String,
8856        details: Option<RawJson>,
8857    ) -> (String, String, Option<String>, Option<Value>) {
8858        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8859        let details = details
8860            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8861            .filter(Value::is_object);
8862        (code, message, recommended_action, details)
8863    }
8864
8865    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8866    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8867    /// replica cursor, the echo digest and the digest of the list the server
8868    /// now stores.
8869    fn finish_live_section(
8870        &mut self,
8871        status: LiveStatus,
8872        reason_code: &str,
8873        through_seq: i64,
8874        echo_digest: Option<[u8; 32]>,
8875        registration_digest: &str,
8876        report: &mut SyncReport,
8877    ) -> Result<(), String> {
8878        let rollback = self.capture();
8879        let resets = report.resets.len();
8880        self.begin_observation("syncular_live_end")?;
8881        let finished = (|| {
8882            let mut batch = ChangeAccumulator::default();
8883            let before = self.stored(replica_cursor(&self.conn))?;
8884            let mut cursor = if status == LiveStatus::Unregistered {
8885                before
8886            } else {
8887                -1
8888            };
8889            let joined = before.max(through_seq);
8890            for unit in self.stored(load_units(&self.conn))? {
8891                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8892                    continue;
8893                }
8894                match status {
8895                    // §4.6 replica reset: every live unit bootstraps again;
8896                    // post-bootstrap units keep their cursors.
8897                    LiveStatus::Reset if unit.cursor.is_none() => {
8898                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8899                            batch.window(&base_key, &value);
8900                        }
8901                        report.resets.push(unit.id.clone());
8902                        self.stored(save_unit(
8903                            &self.conn,
8904                            &UnitRecord {
8905                                reason_code: Some(reason_code.to_owned()),
8906                                cursor: Some(-1),
8907                                seeded: Vec::new(),
8908                                ..unit
8909                            },
8910                        ))?;
8911                    }
8912                    // §4.5 join rule J.
8913                    LiveStatus::Active => {
8914                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8915                        if next >= joined {
8916                            cursor = joined;
8917                        }
8918                        if unit.cursor.is_some() {
8919                            self.stored(save_unit(
8920                                &self.conn,
8921                                &UnitRecord {
8922                                    cursor: (next < joined).then_some(next),
8923                                    ..unit
8924                                },
8925                            ))?;
8926                        }
8927                    }
8928                    _ => {}
8929                }
8930            }
8931            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
8932            self.set_meta(
8933                ECHO_DIGEST_KEY,
8934                &echo_digest
8935                    .map(|digest| bytes_to_hex(&digest))
8936                    .unwrap_or_default(),
8937            )?;
8938            if status == LiveStatus::Unregistered {
8939                self.delete_meta(REGISTERED_DIGEST_KEY)?;
8940            } else {
8941                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
8942            }
8943            self.finish_observation("syncular_live_end", batch)
8944        })();
8945        if finished.is_err() {
8946            self.rollback_observation("syncular_live_end");
8947            self.restore(rollback);
8948            report.resets.truncate(resets);
8949        }
8950        finished
8951    }
8952
8953    // -- push results (§6.3, §7.2) ------------------------------------------------
8954
8955    fn handle_push_results(
8956        &mut self,
8957        frames: &[Frame],
8958        pushed_ids: &HashSet<&str>,
8959        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
8960        report: &mut SyncReport,
8961        last_final_push_result_id: Option<&str>,
8962    ) -> Result<(), String> {
8963        self.begin_observation("syncular_push_results")?;
8964        let rollback = self.capture();
8965        let conflict_count = self.conflicts.len();
8966        let applied_count = report.applied.len();
8967        let rejected_count = report.rejected.len();
8968        let retryable_count = report.retryable.len();
8969        let mut drained = Vec::new();
8970        let mut batch = ChangeAccumulator::default();
8971        let persisted = (|| {
8972            for frame in frames {
8973                let Frame::PushResult {
8974                    client_commit_id,
8975                    status,
8976                    results,
8977                    commit_seq,
8978                } = frame
8979                else {
8980                    return Err("unexpected acknowledgement frame".to_owned());
8981                };
8982                if !pushed_ids.contains(client_commit_id.as_str()) {
8983                    continue;
8984                }
8985                let Some(index) = self
8986                    .outbox
8987                    .iter()
8988                    .position(|commit| commit.client_commit_id == *client_commit_id)
8989                else {
8990                    continue;
8991                };
8992                let status = *status;
8993                let rejection_details = rejection_details_by_commit.get(client_commit_id);
8994                let operations = self.outbox[index].ops.clone();
8995                match status {
8996                    PushStatus::Applied | PushStatus::Cached => {
8997                        // §7.2: a lost ack replays as `cached` — proceed as if the
8998                        // ack had arrived.
8999                        let sequence =
9000                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9001                        let affected: HashSet<_> = operations
9002                            .iter()
9003                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
9004                            .collect();
9005                        let prior = self.acknowledged_operations(Some(&affected), None)?;
9006                        for (op_index, operation) in operations.iter().enumerate() {
9007                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9008                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9009                            if delivered {
9010                                continue;
9011                            }
9012                            let table = self
9013                                .schema
9014                                .table(&operation.table)
9015                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9016                            let mut intended: Map<String, Value> = self
9017                                .stored_row(table, &operation.row_id, true)?
9018                                .map(|(row, _)| {
9019                                    table
9020                                        .columns
9021                                        .iter()
9022                                        .zip(&row)
9023                                        .map(|(column, value)| {
9024                                            (column.name.clone(), column_value_to_json(value))
9025                                        })
9026                                        .collect()
9027                                })
9028                                .unwrap_or_default();
9029                            let mut scope_image = intended.clone();
9030                            for op in prior
9031                                .iter()
9032                                .chain(
9033                                    self.failed_commits
9034                                        .iter()
9035                                        .flat_map(|(commit, _)| &commit.ops),
9036                                )
9037                                .chain(
9038                                    self.outbox
9039                                        .iter()
9040                                        .take(index)
9041                                        .flat_map(|commit| &commit.ops),
9042                                )
9043                                .chain(operations.iter().take(op_index + 1))
9044                            {
9045                                if op.table != operation.table || op.row_id != operation.row_id {
9046                                    continue;
9047                                }
9048                                if let Some(values) = &op.values {
9049                                    intended.extend(values.clone());
9050                                } else {
9051                                    scope_image = intended.clone();
9052                                    intended.clear();
9053                                }
9054                            }
9055                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9056                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9057                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9058                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9059                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9060                        }
9061                        let journal_results = results
9062                            .iter()
9063                            .map(|result| {
9064                                let op_index = match result {
9065                                    OpResult::Applied { op_index }
9066                                    | OpResult::Conflict { op_index, .. }
9067                                    | OpResult::Error { op_index, .. } => *op_index,
9068                                };
9069                                CommitOperationOutcome::Applied { op_index }
9070                            })
9071                            .collect::<Vec<_>>();
9072                        let outcome_status = if status == PushStatus::Applied {
9073                            CommitOutcomeStatus::Applied
9074                        } else {
9075                            CommitOutcomeStatus::Cached
9076                        };
9077                        let persisted = self
9078                            .persist_commit_outcome(
9079                                client_commit_id,
9080                                outcome_status,
9081                                &journal_results,
9082                                None,
9083                            )
9084                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9085                        persisted?;
9086                        self.sync_needed = true;
9087                        batch.status = true;
9088                        batch.outcomes = true;
9089                    }
9090                    PushStatus::Rejected => {
9091                        if results.iter().any(|result| {
9092                            matches!(
9093                                result,
9094                                OpResult::Error {
9095                                    code,
9096                                    retryable: true,
9097                                    ..
9098                                } if code == "sync.idempotency_cache_miss"
9099                            )
9100                        }) {
9101                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9102                            // exact commit queued for an identical retry.
9103                            report.retryable.push(client_commit_id.to_owned());
9104                            continue;
9105                        }
9106
9107                        let mut journal_results = Vec::with_capacity(results.len());
9108                        let mut conflicts = Vec::new();
9109                        let mut rejections = Vec::new();
9110                        for result in results {
9111                            match result {
9112                                OpResult::Applied { op_index } => {
9113                                    journal_results.push(CommitOperationOutcome::Applied {
9114                                        op_index: *op_index,
9115                                    });
9116                                }
9117                                OpResult::Conflict {
9118                                    op_index,
9119                                    code,
9120                                    message,
9121                                    server_version,
9122                                    server_row,
9123                                    conflict_columns: conflict_columns_bitmap,
9124                                } => {
9125                                    let operation = operations
9126                                        .get(*op_index as usize)
9127                                        .map(CommitOperation::from);
9128                                    let (table, row_id) = operation
9129                                        .as_ref()
9130                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9131                                        .unwrap_or_default();
9132                                    let server_row_json = self
9133                                        .schema
9134                                        .table(&table)
9135                                        .and_then(|t| {
9136                                            decode_row_bytes(t, server_row, &self.encryption)
9137                                                .ok()
9138                                                .map(|(row, _)| (t, row))
9139                                        })
9140                                        .map(|(t, row)| {
9141                                            let mut map = Map::new();
9142                                            for (i, column) in t.columns.iter().enumerate() {
9143                                                map.insert(
9144                                                    column.name.clone(),
9145                                                    column_value_to_json(
9146                                                        row.get(i).unwrap_or(&None),
9147                                                    ),
9148                                                );
9149                                            }
9150                                            map
9151                                        })
9152                                        .unwrap_or_default();
9153                                    // §6.3: decode the presence-layout bitmap
9154                                    // into the marked column names.
9155                                    let conflict_columns = self
9156                                        .schema
9157                                        .table(&table)
9158                                        .map(|t| {
9159                                            t.columns
9160                                                .iter()
9161                                                .enumerate()
9162                                                .filter(|(index, _)| {
9163                                                    conflict_columns_bitmap
9164                                                        .get(index / 8)
9165                                                        .is_some_and(|byte| {
9166                                                            byte & (1 << (index % 8)) != 0
9167                                                        })
9168                                                })
9169                                                .map(|(_, column)| column.name.clone())
9170                                                .collect::<Vec<_>>()
9171                                        })
9172                                        .unwrap_or_default();
9173                                    let conflict = ConflictRecord {
9174                                        client_commit_id: client_commit_id.to_owned(),
9175                                        op_index: *op_index,
9176                                        table,
9177                                        row_id,
9178                                        code: code.clone(),
9179                                        message: message.clone(),
9180                                        server_version: *server_version,
9181                                        server_row: server_row_json,
9182                                        conflict_columns,
9183                                        operation,
9184                                    };
9185                                    journal_results.push(CommitOperationOutcome::Conflict {
9186                                        conflict: conflict.clone(),
9187                                    });
9188                                    conflicts.push(conflict);
9189                                }
9190                                OpResult::Error {
9191                                    op_index,
9192                                    code,
9193                                    message,
9194                                    retryable,
9195                                } => {
9196                                    let rejection = RejectionRecord {
9197                                        client_commit_id: client_commit_id.to_owned(),
9198                                        op_index: *op_index,
9199                                        code: code.clone(),
9200                                        message: message.clone(),
9201                                        retryable: *retryable,
9202                                        details: rejection_details
9203                                            .and_then(|details| details.get(op_index))
9204                                            .cloned(),
9205                                        operation: operations
9206                                            .get(*op_index as usize)
9207                                            .map(CommitOperation::from),
9208                                    };
9209                                    journal_results.push(CommitOperationOutcome::Error {
9210                                        rejection: rejection.clone(),
9211                                    });
9212                                    rejections.push(rejection);
9213                                }
9214                            }
9215                        }
9216                        if self.retain_failed_commits {
9217                            self.retain_failed_commit(client_commit_id, &operations)?;
9218                        }
9219                        let outcome_status = if conflicts.is_empty() {
9220                            CommitOutcomeStatus::Rejected
9221                        } else {
9222                            CommitOutcomeStatus::Conflict
9223                        };
9224                        let persisted = self
9225                            .persist_commit_outcome(
9226                                client_commit_id,
9227                                outcome_status,
9228                                &journal_results,
9229                                Some(&operations),
9230                            )
9231                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9232                        persisted?;
9233                        batch.conflicts = !conflicts.is_empty();
9234                        batch.rejections = !rejections.is_empty();
9235                        batch.status = true;
9236                        batch.outcomes = true;
9237                        self.conflicts.extend(conflicts);
9238                        self.rejections.extend(rejections);
9239                    }
9240                }
9241                drained.push(self.outbox.remove(index));
9242                if batch.status {
9243                    for operation in &operations {
9244                        if !self.record_row_scopes(
9245                            &mut batch,
9246                            &operation.table,
9247                            &operation.row_id,
9248                            false,
9249                        ) {
9250                            batch.table(&operation.table);
9251                        }
9252                    }
9253                }
9254                match status {
9255                    PushStatus::Applied | PushStatus::Cached => {
9256                        report.applied.push(client_commit_id.to_owned())
9257                    }
9258                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9259                }
9260            }
9261            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9262                self.prune_commit_outcomes()?;
9263            }
9264            self.reconcile_rows(&drained)?;
9265            self.rebuild_overlay_if_dirty()?;
9266            self.finish_observation("syncular_push_results", batch)
9267        })();
9268        if let Err(error) = persisted {
9269            self.rollback_observation("syncular_push_results");
9270            self.restore(rollback);
9271            report.applied.truncate(applied_count);
9272            report.rejected.truncate(rejected_count);
9273            report.retryable.truncate(retryable_count);
9274            return Err(error);
9275        }
9276        if !rollback.sync_needed && self.sync_needed {
9277            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9278        }
9279        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9280        Ok(())
9281    }
9282
9283    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9284        status != PushStatus::Rejected
9285            || !results.iter().any(|result| {
9286                matches!(
9287                    result,
9288                    OpResult::Error {
9289                        code,
9290                        retryable: true,
9291                        ..
9292                    } if code == "sync.idempotency_cache_miss"
9293                )
9294            })
9295    }
9296
9297    // -- unit sections and the live section ------------------------------------------
9298
9299    /// One unit section (§4.4): apply its segments, then persist its
9300    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9301    /// resume token). A section of a unit the app released while the round was
9302    /// in flight is skipped; a fail-closed first page fails the unit and skips
9303    /// the rest of its section (§5.6).
9304    fn process_section(
9305        &mut self,
9306        transport: &mut dyn Transport,
9307        start: &SectionStart,
9308        body: Vec<Frame>,
9309        sub_end: Option<(i64, Option<String>)>,
9310        meta: &RequestMeta,
9311        report: &mut SyncReport,
9312    ) -> Result<(), SectionError> {
9313        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9314        let unit = self
9315            .stored(get_unit(&self.conn, &start.id))
9316            .map_err(storage)?
9317            .filter(|unit| unit.status == UnitStatus::Active);
9318        // §7.7: a section boundary ends the previous payload.
9319        self.progress.update(|p| {
9320            p.phase = ProgressPhase::Import;
9321            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9322            p.table = None;
9323            p.segment_id = None;
9324            p.bytes_received = 0;
9325            p.bytes_total = None;
9326            p.rows_processed = 0;
9327            p.rows_total = None;
9328            p.import_write_ms = None;
9329            p.import_commit_ms = None;
9330        });
9331        let Some(unit) = unit else {
9332            return Ok(());
9333        };
9334        let echo: Vec<EchoRecord> = start
9335            .fenced
9336            .iter()
9337            .map(|record| EchoRecord {
9338                table: record.table.clone(),
9339                status: record.status.name().to_owned(),
9340                scopes: record.scopes.iter().cloned().collect(),
9341            })
9342            .collect();
9343        // §5.6: the applied-segment record ends when the presented cursor or
9344        // token changes.
9345        if self
9346            .applied_segments
9347            .get(&unit.id)
9348            .is_some_and(|(cursor, token, _)| {
9349                (*cursor, token) != (unit.cursor, &unit.bootstrap_state)
9350            })
9351        {
9352            self.applied_segments.remove(&unit.id);
9353        }
9354        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9355        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9356            BTreeSet::new()
9357        } else {
9358            unit.seeded.iter().cloned().collect()
9359        };
9360        let applied = (|| {
9361            for frame in body {
9362                match frame {
9363                    Frame::SegmentInline { payload } => {
9364                        let segment = decode_rows_segment(&payload).map_err(|e| {
9365                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9366                        })?;
9367                        self.progress.update(|p| {
9368                            p.phase = ProgressPhase::Import;
9369                            p.unit_id = Some(unit.id.clone());
9370                            p.table = Some(segment.table.clone());
9371                            p.segment_id = None;
9372                            p.bytes_received = payload.len() as u64;
9373                            p.bytes_total = Some(payload.len() as u64);
9374                            p.rows_processed = 0;
9375                            p.rows_total =
9376                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9377                            p.import_write_ms = None;
9378                            p.import_commit_ms = None;
9379                        });
9380                        let (pair, clear, effective) =
9381                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9382                        report.segment_rows_applied +=
9383                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9384                        seeded.insert(segment.table);
9385                    }
9386                    Frame::SegmentRef {
9387                        segment_id,
9388                        byte_length,
9389                        media_type,
9390                        table,
9391                        row_count,
9392                        as_of_commit_seq,
9393                        scope_digest,
9394                        row_cursor,
9395                        next_row_cursor,
9396                        url,
9397                        url_expires_at_ms,
9398                        ..
9399                    } => {
9400                        // §4.2: reject a descriptor whose mediaType was not
9401                        // advertised — never skip or guess.
9402                        let advertised = match media_type {
9403                            MediaType::Rows => {
9404                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9405                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9406                            }
9407                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9408                        };
9409                        if !advertised {
9410                            return Err(SectionError::Abort(
9411                                "sync.invalid_request".to_owned(),
9412                                format!(
9413                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9414                                    media_type.name()
9415                                ),
9416                            ));
9417                        }
9418                        // §5.6 applied-segment rule.
9419                        if meta
9420                            .applied
9421                            .get(&unit.id)
9422                            .is_some_and(|ids| ids.contains(&segment_id))
9423                        {
9424                            seeded.insert(table);
9425                            continue;
9426                        }
9427                        if meta
9428                            .deferred
9429                            .contains(&(unit.id.clone(), segment_id.clone()))
9430                        {
9431                            return Err(SectionError::Deferred);
9432                        }
9433                        self.progress.update(|p| {
9434                            p.phase = ProgressPhase::Download;
9435                            p.unit_id = Some(unit.id.clone());
9436                            p.table = Some(table.clone());
9437                            p.segment_id = Some(segment_id.clone());
9438                            p.bytes_received = 0;
9439                            p.bytes_total = u64::try_from(byte_length).ok();
9440                            p.rows_processed = 0;
9441                            p.rows_total = u64::try_from(row_count).ok();
9442                            p.import_write_ms = None;
9443                            p.import_commit_ms = None;
9444                        });
9445                        let observer = self.progress.clone();
9446                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9447                        let bytes = if let Some(url) = url {
9448                            // §5.4: a url-carrying descriptor MUST be fetched
9449                            // from that URL; failure invalidates the whole
9450                            // descriptor (no fall-through to §5.5 — re-pull
9451                            // recovers, §1.4 rule 5).
9452                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9453                                return Err(SectionError::Abort(
9454                                    "sync.invalid_request".to_owned(),
9455                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9456                                        .to_owned(),
9457                                ));
9458                            }
9459                            // §5.4: MUST NOT start a fetch at/past expiry.
9460                            if url_expires_at_ms.is_some_and(|exp| {
9461                                exp <= transport
9462                                    .segment_fetch_started_at(&url)
9463                                    .unwrap_or_else(|| self.clock_now_ms())
9464                            }) {
9465                                return Err(SectionError::Abort(
9466                                    "sync.segment_expired".to_owned(),
9467                                    format!(
9468                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9469                                    ),
9470                                ));
9471                            }
9472                            transport
9473                                .fetch_url(
9474                                    &url,
9475                                    u64::try_from(byte_length).unwrap_or(0),
9476                                    &mut on_progress,
9477                                )
9478                                .map_err(|e| self.segment_fetch_failed(e))?
9479                        } else {
9480                            // §5.5: the owning unit's scope map, each value a one-element list.
9481                            let requested_scopes_json =
9482                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9483                            transport
9484                                .download_segment(
9485                                    &SegmentRequest {
9486                                        segment_id: segment_id.clone(),
9487                                        table: table.clone(),
9488                                        requested_scopes_json,
9489                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9490                                    },
9491                                    &mut on_progress,
9492                                )
9493                                .map_err(|e| {
9494                                    // §5.5: access changed since the response;
9495                                    // the next round's SUB_START settles the unit.
9496                                    if e.code == "sync.forbidden" {
9497                                        SectionError::Refused
9498                                    } else {
9499                                        self.segment_fetch_failed(e)
9500                                    }
9501                                })?
9502                        };
9503                        // §5.4: the descriptor's byteLength caps the body before
9504                        // §5.1 verifies the content address; the client discards
9505                        // the segment and re-pulls on either failure.
9506                        if bytes.len() as i64 > byte_length {
9507                            self.schedule_background_retry();
9508                            return Err(SectionError::Abort(
9509                                "sync.invalid_request".to_owned(),
9510                                "segment exceeds its byteLength".to_owned(),
9511                            ));
9512                        }
9513                        on_progress(bytes.len() as u64);
9514                        let digest = Sha256::digest(&bytes);
9515                        let expected = segment_id
9516                            .strip_prefix("sha256:")
9517                            .unwrap_or(segment_id.as_str());
9518                        if bytes_to_hex(&digest) != expected {
9519                            // §5.1: the client discards the segment and re-pulls.
9520                            self.schedule_background_retry();
9521                            return Err(SectionError::Abort(
9522                                "sync.invalid_request".to_owned(),
9523                                "segment bytes do not match the content address (§5.1)".to_owned(),
9524                            ));
9525                        }
9526                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9527                        if media_type == MediaType::Sqlite {
9528                            // §5.3: a range image keeps its position in the resume
9529                            // token; a descriptor row cursor is invalid.
9530                            if row_cursor.is_some() || next_row_cursor.is_some() {
9531                                return Err(SectionError::Abort(
9532                                    "sync.invalid_request".to_owned(),
9533                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9534                                        .to_owned(),
9535                                ));
9536                            }
9537                            let (pair, clear, effective) =
9538                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9539                            report.segment_rows_applied += self.apply_sqlite_segment(
9540                                &pair,
9541                                &bytes,
9542                                clear,
9543                                &effective,
9544                                &unit.id,
9545                                row_count,
9546                                as_of_commit_seq,
9547                                &scope_digest,
9548                            )?;
9549                            seeded.insert(table);
9550                        } else {
9551                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9552                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9553                            })?;
9554                            let (pair, clear, effective) =
9555                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9556                            report.segment_rows_applied +=
9557                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9558                            seeded.insert(segment.table);
9559                        }
9560                        self.applied_segments
9561                            .entry(unit.id.clone())
9562                            .or_insert_with(|| {
9563                                (unit.cursor, unit.bootstrap_state.clone(), BTreeSet::new())
9564                            })
9565                            .2
9566                            .insert(segment_id);
9567                    }
9568                    _ => {
9569                        return Err(SectionError::Abort(
9570                            "sync.invalid_request".to_owned(),
9571                            "unexpected frame inside a unit section".to_owned(),
9572                        ));
9573                    }
9574                }
9575            }
9576            Ok(())
9577        })();
9578        match applied {
9579            Ok(()) => {}
9580            // The unit keeps catching up with its cursor unpersisted.
9581            Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9582            Err(SectionError::FailClosed) => {
9583                // §5.6: the failure is unit-local; the rest of the response
9584                // still applies and the failed unit leaves the list.
9585                self.begin_observation("syncular_section_failure")
9586                    .map_err(storage)?;
9587                let failed = (|| {
9588                    let mut batch = ChangeAccumulator::default();
9589                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9590                        batch.window(&base_key, &value);
9591                    }
9592                    batch.status = true;
9593                    self.stored(save_unit(
9594                        &self.conn,
9595                        &UnitRecord {
9596                            status: UnitStatus::Failed,
9597                            reason_code: Some("sync.scope_revoked".to_owned()),
9598                            ..unit.clone()
9599                        },
9600                    ))?;
9601                    self.finish_observation("syncular_section_failure", batch)
9602                })();
9603                if let Err(message) = failed {
9604                    self.rollback_observation("syncular_section_failure");
9605                    return Err(storage(message));
9606                }
9607                report.failed.push(unit.id);
9608                return Ok(());
9609            }
9610            Err(error) => return Err(error),
9611        }
9612        let Some((next_cursor, bootstrap_state)) = sub_end else {
9613            return Ok(());
9614        };
9615        let rollback = self.capture();
9616        let counts = (
9617            report.resets.len(),
9618            report.revoked.len(),
9619            report.failed.len(),
9620        );
9621        self.begin_observation("syncular_sub_end")
9622            .map_err(storage)?;
9623        let finished = self
9624            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9625            .and_then(|batch| {
9626                self.rebuild_overlay_if_dirty()?;
9627                self.finish_observation("syncular_sub_end", batch)
9628            });
9629        if let Err(message) = finished {
9630            self.rollback_observation("syncular_sub_end");
9631            self.restore(rollback);
9632            report.resets.truncate(counts.0);
9633            report.revoked.truncate(counts.1);
9634            report.failed.truncate(counts.2);
9635            return Err(storage(message));
9636        }
9637        self.applied_segments.remove(&unit.id);
9638        Ok(())
9639    }
9640
9641    /// §5.6: the table, first-page flag and effective scopes of one segment
9642    /// of `unit`'s section.
9643    #[allow(clippy::type_complexity)]
9644    fn segment_pair(
9645        &self,
9646        unit: &UnitRecord,
9647        echo: &[EchoRecord],
9648        seeded: &BTreeSet<String>,
9649        name: &str,
9650    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9651        let table = self.schema.table(name).cloned().ok_or_else(|| {
9652            SectionError::Abort(
9653                "sync.schema_mismatch".to_owned(),
9654                format!("segment table {name:?} is not in the client schema"),
9655            )
9656        })?;
9657        let effective = covers(&table, &unit.scopes)
9658            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9659            .flatten()
9660            .ok_or_else(|| {
9661                SectionError::Abort(
9662                    "sync.invalid_request".to_owned(),
9663                    "a segment names no active pair of its unit".to_owned(),
9664                )
9665            })?;
9666        Ok((table, !seeded.contains(name), effective))
9667    }
9668
9669    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9670    /// caller's observation: a reset discards cursor, resume token and
9671    /// first-page record and keeps the echo; a revoked pair purges by its last
9672    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9673    /// fenced record that turns active again re-bootstraps the unit, and a
9674    /// bootstrap section persists its pin, resume token and seeded tables.
9675    fn finish_section(
9676        &mut self,
9677        start: &SectionStart,
9678        echo: Vec<EchoRecord>,
9679        next_cursor: i64,
9680        bootstrap_state: Option<String>,
9681        seeded: BTreeSet<String>,
9682        report: &mut SyncReport,
9683    ) -> Result<ChangeAccumulator, String> {
9684        #[cfg(feature = "bench-internals")]
9685        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9686        let mut batch = ChangeAccumulator::default();
9687        // §4.4: SUB_END writes only into a unit that is still registered.
9688        let Some(unit) = self
9689            .stored(get_unit(&self.conn, &start.id))?
9690            .filter(|unit| unit.status == UnitStatus::Active)
9691        else {
9692            return Ok(batch);
9693        };
9694        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9695            batch.window(&base_key, &value);
9696        }
9697        if start.status == SubStatus::Reset {
9698            // §4.6: staleness, never a purge; the rows stay until the
9699            // bootstrap replaces them.
9700            report.resets.push(unit.id.clone());
9701            self.stored(save_unit(
9702                &self.conn,
9703                &UnitRecord {
9704                    reason_code: Some(start.reason_code.clone()),
9705                    cursor: Some(-1),
9706                    bootstrap_state: None,
9707                    seeded: Vec::new(),
9708                    ..unit
9709                },
9710            ))?;
9711            return Ok(batch);
9712        }
9713        let revoked = start.status == SubStatus::Revoked;
9714        let mut units = None;
9715        let mut failed = false;
9716        let mut rearmed = false;
9717        for table in self.schema.tables.clone() {
9718            if !covers(&table, &unit.scopes) {
9719                continue;
9720            }
9721            let last = unit
9722                .echo
9723                .as_deref()
9724                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9725            let next = (!revoked)
9726                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9727                .flatten();
9728            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9729            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9730            // §3.3: purge a pair the response revokes, keyed on the echo the
9731            // client persisted while the pair was active.
9732            if let (Some(last), None) = (&last, &next) {
9733                let units = match units {
9734                    Some(ref units) => units,
9735                    None => units.insert(self.stored(load_units(&self.conn))?),
9736                };
9737                let owners = unit_owners(units, &table, &unit.id);
9738                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9739            }
9740        }
9741        if revoked || failed {
9742            if revoked {
9743                report.revoked.push(unit.id.clone());
9744            }
9745            if failed {
9746                report.failed.push(unit.id.clone());
9747            }
9748            batch.status = true;
9749            self.stored(save_unit(
9750                &self.conn,
9751                &UnitRecord {
9752                    status: if failed {
9753                        UnitStatus::Failed
9754                    } else {
9755                        UnitStatus::Revoked
9756                    },
9757                    reason_code: Some(if failed {
9758                        "sync.scope_revoked".to_owned()
9759                    } else {
9760                        start.reason_code.clone()
9761                    }),
9762                    cursor: Some(-1),
9763                    bootstrap_state: None,
9764                    seeded: Vec::new(),
9765                    echo: None,
9766                    ..unit
9767                },
9768            ))?;
9769            return Ok(batch);
9770        }
9771        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9772        let (cursor, bootstrap_state, seeded) = if rearmed {
9773            (Some(-1), None, Vec::new())
9774        } else if start.bootstrap {
9775            let seeded = if bootstrap_state.is_none() {
9776                Vec::new()
9777            } else {
9778                seeded.into_iter().collect()
9779            };
9780            (Some(next_cursor), bootstrap_state, seeded)
9781        } else {
9782            (
9783                unit.cursor,
9784                unit.bootstrap_state.clone(),
9785                unit.seeded.clone(),
9786            )
9787        };
9788        self.stored(save_unit(
9789            &self.conn,
9790            &UnitRecord {
9791                reason_code: None,
9792                echo: Some(echo.clone()),
9793                cursor,
9794                bootstrap_state,
9795                seeded,
9796                ..unit.clone()
9797            },
9798        ))?;
9799        if !completed {
9800            return Ok(batch);
9801        }
9802        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9803        // at or below its pin.
9804        for table in self.schema.tables.clone() {
9805            let Some(effective) = covers(&table, &unit.scopes)
9806                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9807                .flatten()
9808            else {
9809                continue;
9810            };
9811            let mut statement = self
9812                .conn
9813                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9814                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9815            let rows = statement
9816                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9817                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9818                })
9819                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9820            let mut covered = HashSet::new();
9821            for row in rows {
9822                let (id, intent) =
9823                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9824                let values: Map<String, Value> =
9825                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9826                if owns_local_row(&table, &effective, &[], &values) {
9827                    covered.insert(id);
9828                }
9829            }
9830            drop(statement);
9831            for id in covered {
9832                self.conn
9833                    .execute(
9834                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9835                        rusqlite::params![table.name, id, next_cursor],
9836                    )
9837                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9838                self.overlay_dirty.table(&table.name);
9839                batch.table(&table.name);
9840            }
9841        }
9842        Ok(batch)
9843    }
9844
9845    /// §3.3: purge one revoked pair inside the caller's observation: delete
9846    /// the rows matching its last effective scopes that no owner holds, drop
9847    /// the retained intent and every pending commit that provably writes into
9848    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9849    /// table has no local scope-column mapping (fail closed: nothing deleted).
9850    fn purge_pair(
9851        &mut self,
9852        batch: &mut ChangeAccumulator,
9853        table: &TableSchema,
9854        last: &[(String, Vec<String>)],
9855        owners: &[Vec<(String, Vec<String>)>],
9856    ) -> Result<bool, String> {
9857        if local_scope_predicate(table, last, owners).is_err() {
9858            return Ok(false);
9859        }
9860        for commit in self.drop_failed_commits(|name, values| {
9861            name == table.name && owns_local_row(table, last, owners, values)
9862        })? {
9863            batch.conflicts = true;
9864            batch.rejections = true;
9865            batch.outcomes = true;
9866            for operation in commit.ops {
9867                batch.table(&operation.table);
9868            }
9869        }
9870        if !self.purge_scope_rows(&table.name, last, owners)? {
9871            return Err("storage.failed: the revoked scope purge failed".to_owned());
9872        }
9873        self.record_scope_map(batch, &table.name, last);
9874        if self.drop_doomed_outbox(table, last, owners)? {
9875            batch.status = true;
9876            batch.rejections = true;
9877            batch.outcomes = true;
9878        }
9879        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9880        self.delete_unreferenced_cached_blobs()?;
9881        Ok(true)
9882    }
9883
9884    /// §4.5 client application: apply one live-section or delta `COMMIT`
9885    /// with only the changes that match the persisted echo of a receiving
9886    /// unit; every applied change then goes through the version guard.
9887    /// Returns whether any change of the frame applied.
9888    fn apply_live_commit(
9889        &mut self,
9890        receivers: &[UnitRecord],
9891        tables: &[String],
9892        changes: Vec<ssp2::model::Change>,
9893        commit_seq: i64,
9894        source: AppliedServerCommitSource,
9895    ) -> Result<bool, TransportError> {
9896        let changes: Vec<ssp2::model::Change> = changes
9897            .into_iter()
9898            .filter(|change| {
9899                // An unknown table fails loud in the apply below.
9900                let Some(table) = tables
9901                    .get(change.table_index as usize)
9902                    .and_then(|name| self.schema.table(name))
9903                else {
9904                    return true;
9905                };
9906                receivers.iter().any(|unit| {
9907                    covers(table, &unit.scopes)
9908                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9909                            |scopes| {
9910                                scopes.iter().all(|(variable, values)| {
9911                                    change
9912                                        .scopes
9913                                        .iter()
9914                                        .find(|(key, _)| key == variable)
9915                                        .is_some_and(|(_, value)| values.contains(value))
9916                                })
9917                            },
9918                        )
9919                })
9920            })
9921            .collect();
9922        if changes.is_empty() {
9923            return Ok(false);
9924        }
9925        let affected_tables = changes
9926            .iter()
9927            .filter_map(|change| tables.get(change.table_index as usize))
9928            .cloned()
9929            .collect::<BTreeSet<_>>();
9930        let previous_batches = self.change_queue.len();
9931        self.apply_commit_frame(tables, &changes, commit_seq)?;
9932        if self.change_queue.len() > previous_batches {
9933            if let Some(batch) = self.change_queue.back_mut() {
9934                batch
9935                    .applied_commits
9936                    .extend(
9937                        affected_tables
9938                            .into_iter()
9939                            .map(|table| AppliedServerCommit {
9940                                table,
9941                                commit_seq,
9942                                source,
9943                            }),
9944                    );
9945            }
9946        }
9947        Ok(true)
9948    }
9949
9950    // Both pull and realtime frames use the same durable observation boundary.
9951    fn apply_commit_frame(
9952        &mut self,
9953        tables: &[String],
9954        changes: &[ssp2::model::Change],
9955        sequence: i64,
9956    ) -> Result<(), TransportError> {
9957        #[cfg(feature = "bench-internals")]
9958        let _phase = self.benchmark_phases.start(Phase::CommitApply);
9959        let rollback = self.capture();
9960        self.coalesce_unit(changes.len())
9961            .and_then(|()| self.begin_observation("syncular_commit"))
9962            .map_err(|message| TransportError::new("storage.failed", message))?;
9963        let applied = (|| {
9964            let mut batch = ChangeAccumulator::default();
9965            self.record_commit_changes(&mut batch, tables, changes);
9966            // §7.1 chained base: a change names the version an acknowledged own
9967            // commit wrote; pending upserts chained on it move to that version.
9968            for change in changes {
9969                let Some(table) = tables.get(change.table_index as usize) else {
9970                    continue;
9971                };
9972                let acknowledged = self
9973                    .conn
9974                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
9975                    .and_then(|mut statement| {
9976                        statement
9977                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
9978                            .collect::<Result<Vec<_>, _>>()
9979                    })
9980                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
9981                    .iter()
9982                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
9983                    .collect::<Result<Vec<_>, _>>()
9984                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
9985                for index in 0..self.outbox.len() {
9986                    if !self.outbox[index]
9987                        .ops
9988                        .iter()
9989                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
9990                    {
9991                        continue;
9992                    }
9993                    let commit = Arc::make_mut(&mut self.outbox[index]);
9994                    for op in &mut commit.ops {
9995                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
9996                            op.base_version = Some(change.row_version);
9997                        }
9998                    }
9999                    let ops = serde_json::to_string(
10000                        &commit
10001                            .ops
10002                            .iter()
10003                            .map(CommitOperation::from)
10004                            .collect::<Vec<_>>(),
10005                    )
10006                    .map_err(|_| {
10007                        TransportError::new(
10008                            "storage.failed",
10009                            "sync.local_corrupt: unencodable outbox".to_owned(),
10010                        )
10011                    })?;
10012                    let id = commit.client_commit_id.clone();
10013                    self.conn
10014                        .execute(
10015                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10016                            rusqlite::params![ops, id],
10017                        )
10018                        .map_err(|error| {
10019                            TransportError::new(
10020                                "storage.failed",
10021                                Self::sqlite_failure(&self.storage_failure, error),
10022                            )
10023                        })?;
10024                }
10025            }
10026            {
10027                let storage_failed = |error| {
10028                    TransportError::new(
10029                        "storage.failed",
10030                        Self::sqlite_failure(&self.storage_failure, error),
10031                    )
10032                };
10033                let pinned: HashSet<(&str, &str)> = self
10034                    .outbox
10035                    .iter()
10036                    .flat_map(|commit| &commit.ops)
10037                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10038                    .collect();
10039                for change in changes {
10040                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10041                        TransportError::new(
10042                            "sync.invalid_request",
10043                            "change tableIndex out of range".to_owned(),
10044                        )
10045                    })?;
10046                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10047                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10048                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10049                            .map_err(storage_failed)?;
10050                    }
10051                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10052                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10053                        .map_err(storage_failed)?;
10054                }
10055            }
10056            self.apply_commit_changes(tables, changes)?;
10057            self.rebuild_overlay_if_dirty()
10058                .map_err(|message| TransportError::new("storage.failed", message))?;
10059            self.finish_observation("syncular_commit", batch)
10060                .map_err(|message| TransportError::new("storage.failed", message))
10061        })();
10062        if applied.is_err() {
10063            self.rollback_observation("syncular_commit");
10064            self.restore(rollback);
10065        }
10066        applied
10067    }
10068
10069    fn apply_commit_changes(
10070        &mut self,
10071        tables: &[String],
10072        changes: &[ssp2::model::Change],
10073    ) -> Result<(), TransportError> {
10074        // A clean overlay can reconcile changed primary keys independently
10075        // when their tables have no secondary unique constraints. Such a
10076        // constraint can change whether another row's write succeeds, so a
10077        // table with one and any local intent takes the table rebuild.
10078        let changed_tables: BTreeSet<&str> = changes
10079            .iter()
10080            .filter_map(|change| tables.get(change.table_index as usize))
10081            .map(String::as_str)
10082            .collect();
10083        let mut mirror_visible = !self.overlay_dirty.get();
10084        for name in &changed_tables {
10085            if mirror_visible
10086                && self
10087                    .schema
10088                    .table(name)
10089                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10090            {
10091                let (pending, protected) = self
10092                    .table_intent(name)
10093                    .map_err(|message| TransportError::new("storage.failed", message))?;
10094                mirror_visible = !pending && !protected;
10095            }
10096        }
10097        for change in changes {
10098            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10099                TransportError::new(
10100                    "sync.invalid_request",
10101                    "change tableIndex out of range".to_owned(),
10102                )
10103            })?;
10104            let table = self.schema.table(table_name).ok_or_else(|| {
10105                TransportError::new(
10106                    "sync.schema_mismatch",
10107                    format!("change targets unknown table {table_name:?}"),
10108                )
10109            })?;
10110            let landed = match change.op {
10111                Op::Upsert => {
10112                    let payload = change.row.as_ref().ok_or_else(|| {
10113                        TransportError::new(
10114                            "sync.invalid_request",
10115                            "upsert change without row payload".to_owned(),
10116                        )
10117                    })?;
10118                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10119                    let (row, sealed) = {
10120                        #[cfg(feature = "bench-internals")]
10121                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10122                        // §5.11: a decrypt failure keeps its own identity.
10123                        decode_row_bytes(table, payload, &self.encryption)?
10124                    };
10125                    let table_name = table.name.clone();
10126                    // §4.5 version guard: only a newer upsert lands.
10127                    let landed = self
10128                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10129                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10130                    if mirror_visible && landed {
10131                        self.write_row(
10132                            &visible_table(&table_name),
10133                            &table_name,
10134                            &row,
10135                            change.row_version,
10136                            sealed.as_ref(),
10137                        )
10138                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10139                    }
10140                    landed
10141                }
10142                // §4.5 version guard: a delete at or below the local version
10143                // is skipped, a delete of an absent row is a no-op.
10144                Op::Delete => self
10145                    .delete_base_row(table_name, &change.row_id, change.row_version)
10146                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10147            };
10148            // The change dropped the row's acknowledged intent either way
10149            // (§7.1), so the overlay of a skipped change rebuilds from the
10150            // unchanged base.
10151            self.overlay_dirty.table(table_name);
10152            if mirror_visible && !(landed && change.op == Op::Upsert) {
10153                let predicate = row_id_predicate(table);
10154                let visible = visible_table(table_name);
10155                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10156                if !landed {
10157                    statements.push(format!(
10158                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10159                        base_table(table_name)
10160                    ));
10161                }
10162                for statement in &statements {
10163                    self.conn
10164                        .prepare_cached(statement)
10165                        .and_then(|mut statement| {
10166                            statement.execute(rusqlite::params![change.row_id])
10167                        })
10168                        .map_err(|error| {
10169                            TransportError::new("sync.invalid_request", error.to_string())
10170                        })?;
10171                }
10172            }
10173        }
10174        if mirror_visible {
10175            let changed_rows: HashSet<(&str, &str)> = changes
10176                .iter()
10177                .map(|change| {
10178                    (
10179                        tables[change.table_index as usize].as_str(),
10180                        change.row_id.as_str(),
10181                    )
10182                })
10183                .collect();
10184            self.replay_overlay(&changed_tables, Some(&changed_rows))
10185                .map_err(|message| TransportError::new("storage.failed", message))?;
10186            self.overlay_dirty.set(false);
10187        }
10188        Ok(())
10189    }
10190
10191    /// §5.6 segment application: validate against the generated schema,
10192    /// clear the pair's scope on the table's first page (fail closed
10193    /// without a mapping), then upsert each row with its segment-carried
10194    /// server version under the §4.5 version guard (§5.2).
10195    fn apply_segment(
10196        &mut self,
10197        table: &TableSchema,
10198        segment: &RowsSegment,
10199        first_page: bool,
10200        effective: &[(String, Vec<String>)],
10201        unit_id: &str,
10202    ) -> Result<u32, SectionError> {
10203        // §5.2: the column table validates against the generated schema —
10204        // order, names, types, nullability; mismatch is fatal. §5.11: the
10205        // server sends the WIRE types (bytes for an encrypted column), so
10206        // validate against wire_columns.
10207        let column = if segment.columns.len() != table.wire_columns.len() {
10208            Some(Value::from("count"))
10209        } else {
10210            segment
10211                .columns
10212                .iter()
10213                .zip(table.wire_columns.iter())
10214                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10215                .map(Value::from)
10216        };
10217        if segment.table != table.name
10218            || segment.schema_version != self.schema.version
10219            || column.is_some()
10220        {
10221            let mut details = Map::new();
10222            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10223            if let Some(column) = column {
10224                details.insert("column".to_owned(), column);
10225            }
10226            return Err(SectionError::Transport(TransportError {
10227                code: "sync.schema_mismatch".to_owned(),
10228                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10229                details: Some(Value::Object(details)),
10230            }));
10231        }
10232        let mut applied = 0u32;
10233        let empty = Vec::new();
10234        let blocks = if segment.blocks.is_empty() {
10235            std::slice::from_ref(&empty)
10236        } else {
10237            &segment.blocks
10238        };
10239        for (index, block) in blocks.iter().enumerate() {
10240            let clear = first_page && index == 0;
10241            if !clear && block.is_empty() {
10242                continue;
10243            }
10244            let rollback = self.capture();
10245            let (pending, protected) = self
10246                .table_intent(&table.name)
10247                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10248            let reconcile_pending =
10249                !protected && pending && table.indexes.iter().any(|index| index.unique);
10250            let mirror_visible = !protected && !reconcile_pending;
10251            self.coalesce_unit(block.len())
10252                .and_then(|()| self.begin_observation("syncular_segment_block"))
10253                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10254            let outcome = (|| {
10255                self.rebuild_overlay_if_dirty()
10256                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10257                let mut changed_keys = Vec::new();
10258                let mut batch = ChangeAccumulator::default();
10259                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10260                    batch.table(&table.name);
10261                }
10262                for (row_index, row) in block.iter().enumerate() {
10263                    let decrypted;
10264                    let mut sealed = None;
10265                    let values = if table.has_encrypted_columns() {
10266                        let mut values = row.values.clone();
10267                        sealed = crate::values::decrypt_segment_row(
10268                            table,
10269                            &mut values,
10270                            &self.encryption,
10271                        )
10272                        .map_err(SectionError::Transport)?;
10273                        decrypted = values;
10274                        &decrypted
10275                    } else {
10276                        &row.values
10277                    };
10278                    if !self
10279                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10280                        .map_err(|message| {
10281                            SectionError::Abort("sync.invalid_request".into(), message)
10282                        })?
10283                    {
10284                        continue;
10285                    }
10286                    if reconcile_pending {
10287                        changed_keys.push(
10288                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10289                                |message| SectionError::Abort("storage.failed".into(), message),
10290                            )?,
10291                        );
10292                    }
10293                    if mirror_visible {
10294                        self.write_row(
10295                            &visible_table(&table.name),
10296                            &table.name,
10297                            values,
10298                            row.server_version,
10299                            sealed.as_ref(),
10300                        )
10301                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10302                    }
10303                    let processed = u64::from(applied) + row_index as u64 + 1;
10304                    if processed.is_multiple_of(1024) {
10305                        self.progress.update(|p| p.rows_processed = processed);
10306                    }
10307                }
10308                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10309                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10310                self.finish_observation("syncular_segment_block", batch)
10311                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10312            })();
10313            if let Err(error) = outcome {
10314                self.rollback_observation("syncular_segment_block");
10315                self.restore(rollback);
10316                return Err(error);
10317            }
10318            applied += block.len() as u32;
10319            self.progress
10320                .update(|p| p.rows_processed = u64::from(applied));
10321            std::thread::yield_now();
10322        }
10323        self.progress
10324            .update(|p| p.rows_processed = u64::from(applied));
10325        Ok(applied)
10326    }
10327
10328    /// §5.3 sqlite-image application: validate the in-file metadata
10329    /// against the descriptor, validate column names/order against the
10330    /// generated schema, run the §5.6 first-page clear when fresh, then
10331    /// replace-or-upsert every image row with its `_syncular_version`.
10332    /// Mechanics: the image lands in a temp file read through a second
10333    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10334    /// Each image chunk commits its writes and revision before readers continue.
10335    #[allow(clippy::too_many_arguments)]
10336    fn apply_sqlite_segment(
10337        &mut self,
10338        table: &TableSchema,
10339        bytes: &[u8],
10340        first_page: bool,
10341        effective: &[(String, Vec<String>)],
10342        unit_id: &str,
10343        row_count: i64,
10344        as_of_commit_seq: i64,
10345        scope_digest: &str,
10346    ) -> Result<u32, SectionError> {
10347        let invalid = |detail: &str| {
10348            SectionError::Abort(
10349                "sync.invalid_request".to_owned(),
10350                format!("sqlite segment rejected: {detail} (§5.3)"),
10351            )
10352        };
10353        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10354        // one for a table with encrypted columns.
10355        if table.has_encrypted_columns() {
10356            return Err(invalid("table has encrypted columns"));
10357        }
10358
10359        // The image lands beside the replica so it shares the replica's
10360        // directory permissions and volume; an in-memory replica has no
10361        // directory and uses the system temp dir. The file is owner-only.
10362        let directory = self
10363            .previous_version_replica_path()
10364            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10365            .unwrap_or_else(std::env::temp_dir);
10366        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10367        let outcome = (|| {
10368            let mut options = std::fs::OpenOptions::new();
10369            options.write(true).create_new(true);
10370            #[cfg(unix)]
10371            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10372            std::io::Write::write_all(
10373                &mut options
10374                    .open(&path)
10375                    .map_err(|_| invalid("image temp file write failed"))?,
10376                bytes,
10377            )
10378            .map_err(|_| invalid("image temp file write failed"))?;
10379            let img = rusqlite::Connection::open_with_flags(
10380                &path,
10381                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10382            )
10383            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10384            self.apply_sqlite_image(
10385                &img,
10386                table,
10387                first_page,
10388                effective,
10389                unit_id,
10390                row_count,
10391                as_of_commit_seq,
10392                scope_digest,
10393            )
10394        })();
10395        let _ = std::fs::remove_file(&path);
10396        outcome
10397    }
10398
10399    #[allow(clippy::too_many_arguments)]
10400    fn apply_sqlite_image(
10401        &mut self,
10402        img: &rusqlite::Connection,
10403        table: &crate::schema::TableSchema,
10404        first_fresh_page: bool,
10405        effective: &[(String, Vec<String>)],
10406        unit_id: &str,
10407        row_count: i64,
10408        as_of_commit_seq: i64,
10409        scope_digest: &str,
10410    ) -> Result<u32, SectionError> {
10411        let invalid = |detail: String| {
10412            SectionError::Abort(
10413                "sync.invalid_request".to_owned(),
10414                format!("sqlite segment rejected: {detail} (§5.3)"),
10415            )
10416        };
10417
10418        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10419        //    one row).
10420        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10421        let meta: MetaRow = img
10422            .query_row(
10423                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10424                        \"scopeDigest\", \"rowCount\",
10425                        (SELECT count(*) FROM _syncular_segment)
10426                 FROM _syncular_segment",
10427                [],
10428                |row| {
10429                    Ok((
10430                        row.get(0)?,
10431                        row.get(1)?,
10432                        row.get(2)?,
10433                        row.get(3)?,
10434                        row.get(4)?,
10435                        row.get(5)?,
10436                        row.get(6)?,
10437                    ))
10438                },
10439            )
10440            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10441        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10442        if meta_count != 1 {
10443            return Err(invalid(format!(
10444                "_syncular_segment must contain exactly one row, found {meta_count}"
10445            )));
10446        }
10447        if format != 1 {
10448            return Err(invalid(format!("format {format}")));
10449        }
10450        if meta_table != table.name {
10451            return Err(invalid(format!("image table {meta_table:?}")));
10452        }
10453        if schema_version != i64::from(self.schema.version) {
10454            return Err(invalid(format!("schemaVersion {schema_version}")));
10455        }
10456        if pin != as_of_commit_seq {
10457            return Err(invalid(format!("asOfCommitSeq {pin}")));
10458        }
10459        if digest != scope_digest {
10460            return Err(invalid("scopeDigest mismatch".to_owned()));
10461        }
10462        if meta_rows != row_count {
10463            return Err(invalid(format!("rowCount {meta_rows}")));
10464        }
10465
10466        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10467        let mut names: Vec<String> = Vec::new();
10468        let mut primary_keys: Vec<String> = Vec::new();
10469        {
10470            let mut stmt = img
10471                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10472                .map_err(|_| invalid("image data table missing".to_owned()))?;
10473            let mut rows = stmt
10474                .query([])
10475                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10476            while let Some(row) = rows
10477                .next()
10478                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10479            {
10480                if row
10481                    .get::<_, i64>(5)
10482                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10483                    > 0
10484                {
10485                    primary_keys.push(row.get(1).map_err(|error| {
10486                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10487                    })?);
10488                }
10489                names.push(
10490                    row.get::<_, String>(1)
10491                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10492                );
10493            }
10494        }
10495        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10496        expected.push("_syncular_version");
10497        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10498            return Err(SectionError::Transport(TransportError {
10499                code: "sync.schema_mismatch".to_owned(),
10500                message: "sqlite segment columns do not match the generated schema (§5.3)"
10501                    .to_owned(),
10502                details: Some(serde_json::json!({ "table": table.name })),
10503            }));
10504        }
10505
10506        if primary_keys != vec![table.primary_key.clone()] {
10507            return Err(invalid(
10508                "image primary key does not match generated schema".into(),
10509            ));
10510        }
10511        let actual_count: i64 = img
10512            .query_row(
10513                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10514                [],
10515                |row| row.get(0),
10516            )
10517            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10518        if actual_count != row_count {
10519            return Err(invalid("image row count does not match descriptor".into()));
10520        }
10521        let insert = self.insert_row_sql(&base_table(&table.name), table);
10522        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10523        let column_list = names
10524            .iter()
10525            .map(|n| quote_ident(n))
10526            .collect::<Vec<_>>()
10527            .join(", ");
10528        let mut stmt = img
10529            .prepare(&format!(
10530                "SELECT {column_list} FROM {} ORDER BY {}",
10531                quote_ident(&table.name),
10532                quote_ident(&table.primary_key)
10533            ))
10534            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10535        let mut rows = stmt
10536            .query([])
10537            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10538        let mut applied = 0u32;
10539        let mut limit = if first_fresh_page {
10540            1024
10541        } else {
10542            self.image_chunk_rows
10543                .get(&table.name)
10544                .copied()
10545                .unwrap_or(1024)
10546        };
10547        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10548        loop {
10549            let clear = applied == 0 && first_fresh_page;
10550            let rollback = self.capture();
10551            let (pending, protected) = self
10552                .table_intent(&table.name)
10553                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10554            let reconcile_pending =
10555                !protected && pending && table.indexes.iter().any(|index| index.unique);
10556            let mirror_visible = !protected && !reconcile_pending;
10557            // §5.3: a table's first chunk holds 1,024 rows; a later range image
10558            // starts at the size its previous range reached. A later chunk holds at
10559            // least 1,024 rows, at most twice the rows of the one before, at
10560            // most 65,536, and at most the rows the one before applied per
10561            // 250 ms from its start to its COMMIT; a chunk that replays local
10562            // writes holds at most 1,024.
10563            let chunk = if pending || protected {
10564                limit.min(1024)
10565            } else {
10566                limit
10567            };
10568            self.coalesce_unit(chunk as usize)
10569                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10570                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10571            let started = std::time::Instant::now();
10572            let result = (|| {
10573                self.rebuild_overlay_if_dirty()
10574                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10575                let mut changed_keys = Vec::new();
10576                let mut batch = ChangeAccumulator::default();
10577                if clear && self.clear_scope(table, effective, unit_id)? {
10578                    batch.table(&table.name);
10579                }
10580                let mut processed = 0u32;
10581                {
10582                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10583                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10584                    })?;
10585                    let mut mirror = if mirror_visible {
10586                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10587                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10588                        })?)
10589                    } else {
10590                        None
10591                    };
10592                    for _ in 0..chunk {
10593                        let Some(row) = rows.next().map_err(|error| {
10594                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10595                        })?
10596                        else {
10597                            break;
10598                        };
10599                        for (i, column) in table.columns.iter().enumerate() {
10600                            let cell = row.get_ref(i).map_err(|error| {
10601                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10602                            })?;
10603                            let param = image_cell_param(column, cell).map_err(invalid)?;
10604                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10605                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10606                            })?;
10607                            if let Some(mirror) = &mut mirror {
10608                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10609                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10610                                })?;
10611                            }
10612                        }
10613                        let version_index = table.columns.len();
10614                        let version: i64 = row.get(version_index).map_err(|error| {
10615                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10616                        })?;
10617                        if version < 1 {
10618                            return Err(invalid("image row version must be positive".into()));
10619                        }
10620                        ins.raw_bind_parameter(version_index + 1, version)
10621                            .map_err(|error| {
10622                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10623                            })?;
10624                        // §4.5 version guard: the base upsert decides.
10625                        let landed = ins.raw_execute().map_err(|error| {
10626                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10627                        })? > 0;
10628                        processed += 1;
10629                        if processed.is_multiple_of(1024) && processed < chunk {
10630                            self.progress
10631                                .update(|p| p.rows_processed = u64::from(applied + processed));
10632                        }
10633                        if !landed {
10634                            continue;
10635                        }
10636                        if let Some(mirror) = &mut mirror {
10637                            mirror
10638                                .raw_bind_parameter(version_index + 1, version)
10639                                .map_err(|error| {
10640                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10641                                })?;
10642                            mirror.raw_execute().map_err(|error| {
10643                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10644                            })?;
10645                        }
10646                        if reconcile_pending {
10647                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10648                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10649                            )?);
10650                        }
10651                    }
10652                }
10653                if processed > 0 {
10654                    batch.table(&table.name);
10655                    self.overlay_dirty.table(&table.name);
10656                }
10657                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10658                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10659                let written = std::time::Instant::now();
10660                self.finish_observation("syncular_image_chunk", batch)
10661                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10662                Ok((processed, written))
10663            })();
10664            let (processed, written) = match result {
10665                Ok(done) => done,
10666                Err(error) => {
10667                    self.rollback_observation("syncular_image_chunk");
10668                    self.restore(rollback);
10669                    return Err(error);
10670                }
10671            };
10672            // The chunk commits before the yield, also inside a coalesced apply.
10673            let coalesced = self.coalesced.is_some();
10674            if coalesced {
10675                self.commit_coalesced_apply()
10676                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10677            }
10678            write += written - started;
10679            commit += written.elapsed();
10680            applied += processed;
10681            let last = processed < chunk || i64::from(applied) == actual_count;
10682            // A file replica's WAL keeps its high-water size until truncated.
10683            if last && self.conn.is_autocommit() {
10684                self.conn
10685                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10686                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10687            }
10688            if coalesced {
10689                self.begin_coalesced_apply()
10690                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10691            }
10692            self.progress.update(|p| {
10693                p.rows_processed = u64::from(applied);
10694                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10695                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10696            });
10697            let duration = started.elapsed().as_secs_f64() * 1000.0;
10698            #[cfg(test)]
10699            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10700            std::thread::yield_now();
10701            if last {
10702                break;
10703            }
10704            // A zero duration saturates the cast and leaves the doubling.
10705            limit = (processed * 2)
10706                .min(65_536)
10707                .min((f64::from(processed) * 250.0 / duration) as u32)
10708                .max(1024);
10709            self.image_chunk_rows.insert(table.name.clone(), limit);
10710        }
10711        if i64::from(applied) != actual_count {
10712            return Err(invalid(format!(
10713                "image yielded {applied} rows, expected {actual_count}"
10714            )));
10715        }
10716        Ok(applied)
10717    }
10718
10719    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10720    /// them from the base and replay their remaining intent. A table with a
10721    /// secondary unique index restores and replays every row with intent,
10722    /// since one row's value can admit or defer another row's write.
10723    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10724        let names: BTreeSet<&str> = drained
10725            .iter()
10726            .flat_map(|commit| &commit.ops)
10727            .map(|op| op.table.as_str())
10728            .collect();
10729        for name in names {
10730            let Some(table) = self.schema.table(name) else {
10731                continue;
10732            };
10733            let ops = drained.iter().flat_map(|commit| &commit.ops);
10734            let tables = BTreeSet::from([name]);
10735            let rows: HashSet<_> = ops
10736                .clone()
10737                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10738                .collect();
10739            let unique = table.indexes.iter().any(|index| index.unique);
10740            let acknowledged = if unique {
10741                self.acknowledged_operations(None, Some(&tables))?
10742            } else {
10743                Vec::new()
10744            };
10745            let peers = acknowledged
10746                .iter()
10747                .chain(
10748                    self.failed_commits
10749                        .iter()
10750                        .flat_map(|(commit, _)| &commit.ops),
10751                )
10752                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10753                .filter(|_| unique);
10754            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10755            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10756        }
10757        Ok(())
10758    }
10759
10760    /// Whether `table` has pending outbox intent, and whether it has protected
10761    /// (acknowledged or retained failed) intent.
10762    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10763        let touches = |op: &OutboxOp| op.table == table;
10764        Ok((
10765            self.outbox
10766                .iter()
10767                .flat_map(|commit| &commit.ops)
10768                .any(touches),
10769            self.failed_commits
10770                .iter()
10771                .flat_map(|(commit, _)| &commit.ops)
10772                .any(touches)
10773                || self
10774                    .conn
10775                    .query_row(
10776                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10777                        [table],
10778                        |row| row.get(0),
10779                    )
10780                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10781        ))
10782    }
10783
10784    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10785    /// only pending intent, so a table with protected intent rebuilds; a
10786    /// `restore` restores the imported `keys` and every pending row first.
10787    fn reconcile_import(
10788        &mut self,
10789        table: &TableSchema,
10790        keys: &mut Vec<SqlValue>,
10791        protected: bool,
10792        restore: bool,
10793    ) -> Result<(), String> {
10794        if protected {
10795            return self.rebuild_overlay_if_dirty();
10796        }
10797        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10798        if restore {
10799            self.restore_base_rows(table, keys, pending.clone())?;
10800        }
10801        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10802        self.overlay_dirty.set(false);
10803        Ok(())
10804    }
10805
10806    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10807    /// `table` from the base before the caller replays their intent. The
10808    /// caller owns the transaction and publishes its revision after replay.
10809    fn restore_base_rows<'a>(
10810        &self,
10811        table: &TableSchema,
10812        keys: &mut Vec<SqlValue>,
10813        ops: impl IntoIterator<Item = &'a OutboxOp>,
10814    ) -> Result<(), String> {
10815        let visible = visible_table(&table.name);
10816        let base = base_table(&table.name);
10817        let pk = quote_ident(&table.primary_key);
10818        let predicate = row_id_predicate(table);
10819        let mut lookup = self.conn.prepare_cached(&format!(
10820            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10821        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10822        let mut deleted = HashSet::new();
10823        for op in ops.into_iter().filter(|op| op.table == table.name) {
10824            if op.upsert {
10825                if let Some(values) = &op.values {
10826                    let key = json_to_column_value(
10827                        &table.columns[table.pk_index],
10828                        values.get(&table.primary_key),
10829                    )?;
10830                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10831                }
10832            } else if deleted.insert(&op.row_id) {
10833                keys.extend(
10834                    lookup
10835                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10836                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10837                        .collect::<Result<Vec<_>, _>>()
10838                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10839                );
10840            }
10841        }
10842        // Delete the complete affected set before restoring any base value, so
10843        // old optimistic unique values cannot collide with the new server rows.
10844        for chunk in keys.chunks(400) {
10845            let holes = vec!["?"; chunk.len()].join(",");
10846            self.conn
10847                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10848                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10849                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10850        }
10851        let updates = table
10852            .columns
10853            .iter()
10854            .map(|column| column.name.as_str())
10855            .chain(std::iter::once("_syncular_version"))
10856            .chain(table.sealed_columns().iter().copied())
10857            .filter(|name| *name != table.primary_key)
10858            .map(|name| {
10859                let name = quote_ident(name);
10860                format!("{name}=excluded.{name}")
10861            })
10862            .collect::<Vec<_>>()
10863            .join(",");
10864        for chunk in keys.chunks(400) {
10865            let holes = vec!["?"; chunk.len()].join(",");
10866            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10867                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10868                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10869        }
10870        Ok(())
10871    }
10872
10873    // -- application-authorized local purge ---------------------------------------
10874
10875    fn compile_local_data_purge(
10876        &self,
10877        input: &LocalDataPurgeInput,
10878    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10879        let invalid = |message: String| format!("sync.invalid_request: {message}");
10880        if input.purge_id.is_empty()
10881            || input.purge_id.len() > 128
10882            || !is_local_operation_code_like(&input.purge_id)
10883        {
10884            return Err(invalid(
10885                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10886            ));
10887        }
10888        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10889            return Err(invalid(format!(
10890                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10891            )));
10892        }
10893
10894        let mut deduplicated: BTreeMap<
10895            String,
10896            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10897        > = BTreeMap::new();
10898        for target in &input.targets {
10899            let table = self.schema.table(&target.table).ok_or_else(|| {
10900                invalid(format!(
10901                    "local purge names unknown table {:?}",
10902                    target.table
10903                ))
10904            })?;
10905            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10906                return Err(invalid(format!(
10907                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10908                    target.table
10909                )));
10910            }
10911            let mut selectors = Vec::with_capacity(target.selectors.len());
10912            let mut canonical_selectors = BTreeMap::new();
10913            for (column_name, raw_values) in &target.selectors {
10914                let Some((column_index, column)) = table
10915                    .columns
10916                    .iter()
10917                    .enumerate()
10918                    .find(|(_, column)| column.name == *column_name)
10919                else {
10920                    return Err(invalid(format!(
10921                        "local purge target {:?} names unknown column {:?}",
10922                        target.table, column_name
10923                    )));
10924                };
10925                let encrypted = table
10926                    .encrypted_columns
10927                    .iter()
10928                    .any(|candidate| candidate.index == column_index);
10929                if column.ty != ColumnType::String || encrypted {
10930                    return Err(invalid(format!(
10931                        "local purge selector {:?}.{:?} must be a plaintext string column",
10932                        target.table, column_name
10933                    )));
10934                }
10935                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
10936                    return Err(invalid(format!(
10937                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
10938                        target.table, column_name
10939                    )));
10940                }
10941                let mut values = raw_values.clone();
10942                values.sort();
10943                values.dedup();
10944                if values.iter().any(|value| {
10945                    value.is_empty()
10946                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
10947                        || !is_local_operation_code_like(value)
10948                }) {
10949                    return Err(invalid(format!(
10950                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
10951                    )));
10952                }
10953                selectors.push((column_name.clone(), values.clone()));
10954                canonical_selectors.insert(column_name.clone(), values);
10955            }
10956            let canonical = LocalDataPurgeTarget {
10957                table: target.table.clone(),
10958                selectors: canonical_selectors,
10959            };
10960            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10961            deduplicated.insert(
10962                key,
10963                (
10964                    CompiledLocalDataPurgeTarget {
10965                        table: target.table.clone(),
10966                        selectors,
10967                    },
10968                    canonical,
10969                ),
10970            );
10971        }
10972        let targets = deduplicated
10973            .values()
10974            .map(|(compiled, _)| compiled.clone())
10975            .collect::<Vec<_>>();
10976        let canonical = deduplicated
10977            .values()
10978            .map(|(_, target)| target.clone())
10979            .collect::<Vec<_>>();
10980        let canonical_plan =
10981            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10982        Ok((targets, canonical_plan))
10983    }
10984
10985    fn local_purge_base_row_ids(
10986        &self,
10987        targets: &[CompiledLocalDataPurgeTarget],
10988    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
10989        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
10990        for target in targets {
10991            let table = self
10992                .schema
10993                .table(&target.table)
10994                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
10995            let mut clauses = Vec::with_capacity(target.selectors.len());
10996            let mut params = Vec::new();
10997            for (column, values) in &target.selectors {
10998                clauses.push(format!(
10999                    "{} IN ({})",
11000                    quote_ident(column),
11001                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11002                ));
11003                params.extend(values.iter().cloned().map(SqlValue::Text));
11004            }
11005            let sql = format!(
11006                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11007                quote_ident(&table.primary_key),
11008                base_table(&target.table),
11009                clauses.join(" AND ")
11010            );
11011            let mut statement = self
11012                .conn
11013                .prepare(&sql)
11014                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11015            let rows = statement
11016                .query_map(rusqlite::params_from_iter(params), |row| {
11017                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11018                    render_row_id_json(Some(&value))
11019                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11020                })
11021                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11022            let ids = by_table.entry(target.table.clone()).or_default();
11023            for row in rows {
11024                ids.insert(
11025                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11026                );
11027            }
11028        }
11029        Ok(by_table)
11030    }
11031
11032    fn local_purge_values_match(
11033        target: &CompiledLocalDataPurgeTarget,
11034        values: &Map<String, Value>,
11035    ) -> bool {
11036        target.selectors.iter().all(|(column, allowed)| {
11037            values
11038                .get(column)
11039                .and_then(Value::as_str)
11040                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11041        })
11042    }
11043
11044    /// Apply one host-authorized local security purge. The host MUST gate the
11045    /// corresponding subscriptions first; this primitive owns local SQLite
11046    /// cleanup and never grants/revokes server authority by itself.
11047    pub fn purge_local_data(
11048        &mut self,
11049        input: &LocalDataPurgeInput,
11050    ) -> Result<LocalDataPurgeResult, String> {
11051        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11052        let meta_key = format!("localPurge:{}", input.purge_id);
11053        let applied_plan = self
11054            .conn
11055            .query_row(
11056                "SELECT value FROM _syncular_meta WHERE key = ?1",
11057                rusqlite::params![meta_key],
11058                |row| row.get::<_, String>(0),
11059            )
11060            .optional()
11061            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11062        if let Some(applied_plan) = applied_plan {
11063            if applied_plan != canonical_plan {
11064                return Err(format!(
11065                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11066                    input.purge_id
11067                ));
11068            }
11069            // A retry after a failed rewrite finishes it.
11070            self.erase_purged_pages()?;
11071            return Ok(LocalDataPurgeResult {
11072                already_applied: true,
11073                purged_rows: 0,
11074                dropped_commits: 0,
11075            });
11076        }
11077
11078        // RFC 0005 D7: the container file cannot be a purge target (its name is
11079        // not in the running schema), so purge drops it as a fixed step, before
11080        // any mirror row is touched and unconditionally.
11081        let _ = self.drop_previous_version();
11082
11083        let rollback = self.capture();
11084        self.begin_observation("syncular_local_purge")?;
11085        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11086            let row_ids = self.local_purge_base_row_ids(&targets)?;
11087            let retained_dropped = self.drop_failed_commits(|table, values| {
11088                targets.iter().any(|target| {
11089                    target.table == table && Self::local_purge_values_match(target, values)
11090                })
11091            })?;
11092            let doomed = self
11093                .outbox
11094                .iter()
11095                .filter(|commit| {
11096                    commit.ops.iter().any(|operation| {
11097                        let matching_targets = targets
11098                            .iter()
11099                            .filter(|target| target.table == operation.table)
11100                            .collect::<Vec<_>>();
11101                        if matching_targets.is_empty() {
11102                            return false;
11103                        }
11104                        if row_ids
11105                            .get(&operation.table)
11106                            .is_some_and(|ids| ids.contains(&operation.row_id))
11107                        {
11108                            return true;
11109                        }
11110                        operation.values.as_ref().is_some_and(|values| {
11111                            matching_targets
11112                                .iter()
11113                                .any(|target| Self::local_purge_values_match(target, values))
11114                        })
11115                    })
11116                })
11117                .cloned()
11118                .collect::<Vec<_>>();
11119            let doomed_ids = doomed
11120                .iter()
11121                .map(|commit| commit.client_commit_id.clone())
11122                .collect::<BTreeSet<_>>();
11123            let mut batch = ChangeAccumulator::default();
11124            for commit in &retained_dropped {
11125                batch.conflicts = true;
11126                batch.rejections = true;
11127                batch.outcomes = true;
11128                for operation in &commit.ops {
11129                    batch.table(&operation.table);
11130                }
11131            }
11132            let mut rejections = Vec::new();
11133            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11134            // codes, never its operations, so no purged value survives in it.
11135            for commit in &doomed {
11136                for operation in &commit.ops {
11137                    batch.table(&operation.table);
11138                    self.overlay_dirty.table(&operation.table);
11139                }
11140                let results = (0..commit.ops.len())
11141                    .map(|op_index| {
11142                        let rejection = RejectionRecord {
11143                            client_commit_id: commit.client_commit_id.clone(),
11144                            op_index: op_index as i32,
11145                            code: "client.local_data_purged".to_owned(),
11146                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11147                            retryable: false,
11148                            details: None,
11149                            operation: None,
11150                        };
11151                        rejections.push(rejection.clone());
11152                        CommitOperationOutcome::Error { rejection }
11153                    })
11154                    .collect::<Vec<_>>();
11155                self.persist_commit_outcome(
11156                    &commit.client_commit_id,
11157                    CommitOutcomeStatus::Rejected,
11158                    &results,
11159                    None,
11160                )?;
11161                self.delete_outbox_persisted(&commit.client_commit_id)?;
11162            }
11163            if !doomed.is_empty() {
11164                self.outbox
11165                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11166                self.rejections.extend(rejections);
11167                self.prune_commit_outcomes()?;
11168                batch.status = true;
11169                batch.rejections = true;
11170                batch.outcomes = true;
11171            }
11172
11173            let mut purged_rows = 0usize;
11174            for (table, ids) in &row_ids {
11175                if ids.is_empty() {
11176                    continue;
11177                }
11178                batch.table(table);
11179                self.overlay_dirty.table(table);
11180                for row_id in ids {
11181                    self.delete_base_row(table, row_id, i64::MAX)?;
11182                    purged_rows += 1;
11183                }
11184            }
11185            // §5.11 purge erasure: a journaled outcome with an operation or
11186            // conflict row the purge reaches keeps its identity, status, ids
11187            // and codes only.
11188            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11189                targets.iter().any(|target| {
11190                    target.table == table
11191                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11192                            || values.is_some_and(|values| {
11193                                Self::local_purge_values_match(target, values)
11194                            }))
11195                })
11196            };
11197            let entries = self
11198                .conn
11199                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11200                .and_then(|mut statement| {
11201                    statement
11202                        .query_map([], |row| {
11203                            Ok((
11204                                row.get::<_, String>(0)?,
11205                                row.get::<_, String>(1)?,
11206                                row.get::<_, Option<String>>(2)?,
11207                            ))
11208                        })?
11209                        .collect::<Result<Vec<_>, _>>()
11210                })
11211                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11212            for (client_commit_id, results_json, operations_json) in entries {
11213                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11214                let mut results: Vec<CommitOperationOutcome> =
11215                    serde_json::from_str(&results_json).map_err(corrupt)?;
11216                let operations: Vec<CommitOperation> = operations_json
11217                    .map(|json| serde_json::from_str(&json))
11218                    .transpose()
11219                    .map_err(corrupt)?
11220                    .unwrap_or_default();
11221                let named = operations
11222                    .iter()
11223                    .chain(results.iter().filter_map(|result| match result {
11224                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11225                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11226                        CommitOperationOutcome::Applied { .. } => None,
11227                    }))
11228                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11229                    || results.iter().any(|result| {
11230                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11231                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11232                    });
11233                if !named {
11234                    continue;
11235                }
11236                for result in &mut results {
11237                    match result {
11238                        CommitOperationOutcome::Conflict { conflict } => {
11239                            conflict.server_row = Map::new();
11240                            conflict.conflict_columns.clear();
11241                            conflict.operation = None;
11242                        }
11243                        CommitOperationOutcome::Error { rejection } => {
11244                            rejection.details = None;
11245                            rejection.operation = None;
11246                        }
11247                        CommitOperationOutcome::Applied { .. } => {}
11248                    }
11249                }
11250                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11251                self.conn
11252                    .execute(
11253                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11254                        rusqlite::params![results_json, client_commit_id],
11255                    )
11256                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11257                batch.outcomes = true;
11258            }
11259            self.rebuild_overlay_if_dirty()?;
11260            self.delete_unreferenced_cached_blobs()?;
11261            self.conn
11262                .execute(
11263                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11264                    rusqlite::params![meta_key, canonical_plan],
11265                )
11266                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11267            Ok((
11268                batch,
11269                LocalDataPurgeResult {
11270                    already_applied: false,
11271                    purged_rows,
11272                    dropped_commits: doomed.len(),
11273                },
11274            ))
11275        })()
11276        .and_then(|(batch, result)| {
11277            self.finish_observation("syncular_local_purge", batch)
11278                .map(|()| result)
11279        });
11280        let result = match applied {
11281            Ok(result) => result,
11282            Err(error) => {
11283                self.rollback_observation("syncular_local_purge");
11284                self.restore(rollback);
11285                return Err(error);
11286            }
11287        };
11288        self.cancel_sync_round();
11289        self.erase_purged_pages()?;
11290        Ok(result)
11291    }
11292
11293    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11294    /// database file, and the WAL keeps the frames written before the purge,
11295    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11296    /// rows, and the checkpoint truncates the WAL.
11297    fn erase_purged_pages(&self) -> Result<(), String> {
11298        let erased = self
11299            .conn
11300            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11301            .and_then(|()| {
11302                self.conn
11303                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11304            });
11305        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11306        erased
11307            .and(reset)
11308            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11309    }
11310
11311    /// Application-authorized recovery of the replicated projection. Unlike a
11312    /// local security purge, this retains the entire outbox, device identity,
11313    /// lease, outcomes, unit registrations, and protected bookkeeping.
11314    /// The projection reset, unit rewind, optimistic replay, and
11315    /// idempotency marker share one savepoint for interruption safety.
11316    pub fn rebootstrap_local_data(
11317        &mut self,
11318        input: &LocalDataRebootstrapInput,
11319    ) -> Result<LocalDataRebootstrapResult, String> {
11320        if input.rebootstrap_id.is_empty()
11321            || input.rebootstrap_id.len() > 128
11322            || !is_local_operation_code_like(&input.rebootstrap_id)
11323        {
11324            return Err(
11325                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11326                    .to_owned(),
11327            );
11328        }
11329        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11330        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11331            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11332        })? {
11333            let (retained_commits, reset_subscriptions) =
11334                decode_local_rebootstrap_receipt(&persisted)?;
11335            return Ok(LocalDataRebootstrapResult {
11336                already_applied: true,
11337                retained_commits,
11338                reset_subscriptions,
11339            });
11340        }
11341        if self.stopped || self.schema_floor.is_some() {
11342            return Err(
11343                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11344                    .to_owned(),
11345            );
11346        }
11347
11348        let retained_commits = self.outbox.len();
11349        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11350        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11351        let rollback = self.capture();
11352        self.begin_observation("syncular_local_rebootstrap")?;
11353        let mut batch = ChangeAccumulator::default();
11354        let applied = (|| -> Result<(), String> {
11355            self.run_schema_reset_observed(&mut batch, false)?;
11356            self.conn
11357                .execute(
11358                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11359                    rusqlite::params![meta_key, receipt],
11360                )
11361                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11362            self.sync_needed = true;
11363            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11364            batch.status = true;
11365            Ok(())
11366        })()
11367        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11368        if let Err(error) = applied {
11369            self.rollback_observation("syncular_local_rebootstrap");
11370            self.restore(rollback);
11371            return Err(error);
11372        }
11373
11374        Ok(LocalDataRebootstrapResult {
11375            already_applied: false,
11376            retained_commits,
11377            reset_subscriptions,
11378        })
11379    }
11380
11381    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11382
11383    /// Delete base and visible rows owned by the effective scopes
11384    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11385    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11386    fn purge_scope_rows(
11387        &mut self,
11388        table_name: &str,
11389        effective: &[(String, Vec<String>)],
11390        owners: &[Vec<(String, Vec<String>)>],
11391    ) -> Result<bool, String> {
11392        let Some(table) = self.schema.table(table_name) else {
11393            return Ok(false);
11394        };
11395        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11396            return Ok(false);
11397        };
11398        let Some((predicate, params)) = predicate else {
11399            return Ok(true);
11400        };
11401        self.overlay_dirty.table(table_name);
11402        for source in [base_table(table_name), visible_table(table_name)] {
11403            self.conn
11404                .execute(
11405                    &format!("DELETE FROM {source} WHERE {predicate}"),
11406                    rusqlite::params_from_iter(&params),
11407                )
11408                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11409        }
11410        Ok(true)
11411    }
11412
11413    /// §3.3: drop pending commits whose upserts provably land in the
11414    /// revoked effective scopes and no other owner covers (row ownership)
11415    /// — whole-commit, never per-operation.
11416    fn drop_doomed_outbox(
11417        &mut self,
11418        table: &TableSchema,
11419        effective: &[(String, Vec<String>)],
11420        owners: &[Vec<(String, Vec<String>)>],
11421    ) -> Result<bool, String> {
11422        let doomed: Vec<_> =
11423            self.outbox
11424                .iter()
11425                .filter(|commit| {
11426                    commit.ops.iter().any(|op| {
11427                        op.upsert
11428                            && op.table == table.name
11429                            && op.values.as_ref().is_some_and(|values| {
11430                                owns_local_row(table, effective, owners, values)
11431                            })
11432                    })
11433                })
11434                .cloned()
11435                .collect();
11436        if doomed.is_empty() {
11437            return Ok(false);
11438        }
11439        let mut rejections = Vec::new();
11440        for commit in &doomed {
11441            let results = commit
11442                .ops
11443                .iter()
11444                .enumerate()
11445                .map(|(op_index, operation)| {
11446                    self.overlay_dirty.table(&operation.table);
11447                    let rejection = RejectionRecord {
11448                        client_commit_id: commit.client_commit_id.clone(),
11449                        op_index: op_index as i32,
11450                        code: "sync.scope_revoked".to_owned(),
11451                        message: "the commit was dropped because its effective scope was revoked"
11452                            .to_owned(),
11453                        retryable: false,
11454                        details: None,
11455                        operation: Some(CommitOperation::from(operation)),
11456                    };
11457                    rejections.push(rejection.clone());
11458                    CommitOperationOutcome::Error { rejection }
11459                })
11460                .collect::<Vec<_>>();
11461            self.persist_commit_outcome(
11462                &commit.client_commit_id,
11463                CommitOutcomeStatus::Rejected,
11464                &results,
11465                Some(&commit.ops),
11466            )?;
11467            self.delete_outbox_persisted(&commit.client_commit_id)?;
11468        }
11469        self.prune_commit_outcomes()?;
11470        let doomed_ids = doomed
11471            .iter()
11472            .map(|commit| commit.client_commit_id.as_str())
11473            .collect::<BTreeSet<_>>();
11474        self.outbox
11475            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11476        self.rejections.extend(rejections);
11477        Ok(true)
11478    }
11479
11480    // -- blobs (§5.9) ----------------------------------------------------------------
11481
11482    /// §5.9.7: hash bytes into the content address, cache them, queue the
11483    /// upload (flushed before the next push, B4). Returns the canonical
11484    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11485    /// column value.
11486    pub fn upload_blob(
11487        &mut self,
11488        bytes: &[u8],
11489        media_type: Option<String>,
11490        name: Option<String>,
11491    ) -> Result<Value, String> {
11492        let blob_id = blob_id_for(bytes);
11493        let now = self.clock_now_ms();
11494        let tx = self
11495            .conn
11496            .transaction()
11497            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11498        tx.execute(
11499            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11500             ON CONFLICT(blob_id) DO NOTHING",
11501            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11502        )
11503        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11504        tx.execute(
11505            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11506            rusqlite::params![blob_id, media_type, now],
11507        )
11508        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11509        tx.commit()
11510            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11511        self.enforce_blob_cache_cap();
11512        let mut obj = Map::new();
11513        obj.insert("blobId".to_owned(), Value::from(blob_id));
11514        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11515        if let Some(mt) = media_type {
11516            obj.insert("mediaType".to_owned(), Value::from(mt));
11517        }
11518        if let Some(n) = name {
11519            obj.insert("name".to_owned(), Value::from(n));
11520        }
11521        Ok(Value::Object(obj))
11522    }
11523
11524    /// §5.9.7: resolve an owned blob body without driver encoding. A
11525    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11526    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11527    pub fn fetch_blob_bytes(
11528        &mut self,
11529        transport: &mut dyn Transport,
11530        blob_id_or_ref: &str,
11531    ) -> Result<FetchedBlob, (String, String)> {
11532        let simple = |m: String| ("client.failed".to_owned(), m);
11533        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11534        // blobId declares none.
11535        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11536            (blob_id_or_ref.to_owned(), None)
11537        } else {
11538            let value: Value = serde_json::from_str(blob_id_or_ref)
11539                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11540            (
11541                value
11542                    .get("blobId")
11543                    .and_then(Value::as_str)
11544                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11545                    .to_owned(),
11546                value.get("byteLength").and_then(Value::as_u64),
11547            )
11548        };
11549        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11550            return Ok(cached);
11551        }
11552        if !self.transport_enabled {
11553            return Err(("sync.offline".into(), "transport gate is closed".into()));
11554        }
11555        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11556        // blob.not_found) verbatim so the harness can assert on it. The
11557        // authorized endpoint serves bytes inline OR (always-issue, presign
11558        // configured) a signed url the client fetches directly — no host auth,
11559        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11560        #[cfg(feature = "bench-internals")]
11561        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11562        let bytes = match transport
11563            .blob_download(&blob_id, byte_length)
11564            .map_err(|e| (e.code, e.message))?
11565        {
11566            BlobDownload::Bytes(bytes) => bytes,
11567            BlobDownload::Url {
11568                url,
11569                url_expires_at_ms,
11570            } => {
11571                // §5.9.5: MUST NOT start a fetch at/past expiry.
11572                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11573                    return Err((
11574                        "sync.segment_expired".to_owned(),
11575                        format!(
11576                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11577                        ),
11578                    ));
11579                }
11580                transport
11581                    .fetch_blob_url(&url, byte_length)
11582                    .map_err(|e| (e.code, e.message))?
11583            }
11584        };
11585        #[cfg(feature = "bench-internals")]
11586        drop(download_phase);
11587        #[cfg(feature = "bench-internals")]
11588        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11589        // §5.9.5 inherits §5.1: the declared size caps the body before the
11590        // content address is verified.
11591        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11592            return Err((
11593                "sync.invalid_request".to_owned(),
11594                "blob exceeds its byteLength".to_owned(),
11595            ));
11596        }
11597        if blob_id_for(&bytes) != blob_id {
11598            return Err(simple(format!(
11599                "blob content address mismatch for {blob_id}"
11600            )));
11601        }
11602        #[cfg(feature = "bench-internals")]
11603        drop(validate_phase);
11604        #[cfg(feature = "bench-internals")]
11605        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11606        let now = self.clock_now_ms();
11607        self.conn
11608            .execute(
11609                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11610                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11611            )
11612            .map_err(|e| simple(e.to_string()))?;
11613        self.conn
11614            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11615            .map_err(|e| simple(e.to_string()))?;
11616        self.enforce_blob_cache_cap();
11617        #[cfg(feature = "bench-internals")]
11618        drop(insert_phase);
11619        self.get_cached_blob_bytes(&blob_id)
11620            .map_err(simple)?
11621            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11622    }
11623
11624    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11625        #[cfg(feature = "bench-internals")]
11626        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11627        let mut stmt = self
11628            .conn
11629            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11630            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11631        let mut rows = stmt
11632            .query(rusqlite::params![blob_id])
11633            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11634        if let Some(row) = rows
11635            .next()
11636            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11637        {
11638            let bytes: Vec<u8> = row
11639                .get(0)
11640                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11641            let byte_length: i64 = row
11642                .get(1)
11643                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11644            let media_type: Option<String> = row
11645                .get(2)
11646                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11647            return Ok(Some(FetchedBlob {
11648                blob_id: blob_id.to_owned(),
11649                byte_length,
11650                bytes,
11651                media_type,
11652            }));
11653        }
11654        Ok(None)
11655    }
11656
11657    /// §5.9.7 B4: upload every queued blob before push.
11658    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11659        let pending: Vec<(String, Option<String>)> = {
11660            let mut stmt = self
11661                .conn
11662                .prepare(
11663                    "SELECT blob_id, media_type FROM (
11664                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11665                      UNION ALL
11666                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11667                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11668                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11669                    ) ORDER BY created_at_ms, blob_id",
11670                )
11671                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11672            let rows = stmt
11673                .query_map([], |row| {
11674                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11675                })
11676                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11677            rows.collect::<Result<_, _>>()
11678                .map_err(|error| match error {
11679                    rusqlite::Error::InvalidColumnType(..)
11680                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11681                        "sync.local_corrupt",
11682                        "Pending blob upload metadata is invalid",
11683                    ),
11684                    _ => TransportError::new("client.failed", error.to_string()),
11685                })?
11686        };
11687        let mut uploads = Vec::new();
11688        for (blob_id, media_type) in pending {
11689            let bytes = (|| {
11690                let stored: Option<(SqlValue, SqlValue)> = self
11691                    .conn
11692                    .query_row(
11693                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11694                        rusqlite::params![blob_id],
11695                        |row| Ok((row.get(0)?, row.get(1)?)),
11696                    )
11697                    .optional()
11698                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11699                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11700                    return Err(TransportError::new(
11701                        "sync.local_corrupt",
11702                        "Pending blob upload body is missing or corrupt",
11703                    ));
11704                };
11705                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11706                    return Err(TransportError::new(
11707                        "sync.local_corrupt",
11708                        "Pending blob upload body is missing or corrupt",
11709                    ));
11710                }
11711                Ok(bytes)
11712            })();
11713            let failed = bytes.is_err();
11714            uploads.push(crate::round::PendingUpload {
11715                id: blob_id,
11716                bytes,
11717                media_type,
11718            });
11719            if failed {
11720                break;
11721            }
11722        }
11723        Ok(uploads)
11724    }
11725
11726    fn visible_blob_ids_sql(&self) -> String {
11727        let mut selects = Vec::new();
11728        for table in &self.schema.tables {
11729            for column in table
11730                .columns
11731                .iter()
11732                .filter(|column| column.ty == ColumnType::BlobRef)
11733            {
11734                let identifier = quote_ident(&column.name);
11735                selects.push(format!(
11736                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11737                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11738                     AND json_type({identifier}, '$.blobId') = 'text'",
11739                    quote_ident(&table.name),
11740                ));
11741            }
11742        }
11743        if selects.is_empty() {
11744            "SELECT NULL AS blob_id WHERE 0".to_owned()
11745        } else {
11746            selects.join(" UNION ")
11747        }
11748    }
11749
11750    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11751    /// staged uploads pin bodies directly; no derived refcount is stored.
11752    fn enforce_blob_cache_cap(&self) {
11753        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11754            return;
11755        };
11756        let mut total: i64 = self
11757            .conn
11758            .query_row(
11759                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11760                [],
11761                |row| row.get(0),
11762            )
11763            .unwrap_or(0);
11764        if total <= max_bytes {
11765            return;
11766        }
11767        let query = format!(
11768            "SELECT blob_id, byte_length FROM _syncular_blobs
11769             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11770               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11771               AND blob_id NOT IN ({})
11772             ORDER BY created_at_ms ASC, blob_id ASC",
11773            self.visible_blob_ids_sql(),
11774        );
11775        let candidates: Vec<(String, i64)> = {
11776            let Ok(mut stmt) = self.conn.prepare(&query) else {
11777                return;
11778            };
11779            let Ok(rows) = stmt.query_map([], |row| {
11780                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11781            }) else {
11782                return;
11783            };
11784            rows.filter_map(Result::ok).collect()
11785        };
11786        for (blob_id, byte_length) in candidates {
11787            if total <= max_bytes {
11788                break;
11789            }
11790            let _ = self.conn.execute(
11791                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11792                rusqlite::params![blob_id],
11793            );
11794            total -= byte_length;
11795        }
11796    }
11797
11798    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11799        #[cfg(feature = "bench-internals")]
11800        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11801        if !self.schema_has_blobs() {
11802            return Ok(());
11803        }
11804        self.rebuild_overlay_if_dirty()?;
11805        self.conn
11806            .execute(
11807                &format!(
11808                    "DELETE FROM _syncular_blobs
11809                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11810                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11811                   AND blob_id NOT IN ({})",
11812                    self.visible_blob_ids_sql(),
11813                ),
11814                [],
11815            )
11816            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11817        Ok(())
11818    }
11819
11820    // -- local row storage ----------------------------------------------------------
11821
11822    /// The cached per-table primary-key upsert SQL for `full_table` (see
11823    /// the `insert_sql` field: built once, reused per row).
11824    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11825        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11826            return sql.clone();
11827        }
11828        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11829        columns.push(quote_ident("_syncular_version"));
11830        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11831        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11832        let primary_key = quote_ident(&table.primary_key);
11833        let updates = columns
11834            .iter()
11835            .filter(|column| **column != primary_key)
11836            .map(|column| format!("{column}=excluded.{column}"))
11837            .collect::<Vec<_>>()
11838            .join(", ");
11839        // §4.5 version guard: server state lands in the base table only above
11840        // the stored version; the visible overlay takes every write.
11841        let guard = if full_table == base_table(&table.name) {
11842            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11843        } else {
11844            String::new()
11845        };
11846        let sql = format!(
11847            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11848            columns.join(", "),
11849            placeholders.join(", ")
11850        );
11851        self.insert_sql
11852            .borrow_mut()
11853            .insert(full_table.to_owned(), sql.clone());
11854        sql
11855    }
11856
11857    /// §4.5: write a server row under the version guard; returns whether it
11858    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
11859    fn write_base_row(
11860        &self,
11861        table_name: &str,
11862        row: &Row,
11863        version: i64,
11864        sealed: Option<&SealedRow>,
11865    ) -> Result<bool, String> {
11866        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11867        if landed {
11868            self.overlay_dirty.table(table_name);
11869        }
11870        Ok(landed)
11871    }
11872
11873    fn write_row(
11874        &self,
11875        full_table: &str,
11876        table_name: &str,
11877        row: &Row,
11878        version: i64,
11879        sealed: Option<&SealedRow>,
11880    ) -> Result<bool, String> {
11881        #[cfg(feature = "bench-internals")]
11882        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11883        let table = self
11884            .schema
11885            .table(table_name)
11886            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11887        let sql = self.insert_row_sql(full_table, table);
11888        let mut stmt = self
11889            .conn
11890            .prepare_cached(&sql)
11891            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11892        let params = row
11893            .iter()
11894            .map(RowParam::Cell)
11895            .chain(std::iter::once(RowParam::Version(version)))
11896            .chain(
11897                [
11898                    sealed.map(|sealed| sealed.key_id.as_str()),
11899                    sealed.map(|sealed| sealed.envelopes.as_str()),
11900                ]
11901                .into_iter()
11902                .take(table.sealed_columns().len())
11903                .map(RowParam::Text),
11904            );
11905        stmt.execute(rusqlite::params_from_iter(params))
11906            .map(|changed| changed > 0)
11907            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11908    }
11909
11910    /// §4.5: delete a server row whose stored version lies below `version`;
11911    /// returns whether a row left.
11912    fn delete_base_row(
11913        &self,
11914        table_name: &str,
11915        row_id: &str,
11916        version: i64,
11917    ) -> Result<bool, String> {
11918        let table = self
11919            .schema
11920            .table(table_name)
11921            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11922        let sql = format!(
11923            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
11924            base_table(table_name),
11925            row_id_predicate(table)
11926        );
11927        let mut stmt = self
11928            .conn
11929            .prepare_cached(&sql)
11930            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11931        let landed = stmt
11932            .execute(rusqlite::params![row_id, version])
11933            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11934            > 0;
11935        if landed {
11936            self.overlay_dirty.table(table_name);
11937        }
11938        Ok(landed)
11939    }
11940
11941    fn has_acknowledged_rows(&self) -> Result<bool, String> {
11942        self.conn
11943            .query_row(
11944                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
11945                [],
11946                |row| row.get(0),
11947            )
11948            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11949    }
11950
11951    fn acknowledged_operations(
11952        &self,
11953        rows: Option<&HashSet<(&str, &str)>>,
11954        tables: Option<&BTreeSet<&str>>,
11955    ) -> Result<Vec<OutboxOp>, String> {
11956        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
11957        let mut statement = self
11958            .conn
11959            .prepare_cached(if keys.is_some() {
11960                ACKED_ROWS_BY_ROW
11961            } else {
11962                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
11963            })
11964            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11965        let rows = statement
11966            .query_map(
11967                rusqlite::params![
11968                    keys,
11969                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
11970                ],
11971                |row| {
11972                    #[cfg(test)]
11973                    self.acknowledged_replay_count
11974                        .set(self.acknowledged_replay_count.get() + 1);
11975                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11976                },
11977            )
11978            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11979        rows.map(|row| {
11980            let (op, intent) =
11981                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11982            let op: CommitOperation =
11983                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
11984            let table = self
11985                .schema
11986                .table(&op.table)
11987                .ok_or_else(|| "sync.unknown_table".to_owned())?;
11988            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
11989                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
11990            } else {
11991                op.values.clone()
11992            };
11993            Ok(OutboxOp {
11994                values,
11995                ..op.into()
11996            })
11997        })
11998        .collect()
11999    }
12000
12001    fn drop_acknowledged_rows(
12002        &self,
12003        matches: impl Fn(&str, &Map<String, Value>) -> bool,
12004    ) -> Result<Vec<OutboxCommit>, String> {
12005        let mut statement = self
12006            .conn
12007            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12008            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12009        let rows = statement
12010            .query_map([], |row| {
12011                Ok((
12012                    row.get::<_, String>(0)?,
12013                    row.get::<_, String>(1)?,
12014                    row.get::<_, String>(2)?,
12015                    row.get::<_, String>(3)?,
12016                ))
12017            })
12018            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12019        let mut doomed = HashSet::new();
12020        for row in rows {
12021            let (id, table, intent, row_id) =
12022                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12023            let values =
12024                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12025            let base_matches = self
12026                .schema
12027                .table(&table)
12028                .map(|schema| self.stored_row(schema, &row_id, true))
12029                .transpose()?
12030                .flatten()
12031                .is_some_and(|(base, _)| {
12032                    let schema = self.schema.table(&table).expect("acknowledged table");
12033                    let values = schema
12034                        .columns
12035                        .iter()
12036                        .zip(base.iter())
12037                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12038                        .collect();
12039                    matches(&table, &values)
12040                });
12041            if matches(&table, &values) || base_matches {
12042                doomed.insert(id);
12043            }
12044        }
12045        let mut dropped = Vec::new();
12046        for id in doomed {
12047            let mut statement = self
12048                .conn
12049                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12050                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12051            let operations = statement
12052                .query_map([&id], |row| row.get::<_, String>(0))
12053                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12054            let mut ops = Vec::new();
12055            for operation in operations {
12056                let json = operation
12057                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12058                let op: CommitOperation =
12059                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12060                self.overlay_dirty.table(&op.table);
12061                ops.push(OutboxOp {
12062                    table: op.table,
12063                    row_id: op.row_id,
12064                    upsert: op.op == "upsert",
12065                    base_version: op.base_version,
12066                    values: op.values,
12067                    insert: op.insert,
12068                });
12069            }
12070            dropped.push(OutboxCommit {
12071                client_commit_id: id.clone(),
12072                ops,
12073            });
12074            self.conn
12075                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12076                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12077        }
12078        Ok(dropped)
12079    }
12080
12081    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12082    /// the outbox changed since the last rebuild (the no-op sync round).
12083    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12084        if self.overlay_dirty.get() {
12085            self.rebuild_overlay()?;
12086        }
12087        Ok(())
12088    }
12089
12090    /// §7.1: local reads see outbox state applied optimistically — rebuild
12091    /// every visible table as (base server state) + (pending outbox replay
12092    /// on top). Optimistic rows carry version `-1`.
12093    fn rebuild_overlay(&mut self) -> Result<(), String> {
12094        #[cfg(feature = "bench-internals")]
12095        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12096        #[cfg(test)]
12097        self.overlay_rebuild_count
12098            .set(self.overlay_rebuild_count.get() + 1);
12099        let dirty = self.overlay_dirty.snapshot();
12100        let tables: Vec<_> = self
12101            .schema
12102            .tables
12103            .iter()
12104            .filter(|table| {
12105                dirty
12106                    .as_ref()
12107                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12108            })
12109            .cloned()
12110            .collect();
12111        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12112        self.begin_observation("syncular_overlay")?;
12113        let rebuilt = (|| {
12114            for table in &tables {
12115                for index in &table.fts_indexes {
12116                    self.drop_fts_triggers(index)?;
12117                }
12118                let visible = visible_table(&table.name);
12119                let base = base_table(&table.name);
12120                self.conn
12121                    .execute_batch(&format!(
12122                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12123                    ))
12124                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12125            }
12126            self.replay_overlay(&names, None)?;
12127            for table in &tables {
12128                for index in &table.fts_indexes {
12129                    self.rebuild_fts_projection(table, index)?;
12130                    self.create_fts_triggers(table, index)?;
12131                }
12132            }
12133            self.conn
12134                .execute_batch("RELEASE syncular_overlay")
12135                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12136            Ok(())
12137        })();
12138        if rebuilt.is_err() {
12139            self.conn
12140                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12141                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12142        } else {
12143            self.overlay_dirty.set(false);
12144        }
12145        rebuilt
12146    }
12147
12148    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12149    /// that order, for the operations of `tables` on `rows` (every row for
12150    /// `None`) over their current visible state.
12151    fn replay_overlay(
12152        &self,
12153        tables: &BTreeSet<&str>,
12154        rows: Option<&HashSet<(&str, &str)>>,
12155    ) -> Result<(), String> {
12156        let replays = |op: &&OutboxOp| {
12157            tables.contains(op.table.as_str())
12158                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12159        };
12160        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12161        for (commit, initial) in self.failed_commits.iter() {
12162            for operation in commit.ops.iter().filter(replays) {
12163                let Some(table) = self.schema.table(&operation.table) else {
12164                    continue;
12165                };
12166                if operation.upsert
12167                    && self.visible_row(table, &operation.row_id)?.is_none()
12168                    && operation.presented_values(table).is_some_and(|values| {
12169                        table
12170                            .columns
12171                            .iter()
12172                            .all(|column| values.contains_key(&column.name))
12173                    })
12174                {
12175                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12176                        .expect("row key serialization");
12177                    if let Some(Value::Object(values)) = initial.get(&key) {
12178                        self.apply_outbox_ops(&[OutboxOp {
12179                            values: Some(values.clone()),
12180                            ..operation.clone()
12181                        }])?;
12182                    }
12183                }
12184            }
12185            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12186        }
12187        self.apply_outbox_ops(
12188            self.outbox
12189                .iter()
12190                .flat_map(|commit| &commit.ops)
12191                .filter(replays),
12192        )
12193    }
12194
12195    /// §7.1 replay: every operation is applied over the current visible
12196    /// state. An operation that no longer applies (a server row now holds its
12197    /// unique value) stays pending and invisible until a later replay admits
12198    /// it or the server answers its push.
12199    fn apply_outbox_ops<'a>(
12200        &self,
12201        ops: impl IntoIterator<Item = &'a OutboxOp>,
12202    ) -> Result<(), String> {
12203        #[cfg(feature = "bench-internals")]
12204        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12205        for op in ops {
12206            #[cfg(feature = "bench-internals")]
12207            if let Some(phase) = &mut phase {
12208                phase.unit();
12209            }
12210            if op.upsert {
12211                if let Some((table, values)) = self
12212                    .schema
12213                    .table(&op.table)
12214                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12215                    .and_then(|table| Some((table, op.presented_values(table)?)))
12216                {
12217                    let mut intended: Map<String, Value> = self
12218                        .visible_row(table, &op.row_id)?
12219                        .map(|(row, _)| {
12220                            table
12221                                .columns
12222                                .iter()
12223                                .zip(&row)
12224                                .map(|(column, value)| {
12225                                    (column.name.clone(), column_value_to_json(value))
12226                                })
12227                                .collect()
12228                        })
12229                        .unwrap_or_default();
12230                    intended.extend(values.into_owned());
12231                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12232                        continue;
12233                    }
12234                }
12235            }
12236            self.apply_outbox_op(op)?;
12237        }
12238        Ok(())
12239    }
12240
12241    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12242        let Some(table) = self.schema.table(&op.table) else {
12243            return Ok(());
12244        };
12245        let visible = visible_table(&table.name);
12246        if !op.upsert {
12247            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12248            self.conn
12249                .prepare_cached(&sql)
12250                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12251                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12252            return Ok(());
12253        }
12254        // §7.1: an insert-intent operation that lacks a non-nullable column
12255        // cannot encode; it stays invisible until the send-time prepass drops
12256        // it as sync.outbox_incompatible (§7.4.4).
12257        let Some(values) = op.presented_values(table) else {
12258            return Ok(());
12259        };
12260        // §7.1: the overlay applies the operation's PRESENT columns
12261        // over the current local row. A partial operation over an
12262        // absent local row leaves it absent — the server answers it
12263        // with sync.row_deleted or sync.row_missing (§6.2). An
12264        // insert-intent operation presents every column.
12265        let full = table
12266            .columns
12267            .iter()
12268            .all(|column| values.contains_key(&column.name));
12269        let local = self.visible_row(table, &op.row_id)?;
12270        if local.is_none() && !full {
12271            return Ok(());
12272        }
12273        let mut row: Row = Vec::with_capacity(table.columns.len());
12274        for (index, column) in table.columns.iter().enumerate() {
12275            let incoming = match values.get(&column.name) {
12276                Some(raw) => json_to_column_value(
12277                    if raw.get("$bytes").is_some()
12278                        && table
12279                            .encrypted_columns
12280                            .iter()
12281                            .any(|encrypted| encrypted.index == index)
12282                    {
12283                        &table.wire_columns[index]
12284                    } else {
12285                        column
12286                    },
12287                    Some(raw),
12288                ),
12289                None => Ok(local
12290                    .as_ref()
12291                    .and_then(|(values, _)| values.get(index).cloned())
12292                    .flatten()),
12293            };
12294            row.push(incoming.map_err(|_| {
12295                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12296            })?);
12297        }
12298        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12299        self.write_row(&visible, &table.name, &row, version, None)
12300            .map(|_| ())
12301            .map_err(|error| {
12302                if error.starts_with("UNIQUE constraint failed:") {
12303                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12304                } else {
12305                    error
12306                }
12307            })
12308    }
12309
12310    /// Whether the row is present in the overlay. Existence reads only the
12311    /// key: locked encrypted values stay opaque, and decoding the complete row
12312    /// would reject a valid plain write.
12313    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12314        self.conn
12315            .query_row(
12316                &format!(
12317                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12318                    visible_table(&table.name),
12319                    row_id_predicate(table)
12320                ),
12321                rusqlite::params![row_id],
12322                |row| row.get(0),
12323            )
12324            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12325    }
12326
12327    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12328    /// over ciphertext this device cannot see.
12329    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12330        if !table.has_encrypted_columns() {
12331            return Ok(());
12332        }
12333        let key_id: Option<String> = self
12334            .conn
12335            .query_row(
12336                &format!(
12337                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12338                    visible_table(&table.name),
12339                    row_id_predicate(table)
12340                ),
12341                rusqlite::params![row_id],
12342                |row| row.get(0),
12343            )
12344            .optional()
12345            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12346            .flatten();
12347        match key_id {
12348            None => Ok(()),
12349            Some(key_id) => Err(ClientError {
12350                code: "client.encryption_key_missing".to_owned(),
12351                message: "the row is sealed under a key this device does not hold".to_owned(),
12352                details: Some(json!({ "table": table.name, "keyId": key_id })),
12353                retryable: false,
12354            }),
12355        }
12356    }
12357
12358    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12359    /// keyring now holds into its columns and clear both reserved columns,
12360    /// one change batch per table. `client.decrypt_failed` fails the caller.
12361    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12362        let storage = |message: String| TransportError::new("storage.failed", message);
12363        let tables: Vec<TableSchema> = self
12364            .schema
12365            .tables
12366            .iter()
12367            .filter(|table| table.has_encrypted_columns())
12368            .cloned()
12369            .collect();
12370        for table in &tables {
12371            let base = base_table(&table.name);
12372            let pk = quote_ident(&table.primary_key);
12373            let sealed: Vec<(SqlValue, i64, String)> = self
12374                .conn
12375                .prepare_cached(&format!(
12376                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12377                ))
12378                .and_then(|mut statement| {
12379                    statement
12380                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12381                        .collect()
12382                })
12383                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12384            let mut revealed = Vec::new();
12385            for (key, version, envelopes) in sealed {
12386                if let Some(row) =
12387                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12388                {
12389                    revealed.push((key, version, row));
12390                }
12391            }
12392            if revealed.is_empty() {
12393                continue;
12394            }
12395            self.begin_observation("syncular_reveal").map_err(storage)?;
12396            let outcome = (|| {
12397                let assignments = table
12398                    .encrypted_columns
12399                    .iter()
12400                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12401                    .collect::<Vec<_>>()
12402                    .join(",");
12403                let mut statement = self
12404                    .conn
12405                    .prepare_cached(&format!(
12406                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12407                    ))
12408                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12409                for (key, version, row) in &revealed {
12410                    let mut params = table
12411                        .encrypted_columns
12412                        .iter()
12413                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12414                        .collect::<Result<Vec<_>, _>>()?;
12415                    params.push(key.clone());
12416                    params.push(SqlValue::Integer(*version));
12417                    statement
12418                        .execute(rusqlite::params_from_iter(params))
12419                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12420                }
12421                drop(statement);
12422                self.overlay_dirty.table(&table.name);
12423                self.rebuild_overlay()?;
12424                let mut batch = ChangeAccumulator::default();
12425                batch.table(&table.name);
12426                self.finish_observation("syncular_reveal", batch)
12427            })();
12428            if outcome.is_err() {
12429                self.rollback_observation("syncular_reveal");
12430            }
12431            outcome.map_err(storage)?;
12432        }
12433        Ok(())
12434    }
12435
12436    /// The visible (optimistic) row's values plus version, or `None` when the
12437    /// row is absent from the overlay.
12438    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12439        self.stored_row(table, row_id, false)
12440    }
12441
12442    fn stored_row(
12443        &self,
12444        table: &TableSchema,
12445        row_id: &str,
12446        base: bool,
12447    ) -> Result<Option<(Row, i64)>, String> {
12448        let sql = format!(
12449            "SELECT * FROM {} WHERE {}",
12450            if base {
12451                base_table(&table.name)
12452            } else {
12453                visible_table(&table.name)
12454            },
12455            row_id_predicate(table)
12456        );
12457        let mut stmt = self
12458            .conn
12459            .prepare_cached(&sql)
12460            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12461        let mut rows = stmt
12462            .query(rusqlite::params![row_id])
12463            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12464        let Some(row) = rows
12465            .next()
12466            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12467        else {
12468            return Ok(None);
12469        };
12470        let mut values = Vec::with_capacity(table.columns.len());
12471        for (index, column) in table.columns.iter().enumerate() {
12472            let raw = row
12473                .get_ref(index)
12474                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12475            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12476                if table
12477                    .encrypted_columns
12478                    .iter()
12479                    .any(|encrypted| encrypted.index == index)
12480                {
12481                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12482                    continue;
12483                }
12484            }
12485            let json = sql_ref_to_json(column, raw);
12486            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12487                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12488            })?);
12489        }
12490        let version: i64 = row
12491            .get(table.columns.len())
12492            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12493        Ok(Some((values, version)))
12494    }
12495
12496    // -- realtime (§8) ---------------------------------------------------------------
12497
12498    /// §8.8: the configured realtime policy.
12499    #[must_use]
12500    pub fn realtime_policy(&self) -> RealtimePolicy {
12501        self.realtime_policy
12502    }
12503
12504    /// §8.8: the explicit realtime availability state.
12505    #[must_use]
12506    pub fn realtime_state(&self) -> RealtimeState {
12507        self.realtime_state
12508    }
12509
12510    /// §8.8: set the realtime policy. Refused while the socket is connected:
12511    /// a policy change never silently re-routes an ownership state.
12512    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12513        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12514            return Err(
12515                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12516                    .to_owned(),
12517            );
12518        }
12519        self.realtime_policy = policy;
12520        match policy {
12521            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12522            RealtimePolicy::Required | RealtimePolicy::Optional
12523                if self.realtime_state == RealtimeState::Disabled =>
12524            {
12525                self.set_realtime_state(RealtimeState::Disconnected, None);
12526            }
12527            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12528        }
12529        Ok(())
12530    }
12531
12532    /// Record an availability transition, keeping the reason code bounded and
12533    /// code-like (§7.6) and clearing it where it cannot apply.
12534    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12535        self.realtime_state = state;
12536        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12537        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12538            self.realtime_reason_code = None;
12539            self.realtime_retry_delay_ms = None;
12540        }
12541    }
12542
12543    #[must_use]
12544    fn realtime_connected(&self) -> bool {
12545        self.realtime_state == RealtimeState::Connected
12546    }
12547
12548    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12549        if !self.transport_enabled {
12550            return Err("sync.offline: transport gate is closed".into());
12551        }
12552        if self.realtime_policy == RealtimePolicy::Off {
12553            return Err(
12554                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12555            );
12556        }
12557        if self.realtime_connected() {
12558            return Ok(());
12559        }
12560        self.realtime_state = RealtimeState::Connecting;
12561        match transport.realtime_connect_for_client(&self.client_id) {
12562            Ok(()) => {
12563                self.set_realtime_state(RealtimeState::Connected, None);
12564                Ok(())
12565            }
12566            Err(error) => {
12567                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12568                Err(format!("{}: {}", error.code, error.message))
12569            }
12570        }
12571    }
12572
12573    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12574        self.cancel_sync_round();
12575        // A deliberately disconnected or policy-disabled client has nothing
12576        // to release; a `lost` socket still does (the host owns the handle).
12577        if self.realtime_state == RealtimeState::Disconnected
12578            || self.realtime_state == RealtimeState::Disabled
12579        {
12580            return;
12581        }
12582        let _ = transport.realtime_close();
12583        self.set_realtime_state(
12584            if self.realtime_policy == RealtimePolicy::Off {
12585                RealtimeState::Disabled
12586            } else {
12587                RealtimeState::Disconnected
12588            },
12589            None,
12590        );
12591        self.presence.clear(); // §8.6.1: presence is per-connection
12592    }
12593
12594    /// §8.8: the host observed the socket end without disconnecting it (a
12595    /// peer close, an I/O error, a failed send; the native transport reports
12596    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12597    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12598    /// one background retry intent so the host reconnects and catches up.
12599    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12600        if self.realtime_state != RealtimeState::Connected {
12601            return;
12602        }
12603        self.cancel_sync_round();
12604        let _ = transport.realtime_close();
12605        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12606        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12607        self.presence.clear();
12608    }
12609
12610    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12611    /// document for `scope_key`. Requires a live socket; the document is
12612    /// ephemeral (lost on disconnect). Authorization is the connection's
12613    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12614    /// server with `presence.forbidden`.
12615    pub fn set_presence(
12616        &mut self,
12617        transport: &mut dyn Transport,
12618        scope_key: &str,
12619        doc: Option<&Value>,
12620    ) -> Result<(), String> {
12621        if !self.transport_enabled {
12622            return Err("sync.offline: transport gate is closed".into());
12623        }
12624        if !self.realtime_connected() {
12625            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12626        }
12627        let text = encode_presence_publish(scope_key, doc);
12628        transport
12629            .realtime_send(&text)
12630            .map_err(|e| format!("{}: {}", e.code, e.message))
12631    }
12632
12633    /// §8.6: the peers currently present on a scope key (ephemeral).
12634    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12635        self.presence
12636            .get(scope_key)
12637            .map(|peers| peers.values().cloned().collect())
12638            .unwrap_or_default()
12639    }
12640
12641    /// §8.6 apply an inbound presence fanout to the local map.
12642    fn apply_presence(
12643        &mut self,
12644        scope_key: String,
12645        kind: Option<PresenceKind>,
12646        actor_id: Option<String>,
12647        client_id: Option<String>,
12648        doc: Option<Value>,
12649        error: Option<String>,
12650    ) {
12651        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12652        // it as a `presence` event carrying the server's `error` code.
12653        if error.is_some() {
12654            return;
12655        }
12656        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12657            return;
12658        };
12659        let peer_key = format!("{actor_id} {client_id}");
12660        match kind {
12661            PresenceKind::Leave => {
12662                if let Some(peers) = self.presence.get_mut(&scope_key) {
12663                    peers.remove(&peer_key);
12664                    if peers.is_empty() {
12665                        self.presence.remove(&scope_key);
12666                    }
12667                }
12668            }
12669            _ => {
12670                let doc = match doc {
12671                    Some(Value::Object(_)) => doc.unwrap(),
12672                    _ => return,
12673                };
12674                self.presence.entry(scope_key).or_default().insert(
12675                    peer_key,
12676                    PresencePeer {
12677                        actor_id,
12678                        client_id,
12679                        doc,
12680                    },
12681                );
12682            }
12683        }
12684    }
12685
12686    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12687    pub fn on_realtime_text(&mut self, text: &str) {
12688        if self.stopped || !self.transport_enabled || self.security_preflight {
12689            return;
12690        }
12691        match parse_control(text) {
12692            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12693                if requires_sync {
12694                    // §8.1: pull before trusting the socket for continuity.
12695                    self.set_sync_needed(true, true);
12696                }
12697            }
12698            Ok(ControlMessage::Presence {
12699                scope_key,
12700                kind,
12701                actor_id,
12702                client_id,
12703                doc,
12704                error,
12705                ..
12706            }) => {
12707                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12708            }
12709            Ok(ControlMessage::Wake { .. }) => {
12710                // §8.3: any wake-up means "run a pull soon", never data.
12711                self.set_sync_needed(true, true);
12712            }
12713            _ => {}
12714        }
12715    }
12716
12717    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12718    /// a pull response per section; an unapplied delta is a wake-up.
12719    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12720    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12721        let mut transport = crate::round::DownloadResults::default();
12722        self.on_realtime_binary(&mut transport, bytes);
12723        transport.controls
12724    }
12725
12726    /// §8.2: a delta applies exactly like a live section and sets the replica
12727    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12728    /// duplicate. A delta the client does not apply (no live unit, a status
12729    /// other than `active`, a decode or apply failure) is a drop and raises
12730    /// sync-needed.
12731    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12732        if self.stopped || !self.transport_enabled || self.security_preflight {
12733            return;
12734        }
12735        #[cfg(feature = "bench-internals")]
12736        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12737        let message = match decode_message(bytes) {
12738            Ok(m) if m.msg_kind == MsgKind::Response => m,
12739            _ => {
12740                self.set_sync_needed(true, true);
12741                return;
12742            }
12743        };
12744        #[cfg(feature = "bench-internals")]
12745        drop(decode_phase);
12746        // §8.2: a delta's header means what a pull response's header means. A
12747        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12748        if let Some(Frame::RespHeader {
12749            required_schema_version: Some(required),
12750            latest_schema_version,
12751            ..
12752        }) = message.frames.first()
12753        {
12754            let floor = SchemaFloor {
12755                required_schema_version: Some(*required),
12756                latest_schema_version: *latest_schema_version,
12757            };
12758            if self.set_schema_floor(Some(floor)).is_err() {
12759                self.set_sync_needed(true, true);
12760            }
12761            return;
12762        }
12763        let through = message.frames.iter().find_map(|frame| match frame {
12764            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12765            _ => None,
12766        });
12767        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12768        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12769                reset_required: false,
12770                log_epoch,
12771                ..
12772            }) if Some(log_epoch) == epoch.as_ref());
12773        let mut applied: Option<i64> = None;
12774        dropped |= self.begin_coalesced_apply().is_err();
12775        let mut receivers: Option<Vec<UnitRecord>> = None;
12776        for frame in message.frames {
12777            if dropped {
12778                break;
12779            }
12780            match frame {
12781                Frame::LiveStart { status, .. } => {
12782                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12783                        dropped = true;
12784                        break;
12785                    };
12786                    if status != LiveStatus::Active || cursor < 0 {
12787                        dropped = true;
12788                    } else if !through.is_some_and(|through| through <= cursor) {
12789                        // A catching-up unit receives no delta (§8.2).
12790                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12791                            units
12792                                .into_iter()
12793                                .filter(|unit| {
12794                                    unit.status == UnitStatus::Active
12795                                        && unit.echo.is_some()
12796                                        && unit.cursor.is_none()
12797                                })
12798                                .collect()
12799                        });
12800                        dropped = receivers.is_none();
12801                    }
12802                }
12803                Frame::Commit {
12804                    commit_seq,
12805                    tables,
12806                    changes,
12807                    ..
12808                } => {
12809                    let Some(receivers) = receivers.clone() else {
12810                        continue;
12811                    };
12812                    dropped = self
12813                        .apply_live_commit(
12814                            &receivers,
12815                            &tables,
12816                            changes,
12817                            commit_seq,
12818                            AppliedServerCommitSource::Delta,
12819                        )
12820                        .is_err();
12821                }
12822                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12823                    dropped = self
12824                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12825                        .is_err();
12826                    applied = Some(through_seq);
12827                }
12828                Frame::Error { .. } => dropped = true,
12829                _ => {}
12830            }
12831        }
12832        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12833        // the replica cursor, so the delta counts as a drop.
12834        if self.finish_coalesced_apply().is_err() || dropped {
12835            applied = None;
12836            dropped = true;
12837        }
12838        if let Some(cursor) = applied {
12839            // §8.2 ack point: the delta's throughSeq.
12840            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12841            let _ = transport.realtime_send(&ack);
12842        }
12843        if dropped {
12844            self.set_sync_needed(true, true);
12845        }
12846    }
12847
12848    /// §8.2 ack point after a pull round on a live connection: the replica
12849    /// cursor while it is at least `0`.
12850    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12851        if !self.realtime_connected() {
12852            return;
12853        }
12854        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12855            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12856            let _ = transport.realtime_send(&ack);
12857        }
12858    }
12859}
12860
12861#[cfg(test)]
12862mod previous_version_wiring_tests;
12863
12864#[cfg(test)]
12865mod identity_tests;
12866
12867#[cfg(test)]
12868mod insert_intent_tests;
12869
12870#[cfg(test)]
12871mod robustness_tests;
12872
12873#[cfg(test)]
12874mod range_image_tests;