Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127    version: u8,
128    retained_commits: u64,
129    reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137    retained_commits: usize,
138    reset_subscriptions: usize,
139) -> Result<String, String> {
140    let retained_commits =
141        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    let reset_subscriptions =
143        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145        return Err(invalid_local_rebootstrap_receipt());
146    }
147    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149        retained_commits,
150        reset_subscriptions,
151    })
152    .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156    // Pre-0.15.36 markers proved application but did not retain the receipt.
157    if value == "v1" {
158        return Ok((0, 0));
159    }
160    let receipt: PersistedLocalDataRebootstrapReceipt =
161        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165    {
166        return Err(invalid_local_rebootstrap_receipt());
167    }
168    Ok((
169        usize::try_from(receipt.retained_commits)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171        usize::try_from(receipt.reset_subscriptions)
172            .map_err(|_| invalid_local_rebootstrap_receipt())?,
173    ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179/// Internal state for hosts that perform network I/O outside the client owner.
180#[doc(hidden)]
181pub struct CurrentBasePreparation {
182    client_commit_id: String,
183    sequence: i64,
184    units: Vec<UnitRecord>,
185    transport_epoch: u64,
186    registration_epoch: u64,
187    security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195    upsert: bool,
196    table: String,
197    row_id: String,
198    base_version: Option<i64>,
199    /// Schema-agnostic local form (§0): driver JSON values, encoded with
200    /// the current codec at send time. A key's presence IS the §6.1 sparse
201    /// presence set — both the overlay and the wire payload derive from it.
202    values: Option<Map<String, Value>>,
203    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
204    /// absent from the visible replica. Replay and send-time encoding present
205    /// every current-schema column; a column `values` lacks is NULL.
206    insert: bool,
207}
208
209impl OutboxOp {
210    /// The values this upsert presents under the current `table` (§7.1). An
211    /// insert-intent operation presents every column, a missing one as NULL;
212    /// `None` when a missing column is non-nullable, which leaves the commit
213    /// unencodable (§7.4.4), or when the operation carries no values.
214    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215        let values = self.values.as_ref()?;
216        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217            return Some(Cow::Borrowed(values));
218        }
219        full_row_values(table, values.clone()).ok().map(Cow::Owned)
220    }
221}
222
223impl From<CommitOperation> for OutboxOp {
224    fn from(operation: CommitOperation) -> Self {
225        Self {
226            upsert: operation.op == "upsert",
227            table: operation.table,
228            row_id: operation.row_id,
229            base_version: operation.base_version,
230            values: operation.values,
231            insert: operation.insert,
232        }
233    }
234}
235
236/// §7.5 static authoring refusal; the table and column travel in details.
237fn invalid_authoring(message: &str, details: Value) -> ClientError {
238    ClientError {
239        code: "sync.invalid_request".to_owned(),
240        message: message.to_owned(),
241        details: Some(details),
242        retryable: false,
243    }
244}
245
246/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
247/// row and presents only columns `earlier` presents.
248fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249    // Only an upsert carries values, and it always carries its primary key.
250    earlier.row_id == later.row_id
251        && earlier.table == later.table
252        && later.base_version.is_some_and(|base| base > 0)
253        && earlier.base_version == later.base_version
254        && later.values.as_ref().is_some_and(|values| {
255            values.keys().all(|column| {
256                earlier
257                    .values
258                    .as_ref()
259                    .is_some_and(|earlier| earlier.contains_key(column))
260            })
261        })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265    fn from(operation: &OutboxOp) -> Self {
266        Self {
267            table: operation.table.clone(),
268            row_id: operation.row_id.clone(),
269            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270            base_version: operation.base_version,
271            values: operation.values.clone(),
272            insert: operation.insert,
273        }
274    }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279    client_commit_id: String,
280    ops: Vec<OutboxOp>,
281}
282
283/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
284/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
285/// can never disagree about which commits cannot re-encode.
286fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287    commit
288        .ops
289        .iter()
290        .filter(|operation| operation.upsert)
291        .map(|operation| {
292            (
293                operation.table.as_str(),
294                operation
295                    .values
296                    .as_ref()
297                    .map(|values| values.keys().map(String::as_str).collect())
298                    .unwrap_or_default(),
299                operation.insert,
300            )
301        })
302        .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309    table: String,
310    selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314    sequence: i64,
315    client_commit_id: String,
316    status: String,
317    recorded_at_ms: i64,
318    results_json: String,
319    operations_json: Option<String>,
320    resolution: String,
321    resolved_at_ms: Option<i64>,
322    replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327        Ok(Self {
328            sequence: row.get(0)?,
329            client_commit_id: row.get(1)?,
330            status: row.get(2)?,
331            recorded_at_ms: row.get(3)?,
332            results_json: row.get(4)?,
333            operations_json: row.get(5)?,
334            resolution: row.get(6)?,
335            resolved_at_ms: row.get(7)?,
336            replacement_client_commit_id: row.get(8)?,
337        })
338    }
339}
340
341/// Section outcome distinguishing the §5.6 unit-local fail-closed
342/// path from a round-aborting failure (§1.4 rule 5).
343/// §4.4 a unit section's `SUB_START`.
344struct SectionStart {
345    id: String,
346    status: SubStatus,
347    reason_code: String,
348    bootstrap: bool,
349    fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353    FailClosed,
354    /// §5.5: the server refused a segment with `sync.forbidden`.
355    Refused,
356    /// §5.3 client memory: the round deferred this image's body to a later
357    /// pull of the same resume token.
358    Deferred,
359    Abort(String, String),
360    Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365    pushed_ids: Vec<String>,
366    /// Hex of the registration digest this pull carried; the server stores
367    /// that list once the round completes (§4.1).
368    registration_digest: String,
369    pub(crate) accept: u8,
370    /// §6.1 splitBatch: outbox commits held back from THIS request because
371    /// the running operation count reached the push cap — the next round
372    /// pushes them (`sync_needed` stays set while any remain).
373    deferred_commits: usize,
374    /// §5.6 applied-segment rule: per unit presenting its recorded cursor
375    /// and token, the segment ids neither downloaded nor applied again.
376    pub(crate) applied: BTreeMap<String, BTreeSet<String>>,
377    /// §5.3 client memory: (unit, segment id) of the first image whose body
378    /// the round did not fetch because the batch already holds two images.
379    /// That unit's section stops there without persisting `SUB_END`.
380    pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385    /// None means table-wide; Some is the exact scoped domain.
386    tables: BTreeMap<String, Option<BTreeSet<String>>>,
387    windows: BTreeMap<String, BTreeSet<String>>,
388    status: bool,
389    conflicts: bool,
390    rejections: bool,
391    outcomes: bool,
392}
393
394impl ChangeAccumulator {
395    fn table(&mut self, table: &str) {
396        self.tables.insert(table.to_owned(), None);
397    }
398
399    fn scope(&mut self, table: &str, key: String) {
400        match self.tables.get_mut(table) {
401            Some(None) => {}
402            Some(Some(keys)) => {
403                keys.insert(key);
404            }
405            None => {
406                self.tables
407                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
408            }
409        }
410    }
411
412    fn window(&mut self, base_key: &str, value: &str) {
413        self.windows
414            .entry(base_key.to_owned())
415            .or_default()
416            .insert(value.to_owned());
417    }
418
419    fn touched(&self) -> bool {
420        !self.tables.is_empty()
421            || !self.windows.is_empty()
422            || self.status
423            || self.conflicts
424            || self.rejections
425            || self.outcomes
426    }
427}
428
429/// §6.1: the server caps total operations per request (reference default
430/// 500) and rejects the whole batch with `sync.too_many_operations`; the
431/// client "splits and retries". Splitting happens at build time: commits are
432/// included IN ORDER until the operation budget is spent, the rest wait for
433/// the next round.
434const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441/// Dirty visible tables; `None` means a whole-replica recovery/reset.
442struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445    fn clean() -> Self {
446        Self(RefCell::new(Some(BTreeSet::new())))
447    }
448    fn get(&self) -> bool {
449        self.0
450            .borrow()
451            .as_ref()
452            .is_none_or(|tables| !tables.is_empty())
453    }
454    fn set(&self, dirty: bool) {
455        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456    }
457    fn table(&self, table: &str) {
458        if let Some(tables) = self.0.borrow_mut().as_mut() {
459            tables.insert(table.to_owned());
460        }
461    }
462    fn snapshot(&self) -> Option<BTreeSet<String>> {
463        self.0.borrow().clone()
464    }
465    fn restore(&self, tables: Option<BTreeSet<String>>) {
466        *self.0.borrow_mut() = tables;
467    }
468}
469
470/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
471/// write after a capture copies it once.
472#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476    type Target = T;
477    fn deref(&self) -> &T {
478        &self.0
479    }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483    fn deref_mut(&mut self) -> &mut T {
484        Arc::make_mut(&mut self.0)
485    }
486}
487
488impl<T> From<T> for Shared<T> {
489    fn from(value: T) -> Self {
490        Self(Arc::new(value))
491    }
492}
493
494/// The in-memory mirrors of durable state, captured before a local
495/// transaction and restored when it rolls back. Queues only grow inside a
496/// transaction, so their lengths suffice.
497#[derive(Debug, Clone)]
498struct Rollback {
499    outbox: Shared<Vec<Arc<OutboxCommit>>>,
500    failed_commits: Shared<Vec<FailedCommit>>,
501    conflicts: Shared<Vec<ConflictRecord>>,
502    rejections: Shared<Vec<RejectionRecord>>,
503    schema_floor: Option<SchemaFloor>,
504    lease_state: Option<LeaseState>,
505    stopped: bool,
506    upgrading: bool,
507    sync_needed: bool,
508    active_round: Option<uuid::Uuid>,
509    overlay_dirty: Option<BTreeSet<String>>,
510    change_queue: usize,
511    sync_intent_queue: usize,
512    last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516    #[cfg(feature = "bench-internals")]
517    benchmark_phases: Recorder,
518    conn: Connection,
519    schema: ClientSchema,
520    client_id: String,
521    limits: ClientLimits,
522    outbox: Shared<Vec<Arc<OutboxCommit>>>,
523    failed_commits: Shared<Vec<FailedCommit>>,
524    retain_failed_commits: bool,
525    conflicts: Shared<Vec<ConflictRecord>>,
526    rejections: Shared<Vec<RejectionRecord>>,
527    schema_floor: Option<SchemaFloor>,
528    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
529    lease_state: Option<LeaseState>,
530    /// §1.6: the schema-floor response stops syncing until an upgrade.
531    stopped: bool,
532    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
533    upgrading: bool,
534    /// §8.4 coalesced sync-needed signal.
535    sync_needed: bool,
536    /// §8.8: how sync rounds treat the realtime binding, and its exact
537    /// availability state with the evidence behind a lost or refused one.
538    realtime_policy: RealtimePolicy,
539    realtime_state: RealtimeState,
540    realtime_reason_code: Option<String>,
541    realtime_retry_delay_ms: Option<u64>,
542    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
543    presence: HashMap<String, HashMap<String, PresencePeer>>,
544    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check; the
545    /// host may pin it (conformance runs on a virtual clock).
546    now_ms: Option<i64>,
547    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
548    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
549    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
550    encryption: crate::values::EncryptionConfig,
551    /// Fail-closed host bootstrap gate. While set, command hosts permit only
552    /// status/lifecycle inspection and an exact authorized local purge.
553    security_preflight: bool,
554    security_epoch: u64,
555    transport_enabled: bool,
556    transport_epoch: u64,
557    /// Raised by every registration change; a current-base proof spanning
558    /// one reports missing coverage.
559    registration_epoch: u64,
560    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
561    /// means the feature is off; the descriptor write and the container orphan
562    /// sweep still happen.
563    previous_version: Option<PreviousVersionContextConfig>,
564    /// Per-table primary-key upsert SQL, built once per (full table name) —
565    /// the row write path runs per row during bootstrap (§5.6), so the SQL
566    /// string (and, via `prepare_cached`, its compiled statement) is reused
567    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
568    /// schema reset (the column lists may have changed).
569    insert_sql: RefCell<HashMap<String, String>>,
570    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
571    /// reset and restart explicitly invalidate every table.
572    overlay_dirty: OverlayDirty,
573    // First SQLite failure of the current operation, before savepoint cleanup.
574    storage_failure: RefCell<Option<QueryReadFailure>>,
575    /// Test-only structural performance signal: response processing must not
576    /// turn a batch of acknowledgements into one full overlay rebuild each.
577    #[cfg(test)]
578    overlay_rebuild_count: Cell<usize>,
579    #[cfg(test)]
580    acknowledged_replay_count: Cell<usize>,
581    /// Test-only structural performance signal: outcome retention is enforced
582    /// once per response rather than once per acknowledgement.
583    #[cfg(test)]
584    outcome_prune_count: Cell<usize>,
585    /// Test-only pinned duration of every §5.3 image chunk, in ms.
586    #[cfg(test)]
587    image_chunk_ms: Cell<Option<f64>>,
588    /// Exact observer-transaction output drained by command/FFI hosts.
589    progress: ProgressObserver,
590    change_queue: VecDeque<ClientChangeBatch>,
591    sync_intent_queue: VecDeque<SyncIntent>,
592    /// Explicit exponential retry policy for transient transport failures.
593    retry_delay_ms: u64,
594    /// §7.7: the background retry the current round scheduled, if any.
595    round_retry_delay_ms: Option<u64>,
596    /// §5.6 applied-segment rule: per unit, the cursor and resume token it
597    /// presented and the `SEGMENT_REF`s applied completely in sections whose
598    /// `SUB_END` did not persist.
599    #[allow(clippy::type_complexity)]
600    applied_segments: HashMap<String, (Option<i64>, Option<String>, BTreeSet<String>)>,
601    /// §5.3: a table's next image chunk size, carried from one range image
602    /// of the table to the next.
603    image_chunk_rows: HashMap<String, u32>,
604    /// A round whose response never completed (§1.5): the client keeps its
605    /// code and reports it on every following request until a round receives a
606    /// complete, decodable response. The server already recorded the round, so
607    /// nothing else can tell it the round failed on this side.
608    round_failure: Option<String>,
609    active_round: Option<uuid::Uuid>,
610    last_round: Option<DiagnosticLastRound>,
611    last_change: Option<DiagnosticLastChange>,
612    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
613    query_failures: Vec<DiagnosticQueryFailure>,
614    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
615    coalesced: Option<CoalescedApply>,
616    /// A coalesced apply transaction of the current response failed; every
617    /// later observation of that response fails until it ends.
618    coalesced_failure: Option<String>,
619}
620
621/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
622/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
623/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
624struct CoalescedApply {
625    rows: usize,
626    rollback: Rollback,
627}
628
629const COALESCED_APPLY_ROWS: usize = 1024;
630
631pub(crate) fn quote_ident(name: &str) -> String {
632    format!("\"{}\"", name.replace('"', "\"\""))
633}
634/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
635/// previous-version module reads and writes the SAME records as the client
636/// instead of growing a parallel meta layer.
637pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
638    conn.query_row(
639        "SELECT value FROM _syncular_meta WHERE key = ?1",
640        rusqlite::params![key],
641        |row| row.get::<_, String>(0),
642    )
643    .optional()
644}
645
646pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
647    conn.execute(
648        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
649        rusqlite::params![key, value],
650    )
651    .map(|_| ())
652}
653
654pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
655    conn.execute(
656        "DELETE FROM _syncular_meta WHERE key = ?1",
657        rusqlite::params![key],
658    )
659    .map(|_| ())
660}
661
662/// Seek by the existing key while retaining the exact legacy text match.
663fn row_id_predicate(table: &TableSchema) -> String {
664    let key = quote_ident(&table.primary_key);
665    let text_match = format!("CAST({key} AS TEXT) = ?1");
666    match table.columns[table.pk_index].ty {
667        ColumnType::String | ColumnType::Json => {
668            format!("{key} = ?1 AND {text_match}")
669        }
670        // Native columns have no declared SQLite affinity. Match the stored
671        // integer bind type explicitly. Unary + removes expression affinity so
672        // SQLite can seek the typeless key; the text check preserves exact IDs.
673        ColumnType::Integer | ColumnType::Boolean => {
674            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
675        }
676        // REAL-to-text conversion can round distinct values to the same text.
677        // Narrowing those matches by numeric equality would change behavior.
678        _ => text_match,
679    }
680}
681
682/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
683/// `effective` and that no entry of `owners` matches. `owners` holds the
684/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
685/// Scope values bind by the column's declared type, as `row_id_predicate`
686/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
687/// is a missing local scope-column mapping (the fail-closed case).
688fn local_scope_predicate(
689    table: &TableSchema,
690    effective: &[(String, Vec<String>)],
691    owners: &[Vec<(String, Vec<String>)>],
692) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
693    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
694        return Ok(None);
695    }
696    let column = |variable: &str| {
697        let name = table.scope_column(variable).ok_or(())?;
698        let column = table
699            .columns
700            .iter()
701            .find(|column| column.name == name)
702            .ok_or(())?;
703        Ok((quote_ident(name), column.ty == ColumnType::Integer))
704    };
705    let mut clauses = Vec::new();
706    let mut params = Vec::new();
707    for (variable, values) in effective {
708        let (key, integer) = column(variable)?;
709        let holes = vec!["?"; values.len()].join(", ");
710        clauses.push(if integer {
711            params.extend(values.iter().cloned().map(SqlValue::Text));
712            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
713            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
714        } else {
715            format!("{key} IN ({holes})")
716        });
717        params.extend(values.iter().cloned().map(SqlValue::Text));
718    }
719    // Owners bind as one JSON parameter per variable set, so the statement
720    // keeps its size at any owner count; owners of one variable bind the
721    // union of their values. An owner holding other values of a variable
722    // `effective` names owns none of its rows and drops out. Values compare
723    // as text, the exact semantics of the effective clauses.
724    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
725    for owner in owners {
726        if owner.is_empty()
727            || owner.iter().any(|(variable, values)| {
728                values.is_empty()
729                    || effective.iter().any(|(name, wanted)| {
730                        name == variable && wanted.iter().all(|value| !values.contains(value))
731                    })
732            })
733        {
734            continue;
735        }
736        let mut owner: Vec<_> = owner.iter().collect();
737        owner.sort_by(|a, b| a.0.cmp(&b.0));
738        groups
739            .entry(
740                owner
741                    .iter()
742                    .map(|(variable, _)| variable.as_str())
743                    .collect(),
744            )
745            .or_default()
746            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
747    }
748    let within = |variable: &str, list: &str| {
749        let (key, integer) = column(variable)?;
750        let key = if integer {
751            format!("CAST({key} AS TEXT)")
752        } else {
753            key
754        };
755        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
756    };
757    for (variables, tuples) in groups {
758        if let [variable] = variables[..] {
759            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
760            params.push(SqlValue::Text(
761                Value::from_iter(values.into_iter().cloned()).to_string(),
762            ));
763            clauses.push(format!("NOT {}", within(variable, "?")?));
764        } else {
765            params.push(SqlValue::Text(
766                serde_json::to_string(&tuples).expect("owner tuples serialize"),
767            ));
768            let matches = variables
769                .iter()
770                .enumerate()
771                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
772                .collect::<Result<Vec<_>, ()>>()?;
773            clauses.push(format!(
774                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
775                matches.join(" AND ")
776            ));
777        }
778    }
779    Ok(Some((clauses.join(" AND "), params)))
780}
781
782/// [`local_scope_predicate`] over decoded row values (retained rows).
783fn owns_local_row(
784    table: &TableSchema,
785    effective: &[(String, Vec<String>)],
786    owners: &[Vec<(String, Vec<String>)>],
787    values: &Map<String, Value>,
788) -> bool {
789    let matches = |scopes: &[(String, Vec<String>)]| {
790        !scopes.is_empty()
791            && scopes.iter().all(|(variable, allowed)| {
792                table
793                    .scope_column(variable)
794                    .and_then(|column| values.get(column))
795                    .filter(|value| !value.is_null())
796                    .is_some_and(|value| {
797                        allowed.contains(
798                            &value
799                                .as_str()
800                                .map_or_else(|| value.to_string(), str::to_owned),
801                        )
802                    })
803            })
804    };
805    matches(effective) && !owners.iter().any(|owner| matches(owner))
806}
807
808fn is_local_operation_code_like(value: &str) -> bool {
809    let bytes = value.as_bytes();
810    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
811        && bytes
812            .iter()
813            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
814}
815
816fn base_table(name: &str) -> String {
817    quote_ident(&format!("_syncular_base_{name}"))
818}
819
820/// §4.8 deferred eviction record: (unit id, table, effective scope map).
821type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
822
823/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
824fn window_base_key(base: &WindowBase) -> String {
825    format!(
826        "{}\0{}",
827        base.variable,
828        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
829    )
830}
831
832/// The unit of one window value: the fixed scopes plus the value.
833fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
834    let mut scopes = base.fixed_scopes.clone();
835    scopes.insert(base.variable.clone(), value.to_owned());
836    scopes
837}
838
839/// A unit's scope map as the codec and the digests take it.
840fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
841    let mut pairs: Vec<(String, String)> = scopes
842        .iter()
843        .map(|(variable, value)| (variable.clone(), value.clone()))
844        .collect();
845    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
846    pairs
847}
848
849/// §4.1: the unit covers every table declaring all of its variables.
850fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
851    scopes
852        .keys()
853        .all(|variable| table.scope_column(variable).is_some())
854}
855
856/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
857/// scope map as one-element lists plus the fenced lists. `None` when the
858/// fence revoked the pair.
859fn pair_scopes(
860    scopes: &UnitScopes,
861    table: &str,
862    echo: Option<&[EchoRecord]>,
863) -> Option<Vec<(String, Vec<String>)>> {
864    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
865    if fenced.is_some_and(|record| record.status == "revoked") {
866        return None;
867    }
868    let mut effective: BTreeMap<String, Vec<String>> = fenced
869        .map(|record| record.scopes.clone())
870        .unwrap_or_default();
871    for (variable, value) in scopes {
872        effective.insert(variable.clone(), vec![value.clone()]);
873    }
874    Some(effective.into_iter().collect())
875}
876
877/// §3.3 row ownership: the pair scopes of every owner of `table` except
878/// `except_id`. An owner is active with a persisted echo and live or
879/// post-bootstrap, or bootstrapping with a segment of the table in its
880/// current bootstrap.
881fn unit_owners(
882    units: &[UnitRecord],
883    table: &TableSchema,
884    except_id: &str,
885) -> Vec<Vec<(String, Vec<String>)>> {
886    units
887        .iter()
888        .filter(|unit| {
889            unit.id != except_id
890                && unit.status == UnitStatus::Active
891                && unit.echo.is_some()
892                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
893                && covers(table, &unit.scopes)
894        })
895        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
896        .collect()
897}
898
899/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
900fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
901    scopes
902        .iter()
903        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
904        .collect()
905}
906
907/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
908/// It joins through the (tbl, id) index: a scan per acknowledgement would
909/// make draining a batch quadratic.
910const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
911
912const UNIT_COLUMNS: &str =
913    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
914
915/// A unit row; a field that does not parse is a conversion failure, which
916/// both read paths classify as `sync.local_corrupt`.
917fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
918    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
919        serde_json::from_str(raw).map_err(|error| {
920            rusqlite::Error::FromSqlConversionFailure(
921                index,
922                rusqlite::types::Type::Text,
923                Box::new(error),
924            )
925        })
926    }
927    Ok(UnitRecord {
928        id: row.get(0)?,
929        scopes: json(1, &row.get::<_, String>(1)?)?,
930        explicit: row.get::<_, i64>(2)? == 1,
931        status: json(
932            3,
933            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
934        )?,
935        reason_code: row.get(4)?,
936        cursor: row.get(5)?,
937        bootstrap_state: row.get(6)?,
938        seeded: json(7, &row.get::<_, String>(7)?)?,
939        echo: row
940            .get::<_, Option<String>>(8)?
941            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
942            .transpose()?
943            .map(|echo| {
944                if echo
945                    .iter()
946                    .all(|record| record.status == "active" || record.status == "revoked")
947                {
948                    Ok(echo)
949                } else {
950                    Err(rusqlite::Error::FromSqlConversionFailure(
951                        8,
952                        rusqlite::types::Type::Text,
953                        "echo status is neither active nor revoked".into(),
954                    ))
955                }
956            })
957            .transpose()?,
958    })
959}
960
961/// Every unit in registration order (§4.1).
962fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
963    conn.prepare_cached(&format!(
964        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
965    ))?
966    .query_map([], unit_from_row)?
967    .collect()
968}
969
970fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
971    conn.query_row(
972        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
973        [id],
974        unit_from_row,
975    )
976    .optional()
977}
978
979/// Insert or update in place: an update keeps the registration order.
980fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
981    conn.prepare_cached(&format!(
982        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
983         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
984           reason_code=excluded.reason_code, cursor=excluded.cursor,
985           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo"
986    ))?
987    .execute(rusqlite::params![
988        unit.id,
989        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
990        i64::from(unit.explicit),
991        unit.status.name(),
992        unit.reason_code,
993        unit.cursor,
994        unit.bootstrap_state,
995        serde_json::to_string(&unit.seeded).expect("table list serializes"),
996        unit.echo
997            .as_ref()
998            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
999    ])
1000    .map(|_| ())
1001}
1002
1003/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1004fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1005    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1006        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1007        .collect()
1008}
1009
1010/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1011/// registrations and discard the synced state of every unit outside `keep`,
1012/// so the next round sends the list and fresh-bootstraps those units. The
1013/// replica cursor survives only while a kept unit is live. The caller owns
1014/// the transaction.
1015fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1016    conn.execute(
1017        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1018           bootstrap_state=NULL, seeded='[]', echo=NULL
1019         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1020        [serde_json::to_string(keep).expect("unit ids serialize")],
1021    )?;
1022    conn.execute_batch(
1023        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1024         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1025           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1026    )
1027}
1028
1029/// §4.5: the replica cursor, `-1` while no unit is live.
1030fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1031    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1032        raw.parse().map_err(|error| {
1033            rusqlite::Error::FromSqlConversionFailure(
1034                0,
1035                rusqlite::types::Type::Text,
1036                Box::new(error),
1037            )
1038        })
1039    })
1040}
1041
1042fn visible_table(name: &str) -> String {
1043    quote_ident(name)
1044}
1045
1046const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1047
1048/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1049/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1050fn is_synced_table_name(name: &str) -> bool {
1051    if name.starts_with("sqlite_") {
1052        return false;
1053    }
1054    if name.starts_with("_syncular_base_") {
1055        return true; // the base half of a synced table pair
1056    }
1057    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1058    !name.starts_with("_syncular_")
1059}
1060
1061/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1062fn blob_id_for(bytes: &[u8]) -> String {
1063    let digest = Sha256::digest(bytes);
1064    format!("sha256:{}", bytes_to_hex(&digest))
1065}
1066
1067/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1068/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1069/// per row on the §5.6 bootstrap path), scalars bind owned.
1070enum RowParam<'a> {
1071    Cell(&'a Option<ColumnValue>),
1072    Version(i64),
1073    Text(Option<&'a str>),
1074}
1075
1076impl rusqlite::ToSql for RowParam<'_> {
1077    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1078        Ok(match self {
1079            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1080            RowParam::Text(text) => match text {
1081                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1082                None => ToSqlOutput::Owned(SqlValue::Null),
1083            },
1084            RowParam::Cell(cell) => match cell {
1085                None => ToSqlOutput::Owned(SqlValue::Null),
1086                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1087                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1088                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1089                Some(ColumnValue::Boolean(b)) => {
1090                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1091                }
1092                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1093                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1094                }
1095                // §5.10: crdt bytes store as BLOB, like bytes.
1096                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1097                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1098                }
1099            },
1100        })
1101    }
1102}
1103
1104// Retain a typed primary key across the current import chunk without copying row payloads.
1105fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1106    match param.to_sql().map_err(|error| error.to_string())? {
1107        ToSqlOutput::Owned(value) => Ok(value),
1108        ToSqlOutput::Borrowed(value) => {
1109            SqlValue::try_from(value).map_err(|error| error.to_string())
1110        }
1111        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1112    }
1113}
1114
1115/// §5.3 image cell → bind parameter, strict per the declared column type
1116/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1117/// nullable). Mismatches are image-producer violations. Returns the cell
1118/// borrowed when the stored representation already matches what the row
1119/// codec would write, or the normalized scalar (boolean → 0/1, float from
1120/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1121/// old convert-then-insert path without allocating per cell.
1122fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1123    use ssp2::segment::ColumnType;
1124    let mismatch = || {
1125        Err(format!(
1126            "image column {:?} holds a value of the wrong type",
1127            column.name
1128        ))
1129    };
1130    match value {
1131        ValueRef::Null => {
1132            if !column.nullable {
1133                return Err(format!(
1134                    "image column {:?} is NULL but not nullable",
1135                    column.name
1136                ));
1137            }
1138            Ok(ToSqlOutput::Owned(SqlValue::Null))
1139        }
1140        ValueRef::Integer(i) => match column.ty {
1141            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1142            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1143            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1144            _ => mismatch(),
1145        },
1146        ValueRef::Real(_) => match column.ty {
1147            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1148            _ => mismatch(),
1149        },
1150        ValueRef::Text(t) => {
1151            std::str::from_utf8(t)
1152                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1153            match column.ty {
1154                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1155                    Ok(ToSqlOutput::Borrowed(value))
1156                }
1157                _ => mismatch(),
1158            }
1159        }
1160        ValueRef::Blob(_) => match column.ty {
1161            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1162            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1163            _ => mismatch(),
1164        },
1165    }
1166}
1167
1168pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1169    use rusqlite::types::ValueRef;
1170    match value {
1171        ValueRef::Null => Value::Null,
1172        ValueRef::Integer(i) => match column.ty {
1173            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1174            ssp2::segment::ColumnType::Float => {
1175                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1176            }
1177            _ => Value::from(i),
1178        },
1179        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1180        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1181        ValueRef::Blob(b) => {
1182            let mut map = Map::new();
1183            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1184            Value::Object(map)
1185        }
1186    }
1187}
1188
1189/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1190/// Objects are accepted in lossless `{"$bytes": hex}` and
1191/// `{"$bigint": decimal}` envelope forms.
1192fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1193    Ok(match value {
1194        Value::Null => SqlValue::Null,
1195        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1196        Value::Number(n) => {
1197            if let Some(i) = n.as_i64() {
1198                SqlValue::Integer(i)
1199            } else if let Some(f) = n.as_f64() {
1200                SqlValue::Real(f)
1201            } else {
1202                return Err(format!("query param number {n} is out of range"));
1203            }
1204        }
1205        Value::String(s) => SqlValue::Text(s.clone()),
1206        Value::Object(_) => {
1207            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1208                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1209            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1210                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1211                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1212                })?)
1213            } else {
1214                return Err(
1215                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1216                        .to_owned(),
1217                );
1218            }
1219        }
1220        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1221    })
1222}
1223
1224/// Map a rusqlite value with no schema column to consult (arbitrary query
1225/// output): integers/reals/text pass through by stored affinity, blobs ride
1226/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1227/// schema column type to recover booleans/floats/json.
1228fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1229    use rusqlite::types::ValueRef;
1230    match value {
1231        ValueRef::Null => Value::Null,
1232        ValueRef::Integer(i) => {
1233            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1234            // ordinary UI-sized integers ergonomic and envelope only values
1235            // beyond JavaScript's safe range.
1236            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1237            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1238                Value::from(i)
1239            } else {
1240                let mut map = Map::new();
1241                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1242                Value::Object(map)
1243            }
1244        }
1245        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1246        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1247        ValueRef::Blob(b) => {
1248            let mut map = Map::new();
1249            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1250            Value::Object(map)
1251        }
1252    }
1253}
1254
1255/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1256fn query_connection(
1257    conn: &Connection,
1258    sql: &str,
1259    params: &[QueryValue],
1260) -> Result<Vec<QueryRow>, QueryReadFailure> {
1261    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1262    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1263    let bound: Vec<SqlValue> = params
1264        .iter()
1265        .map(json_param_to_sql)
1266        .collect::<Result<_, _>>()?;
1267    // §2.4: an unaliased physical version column stays engine-internal, so a
1268    // `SELECT *` row feeds straight back into mutation values. An explicit
1269    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1270    // §5.11 sealed columns follow the same rule.
1271    let columns: Vec<(usize, String)> = stmt
1272        .column_names()
1273        .enumerate()
1274        .filter(|(_, name)| {
1275            !matches!(
1276                *name,
1277                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1278            )
1279        })
1280        .map(|(i, name)| (i, name.to_owned()))
1281        .collect();
1282    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1283        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1284    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1285    let mut out = Vec::new();
1286    while let Some(row) = sql_rows.next()? {
1287        let mut record = Map::new();
1288        for (i, name) in &columns {
1289            let value = row.get_ref(*i)?;
1290            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1291        }
1292        out.push(record);
1293    }
1294    Ok(out)
1295}
1296
1297fn persisted_window_state(
1298    conn: &Connection,
1299    base: &WindowBase,
1300) -> Result<WindowState, QueryReadFailure> {
1301    let mut stmt = conn.prepare(
1302        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1303           FROM _syncular_windows AS windows
1304           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1305          WHERE windows.base = ?1
1306          ORDER BY windows.unit ASC",
1307    )?;
1308    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1309        Ok((
1310            row.get::<_, String>(0)?,
1311            row.get::<_, Option<String>>(1)?,
1312            row.get::<_, Option<i64>>(2)?,
1313            row.get::<_, Option<String>>(3)?,
1314        ))
1315    })?;
1316    let mut units = Vec::new();
1317    let mut pending = Vec::new();
1318    for row in rows {
1319        let (unit, status, cursor, bootstrap_state) = row?;
1320        // §4.8: complete iff registered, active and not bootstrapping.
1321        if status.as_deref() != Some("active")
1322            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1323        {
1324            pending.push(unit.clone());
1325        }
1326        units.push(unit);
1327    }
1328    Ok(WindowState { units, pending })
1329}
1330
1331/// One read-only statement in a §7.5 snapshot read.
1332#[derive(Debug, Clone, Copy)]
1333pub struct SnapshotStatement<'a> {
1334    pub sql: &'a str,
1335    pub params: &'a [Value],
1336}
1337
1338/// A §7.5 atomic read request: read-only statements, window coverage, unit
1339/// catch-up states, and commit deliveries, all resolved from one SQLite read
1340/// transaction and one revision.
1341#[derive(Debug, Clone, Default)]
1342pub struct SnapshotReadRequest<'a> {
1343    pub statements: Vec<SnapshotStatement<'a>>,
1344    pub coverage: &'a [WindowCoverage],
1345    pub units: Vec<String>,
1346    pub commit_ids: Vec<String>,
1347}
1348
1349#[cfg(test)]
1350thread_local! {
1351    /// Test-only rendezvous that runs between the statements of one snapshot
1352    /// read. A test uses it to commit on another connection mid-read and prove
1353    /// the read transaction's snapshot and revision stay stable.
1354    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1355        std::cell::RefCell::new(None);
1356}
1357
1358#[cfg(test)]
1359fn run_snapshot_read_hook() {
1360    SNAPSHOT_READ_HOOK.with(|slot| {
1361        if let Some(hook) = slot.borrow_mut().take() {
1362            hook();
1363        }
1364    });
1365}
1366
1367/// §7.5: resolve a request over one SQLite read transaction. The revision,
1368/// every statement's rows, coverage, catch-up states, and deliveries are
1369/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1370/// read back and releases the connection.
1371fn snapshot_read_connection(
1372    conn: &Connection,
1373    request: &SnapshotReadRequest<'_>,
1374) -> Result<SnapshotRead, QueryReadFailure> {
1375    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1376    let corrupt = |message: &str| QueryReadFailure {
1377        code: Some("sync.local_corrupt"),
1378        sqlite_code: None,
1379        sqlite_message: None,
1380        rollback_failure: None,
1381        message: message.to_owned(),
1382    };
1383    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1384        rusqlite::Error::Utf8Error(..)
1385        | rusqlite::Error::InvalidColumnType(..)
1386        | rusqlite::Error::FromSqlConversionFailure(..)
1387        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1388        _ => QueryReadFailure::from(error),
1389    };
1390    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1391        // The durable revision as a canonical decimal `u64`; a missing marker
1392        // is the legacy zero, and any other non-canonical value is corrupt.
1393        let revision: Option<String> = conn
1394            .query_row(
1395                "SELECT value FROM _syncular_meta WHERE key = ?1",
1396                rusqlite::params![LOCAL_REVISION_KEY],
1397                |row| row.get(0),
1398            )
1399            .optional()
1400            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1401        let revision = match revision {
1402            None => "0".to_owned(),
1403            Some(value)
1404                if value == "0"
1405                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1406                        && value.bytes().all(|b| b.is_ascii_digit())
1407                        && value.parse::<u64>().is_ok()) =>
1408            {
1409                value
1410            }
1411            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1412        };
1413        let mut queries = Vec::with_capacity(request.statements.len());
1414        for statement in &request.statements {
1415            queries.push(query_connection(conn, statement.sql, statement.params)?);
1416            #[cfg(test)]
1417            run_snapshot_read_hook();
1418        }
1419        let mut pending = Vec::new();
1420        let mut missing = Vec::new();
1421        for requested in request.coverage {
1422            let base_key = window_base_key(&requested.base);
1423            let state = persisted_window_state(conn, &requested.base)?;
1424            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1425                let reference = WindowUnitRef {
1426                    base_key: base_key.clone(),
1427                    unit: unit.clone(),
1428                };
1429                if !state.units.iter().any(|held| held == &unit) {
1430                    missing.push(reference);
1431                } else if state.pending.iter().any(|held| held == &unit) {
1432                    pending.push(reference);
1433                }
1434            }
1435        }
1436        let mut units = Vec::with_capacity(request.units.len());
1437        for id in &request.units {
1438            let exists: bool = conn.query_row(
1439                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1440                [id],
1441                |row| row.get(0),
1442            )?;
1443            if !exists {
1444                units.push(UnitCatchup::Unknown { id: id.clone() });
1445                continue;
1446            }
1447            // Every persisted field is required to be well typed; a corrupt
1448            // record fails typed instead of inventing a state.
1449            let unit = get_unit(conn, id)
1450                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1451                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1452            let cursor = match unit.cursor {
1453                Some(cursor) => cursor,
1454                None => replica_cursor(conn)
1455                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1456            };
1457            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1458                return Err(corrupt("persisted unit state is invalid"));
1459            }
1460            let active = unit.status == UnitStatus::Active;
1461            let pending = unit.bootstrapping();
1462            units.push(UnitCatchup::Known {
1463                id: id.clone(),
1464                status: unit.status,
1465                live: unit.cursor.is_none(),
1466                cursor,
1467                has_resume_token: unit.bootstrap_state.is_some(),
1468                bootstrap_complete: active && !pending,
1469                known_pending_pages: active && pending,
1470                reason_code: unit.reason_code,
1471            });
1472        }
1473        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1474        for id in &request.commit_ids {
1475            let pending: bool = conn.query_row(
1476                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1477                rusqlite::params![id],
1478                |row| row.get(0),
1479            )?;
1480            if pending {
1481                deliveries.push(CommitDelivery::Pending {
1482                    client_commit_id: id.clone(),
1483                });
1484                continue;
1485            }
1486            let row = conn
1487                .query_row(
1488                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1489                            resolution, resolved_at_ms, replacement_client_commit_id
1490                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1491                    rusqlite::params![id],
1492                    StoredCommitOutcomeRow::from_row,
1493                )
1494                .optional()
1495                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1496            deliveries.push(match row {
1497                Some(row) => CommitDelivery::Known {
1498                    client_commit_id: id.clone(),
1499                    outcome: SyncClient::outcome_from_row(row)
1500                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1501                },
1502                None => CommitDelivery::Unknown {
1503                    client_commit_id: id.clone(),
1504                },
1505            });
1506        }
1507        Ok(SnapshotRead {
1508            revision,
1509            queries,
1510            coverage: CoverageSnapshot {
1511                complete: pending.is_empty() && missing.is_empty(),
1512                pending,
1513                missing,
1514            },
1515            units,
1516            deliveries,
1517        })
1518    })();
1519    match result {
1520        Ok(read) => {
1521            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1522            Ok(read)
1523        }
1524        Err(mut error) => {
1525            if let Err(rollback) = conn
1526                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1527            {
1528                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1529            }
1530            Err(error)
1531        }
1532    }
1533}
1534
1535fn snapshot_connection(
1536    conn: &Connection,
1537    sql: &str,
1538    params: &[Value],
1539    coverage: &[WindowCoverage],
1540) -> Result<QuerySnapshot, QueryReadFailure> {
1541    let read = snapshot_read_connection(
1542        conn,
1543        &SnapshotReadRequest {
1544            statements: vec![SnapshotStatement { sql, params }],
1545            coverage,
1546            units: Vec::new(),
1547            commit_ids: Vec::new(),
1548        },
1549    )?;
1550    Ok(QuerySnapshot {
1551        revision: read.revision,
1552        rows: read
1553            .queries
1554            .into_iter()
1555            .next()
1556            .expect("single-query snapshot returns one result"),
1557        coverage: read.coverage,
1558    })
1559}
1560
1561/// A long-lived read-only SQLite sidecar for latency-critical native views.
1562/// Network rounds stay serialized on the mutable core owner, while atomic
1563/// query snapshots use this independent connection and therefore never queue
1564/// behind HTTP/WebSocket latency.
1565pub struct FileQuerySnapshotReader {
1566    path: String,
1567    conn: Option<Connection>,
1568}
1569
1570impl FileQuerySnapshotReader {
1571    #[must_use]
1572    pub fn new(path: impl Into<String>) -> Self {
1573        Self {
1574            path: path.into(),
1575            conn: None,
1576        }
1577    }
1578
1579    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1580        if self.conn.is_none() {
1581            let conn = Connection::open_with_flags(
1582                &self.path,
1583                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1584            )
1585            .map_err(QueryReadFailure::from)?;
1586            conn.busy_timeout(std::time::Duration::from_millis(250))
1587                .map_err(QueryReadFailure::from)?;
1588            self.conn = Some(conn);
1589        }
1590        self.conn
1591            .as_ref()
1592            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1593    }
1594
1595    /// One atomic read over the shared primitive: every statement, coverage,
1596    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1597    pub fn snapshot_read(
1598        &mut self,
1599        request: &SnapshotReadRequest<'_>,
1600    ) -> Result<SnapshotRead, QueryReadFailure> {
1601        snapshot_read_connection(self.connection()?, request)
1602    }
1603
1604    /// The current durable local revision. A host can skip a read when it is
1605    /// unchanged; a corrupt or unreadable marker fails typed.
1606    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1607        Ok(self
1608            .snapshot_read(&SnapshotReadRequest::default())?
1609            .revision)
1610    }
1611
1612    /// One unit's persisted catch-up state (see the §7.5 contract).
1613    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1614        let read = self.snapshot_read(&SnapshotReadRequest {
1615            units: vec![id.to_owned()],
1616            ..Default::default()
1617        })?;
1618        Ok(read
1619            .units
1620            .into_iter()
1621            .next()
1622            .expect("one requested catch-up"))
1623    }
1624
1625    /// One client commit's delivery status: pending, its retained outcome, or
1626    /// an explicit unknown (never held, or pruned by retention).
1627    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1628        let read = self.snapshot_read(&SnapshotReadRequest {
1629            commit_ids: vec![id.to_owned()],
1630            ..Default::default()
1631        })?;
1632        Ok(read
1633            .deliveries
1634            .into_iter()
1635            .next()
1636            .expect("one requested delivery"))
1637    }
1638
1639    /// Hosts report an owned failure, and the first success after it, to the
1640    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1641    pub fn query_snapshot(
1642        &mut self,
1643        sql: &str,
1644        params: &[Value],
1645        coverage: &[WindowCoverage],
1646    ) -> Result<QuerySnapshot, QueryReadFailure> {
1647        snapshot_connection(self.connection()?, sql, params, coverage)
1648    }
1649}
1650
1651fn validate_authority_reads(
1652    schema: &ClientSchema,
1653    declarations: &[crate::api::AuthorityReadDeclaration],
1654) -> Result<(), String> {
1655    let mut seen = BTreeSet::new();
1656    for read in declarations {
1657        let table = schema
1658            .table(&read.table)
1659            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1660        let plain = |name: &str| {
1661            table.columns.iter().enumerate().any(|(i, c)| {
1662                c.name == name
1663                    && name != "authority_version"
1664                    && !name.starts_with("_sync")
1665                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1666                    && matches!(
1667                        c.ty,
1668                        ssp2::segment::ColumnType::String
1669                            | ssp2::segment::ColumnType::Integer
1670                            | ssp2::segment::ColumnType::Float
1671                            | ssp2::segment::ColumnType::Boolean
1672                            | ssp2::segment::ColumnType::Json
1673                    )
1674            })
1675        };
1676        if !seen.insert(&read.table)
1677            || read.columns.is_empty()
1678            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1679            || !read.columns.contains(&table.primary_key)
1680            || !read.columns.iter().all(|c| plain(c))
1681            || read.scopes.is_empty()
1682            || !read.scopes.iter().all(|(v, values)| {
1683                table
1684                    .scope_column(v)
1685                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1686                    && !values.is_empty()
1687                    && values.iter().all(|v| !v.is_empty() && v != "*")
1688            })
1689        {
1690            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1691        }
1692    }
1693    Ok(())
1694}
1695
1696// §7.4.2: every open and recreation checks before any persistent writes.
1697/// A marker read that hit a recognized SQLite storage failure keeps that
1698/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1699/// failure is the invalid-marker `sync.local_corrupt`.
1700fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1701    let failure = QueryReadFailure::from(error);
1702    if failure.code.is_some() {
1703        failure.to_string()
1704    } else {
1705        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1706    }
1707}
1708
1709/// §7.1: reject an invalid configured client limit before any storage is
1710/// opened or written. Zero would wedge the push queue and an out-of-range
1711/// value would differ per target; the TS core enforces the same bounds.
1712fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1713    if limits.outcome_retention_max_entries == Some(0) {
1714        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1715    }
1716    if [
1717        limits.max_push_commits_per_request,
1718        limits.max_push_operations_per_request,
1719        limits.max_push_request_bytes,
1720    ]
1721    .into_iter()
1722    .flatten()
1723    .any(|value| value == 0 || value > u32::MAX as usize)
1724    {
1725        return Err(
1726            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1727        );
1728    }
1729    Ok(())
1730}
1731
1732fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1733    let corrupt =
1734        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1735    let table_type: Option<String> = conn
1736        .query_row(
1737            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1738            [],
1739            |row| row.get(0),
1740        )
1741        .optional()
1742        .map_err(schema_marker_read_failure)?;
1743    let Some(table_type) = table_type else {
1744        return Ok(None);
1745    };
1746    if table_type != "table" {
1747        return Err(corrupt());
1748    }
1749    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1750    // whichever row SQLite returns first, and an older one would drive a
1751    // destructive reset over a newer replica.
1752    let mut statement = conn
1753        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1754        .map_err(schema_marker_read_failure)?;
1755    let mut rows = statement
1756        .query([LOCAL_SCHEMA_VERSION_KEY])
1757        .map_err(schema_marker_read_failure)?;
1758    let value: Option<String> = rows
1759        .next()
1760        .map_err(schema_marker_read_failure)?
1761        .map(|row| row.get(0))
1762        .transpose()
1763        .map_err(schema_marker_read_failure)?;
1764    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1765        return Err(corrupt());
1766    }
1767    let Some(value) = value else {
1768        let has_descriptor = conn
1769            .query_row(
1770                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1771                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1772                |_| Ok(()),
1773            )
1774            .optional()
1775            .map_err(schema_marker_read_failure)?
1776            .is_some();
1777        if has_descriptor {
1778            return Err(corrupt());
1779        }
1780        return Ok(None);
1781    };
1782    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1783        || !value.bytes().all(|b| b.is_ascii_digit())
1784    {
1785        return Err(corrupt());
1786    }
1787    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1788    if version > requested {
1789        return Err(
1790            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1791                .to_owned(),
1792        );
1793    }
1794    Ok(Some(version))
1795}
1796
1797/// A file replica on a browser worker must recover a hot journal before any
1798/// statement reads it. `wasm_vfs` replaces the installed OPFS VFS's reserved
1799/// lock callback, so install it before the connection opens its file.
1800#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1801fn prepare_wasm_replica(path: &str) -> Result<(), String> {
1802    if path.is_empty() || path == ":memory:" {
1803        return Ok(());
1804    }
1805    crate::wasm_vfs::disable_foreign_lock_check()
1806}
1807
1808#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
1809fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
1810    Ok(())
1811}
1812
1813impl SyncClient {
1814    pub fn new_with_identity(
1815        client_id: Option<String>,
1816        schema_json: &Value,
1817        limits: ClientLimits,
1818    ) -> Result<Self, String> {
1819        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1820        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1821        Self::with_connection(resolved, schema_json, limits, conn)
1822    }
1823
1824    pub fn new(
1825        client_id: String,
1826        schema_json: &Value,
1827        limits: ClientLimits,
1828    ) -> Result<Self, String> {
1829        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1830        Self::with_connection(client_id, schema_json, limits, conn)
1831    }
1832
1833    /// Build a client backed by an on-disk SQLite database at `path` — the
1834    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1835    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1836    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1837    /// of the command router's dependency set (the router only holds a path).
1838    pub fn open_path(
1839        client_id: String,
1840        schema_json: &Value,
1841        limits: ClientLimits,
1842        path: &str,
1843    ) -> Result<Self, String> {
1844        validate_client_limits(&limits)?;
1845        parse_schema_json(schema_json)?;
1846        prepare_wasm_replica(path)?;
1847        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1848        Self::with_connection(client_id, schema_json, limits, conn)
1849    }
1850
1851    /// Open a file replica and resolve its client id. Native targets require
1852    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1853    /// accepts the sqlite-wasm worker journal instead.
1854    pub fn open_path_with_identity(
1855        client_id: Option<String>,
1856        schema_json: &Value,
1857        limits: ClientLimits,
1858        path: &str,
1859    ) -> Result<Self, String> {
1860        Self::open_path_with_identity_journal(
1861            client_id,
1862            schema_json,
1863            limits,
1864            path,
1865            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1866        )
1867    }
1868
1869    fn open_path_with_identity_journal(
1870        client_id: Option<String>,
1871        schema_json: &Value,
1872        limits: ClientLimits,
1873        path: &str,
1874        require_wal: bool,
1875    ) -> Result<Self, String> {
1876        validate_client_limits(&limits)?;
1877        // §7.4.2: an invalid requested version is refused before the replica is
1878        // even created, so a rejected constructor leaves no file behind.
1879        parse_schema_json(schema_json)?;
1880        prepare_wasm_replica(path)?;
1881        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1882        // A file-backed native client uses an independent read connection for
1883        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1884        // concurrency mode: a view read never holds a rollback-journal lock
1885        // that delays the client's next commit. The busy timeout must precede
1886        // this replica's own startup transaction; WAL is applied only after
1887        // startup succeeded, because a refused open must leave the replica's
1888        // journal mode and contents untouched (§7.4.2).
1889        conn.busy_timeout(std::time::Duration::from_millis(250))
1890            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1891        // Durability is the caller's contract here, not the VFS default: a
1892        // rollback-journal commit needs FULL before it writes the database
1893        // pages, or a killed worker can leave a half-written image behind.
1894        // WAL below inherits the same setting.
1895        conn.pragma_update(None, "synchronous", "FULL")
1896            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1897        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1898        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1899        // host has no independent snapshot reader. The WAL requirement exists
1900        // for native file replicas, where it is the reader/writer boundary.
1901        if require_wal {
1902            let journal_mode: String = client
1903                .conn
1904                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1905                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1906            if journal_mode != "wal" {
1907                return Err(
1908                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1909                );
1910            }
1911        }
1912        Ok(client)
1913    }
1914
1915    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1916    /// `-shm` sidecars, through the VFS that owns it. Use this after
1917    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1918    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1919    /// pool's slot accounting.
1920    ///
1921    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1922    /// a VFS cannot delete a file another handle still owns. A file that does
1923    /// not exist counts as deleted, so a cleanly closed replica without
1924    /// sidecars resets. The next open
1925    /// creates a fresh replica (new client id) and re-bootstraps from the
1926    /// server. A memory replica has nothing to delete and fails with
1927    /// `sync.invalid_request`.
1928    pub fn reset_replica(path: &str) -> Result<(), String> {
1929        if path.is_empty() || path == ":memory:" {
1930            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1931        }
1932        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1933        // the same one `Connection::open(path)` above selects. `xDelete` is
1934        // that VFS's own function pointer, and `c_name` outlives the call.
1935        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1936        if vfs.is_null() {
1937            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1938        }
1939        let delete = unsafe { (*vfs).xDelete }
1940            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1941        for suffix in ["-wal", "-shm", "-journal", ""] {
1942            let file_name = format!("{path}{suffix}");
1943            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1944                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1945            })?;
1946            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1947            // SQLite library; `c_name` is a valid NUL-terminated path.
1948            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1949            // A cleanly closed WAL replica has no sidecars; a missing file is
1950            // already deleted.
1951            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1952            {
1953                return Err(format!(
1954                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1955                ));
1956            }
1957        }
1958        Ok(())
1959    }
1960
1961    #[must_use]
1962    pub fn client_id(&self) -> &str {
1963        &self.client_id
1964    }
1965
1966    /// Rotate the caller's client id (§1.5) and return the new one. Use it
1967    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
1968    /// resetClientId`): the next round carries the new id. A live realtime
1969    /// socket is closed so the next connection registers with the new id.
1970    /// The replica and its outbox stay in place; wipe them yourself when the
1971    /// stored data belongs to the account that signed out.
1972    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
1973        self.disconnect_realtime(transport);
1974        let client_id = uuid::Uuid::new_v4().to_string();
1975        self.set_meta(CLIENT_ID_KEY, &client_id)?;
1976        self.client_id.clone_from(&client_id);
1977        self.round_failure = None;
1978        Ok(client_id)
1979    }
1980
1981    /// Build a client over a caller-supplied rusqlite connection — the seam a
1982    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
1983    /// with an on-disk database (`Connection::open(path)`) rather than the
1984    /// default `:memory:`. The connection MUST be fresh (no pre-existing
1985    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
1986    /// the same file across process restarts reuses the persisted rows.
1987    pub fn with_connection(
1988        client_id: String,
1989        schema_json: &Value,
1990        limits: ClientLimits,
1991        conn: Connection,
1992    ) -> Result<Self, String> {
1993        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
1994    }
1995
1996    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
1997    /// means a durable replica's persisted `clientId` wins over a fresh one.
1998    fn with_connection_identity(
1999        client_id: Option<String>,
2000        schema_json: &Value,
2001        limits: ClientLimits,
2002        conn: Connection,
2003    ) -> Result<Self, String> {
2004        validate_client_limits(&limits)?;
2005        let schema = parse_schema_json(schema_json)?;
2006        validate_authority_reads(&schema, &limits.authority_reads)?;
2007        // RFC 0005 D8: resolve the previous-version config before the opening
2008        // reset runs — the capture happens inside it. Bad bounds fail loud.
2009        let previous_version = match limits.previous_version_context {
2010            Some(config) => {
2011                config.validate()?;
2012                Some(config)
2013            }
2014            None => None,
2015        };
2016        let mut client = SyncClient {
2017            #[cfg(feature = "bench-internals")]
2018            benchmark_phases: Recorder::default(),
2019            conn,
2020            schema,
2021            client_id: client_id
2022                .clone()
2023                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2024            limits,
2025            outbox: Shared::default(),
2026            failed_commits: Shared::default(),
2027            retain_failed_commits: false,
2028            conflicts: Shared::default(),
2029            rejections: Shared::default(),
2030            schema_floor: None,
2031            lease_state: None,
2032            stopped: false,
2033            upgrading: false,
2034            sync_needed: false,
2035            realtime_policy: RealtimePolicy::Optional,
2036            realtime_state: RealtimeState::Disconnected,
2037            realtime_reason_code: None,
2038            realtime_retry_delay_ms: None,
2039            presence: HashMap::new(),
2040            now_ms: None,
2041            encryption: crate::values::EncryptionConfig::default(),
2042            security_preflight: false,
2043            security_epoch: 0,
2044            transport_enabled: true,
2045            transport_epoch: 0,
2046            registration_epoch: 0,
2047            previous_version,
2048            insert_sql: RefCell::new(HashMap::new()),
2049            overlay_dirty: OverlayDirty::clean(),
2050            storage_failure: RefCell::new(None),
2051            #[cfg(test)]
2052            overlay_rebuild_count: Cell::new(0),
2053            #[cfg(test)]
2054            acknowledged_replay_count: Cell::new(0),
2055            #[cfg(test)]
2056            outcome_prune_count: Cell::new(0),
2057            #[cfg(test)]
2058            image_chunk_ms: Cell::new(None),
2059            progress: ProgressObserver::default(),
2060            change_queue: VecDeque::new(),
2061            sync_intent_queue: VecDeque::new(),
2062            retry_delay_ms: 250,
2063            round_retry_delay_ms: None,
2064            applied_segments: HashMap::new(),
2065            image_chunk_rows: HashMap::new(),
2066            round_failure: None,
2067            active_round: None,
2068            last_round: None,
2069            last_change: None,
2070            query_failures: Vec::new(),
2071            coalesced: None,
2072            coalesced_failure: None,
2073        };
2074        // The row write path leans on the prepared-statement cache (two
2075        // insert statements per synced table, plus the bookkeeping
2076        // statements); size it so a multi-table schema never thrashes.
2077        client
2078            .conn
2079            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2080        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2081        // writer) covers the marker read, the identity check, and every
2082        // bookkeeping/schema write, so a concurrent open that upgraded the
2083        // replica cannot interleave between the guard and the writes, and a
2084        // refusal leaves the replica unchanged.
2085        client
2086            .conn
2087            .execute_batch("BEGIN IMMEDIATE")
2088            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2089        let startup = (|| -> Result<(), String> {
2090            // The marker guard runs inside the write transaction, so the version
2091            // it validates is the version every write below commits under.
2092            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2093            // New app indexes may reference columns that only exist after a
2094            // version-bump reset.
2095            client.create_bookkeeping_tables()?;
2096            match client.get_meta(CLIENT_ID_KEY)? {
2097                Some(existing) => match client_id {
2098                    Some(requested) if existing != requested => {
2099                        return Err(format!(
2100                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2101                        ))
2102                    }
2103                    _ => client.client_id = existing,
2104                },
2105                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2106            }
2107            client.restore_persisted_state()?;
2108            client.failed_commits = client.load_failed_commits()?.into();
2109            match marker {
2110                None => {
2111                    client.create_synced_tables()?;
2112                    client.persist_schema_version()?;
2113                    client.save_unit_scope_schema()?;
2114                }
2115                Some(version) if version == client.schema.version => {
2116                    client.create_synced_tables()?;
2117                    // RFC 0005 D1: the same-version open backfills the descriptor
2118                    // for a database first opened by an unaware binary, so the
2119                    // NEXT bump can capture without needing a schema bump first.
2120                    set_local_schema_descriptor(&client.conn, &client.schema)
2121                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2122                    client.save_unit_scope_schema()?;
2123                }
2124                Some(_) => client.run_schema_reset()?,
2125            }
2126            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2127            // committed in its OWN file while the replica's savepoint never
2128            // released — a crash at that seam leaves a container beside a matching
2129            // OLD marker, and the next open at that version runs no reset. Discard
2130            // it before anything can read it. No-op when no container exists.
2131            client.reconcile_previous_version_at_boot()?;
2132            client.clear_satisfied_persisted_schema_floor()?;
2133            client.prune_unknown_units(false)?;
2134            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2135            // the full startup/reset reconciliation boundary, inside this open
2136            // transaction and after the schema and identity guards, so a commit
2137            // an earlier version accepted is dropped with durable rejection
2138            // evidence before any replay. A table the current schema removed
2139            // keeps the §7.4.4 send-time classification, and an unresolved
2140            // encryption key stays with the send seam because hosts configure
2141            // keys after open.
2142            client.drop_unencodable_outbox(false, None)?;
2143            if marker == Some(client.schema.version)
2144                && (!client.outbox.is_empty()
2145                    || !client.failed_commits.is_empty()
2146                    || client.has_acknowledged_rows()?)
2147            {
2148                // Reconstruct the visible optimistic overlay from the durable
2149                // base plus outbox instead of trusting a process-interrupted
2150                // mirror.
2151                client.overlay_dirty.set(true);
2152                client.rebuild_overlay()?;
2153            }
2154            // Every persisted active unit needs one catch-up pull on
2155            // open: realtime only covers changes after connection, while an
2156            // idempotent setWindow correctly creates no fresh command effect.
2157            // Pending outbox work has the same restart requirement. The core
2158            // owns this intent so native hosts never poll or require an
2159            // application-issued sync().
2160            client.enqueue_startup_sync_if_needed()?;
2161            Ok(())
2162        })();
2163        match startup {
2164            Ok(()) => match client.conn.execute_batch("COMMIT") {
2165                // §4.8: an eviction a crash interrupted finishes before any
2166                // read, with or without a transport; a replica reopened in
2167                // preflight finishes it at activation.
2168                Ok(()) => {
2169                    if !client.security_preflight {
2170                        client.drain_pending_evictions()?;
2171                    }
2172                    Ok(client)
2173                }
2174                Err(error) => {
2175                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2176                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2177                        // Keep a failed rollback's typed storage classification.
2178                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2179                    }
2180                    Err(failure)
2181                }
2182            },
2183            Err(error) => {
2184                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2185                    // Keep a failed rollback's typed storage classification.
2186                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2187                }
2188                Err(error)
2189            }
2190        }
2191    }
2192
2193    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2194    /// against this instead of system time (conformance virtual clock).
2195    pub fn set_now_ms(&mut self, now_ms: i64) {
2196        self.now_ms = Some(now_ms);
2197    }
2198
2199    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2200    /// The command router parses these from the `create` command's
2201    /// `encryption` config (keys as `{$bytes: hex}`).
2202    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2203        self.encryption = encryption;
2204    }
2205
2206    #[must_use]
2207    pub fn security_lifecycle(&self) -> &'static str {
2208        if self.security_preflight {
2209            "preflight"
2210        } else {
2211            "active"
2212        }
2213    }
2214
2215    #[must_use]
2216    pub fn security_preflight(&self) -> bool {
2217        self.security_preflight
2218    }
2219
2220    /// Host-owned, non-persisted network gate. Local operations remain available.
2221    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2222        if self.transport_enabled != enabled {
2223            self.transport_enabled = enabled;
2224            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2225            self.sync_intent_queue.clear();
2226            self.sync_intent_queue
2227                .push_back(if enabled && !self.security_preflight {
2228                    SyncIntent::Interactive
2229                } else {
2230                    SyncIntent::None
2231                });
2232        }
2233        // A captured reply keeps its normal apply/revocation semantics.
2234        if !enabled && self.active_round.is_none() {
2235            self.disconnect_realtime(transport);
2236        }
2237    }
2238
2239    pub fn transport_enabled(&self) -> bool {
2240        self.transport_enabled
2241    }
2242
2243    /// Quarantine this replica: enter the fail-closed gate, release all
2244    /// core-owned key material, and record the gate in the database so
2245    /// reopening re-enters preflight until `activate_security` clears it.
2246    ///
2247    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2248    /// which must NOT leave that durable mark.
2249    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2250        let entering = !self.security_preflight;
2251        self.seal_security_on_teardown();
2252        // Persist the quarantine so it survives handle teardown and restart:
2253        // reopening this replica re-enters preflight until activation clears it.
2254        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2255        if entering {
2256            self.emit_status_batch()?;
2257        }
2258        Ok(())
2259    }
2260
2261    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2262    fn emit_status_batch(&mut self) -> Result<(), String> {
2263        self.begin_observation("syncular_status")?;
2264        let batch = ChangeAccumulator {
2265            status: true,
2266            ..ChangeAccumulator::default()
2267        };
2268        self.finish_observation("syncular_status", batch)
2269            .inspect_err(|_| self.rollback_observation("syncular_status"))
2270    }
2271
2272    /// Release core-owned key material as a host tears the client down,
2273    /// WITHOUT recording a quarantine.
2274    ///
2275    /// Shutting an activated client down is not a quarantine event. Persisting
2276    /// the gate here marks every cleanly closed replica as pending, and the
2277    /// next plain `create` is then refused permanently: the reopen path
2278    /// restores the flag from the marker and the create guard rejects it. The
2279    /// in-memory flag still closes the gate for anything still holding this
2280    /// instance, which is all a teardown barrier needs.
2281    pub fn seal_security_on_teardown(&mut self) {
2282        self.cancel_sync_round();
2283        self.security_preflight = true;
2284        self.security_epoch = self.security_epoch.wrapping_add(1);
2285        self.encryption = crate::values::EncryptionConfig::default();
2286        self.sync_intent_queue.clear();
2287    }
2288
2289    /// Install the post-authentication keyring and release the host loop.
2290    pub fn activate_security(
2291        &mut self,
2292        encryption: crate::values::EncryptionConfig,
2293    ) -> Result<(), String> {
2294        if !self.security_preflight {
2295            return Err(
2296                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2297            );
2298        }
2299        // §4.8: an interrupted eviction finishes before protected reads open.
2300        self.drain_pending_evictions()?;
2301        self.enqueue_startup_sync_if_needed()?;
2302        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2303        self.encryption = encryption;
2304        self.security_preflight = false;
2305        // §5.11: rows sealed under a key this keyring now holds open at once.
2306        self.reveal_sealed_rows()
2307            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2308        self.emit_status_batch()
2309    }
2310
2311    fn clock_now_ms(&self) -> i64 {
2312        self.now_ms.unwrap_or_else(crate::wall_clock_ms)
2313    }
2314
2315    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2316        let exists = |name: &str| {
2317            self.conn
2318                .query_row(
2319                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2320                    [name],
2321                    |row| row.get::<_, bool>(0),
2322                )
2323                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2324        };
2325        let outbox_existed = exists("_syncular_outbox")?;
2326        // Durable client bookkeeping (outbox + units + meta).
2327        self.conn
2328            .execute_batch(
2329                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2330                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2331                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2332                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2333                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2334                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2335                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2336                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2337                   PRIMARY KEY(commit_id, idx));
2338                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2339                   ON _syncular_acked_rows(tbl, id);
2340                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2341                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2342                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2343                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2344                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2345                   client_commit_id TEXT NOT NULL UNIQUE,
2346                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2347                   recorded_at_ms INTEGER NOT NULL,
2348                   results_json TEXT NOT NULL,
2349                   operations_json TEXT,
2350                   resolution TEXT NOT NULL DEFAULT 'active'
2351                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2352                   resolved_at_ms INTEGER,
2353                   replacement_client_commit_id TEXT);
2354                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2355                   ON _syncular_commit_outcomes(resolution, seq);
2356                 CREATE TABLE IF NOT EXISTS _syncular_units (
2357                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2358                   explicit INTEGER NOT NULL DEFAULT 0,
2359                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2360                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2361                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT);
2362                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2363                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2364                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2365                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2366                   PRIMARY KEY (base, unit));
2367                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2368                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2369                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2370                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2371                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2372                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2373                   PRIMARY KEY(commit_id, blob_id));
2374                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2375                   ON _syncular_blob_commit_refs(blob_id);",
2376            )
2377            .map_err(|e| e.to_string())?;
2378        // Migrate an outcome journal created before failed aggregate
2379        // envelopes were retained. Historical rows intentionally stay NULL.
2380        let _ = self
2381            .conn
2382            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2383        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2384            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2385        }
2386        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2387        // a new sequence, so it replaces the id in the same transaction.
2388        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2389            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2390        }
2391        // §9: a replica written under wire version 3 keeps its outbox, journal,
2392        // rows and pending evictions and registers its units fresh. Its
2393        // subscriptions, cursors, echoes, resume tokens and window entries go;
2394        // units left from an earlier version 4 life restart.
2395        if exists("_syncular_window_pending_evict")? {
2396            let invalid: bool = self
2397                .conn
2398                .query_row(
2399                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2400                       WHERE NOT json_valid(effective_scopes)
2401                          OR json_type(effective_scopes) <> 'object'
2402                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2403                               WHERE type <> 'array'
2404                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2405                                       WHERE v.type <> 'text')))",
2406                    [],
2407                    |row| row.get(0),
2408                )
2409                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2410            if invalid {
2411                return Err(
2412                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2413                        .to_owned(),
2414                );
2415            }
2416            self.conn
2417                .execute_batch(
2418                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2419                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2420                     DROP TABLE _syncular_window_pending_evict;
2421                     DROP TABLE IF EXISTS _syncular_subscriptions;
2422                     DELETE FROM _syncular_windows;",
2423                )
2424                .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2425                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2426        }
2427        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2428        // completing a large upload does not rewrite its BLOB record.
2429        if self.schema_has_blobs() {
2430            if exists("_syncular_blobs")? {
2431                let mut statement = self
2432                    .conn
2433                    .prepare("PRAGMA table_info(_syncular_blobs)")
2434                    .map_err(|e| e.to_string())?;
2435                let mut columns = statement
2436                    .query_map([], |row| row.get::<_, String>(1))
2437                    .map_err(|e| e.to_string())?
2438                    .collect::<Result<Vec<_>, _>>()
2439                    .map_err(|e| e.to_string())?;
2440                columns.sort();
2441                let mut expected = vec![
2442                    "blob_id",
2443                    "byte_length",
2444                    "bytes",
2445                    "created_at_ms",
2446                    "media_type",
2447                ];
2448                expected.sort_unstable();
2449                if columns != expected {
2450                    return Err(
2451                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2452                    );
2453                }
2454            } else {
2455                self.conn
2456                    .execute_batch(
2457                        "CREATE TABLE _syncular_blobs(
2458                           blob_id TEXT PRIMARY KEY,
2459                           bytes BLOB NOT NULL,
2460                           byte_length INTEGER NOT NULL,
2461                           media_type TEXT,
2462                           created_at_ms INTEGER NOT NULL);",
2463                    )
2464                    .map_err(|e| e.to_string())?;
2465            }
2466            self.conn
2467                .execute_batch(
2468                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2469                       blob_id TEXT PRIMARY KEY,
2470                       media_type TEXT,
2471                       created_at_ms INTEGER NOT NULL);",
2472                )
2473                .map_err(|e| e.to_string())?;
2474        }
2475        Ok(())
2476    }
2477
2478    /// True iff any synced table declares a `blob_ref` column (§5.9).
2479    fn schema_has_blobs(&self) -> bool {
2480        self.schema
2481            .tables
2482            .iter()
2483            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2484    }
2485
2486    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2487
2488    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2489        meta_get(&self.conn, key)
2490            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2491    }
2492
2493    // The caller holds the startup/reset transaction. Either both schema
2494    // records persist or the enclosing operation fails and rolls back.
2495    fn persist_schema_version(&self) -> Result<(), String> {
2496        self.conn
2497            .execute(
2498                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2499                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2500            )
2501            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2502        set_local_schema_descriptor(&self.conn, &self.schema)
2503            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2504    }
2505
2506    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2507        meta_set(&self.conn, key, value)
2508            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2509    }
2510
2511    fn delete_meta(&self, key: &str) -> Result<(), String> {
2512        meta_delete(&self.conn, key)
2513            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2514    }
2515
2516    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2517    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2518        let mut stmt = self
2519            .conn
2520            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2521            .map_err(|error| error.to_string())?;
2522        let rows = stmt
2523            .query_map([], |row| {
2524                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2525            })
2526            .map_err(|error| error.to_string())?;
2527        let mut commits = Vec::new();
2528        for row in rows {
2529            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2530            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2531                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2532            })?;
2533            let mut ops = Vec::with_capacity(entries.len());
2534            for entry in entries {
2535                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2536                ops.push(OutboxOp {
2537                    upsert: op == "upsert",
2538                    table: entry
2539                        .get("table")
2540                        .and_then(Value::as_str)
2541                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2542                        .to_owned(),
2543                    row_id: entry
2544                        .get("rowId")
2545                        .and_then(Value::as_str)
2546                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2547                        .to_owned(),
2548                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2549                    // A legacy entry's `changedFields` key is ignored:
2550                    // the values map is the presence set (§6.1).
2551                    values: entry.get("values").and_then(Value::as_object).cloned(),
2552                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2553                });
2554            }
2555            commits.push(Arc::new(OutboxCommit {
2556                client_commit_id,
2557                ops,
2558            }));
2559        }
2560        Ok(commits)
2561    }
2562
2563    fn restore_persisted_state(&mut self) -> Result<(), String> {
2564        self.outbox = self.load_outbox()?.into();
2565
2566        self.prune_commit_outcomes()?;
2567        let active = self.commit_outcomes(CommitOutcomeQuery {
2568            active_only: true,
2569            ..CommitOutcomeQuery::default()
2570        })?;
2571        self.conflicts = active
2572            .iter()
2573            .flat_map(|outcome| outcome.results.iter())
2574            .filter_map(|result| match result {
2575                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2576                _ => None,
2577            })
2578            .collect::<Vec<_>>()
2579            .into();
2580        self.rejections = active
2581            .iter()
2582            .flat_map(|outcome| outcome.results.iter())
2583            .filter_map(|result| match result {
2584                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2585                _ => None,
2586            })
2587            .collect::<Vec<_>>()
2588            .into();
2589
2590        self.lease_state = self
2591            .get_meta(LEASE_STATE_KEY)?
2592            .map(|raw| serde_json::from_str(&raw))
2593            .transpose()
2594            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2595        self.schema_floor = self
2596            .get_meta(SCHEMA_FLOOR_KEY)?
2597            .map(|raw| serde_json::from_str(&raw))
2598            .transpose()
2599            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2600        self.stopped = self.schema_floor.is_some();
2601        // A replica left in unactivated preflight reopens gated: the quarantine
2602        // decision persists with the data across handle teardown and restart.
2603        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2604            self.security_preflight = true;
2605        }
2606        Ok(())
2607    }
2608
2609    fn save_unit_scope_schema(&self) -> Result<(), String> {
2610        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2611            .schema
2612            .tables
2613            .iter()
2614            .map(|table| {
2615                (
2616                    table.name.clone(),
2617                    table
2618                        .scope_variables
2619                        .iter()
2620                        .map(|scope| {
2621                            (
2622                                scope.variable.clone(),
2623                                [scope.prefix.clone(), scope.column.clone()],
2624                            )
2625                        })
2626                        .collect(),
2627                )
2628            })
2629            .collect();
2630        self.set_meta(
2631            "subscriptionScopeSchema",
2632            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2633        )?;
2634        let generations: BTreeMap<_, _> = self
2635            .schema
2636            .tables
2637            .iter()
2638            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2639            .collect();
2640        self.set_meta(
2641            TABLE_GENERATIONS_KEY,
2642            &serde_json::to_string(&generations).expect("generations serialize"),
2643        )
2644    }
2645
2646    /// Drop units that cover no table of the running schema, with their window
2647    /// registry rows and pending evictions; never translate scopes. At a bump
2648    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2649    /// on every table it covered before.
2650    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2651        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2652            self.get_meta("subscriptionScopeSchema")?
2653                .map(|raw| serde_json::from_str(&raw))
2654                .transpose()
2655                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2656        } else {
2657            None
2658        };
2659        // A table keeps the variables' meaning when each has the same prefix
2660        // and column as before. An old binary stored no declaration evidence;
2661        // matching names cannot prove unchanged meaning, so the app registers
2662        // again.
2663        let unchanged = |name: &str, variables: &[&String]| {
2664            let (Some(old), Some(table)) = (
2665                previous.as_ref().and_then(|previous| previous.get(name)),
2666                self.schema.table(name),
2667            ) else {
2668                return false;
2669            };
2670            variables.iter().all(|variable| {
2671                table
2672                    .scope_variables
2673                    .iter()
2674                    .find(|scope| &scope.variable == *variable)
2675                    .is_some_and(|scope| {
2676                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2677                    })
2678            })
2679        };
2680        let compatible = |scopes: &UnitScopes| {
2681            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2682                return false;
2683            }
2684            if !bump {
2685                return true;
2686            }
2687            let Some(previous) = &previous else {
2688                return false;
2689            };
2690            previous.iter().all(|(name, old)| {
2691                !scopes.keys().all(|variable| old.contains_key(variable))
2692                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2693            })
2694        };
2695        let mut stale: BTreeSet<String> = self
2696            .stored(load_units(&self.conn))?
2697            .into_iter()
2698            .filter(|unit| !compatible(&unit.scopes))
2699            .map(|unit| unit.id)
2700            .collect();
2701        // A shrunk window's deferred eviction outlives its unit row and keeps
2702        // its per-table scopes. A table the schema no longer has cannot be
2703        // evicted, and at a bump a table whose scope declarations changed must
2704        // not be evicted by the outdated rule.
2705        for (id, table, effective) in self.load_pending_evictions()? {
2706            if self.schema.table(&table).is_none()
2707                || (bump
2708                    && !unchanged(
2709                        &table,
2710                        &effective
2711                            .iter()
2712                            .map(|(variable, _)| variable)
2713                            .collect::<Vec<_>>(),
2714                    ))
2715            {
2716                stale.insert(id);
2717            }
2718        }
2719        for id in &stale {
2720            for sql in [
2721                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2722                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2723                "DELETE FROM _syncular_units WHERE id = ?1",
2724            ] {
2725                self.conn
2726                    .execute(sql, rusqlite::params![id])
2727                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2728            }
2729        }
2730        Ok(())
2731    }
2732
2733    pub fn local_revision(&self) -> Result<u64, String> {
2734        Ok(self
2735            .get_meta(LOCAL_REVISION_KEY)?
2736            .and_then(|value| value.parse().ok())
2737            .unwrap_or(0))
2738    }
2739
2740    /// §7.5: resolve a snapshot read on the mutable owner connection through
2741    /// the same primitive the read-only sidecar uses, so the command surface
2742    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2743    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2744    /// `owner` records its failure (and its next success) in diagnostics.
2745    pub fn snapshot_read(
2746        &mut self,
2747        request: &SnapshotReadRequest<'_>,
2748        owner: Option<QueryOwner<'_>>,
2749    ) -> Result<SnapshotRead, QueryReadFailure> {
2750        let result = snapshot_read_connection(&self.conn, request);
2751        if let Some(owner) = owner {
2752            self.record_query_read(owner, result.as_ref().err());
2753        }
2754        result
2755    }
2756
2757    #[must_use]
2758    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2759        SyncStatusSnapshot {
2760            current_schema_version: self.schema.version,
2761            outbox: self.outbox.len(),
2762            upgrading: self.upgrading,
2763            lease_state: self.lease_state.clone(),
2764            schema_floor: self.schema_floor.clone(),
2765            sync_needed: self.sync_needed,
2766            previous_version_context: self.previous_version_status(),
2767            security_lifecycle: self.security_lifecycle(),
2768        }
2769    }
2770
2771    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2772    /// rollback precondition), read without creating the file.
2773    fn previous_version_status(&self) -> PreviousVersionStatus {
2774        let Some(replica_path) = self.previous_version_replica_path() else {
2775            return PreviousVersionStatus {
2776                present: false,
2777                created_at_ms: None,
2778            };
2779        };
2780        match crate::previous_version::read_previous_version_record(&replica_path) {
2781            Ok(Some(record)) => PreviousVersionStatus {
2782                present: true,
2783                created_at_ms: Some(record.created_at_ms),
2784            },
2785            _ => PreviousVersionStatus {
2786                present: std::path::Path::new(
2787                    &crate::previous_version::previous_version_container_path(&replica_path),
2788                )
2789                .exists(),
2790                created_at_ms: None,
2791            },
2792        }
2793    }
2794
2795    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2796    /// is always `previousVersion`; the container never makes
2797    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2798    /// revocation, coverage completion and the TTL DISCARD the container and
2799    /// close the read with their own reason.
2800    pub fn previous_version_snapshot(
2801        &self,
2802        spec: &PreviousVersionReadSpec,
2803    ) -> Result<PreviousVersionSnapshot, String> {
2804        let now_ms = self.clock_now_ms();
2805        crate::previous_version::previous_version_snapshot(
2806            &self.conn,
2807            self.previous_version_replica_path().as_deref(),
2808            self.previous_version.as_ref(),
2809            PreviousVersionLifecycle {
2810                lease_inactive: self.previous_version_lease_inactive(now_ms),
2811                scope_revoked: self
2812                    .stored(load_units(&self.conn))?
2813                    .iter()
2814                    .any(|unit| unit.status == UnitStatus::Revoked),
2815                coverage_complete: self.previous_version_coverage_complete()?,
2816            },
2817            now_ms,
2818            self.schema.version,
2819            spec,
2820        )
2821    }
2822
2823    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2824    pub fn previous_version_audit(
2825        &self,
2826    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2827        crate::previous_version::stored_previous_version_audit(&self.conn)
2828            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2829    }
2830
2831    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2832    /// and both metadata records, and report whether anything was present.
2833    /// Idempotent: absence is a successful no-op.
2834    pub fn previous_version_discard(
2835        &self,
2836    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2837        self.drop_previous_version()
2838    }
2839
2840    fn drop_previous_version(
2841        &self,
2842    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2843        let Some(replica_path) = self.previous_version_replica_path() else {
2844            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2845                present: false,
2846                discarded: false,
2847            });
2848        };
2849        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2850    }
2851
2852    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2853    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2854        let Some(lease) = &self.lease_state else {
2855            return false;
2856        };
2857        if lease.error_code.is_some() {
2858            return true;
2859        }
2860        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2861    }
2862
2863    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2864    /// bootstrapping. An empty active set is not completion (mirrors the TS
2865    /// predicate exactly).
2866    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2867        let units = self.stored(load_units(&self.conn))?;
2868        let mut active = units
2869            .iter()
2870            .filter(|unit| unit.status == UnitStatus::Active)
2871            .peekable();
2872        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2873    }
2874
2875    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2876    /// apply. A container that no longer has a reason to exist is discarded so
2877    /// the rows do not outlive the window they were kept for.
2878    fn previous_version_lifetime_check(&self) {
2879        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2880            return;
2881        };
2882        let now_ms = self.clock_now_ms();
2883        if self.previous_version_lease_inactive(now_ms) {
2884            let _ = self.drop_previous_version();
2885            return;
2886        }
2887        let Some(replica_path) = self.previous_version_replica_path() else {
2888            return;
2889        };
2890        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2891        else {
2892            return;
2893        };
2894        let Ok(units) = load_units(&self.conn) else {
2895            return;
2896        };
2897        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2898            let _ = self.drop_previous_version();
2899            return;
2900        }
2901        if now_ms - record.created_at_ms > config.max_age_ms {
2902            let _ = self.drop_previous_version();
2903            return;
2904        }
2905        if self.previous_version_coverage_complete() == Ok(true) {
2906            let _ = self.drop_previous_version();
2907        }
2908    }
2909
2910    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2911        let registered = self.stored(load_units(&self.conn))?;
2912        // Both meta values in one statement: the FFI host recomputes
2913        // diagnostics after every command.
2914        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2915            self.conn
2916                .prepare_cached(
2917                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2918                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2919                )
2920                .and_then(|mut statement| {
2921                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2922                        Ok((row.get(0)?, row.get(1)?))
2923                    })
2924                }),
2925        )?;
2926        let mut units = DiagnosticUnits::default();
2927        for unit in &registered {
2928            *match unit.status {
2929                UnitStatus::Revoked => &mut units.revoked,
2930                UnitStatus::Failed => &mut units.failed,
2931                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2932                UnitStatus::Active
2933                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2934                {
2935                    &mut units.reset
2936                }
2937                UnitStatus::Active => &mut units.bootstrapping,
2938            } += 1;
2939        }
2940        units.reason_codes = registered
2941            .iter()
2942            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
2943            .collect::<BTreeSet<_>>()
2944            .into_iter()
2945            .collect();
2946        let captured_at_ms = self.clock_now_ms();
2947        let lease = if let Some(error_code) = self
2948            .lease_state
2949            .as_ref()
2950            .and_then(|state| state.error_code.clone())
2951        {
2952            ClientDiagnosticsLease {
2953                state: "stopped".to_owned(),
2954                expires_at_ms: self
2955                    .lease_state
2956                    .as_ref()
2957                    .and_then(|state| state.expires_at_ms),
2958                error_code: Some(Self::diagnostic_code(&error_code)),
2959            }
2960        } else if let Some(expires_at_ms) = self
2961            .lease_state
2962            .as_ref()
2963            .and_then(|state| state.expires_at_ms)
2964        {
2965            ClientDiagnosticsLease {
2966                state: if expires_at_ms <= captured_at_ms {
2967                    "expired".to_owned()
2968                } else {
2969                    "active".to_owned()
2970                },
2971                expires_at_ms: Some(expires_at_ms),
2972                error_code: None,
2973            }
2974        } else {
2975            ClientDiagnosticsLease {
2976                state: "none".to_owned(),
2977                expires_at_ms: None,
2978                error_code: None,
2979            }
2980        };
2981        let connectivity = match self.last_round.as_ref() {
2982            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
2983            Some(round) if round.status == "succeeded" => "online",
2984            Some(round)
2985                if round.status == "failed"
2986                    && matches!(
2987                        round.cause_kind,
2988                        Some(
2989                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
2990                        )
2991                    ) =>
2992            {
2993                "offline"
2994            }
2995            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
2996            _ => "unknown",
2997        };
2998        Ok(ClientDiagnosticsSnapshot {
2999            version: CLIENT_DIAGNOSTICS_VERSION,
3000            captured_at_ms,
3001            host: ClientDiagnosticsHost {
3002                kind: "direct".to_owned(),
3003                role: "single".to_owned(),
3004                connectivity: connectivity.to_owned(),
3005                realtime: self.realtime_state,
3006                realtime_policy: self.realtime_policy,
3007                realtime_reason_code: self.realtime_reason_code.clone(),
3008                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3009            },
3010            security_lifecycle: self.security_lifecycle().to_owned(),
3011            schema: ClientDiagnosticsSchema {
3012                current_version: self.schema.version,
3013                upgrading: self.upgrading,
3014                required_version: self
3015                    .schema_floor
3016                    .as_ref()
3017                    .and_then(|floor| floor.required_schema_version),
3018                latest_version: self
3019                    .schema_floor
3020                    .as_ref()
3021                    .and_then(|floor| floor.latest_schema_version),
3022            },
3023            replica: ClientDiagnosticsReplica {
3024                local_revision: local_revision
3025                    .and_then(|value| value.parse::<u64>().ok())
3026                    .unwrap_or(0)
3027                    .to_string(),
3028                sync_needed: self.sync_needed,
3029                pending_outbox: self.outbox.len(),
3030            },
3031            lease,
3032            active_units: registered
3033                .iter()
3034                .filter(|unit| unit.status == UnitStatus::Active)
3035                .count(),
3036            units,
3037            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3038                raw.parse()
3039                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3040            })?,
3041            last_round: self.last_round.clone(),
3042            last_change: self.last_change.clone(),
3043            storage: self.diagnostics_storage(),
3044            query_failures: self.query_failures.clone(),
3045        })
3046    }
3047
3048    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3049        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3050            let page_count: i64 = self
3051                .conn
3052                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3053            let page_size: i64 = self
3054                .conn
3055                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3056            // One cached aggregate, since the FFI host recomputes diagnostics
3057            // after every command; every invocation still reads current storage.
3058            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3059                .conn
3060                .prepare_cached(
3061                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3062                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3063                       FROM _syncular_commit_outcomes",
3064                )?
3065                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3066            let blob_bytes = if self.schema_has_blobs() {
3067                self.conn
3068                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3069                    .query_row([], |row| row.get(0))?
3070            } else {
3071                0
3072            };
3073            let pressure = self
3074                .limits
3075                .blob_cache_max_bytes
3076                .is_some_and(|limit| blob_bytes > limit);
3077            Ok(ClientDiagnosticsStorage {
3078                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3079                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3080                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3081                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3082                retained_outcome_entries: Some(outcome_entries.max(0)),
3083                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3084                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3085            })
3086        };
3087        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3088            status: "unreadable".to_owned(),
3089            database_bytes_approx: None,
3090            pending_outbox_bytes_approx: None,
3091            retained_outcome_bytes_approx: None,
3092            retained_outcome_entries: None,
3093            blob_cache_bytes_approx: None,
3094            pressure_reason_code: None,
3095        })
3096    }
3097
3098    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3099        self.change_queue.drain(..).collect()
3100    }
3101
3102    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3103        let intents = self.sync_intent_queue.drain(..).collect();
3104        if self.transport_enabled {
3105            intents
3106        } else {
3107            vec![SyncIntent::None]
3108        }
3109    }
3110
3111    fn schedule_background_retry(&mut self) -> u64 {
3112        let delay_ms = self.retry_delay_ms;
3113        self.sync_intent_queue
3114            .push_back(SyncIntent::Background { delay_ms });
3115        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3116        self.round_retry_delay_ms = Some(delay_ms);
3117        delay_ms
3118    }
3119
3120    fn reset_background_retry(&mut self) {
3121        self.retry_delay_ms = 250;
3122    }
3123
3124    fn retryable_transport_code(code: &str) -> bool {
3125        code == "transport.failed"
3126            || code == "transport.timeout"
3127            || code == "transport.unavailable"
3128            || code == "sync.transport_failed"
3129    }
3130
3131    /// §1.8: the transport reported a refused or unreachable endpoint. The
3132    /// native transport says `connect` (DNS failure, connection refused,
3133    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3134    /// the request on a backoff repairs neither, so no retry is scheduled.
3135    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3136        matches!(
3137            details
3138                .as_ref()
3139                .and_then(|details| details.get("causeKind"))
3140                .and_then(serde_json::Value::as_str),
3141            Some("connect" | "network")
3142        )
3143    }
3144
3145    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3146    /// invisible to the server (the round response arrived whole), so its code
3147    /// rides the following requests like a socket-round failure. A code the
3148    /// frame grammar rejects is not reportable and the previous code stays.
3149    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3150        if ssp2::model::valid_round_failure_code(&error.code) {
3151            self.round_failure = Some(error.code.clone());
3152        }
3153        SectionError::Transport(error)
3154    }
3155
3156    /// A failed round worth a background retry: a transport failure, or a
3157    /// §10.2 code the catalog marks retryable (clients may hardcode the
3158    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3159    /// on the server's `retryable` flag.
3160    fn retryable_failure_code(code: &str) -> bool {
3161        Self::retryable_transport_code(code)
3162            || matches!(
3163                code,
3164                "sync.auth_required"
3165                    | "sync.auth_lease_required"
3166                    | "sync.auth_lease_revoked"
3167                    | "sync.internal_error"
3168                    | "sync.idempotency_cache_miss"
3169                    | "sync.schema_not_ready"
3170                    | "sync.segment_expired"
3171                    | "sync.rate_limited"
3172                    | "sync.websocket_connection_limit"
3173            )
3174    }
3175
3176    fn diagnostic_code(code: &str) -> String {
3177        let valid = !code.is_empty()
3178            && code.len() <= 96
3179            && code.contains('.')
3180            && code
3181                .bytes()
3182                .next()
3183                .is_some_and(|byte| byte.is_ascii_lowercase())
3184            && code.bytes().all(|byte| {
3185                byte.is_ascii_lowercase()
3186                    || byte.is_ascii_digit()
3187                    || matches!(byte, b'.' | b'_' | b'-')
3188            });
3189        if valid {
3190            code.to_owned()
3191        } else {
3192            "client.unknown_failure".to_owned()
3193        }
3194    }
3195
3196    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3197        if self.sync_needed != value {
3198            if self.begin_observation("syncular_status").is_ok() {
3199                self.sync_needed = value;
3200                let batch = ChangeAccumulator {
3201                    status: true,
3202                    ..ChangeAccumulator::default()
3203                };
3204                if self.finish_observation("syncular_status", batch).is_err() {
3205                    self.rollback_observation("syncular_status");
3206                }
3207            } else {
3208                self.sync_needed = value;
3209            }
3210        }
3211        if value && interactive {
3212            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3213        }
3214    }
3215
3216    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3217    /// every other SQLite error keeps its storage classification.
3218    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3219        result.map_err(|error| match error {
3220            rusqlite::Error::FromSqlConversionFailure(..) => {
3221                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3222            }
3223            error => Self::sqlite_failure(&self.storage_failure, error),
3224        })
3225    }
3226
3227    fn sqlite_failure(
3228        storage_failure: &RefCell<Option<QueryReadFailure>>,
3229        error: rusqlite::Error,
3230    ) -> String {
3231        let message = error.to_string();
3232        let failure = QueryReadFailure::from(error);
3233        let code = failure.code;
3234        if code.is_some() && storage_failure.borrow().is_none() {
3235            *storage_failure.borrow_mut() = Some(failure);
3236        }
3237        match code {
3238            Some(code) => format!("{code}: {message}"),
3239            None => message,
3240        }
3241    }
3242
3243    fn begin_observation(&self, name: &str) -> Result<(), String> {
3244        if let Some(failure) = &self.coalesced_failure {
3245            return Err(failure.clone());
3246        }
3247        // SQLite rolls a whole transaction back on some I/O and capacity
3248        // errors; later frames must not commit on their own after that.
3249        if self.coalesced.is_some() && self.conn.is_autocommit() {
3250            return Err("the coalesced apply transaction was rolled back".to_owned());
3251        }
3252        self.conn
3253            .execute_batch(&format!("SAVEPOINT {name}"))
3254            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3255    }
3256
3257    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3258    /// already owns one.
3259    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3260        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3261            return Ok(());
3262        }
3263        self.conn
3264            .execute_batch("BEGIN")
3265            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3266        self.coalesced = Some(CoalescedApply {
3267            rows: 0,
3268            rollback: self.capture(),
3269        });
3270        Ok(())
3271    }
3272
3273    fn capture(&self) -> Rollback {
3274        Rollback {
3275            outbox: self.outbox.clone(),
3276            failed_commits: self.failed_commits.clone(),
3277            conflicts: self.conflicts.clone(),
3278            rejections: self.rejections.clone(),
3279            schema_floor: self.schema_floor.clone(),
3280            lease_state: self.lease_state.clone(),
3281            stopped: self.stopped,
3282            upgrading: self.upgrading,
3283            sync_needed: self.sync_needed,
3284            active_round: self.active_round,
3285            overlay_dirty: self.overlay_dirty.snapshot(),
3286            change_queue: self.change_queue.len(),
3287            sync_intent_queue: self.sync_intent_queue.len(),
3288            last_change: self.last_change.clone(),
3289        }
3290    }
3291
3292    fn restore(&mut self, rollback: Rollback) {
3293        self.outbox = rollback.outbox;
3294        self.failed_commits = rollback.failed_commits;
3295        self.conflicts = rollback.conflicts;
3296        self.rejections = rollback.rejections;
3297        self.schema_floor = rollback.schema_floor;
3298        self.lease_state = rollback.lease_state;
3299        self.stopped = rollback.stopped;
3300        self.upgrading = rollback.upgrading;
3301        self.sync_needed = rollback.sync_needed;
3302        self.active_round = rollback.active_round;
3303        self.overlay_dirty.restore(rollback.overlay_dirty);
3304        self.change_queue.truncate(rollback.change_queue);
3305        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3306        self.last_change = rollback.last_change;
3307    }
3308
3309    /// Account the rows of the next frame or block. A unit that would take the
3310    /// coalesced transaction past the bound commits the earlier units first.
3311    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3312        let Some(open) = &self.coalesced else {
3313            return Ok(());
3314        };
3315        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3316            self.commit_coalesced_apply()?;
3317            self.begin_coalesced_apply()?;
3318        }
3319        if let Some(open) = &mut self.coalesced {
3320            open.rows += rows;
3321        }
3322        Ok(())
3323    }
3324
3325    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3326    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3327    /// change batches and sync intents queued since, and fails the rest of the
3328    /// response.
3329    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3330        let Some(open) = self.coalesced.take() else {
3331            return Ok(());
3332        };
3333        let committed = if self.conn.is_autocommit() {
3334            Err("the coalesced apply transaction was rolled back".to_owned())
3335        } else {
3336            self.conn
3337                .execute_batch("COMMIT")
3338                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3339        };
3340        let Err(error) = committed else {
3341            return Ok(());
3342        };
3343        if !self.conn.is_autocommit() {
3344            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3345                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3346            }
3347        }
3348        self.restore(open.rollback);
3349        self.coalesced_failure = Some(error.clone());
3350        Err(error)
3351    }
3352
3353    /// End the response's coalescing: COMMIT what is open and clear a failure.
3354    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3355        let committed = self.commit_coalesced_apply();
3356        match self.coalesced_failure.take() {
3357            Some(failure) => Err(failure),
3358            None => committed,
3359        }
3360    }
3361
3362    fn rollback_observation(&self, name: &str) {
3363        if let Err(error) = self
3364            .conn
3365            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3366        {
3367            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3368                if failure.rollback_failure.is_none() {
3369                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3370                }
3371            }
3372        }
3373    }
3374
3375    fn finish_observation(
3376        &mut self,
3377        name: &str,
3378        mut batch: ChangeAccumulator,
3379    ) -> Result<(), String> {
3380        #[cfg(feature = "bench-internals")]
3381        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3382        if self.failed_commits.iter().any(|(commit, _)| {
3383            commit
3384                .ops
3385                .iter()
3386                .any(|operation| batch.tables.contains_key(&operation.table))
3387        }) {
3388            batch.outcomes = true;
3389        }
3390        if !batch.touched() {
3391            self.conn
3392                .execute_batch(&format!("RELEASE {name}"))
3393                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3394            return Ok(());
3395        }
3396        let revision = self
3397            .local_revision()?
3398            .checked_add(1)
3399            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3400        self.conn
3401            .execute(
3402                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3403                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3404            )
3405            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3406        let status = batch.status.then(|| self.status_snapshot());
3407        let event = ClientChangeBatch {
3408            revision: revision.to_string(),
3409            tables: batch
3410                .tables
3411                .into_iter()
3412                .map(|(table, scope_keys)| TableChange {
3413                    table,
3414                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3415                })
3416                .collect(),
3417            windows: batch
3418                .windows
3419                .into_iter()
3420                .map(|(base_key, units)| WindowChange {
3421                    base_key,
3422                    units: units.into_iter().collect(),
3423                })
3424                .collect(),
3425            status,
3426            conflicts_changed: batch.conflicts,
3427            rejections_changed: batch.rejections,
3428            outcomes_changed: batch.outcomes,
3429            applied_commits: Vec::new(),
3430        };
3431        self.conn
3432            .execute_batch(&format!("RELEASE {name}"))
3433            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3434        let mut diagnostic_tables = event
3435            .tables
3436            .iter()
3437            .map(|entry| entry.table.clone())
3438            .collect::<BTreeSet<_>>()
3439            .into_iter()
3440            .collect::<Vec<_>>();
3441        // §7.6: the selected variables of the changed window bases.
3442        let mut diagnostic_windows = event
3443            .windows
3444            .iter()
3445            .map(|entry| {
3446                entry
3447                    .base_key
3448                    .split('\0')
3449                    .next()
3450                    .unwrap_or_default()
3451                    .to_owned()
3452            })
3453            .collect::<BTreeSet<_>>()
3454            .into_iter()
3455            .collect::<Vec<_>>();
3456        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3457            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3458        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3459        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3460        self.last_change = Some(DiagnosticLastChange {
3461            revision: event.revision.clone(),
3462            recorded_at_ms: self.clock_now_ms(),
3463            tables: diagnostic_tables,
3464            windows: diagnostic_windows,
3465            domains_truncated,
3466            status_changed: event.status.is_some(),
3467            conflicts_changed: event.conflicts_changed,
3468            rejections_changed: event.rejections_changed,
3469            outcomes_changed: event.outcomes_changed,
3470        });
3471        self.change_queue.push_back(event);
3472        Ok(())
3473    }
3474
3475    fn record_scope_map(
3476        &self,
3477        batch: &mut ChangeAccumulator,
3478        table_name: &str,
3479        scopes: &[(String, Vec<String>)],
3480    ) {
3481        let Some(table) = self.schema.table(table_name) else {
3482            return;
3483        };
3484        // A bulk scoped delete whose map leaves a declared variable unnamed
3485        // cannot list the deleted rows' keys for it: the change is table-wide.
3486        if table.scope_variables.iter().any(|scope| {
3487            !scopes
3488                .iter()
3489                .any(|(variable, _)| *variable == scope.variable)
3490        }) {
3491            batch.table(table_name);
3492            return;
3493        }
3494        for (variable, values) in scopes {
3495            let Some(scope) = table
3496                .scope_variables
3497                .iter()
3498                .find(|scope| &scope.variable == variable)
3499            else {
3500                continue;
3501            };
3502            for value in values {
3503                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3504            }
3505        }
3506    }
3507
3508    /// Record a row's current scope keys from the base or visible table.
3509    fn record_row_scopes(
3510        &self,
3511        batch: &mut ChangeAccumulator,
3512        table_name: &str,
3513        row_id: &str,
3514        base: bool,
3515    ) -> bool {
3516        let Some(table) = self.schema.table(table_name) else {
3517            return false;
3518        };
3519        if table.scope_variables.is_empty() {
3520            return false;
3521        }
3522        let columns = table
3523            .scope_variables
3524            .iter()
3525            .map(|scope| quote_ident(&scope.column))
3526            .collect::<Vec<_>>()
3527            .join(", ");
3528        let full_table = if base {
3529            base_table(table_name)
3530        } else {
3531            visible_table(table_name)
3532        };
3533        let sql = format!(
3534            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3535            row_id_predicate(table)
3536        );
3537        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3538            return false;
3539        };
3540        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3541            let mut values = Vec::with_capacity(table.scope_variables.len());
3542            for index in 0..table.scope_variables.len() {
3543                values.push(row.get::<_, Option<String>>(index)?);
3544            }
3545            Ok(values)
3546        });
3547        let Ok(values) = values else {
3548            return false;
3549        };
3550        let mut recorded = false;
3551        for (scope, value) in table.scope_variables.iter().zip(values) {
3552            if let Some(value) = value {
3553                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3554                recorded = true;
3555            }
3556        }
3557        recorded
3558    }
3559
3560    fn record_commit_changes(
3561        &self,
3562        batch: &mut ChangeAccumulator,
3563        tables: &[String],
3564        changes: &[ssp2::model::Change],
3565    ) {
3566        #[cfg(feature = "bench-internals")]
3567        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3568        for change in changes {
3569            let Some(table_name) = tables.get(change.table_index as usize) else {
3570                continue;
3571            };
3572            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3573            if let Some(table) = self.schema.table(table_name) {
3574                for (variable, value) in &change.scopes {
3575                    if let Some(scope) = table
3576                        .scope_variables
3577                        .iter()
3578                        .find(|scope| &scope.variable == variable)
3579                    {
3580                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3581                        precise = true;
3582                    }
3583                }
3584            }
3585            if !precise {
3586                batch.table(table_name);
3587            }
3588        }
3589    }
3590
3591    /// Whether a base or visible row matches the scope predicate; `false`
3592    /// without a local scope-column mapping. A SQLite failure is an error.
3593    fn scoped_rows_exist(
3594        &self,
3595        table_name: &str,
3596        effective: &[(String, Vec<String>)],
3597        owners: &[Vec<(String, Vec<String>)>],
3598    ) -> Result<bool, String> {
3599        let Some(table) = self.schema.table(table_name) else {
3600            return Ok(false);
3601        };
3602        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3603            return Ok(false);
3604        };
3605        for source in [base_table(table_name), visible_table(table_name)] {
3606            if self
3607                .conn
3608                .query_row(
3609                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3610                    rusqlite::params_from_iter(&params),
3611                    |row| row.get::<_, bool>(0),
3612                )
3613                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3614            {
3615                return Ok(true);
3616            }
3617        }
3618        Ok(false)
3619    }
3620
3621    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3622    /// those another owner covers (§3.3), failing closed without a local
3623    /// scope-column mapping. Owners are read only when a row matches, inside
3624    /// the caller's transaction. Returns whether rows matched.
3625    fn clear_scope(
3626        &mut self,
3627        table: &TableSchema,
3628        effective: &[(String, Vec<String>)],
3629        unit_id: &str,
3630    ) -> Result<bool, SectionError> {
3631        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3632        let matched = self
3633            .scoped_rows_exist(&table.name, effective, &[])
3634            .map_err(storage)?;
3635        let owners = if matched {
3636            unit_owners(
3637                &self.stored(load_units(&self.conn)).map_err(storage)?,
3638                table,
3639                unit_id,
3640            )
3641        } else {
3642            Vec::new()
3643        };
3644        let changed = matched
3645            && (owners.is_empty()
3646                || self
3647                    .scoped_rows_exist(&table.name, effective, &owners)
3648                    .map_err(storage)?);
3649        if !self
3650            .purge_scope_rows(&table.name, effective, &owners)
3651            .map_err(storage)?
3652        {
3653            return Err(SectionError::FailClosed);
3654        }
3655        Ok(changed)
3656    }
3657
3658    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3659    pub fn upgrading(&self) -> bool {
3660        self.upgrading
3661    }
3662
3663    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3664    /// keeping this client's local database (identity, outbox, tables). The
3665    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3666    /// version increases. Recreation runs the same marker guard as opening
3667    /// a durable replica, holds the guard and every write it authorizes under
3668    /// one write transaction, and preserves the current client (compiled
3669    /// schema, outbox, tables) on refusal.
3670    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3671        let new_schema = parse_schema_json(schema_json)?;
3672        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3673        self.conn
3674            .execute_batch("BEGIN IMMEDIATE")
3675            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3676        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3677        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3678        // state (dropped outbox commits, rejections, upgrade and floor flags,
3679        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3680        // fails after the reset succeeded rolls the database back, so every
3681        // one of those fields is captured here and restored with it.
3682        let rollback = self.capture();
3683        let guarded = (|| -> Result<(), String> {
3684            // §7.4.2: the marker is re-read under the write lock the caller
3685            // holds, so a replica another process upgraded meanwhile refuses
3686            // this recreation instead of having its marker and tables
3687            // downgraded.
3688            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3689            self.create_bookkeeping_tables()?;
3690            match marker {
3691                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3692                None => {
3693                    self.create_synced_tables()?;
3694                    self.persist_schema_version()?;
3695                }
3696                _ => {}
3697            }
3698            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3699            // durable outbox the same-version path is about to replay. Only the
3700            // outbox is reloaded: the recovery below reads it, and the caller's
3701            // rollback snapshot already covers this field.
3702            self.outbox = self.load_outbox()?.into();
3703            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3704            // this guarded boundary once the schema and marker agree. A bump went
3705            // through `run_schema_reset`, which performs the same recovery after
3706            // it creates the new physical tables and before its final replay.
3707            self.drop_unencodable_outbox(false, None)?;
3708            if marker == Some(self.schema.version)
3709                && (!self.outbox.is_empty()
3710                    || !self.failed_commits.is_empty()
3711                    || self.has_acknowledged_rows()?)
3712            {
3713                self.overlay_dirty.set(true);
3714                self.rebuild_overlay()?;
3715            }
3716            self.prune_unknown_units(false)?;
3717            self.save_unit_scope_schema()?;
3718            // The conformance recreate is the in-memory equivalent of reopening
3719            // a durable client. Apply the same startup catch-up contract even
3720            // when the schema itself did not change.
3721            self.enqueue_startup_sync_if_needed()?;
3722            Ok(())
3723        })();
3724        let outcome = match guarded {
3725            Ok(()) => self
3726                .conn
3727                .execute_batch("COMMIT")
3728                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3729            Err(error) => Err(error),
3730        };
3731        if let Err(error) = outcome {
3732            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3733                // Keep a failed rollback's typed storage classification.
3734                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3735            }
3736            self.schema = prior_schema;
3737            self.restore(rollback);
3738            // Prepared insert statements were derived from the reverted schema.
3739            self.insert_sql.borrow_mut().clear();
3740            return Err(error);
3741        }
3742        if !self.security_preflight {
3743            self.drain_pending_evictions()?;
3744        }
3745        Ok(())
3746    }
3747
3748    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3749        let startup_work = !self.stopped
3750            && (!self.outbox.is_empty()
3751                || self.has_acknowledged_rows()?
3752                || !self.load_pending_evictions()?.is_empty()
3753                || self
3754                    .stored(load_units(&self.conn))?
3755                    .iter()
3756                    .any(|unit| unit.status == UnitStatus::Active));
3757        if startup_work {
3758            self.sync_needed = true;
3759            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3760        }
3761        Ok(())
3762    }
3763
3764    /// A native client persists schema-floor stops across process restarts.
3765    /// Once the running app already satisfies that floor, the persisted stop
3766    /// is only stale evidence from an older server round (for example, the
3767    /// server was restarted after catching up to an app that was ahead).
3768    /// Clear it and let the normal startup pull re-negotiate. If the server is
3769    /// still incompatible it will return the floor again in that first round.
3770    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3771        let satisfied = self
3772            .schema_floor
3773            .as_ref()
3774            .and_then(|floor| floor.required_schema_version)
3775            .is_some_and(|required| self.schema.version >= required);
3776        if !satisfied {
3777            return Ok(());
3778        }
3779        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3780        self.schema_floor = None;
3781        self.stopped = false;
3782        Ok(())
3783    }
3784
3785    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3786    /// path already swept it; this covers a same-version or fresh-install open,
3787    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3788    /// an aware binary can apply.
3789    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3790        let Some(replica_path) = self.previous_version_replica_path() else {
3791            return Ok(());
3792        };
3793        let max_age_ms = self
3794            .previous_version
3795            .filter(|config| config.enabled)
3796            .map(|config| config.max_age_ms);
3797        reconcile_previous_version_at_boot(
3798            &self.conn,
3799            &replica_path,
3800            self.schema.version,
3801            max_age_ms,
3802            self.clock_now_ms(),
3803        )
3804    }
3805
3806    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3807    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3808    /// has no plain sibling path — so the feature resolves as off there.
3809    fn previous_version_replica_path(&self) -> Option<String> {
3810        let path = self.conn.path()?;
3811        if path.is_empty() || path.starts_with("file:") {
3812            return None;
3813        }
3814        Some(path.to_owned())
3815    }
3816
3817    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3818    /// and leaseState. Drops/recreates every synced table from the new
3819    /// schema, resets unit sync state (keeping registrations), clears
3820    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3821    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3822    fn run_schema_reset(&mut self) -> Result<(), String> {
3823        self.begin_observation("syncular_schema_reset")?;
3824        let rollback = self.capture();
3825        let mut batch = ChangeAccumulator::default();
3826        let result = self
3827            .run_schema_reset_observed(&mut batch, true)
3828            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3829        if let Err(error) = result {
3830            self.rollback_observation("syncular_schema_reset");
3831            self.restore(rollback);
3832            return Err(error);
3833        }
3834        Ok(())
3835    }
3836
3837    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3838        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3839            self.begin_observation("syncular_epoch_acquisition")?;
3840            let rollback = self.capture();
3841            if let Err(error) = self.conn.execute(
3842                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3843                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3844            ) {
3845                self.rollback_observation("syncular_epoch_acquisition");
3846                return Err(Self::sqlite_failure(&self.storage_failure, error));
3847            }
3848            self.sync_needed = true;
3849            let batch = ChangeAccumulator {
3850                status: true,
3851                ..ChangeAccumulator::default()
3852            };
3853            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3854                self.rollback_observation("syncular_epoch_acquisition");
3855                self.restore(rollback);
3856                return Err(error);
3857            }
3858            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3859            return Ok(Vec::new());
3860        }
3861        let resets = self
3862            .stored(load_units(&self.conn))?
3863            .into_iter()
3864            .map(|unit| unit.id)
3865            .collect();
3866        self.begin_observation("syncular_log_epoch_reset")?;
3867        let rollback = self.capture();
3868        let mut batch = ChangeAccumulator::default();
3869        let result = self
3870            .run_schema_reset_observed(&mut batch, false)
3871            .and_then(|()| {
3872                self.conn
3873                    .execute(
3874                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3875                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3876                    )
3877                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3878                self.sync_needed = true;
3879                batch.status = true;
3880                self.finish_observation("syncular_log_epoch_reset", batch)
3881            });
3882        if let Err(error) = result {
3883            self.rollback_observation("syncular_log_epoch_reset");
3884            self.restore(rollback);
3885            return Err(error);
3886        }
3887        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3888        Ok(resets)
3889    }
3890
3891    fn run_schema_reset_observed(
3892        &mut self,
3893        batch: &mut ChangeAccumulator,
3894        capture: bool,
3895    ) -> Result<(), String> {
3896        // §7.4.2: the reset is the only path that wipes local tables for a
3897        // version bump, and it rewrites the marker last. Re-read the marker
3898        // under this reset's transaction: a replica another process upgraded
3899        // meanwhile refuses here instead of being reset to this client's
3900        // older schema.
3901        read_local_schema_version(&self.conn, self.schema.version)?;
3902        // §7.4.3: a schema bump keeps every table whose generation equals the
3903        // persisted one; the log-epoch reset keeps nothing. The previous
3904        // declaration evidence decides which units cover only kept tables.
3905        let mut kept = BTreeSet::new();
3906        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3907        if capture {
3908            let persisted: BTreeMap<String, String> = self
3909                .get_meta(TABLE_GENERATIONS_KEY)?
3910                .map(|raw| serde_json::from_str(&raw))
3911                .transpose()
3912                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3913                .unwrap_or_default();
3914            kept = self
3915                .schema
3916                .tables
3917                .iter()
3918                .filter(|table| {
3919                    table.generation.is_some()
3920                        && table.generation.as_ref() == persisted.get(&table.name)
3921                })
3922                .map(|table| table.name.clone())
3923                .collect();
3924            previous_declarations = self
3925                .get_meta("subscriptionScopeSchema")?
3926                .map(|raw| serde_json::from_str(&raw))
3927                .transpose()
3928                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
3929        }
3930        // Kept tables keep both halves, their FTS projections with the FTS5
3931        // shadow tables and id mapping, and their row-level bookkeeping.
3932        let mut kept_objects = BTreeSet::new();
3933        for table in self
3934            .schema
3935            .tables
3936            .iter()
3937            .filter(|table| kept.contains(&table.name))
3938        {
3939            kept_objects.insert(table.name.clone());
3940            kept_objects.insert(format!("_syncular_base_{}", table.name));
3941            for index in &table.fts_indexes {
3942                kept_objects.insert(index.name.clone());
3943                for shadow in ["data", "idx", "content", "docsize", "config"] {
3944                    kept_objects.insert(format!("{}_{shadow}", index.name));
3945                }
3946            }
3947        }
3948        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
3949        for sql in [
3950            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
3951            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
3952        ] {
3953            self.conn
3954                .execute(sql, [&kept_json])
3955                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3956        }
3957        self.cancel_sync_round();
3958        self.upgrading = true;
3959        batch.status = true;
3960        // RFC 0005 D6: classify the pending outbox against the NEW compiled
3961        // schema BEFORE the wipe, and record the advisory audit. This pass
3962        // drops nothing, so the audit sees the original pending intent; the
3963        // §7.4.4 send-time drop and the intrinsic value recovery later in this
3964        // reset are the paths that remove a commit. The classification is
3965        // shared with the send-time drop so the two cannot disagree about what
3966        // is incompatible.
3967        if self.previous_version.is_some_and(|config| config.enabled) {
3968            let previous_version = self
3969                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
3970                .and_then(|value| value.parse::<i32>().ok())
3971                .unwrap_or(self.schema.version);
3972            let pending = self
3973                .outbox
3974                .iter()
3975                .map(|commit| PendingCommitAudit {
3976                    commit_id: &commit.client_commit_id,
3977                    operations: commit_audit_operations(commit),
3978                })
3979                .collect::<Vec<_>>();
3980            crate::previous_version::write_previous_version_audit(
3981                &self.conn,
3982                &crate::previous_version::build_previous_version_audit(
3983                    &self.schema,
3984                    &pending,
3985                    previous_version,
3986                    self.schema.version,
3987                    self.clock_now_ms(),
3988                ),
3989            )?;
3990        }
3991        // RFC 0005 D5: the container lives in its own FILE, so the reset can
3992        // never reach it. Sweep first (unconditional — also on the log-epoch
3993        // reset, which captures nothing), then capture BEFORE the wipe drops
3994        // any row. A refused capture records its reason and leaves no file.
3995        if let Some(replica_path) = self.previous_version_replica_path() {
3996            sweep_previous_version_container(&self.conn, &replica_path)?;
3997            if capture {
3998                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
3999                    let previous_version = self
4000                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4001                        .and_then(|value| value.parse::<i32>().ok())
4002                        .unwrap_or(self.schema.version);
4003                    capture_previous_version_from_replica(
4004                        &self.conn,
4005                        &replica_path,
4006                        previous_version,
4007                        self.schema.version,
4008                        &config,
4009                        self.clock_now_ms(),
4010                    )?;
4011                }
4012            }
4013        }
4014        for table in self
4015            .schema
4016            .tables
4017            .iter()
4018            .filter(|table| !kept.contains(&table.name))
4019        {
4020            batch.table(&table.name);
4021        }
4022        for (base_key, value) in self.load_window_registry()? {
4023            batch.window(&base_key, &value);
4024        }
4025        if capture {
4026            self.prune_unknown_units(true)?;
4027        }
4028        // The per-table insert SQL is derived from the OLD column lists.
4029        self.insert_sql.borrow_mut().clear();
4030        self.overlay_dirty.set(true);
4031        // Drop every synced table (base + visible) that currently exists —
4032        // discovered from sqlite_master so a bump that adds/removes tables is
4033        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4034        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4035        // so they are matched explicitly, not by the bookkeeping filter.
4036        // Drop virtual tables first so SQLite removes every FTS shadow table
4037        // atomically instead of the generic discovery tearing it apart.
4038        let virtual_tables: Vec<String> = {
4039            let mut stmt = self
4040                .conn
4041                .prepare(
4042                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4043                )
4044                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4045            let rows = stmt
4046                .query_map([], |row| row.get::<_, String>(0))
4047                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4048            rows.filter_map(Result::ok)
4049                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4050                .collect()
4051        };
4052        for name in virtual_tables {
4053            self.conn
4054                .execute_batch(&format!(
4055                    "DROP TABLE IF EXISTS {}",
4056                    quote_ident(&format!("_syncular_fts_{name}"))
4057                ))
4058                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4059            self.conn
4060                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4061                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4062        }
4063        let existing: Vec<String> = {
4064            let mut stmt = self
4065                .conn
4066                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4067                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4068            let rows = stmt
4069                .query_map([], |row| row.get::<_, String>(0))
4070                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4071            rows.filter_map(Result::ok)
4072                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4073                .collect()
4074        };
4075        for name in &existing {
4076            if let Some(table) = name.strip_prefix("_syncular_base_") {
4077                batch.table(table);
4078            } else if !name.starts_with("_syncular_") {
4079                batch.table(name);
4080            }
4081        }
4082        for name in existing {
4083            self.conn
4084                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4085                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4086        }
4087        // Recreate the synced tables from the NEW schema.
4088        self.create_synced_tables()?;
4089        // Reset the sync state of every unit that covers a table the reset
4090        // did not keep, under the previous declarations or the new schema,
4091        // keeping the registration.
4092        let keep_units = match &previous_declarations {
4093            Some(previous) if !kept.is_empty() => self
4094                .stored(load_units(&self.conn))?
4095                .into_iter()
4096                .filter(|unit| {
4097                    previous.iter().all(|(name, old)| {
4098                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4099                    }) && self
4100                        .schema
4101                        .tables
4102                        .iter()
4103                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4104                })
4105                .map(|unit| unit.id)
4106                .collect(),
4107            _ => BTreeSet::new(),
4108        };
4109        reset_units(&self.conn, &keep_units)
4110            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4111        // The stop state is over: this client now ships a servable schema.
4112        self.stopped = false;
4113        self.schema_floor = None;
4114        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4115        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4116        self.persist_schema_version()?;
4117        self.save_unit_scope_schema()?;
4118        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4119        // original pending intent; the synced tables now carry the new schema, so
4120        // recover intrinsically codec-invalid persisted values here, before the
4121        // final replay. A pending upsert that names a column the new schema
4122        // removed stays pending for the send-time `drop_incompatible_outbox`
4123        // prepass. The reset wrapper snapshots and restores the outbox,
4124        // rejections, and change queues, so a failed commit rolls this recovery
4125        // back with the rest of the reset.
4126        // §7.4.4: the enclosing reset observation owns the publication, so the
4127        // recovery contributes its rejection, outcome, and table domains to it.
4128        self.drop_unencodable_outbox(false, Some(batch))?;
4129        // Re-apply the surviving outbox optimistically over the empty tables.
4130        self.rebuild_overlay()?;
4131        Ok(())
4132    }
4133
4134    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4135    /// column the current schema lacks (or a removed table) cannot be encoded
4136    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4137    /// encodable and is validated by the server), nor can an insert-intent
4138    /// upsert that lacks a non-nullable column (§7.1, the audit's
4139    /// `missing-required-column`). Drop the commit, undo
4140    /// its optimistic projection, and raise a client-local
4141    /// `sync.outbox_incompatible` rejection.
4142    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4143        let incompatible = self
4144            .outbox
4145            .iter()
4146            .filter(|commit| {
4147                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4148            })
4149            .cloned()
4150            .collect::<Vec<_>>();
4151        if incompatible.is_empty() {
4152            return Ok(false);
4153        }
4154        self.drop_outbox_commits(
4155            &incompatible,
4156            OUTBOX_INCOMPATIBLE_CODE,
4157            "the persisted commit cannot encode under the current schema",
4158        )?;
4159        Ok(true)
4160    }
4161
4162    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4163    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4164    /// remove them from the outbox, and undo their optimistic rows.
4165    fn drop_outbox_commits(
4166        &mut self,
4167        incompatible: &[Arc<OutboxCommit>],
4168        code: &str,
4169        message: &str,
4170    ) -> Result<(), String> {
4171        // §7.4.4: one observation covers the durable outcome writes, the outbox
4172        // removal, the in-memory drop, and the overlay rebuild, so a storage
4173        // fault leaves the replica and the client unchanged and publishes no
4174        // change; a successful drop publishes exactly one batch carrying the
4175        // rejections, outcomes, status, and the tables whose projection changed.
4176        let rollback = self.capture();
4177        let owns_transaction = self.conn.is_autocommit();
4178        self.begin_observation("syncular_outbox_drop")?;
4179        let mut batch = ChangeAccumulator::default();
4180        let dropped = (|| -> Result<(), String> {
4181            let mut rejections = Vec::new();
4182            for commit in incompatible {
4183                let results = commit
4184                    .ops
4185                    .iter()
4186                    .enumerate()
4187                    .map(|(op_index, operation)| {
4188                        let rejection = RejectionRecord {
4189                            client_commit_id: commit.client_commit_id.clone(),
4190                            op_index: op_index as i32,
4191                            code: code.to_owned(),
4192                            message: message.to_owned(),
4193                            retryable: false,
4194                            details: None,
4195                            operation: Some(CommitOperation::from(operation)),
4196                        };
4197                        rejections.push(rejection.clone());
4198                        CommitOperationOutcome::Error { rejection }
4199                    })
4200                    .collect::<Vec<_>>();
4201                self.persist_commit_outcome(
4202                    &commit.client_commit_id,
4203                    CommitOutcomeStatus::Rejected,
4204                    &results,
4205                    Some(&commit.ops),
4206                )?;
4207                self.delete_outbox_persisted(&commit.client_commit_id)?;
4208                // A removed table has no local projection to invalidate; every
4209                // surviving table the dropped commit touched does.
4210                for operation in &commit.ops {
4211                    if self.schema.table(&operation.table).is_some() {
4212                        batch.table(&operation.table);
4213                    }
4214                }
4215            }
4216            self.prune_commit_outcomes()?;
4217            let incompatible_ids = incompatible
4218                .iter()
4219                .map(|commit| commit.client_commit_id.as_str())
4220                .collect::<BTreeSet<_>>();
4221            self.outbox
4222                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4223            self.rejections.extend(rejections);
4224            // The dropped commits' purely-optimistic rows are undone: the
4225            // visible projection is re-derived from the surviving base + outbox.
4226            self.overlay_dirty.set(true);
4227            self.rebuild_overlay()?;
4228            batch.status = true;
4229            batch.rejections = true;
4230            batch.outcomes = true;
4231            Ok(())
4232        })();
4233        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4234        if let Err(error) = result {
4235            // An outer RELEASE is the COMMIT. Roll back that transaction
4236            // directly: retrying RELEASE after ROLLBACK TO still needs the
4237            // reader's lock. A nested observation only rolls back its savepoint.
4238            if owns_transaction {
4239                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4240                    let mut failure = self.storage_failure.borrow_mut();
4241                    let failure =
4242                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4243                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4244                }
4245            } else {
4246                self.rollback_observation("syncular_outbox_drop");
4247            }
4248            self.restore(rollback);
4249            return Err(error);
4250        }
4251        Ok(())
4252    }
4253
4254    /// §7.4.3: (re)create the base + visible table pair for every synced
4255    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4256    fn create_synced_tables(&self) -> Result<(), String> {
4257        for table in &self.schema.tables {
4258            // The base half + the visible half form the synced-table pair. An
4259            // index name is global in SQLite, so the base half's indexes are
4260            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4261            for (full, index_prefix) in [
4262                (base_table(&table.name), "_syncular_base_"),
4263                (visible_table(&table.name), ""),
4264            ] {
4265                let mut cols: Vec<String> =
4266                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4267                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4268                cols.extend(
4269                    table
4270                        .sealed_columns()
4271                        .iter()
4272                        .map(|name| format!("{} TEXT", quote_ident(name))),
4273                );
4274                let sql = format!(
4275                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4276                    cols.join(", "),
4277                    quote_ident(&table.primary_key)
4278                );
4279                self.conn
4280                    .execute(&sql, [])
4281                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4282                // §5.11: a replica created before sealed apply gains the
4283                // reserved columns in place, on both halves in the same order,
4284                // so data, indexes and the FTS projection stay.
4285                for name in table.sealed_columns() {
4286                    let present: bool = self
4287                        .conn
4288                        .query_row(
4289                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4290                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4291                            |row| row.get(0),
4292                        )
4293                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4294                    if !present {
4295                        self.conn
4296                            .execute(
4297                                &format!(
4298                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4299                                    quote_ident(name)
4300                                ),
4301                                [],
4302                            )
4303                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4304                    }
4305                }
4306                // The reveal at every round start reads the sealed rows of the
4307                // base half through this partial index.
4308                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4309                    self.conn
4310                        .execute(
4311                            &format!(
4312                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4313                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4314                            ),
4315                            [],
4316                        )
4317                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4318                }
4319                // Local secondary indexes (CREATE INDEX subset). Created on
4320                // both halves so mirror reads hit an index on either. Runs on
4321                // both the initial create and the §7.4.3 reset recreate path.
4322                for index in &table.indexes {
4323                    let unique = if index.unique { "UNIQUE " } else { "" };
4324                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4325                    let cols_sql = index
4326                        .columns
4327                        .iter()
4328                        .map(|c| quote_ident(c))
4329                        .collect::<Vec<_>>()
4330                        .join(", ");
4331                    let index_sql = format!(
4332                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4333                    );
4334                    self.conn
4335                        .execute(&index_sql, [])
4336                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4337                }
4338            }
4339        }
4340        // FTS exists only on the visible half. It is a local search
4341        // projection, never a synced/base table or a wire-schema member.
4342        for table in &self.schema.tables {
4343            for index in &table.fts_indexes {
4344                self.create_fts_projection(table, index)?;
4345            }
4346        }
4347        Ok(())
4348    }
4349
4350    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4351        let count: i64 = self
4352            .conn
4353            .query_row(
4354                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4355                rusqlite::params![index.name],
4356                |row| row.get(0),
4357            )
4358            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4359        Ok(count > 0)
4360    }
4361
4362    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4363        for suffix in ["ai", "ad", "au"] {
4364            self.conn
4365                .execute(
4366                    &format!(
4367                        "DROP TRIGGER IF EXISTS {}",
4368                        quote_ident(&format!("{}_{suffix}", index.name))
4369                    ),
4370                    [],
4371                )
4372                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4373        }
4374        Ok(())
4375    }
4376
4377    fn create_fts_triggers(
4378        &self,
4379        table: &TableSchema,
4380        index: &FtsIndexSchema,
4381    ) -> Result<(), String> {
4382        let fts = quote_ident(&index.name);
4383        let source = visible_table(&table.name);
4384        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4385        let pk = quote_ident(&table.primary_key);
4386        let projection_columns = std::iter::once(source_id.clone())
4387            .chain(index.columns.iter().map(|column| quote_ident(column)))
4388            .collect::<Vec<_>>()
4389            .join(", ");
4390        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4391            .chain(
4392                index
4393                    .columns
4394                    .iter()
4395                    .map(|column| format!("new.{}", quote_ident(column))),
4396            )
4397            .collect::<Vec<_>>()
4398            .join(", ");
4399        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4400        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4401        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4402        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4403        let bi = quote_ident(&format!("{}_bi", index.name));
4404        let ai = quote_ident(&format!("{}_ai", index.name));
4405        let ad = quote_ident(&format!("{}_ad", index.name));
4406        let au = quote_ident(&format!("{}_au", index.name));
4407        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4408        let sql = format!(
4409            "DROP TRIGGER IF EXISTS {bi};
4410             DROP TRIGGER IF EXISTS {ai};
4411             DROP TRIGGER IF EXISTS {ad};
4412             DROP TRIGGER IF EXISTS {au};
4413             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4414             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4415             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4416             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4417        );
4418        self.conn
4419            .execute_batch(&sql)
4420            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4421    }
4422
4423    fn rebuild_fts_projection(
4424        &self,
4425        table: &TableSchema,
4426        index: &FtsIndexSchema,
4427    ) -> Result<(), String> {
4428        let fts = quote_ident(&index.name);
4429        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4430        let pk = quote_ident(&table.primary_key);
4431        let indexed_columns = index
4432            .columns
4433            .iter()
4434            .map(|column| quote_ident(column))
4435            .collect::<Vec<_>>();
4436        let projection_columns = std::iter::once(source_id.clone())
4437            .chain(indexed_columns.iter().cloned())
4438            .collect::<Vec<_>>()
4439            .join(", ");
4440        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4441        let sql = format!(
4442            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4443            columns = indexed_columns.join(", "),
4444            source = visible_table(&table.name),
4445        );
4446        self.conn
4447            .execute_batch(&sql)
4448            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4449    }
4450
4451    fn create_fts_projection(
4452        &self,
4453        table: &TableSchema,
4454        index: &FtsIndexSchema,
4455    ) -> Result<(), String> {
4456        self.begin_observation("syncular_fts_schema")?;
4457        let result = (|| {
4458            let existed = self.fts_projection_exists(index)?;
4459            let tokenizer = index.tokenize.replace('\'', "''");
4460            let columns = index
4461                .columns
4462                .iter()
4463                .map(|column| quote_ident(column))
4464                .collect::<Vec<_>>()
4465                .join(", ");
4466            let sql = format!(
4467            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4468            fts = quote_ident(&index.name),
4469            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4470        );
4471            self.conn.execute(&sql, []).map_err(|error| {
4472                format!(
4473                    "cannot create local FTS5 projection {:?}: {error}",
4474                    index.name
4475                )
4476            })?;
4477            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4478            let mapped: bool = self
4479                .conn
4480                .query_row(
4481                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4482                    [format!("_syncular_fts_{}", index.name)],
4483                    |row| row.get(0),
4484                )
4485                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4486            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4487            if !mapped && existed {
4488                self.conn
4489                    .execute_batch(&format!(
4490                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4491                        quote_ident(FTS_SOURCE_ID_COLUMN),
4492                        quote_ident(&index.name)
4493                    ))
4494                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4495            }
4496            self.create_fts_triggers(table, index)?;
4497            if !existed {
4498                self.rebuild_fts_projection(table, index)?;
4499            }
4500            Ok(())
4501        })();
4502        if let Err(error) = result {
4503            self.rollback_observation("syncular_fts_schema");
4504            return Err(error);
4505        }
4506        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4507            self.rollback_observation("syncular_fts_schema");
4508            return Err(error.to_string());
4509        }
4510        Ok(())
4511    }
4512
4513    // -- persistence write-through --------------------------------------------
4514
4515    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4516        let ops: Vec<Value> = commit
4517            .ops
4518            .iter()
4519            .map(|op| {
4520                let mut entry = serde_json::json!({
4521                    "op": if op.upsert { "upsert" } else { "delete" },
4522                    "table": op.table,
4523                    "rowId": op.row_id,
4524                    "baseVersion": op.base_version,
4525                    "values": op.values.clone().map(Value::Object),
4526                });
4527                if op.insert {
4528                    entry["insert"] = Value::Bool(true);
4529                }
4530                entry
4531            })
4532            .collect();
4533        self.conn
4534            .execute(
4535                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4536                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4537            )
4538            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4539        for operation in &commit.ops {
4540            let Some(values) = &operation.values else {
4541                continue;
4542            };
4543            let Some(table) = self.schema.table(&operation.table) else {
4544                continue;
4545            };
4546            for column in table
4547                .columns
4548                .iter()
4549                .filter(|column| column.ty == ColumnType::BlobRef)
4550            {
4551                let Some(Value::String(raw)) = values.get(&column.name) else {
4552                    continue;
4553                };
4554                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4555                    continue;
4556                };
4557                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4558                    continue;
4559                };
4560                self.conn
4561                    .execute(
4562                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4563                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4564                        rusqlite::params![commit.client_commit_id, blob_id],
4565                    )
4566                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4567            }
4568        }
4569        Ok(())
4570    }
4571
4572    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4573        self.conn
4574            .execute(
4575                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4576                rusqlite::params![client_commit_id],
4577            )
4578            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4579        self.conn
4580            .execute(
4581                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4582                rusqlite::params![client_commit_id],
4583            )
4584            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4585        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4586            .map(|_| ())
4587            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4588    }
4589
4590    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4591        match status {
4592            CommitOutcomeStatus::Applied => "applied",
4593            CommitOutcomeStatus::Cached => "cached",
4594            CommitOutcomeStatus::Conflict => "conflict",
4595            CommitOutcomeStatus::Rejected => "rejected",
4596        }
4597    }
4598
4599    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4600        match resolution {
4601            CommitOutcomeResolution::Active => "active",
4602            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4603            CommitOutcomeResolution::Superseded => "superseded",
4604            CommitOutcomeResolution::Dismissed => "dismissed",
4605        }
4606    }
4607
4608    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4609        match value {
4610            "applied" => Ok(CommitOutcomeStatus::Applied),
4611            "cached" => Ok(CommitOutcomeStatus::Cached),
4612            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4613            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4614            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4615        }
4616    }
4617
4618    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4619        match value {
4620            "active" => Ok(CommitOutcomeResolution::Active),
4621            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4622            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4623            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4624            _ => Err(format!(
4625                "invalid persisted commit outcome resolution {value:?}"
4626            )),
4627        }
4628    }
4629
4630    fn persist_commit_outcome(
4631        &self,
4632        client_commit_id: &str,
4633        status: CommitOutcomeStatus,
4634        results: &[CommitOperationOutcome],
4635        operations: Option<&[OutboxOp]>,
4636    ) -> Result<(), String> {
4637        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4638        let operations_json = operations
4639            .map(|items| {
4640                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4641            })
4642            .transpose()
4643            .map_err(|error| error.to_string())?;
4644        self.conn
4645            .execute(
4646                "INSERT INTO _syncular_commit_outcomes (
4647                   client_commit_id, status, recorded_at_ms, results_json,
4648                   operations_json, resolution
4649                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4650                rusqlite::params![
4651                    client_commit_id,
4652                    Self::outcome_status_name(status),
4653                    self.clock_now_ms(),
4654                    results_json,
4655                    operations_json
4656                ],
4657            )
4658            .map(|_| ())
4659            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4660    }
4661
4662    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4663    /// the byte-envelope rule refuses cannot be journaled verbatim.
4664    fn stored_cell_is_representable(cell: &Value) -> bool {
4665        cell.is_null()
4666            || cell.is_string()
4667            || cell.is_boolean()
4668            || cell.is_number()
4669            || is_canonical_bytes_envelope(cell)
4670    }
4671
4672    fn stored_row_is_representable(value: &Value) -> bool {
4673        value
4674            .as_object()
4675            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4676    }
4677
4678    fn stored_operation_is_representable(value: &Value) -> bool {
4679        value.get("table").is_some_and(Value::is_string)
4680            && value.get("rowId").is_some_and(Value::is_string)
4681            && value.get("baseVersion").is_none_or(|v| {
4682                v.as_i64()
4683                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4684            })
4685            && match value.get("op").and_then(Value::as_str) {
4686                Some("upsert") => {
4687                    value
4688                        .get("values")
4689                        .is_some_and(Self::stored_row_is_representable)
4690                        && value
4691                            .get("insert")
4692                            .is_none_or(|insert| *insert == Value::Bool(true))
4693                }
4694                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4695                _ => false,
4696            }
4697    }
4698
4699    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4700    /// only when the journal reader accepts the serialized operation, which is the
4701    /// rule `outcome_from_row` applies to this same row later.
4702    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4703        serde_json::to_value(CommitOperation::from(operation))
4704            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4705    }
4706
4707    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4708        let StoredCommitOutcomeRow {
4709            sequence,
4710            client_commit_id,
4711            status,
4712            recorded_at_ms,
4713            results_json,
4714            operations_json,
4715            resolution,
4716            resolved_at_ms,
4717            replacement_client_commit_id,
4718        } = row;
4719        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4720            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4721            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4722        {
4723            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4724        }
4725        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4726        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4727        for result in &stored {
4728            let nested = match result.get("status").and_then(Value::as_str) {
4729                Some("conflict") => result.get("conflict"),
4730                Some("error") => result.get("rejection"),
4731                _ => None,
4732            };
4733            if nested
4734                .and_then(|record| record.get("operation"))
4735                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4736                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4737                    && nested
4738                        .and_then(|record| record.get("serverRow"))
4739                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4740            {
4741                return Err(invalid());
4742            }
4743        }
4744        let results: Vec<CommitOperationOutcome> =
4745            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4746        let has_conflict = results
4747            .iter()
4748            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4749        let has_error = results
4750            .iter()
4751            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4752        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4753        let redacted = results.is_empty() && operations_json.is_none();
4754        if match status {
4755            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4756            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4757            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4758        } {
4759            return Err(invalid());
4760        }
4761        let operations: Option<Vec<CommitOperation>> = operations_json
4762            .map(|raw| {
4763                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4764                if !stored.iter().all(Self::stored_operation_is_representable) {
4765                    return Err(invalid());
4766                }
4767                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4768            })
4769            .transpose()?;
4770        for result in &results {
4771            match result {
4772                CommitOperationOutcome::Conflict { conflict }
4773                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4774                {
4775                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4776                }
4777                CommitOperationOutcome::Error { rejection } => {
4778                    if let Some(details) = &rejection.details {
4779                        let encoded = serde_json::to_string(details).map_err(|_| {
4780                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4781                        })?;
4782                        RejectionDetails::parse(&encoded).map_err(|_| {
4783                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4784                        })?;
4785                    }
4786                }
4787                _ => {}
4788            }
4789        }
4790        Ok(CommitOutcome {
4791            sequence,
4792            client_commit_id,
4793            status,
4794            recorded_at_ms,
4795            results,
4796            operations,
4797            retained_rows: None,
4798            resolution: Self::parse_outcome_resolution(&resolution)?,
4799            resolved_at_ms,
4800            replacement_client_commit_id,
4801        })
4802    }
4803
4804    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4805    fn unique_conflicts(
4806        &self,
4807        table: &TableSchema,
4808        values: &Map<String, Value>,
4809        base: bool,
4810    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4811        let source = if base {
4812            base_table(&table.name)
4813        } else {
4814            visible_table(&table.name)
4815        };
4816        let mut conflicts = Vec::new();
4817        for index in table.indexes.iter().filter(|index| index.unique) {
4818            let mut params = vec![json_param_to_sql(
4819                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4820            )?];
4821            for name in &index.columns {
4822                let column = table
4823                    .columns
4824                    .iter()
4825                    .find(|column| column.name == *name)
4826                    .ok_or("sync.local_corrupt")?;
4827                let cell = json_to_column_value(column, values.get(name))?;
4828                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4829            }
4830            let sql = format!(
4831                "SELECT * FROM {source} WHERE {} != ? AND {}",
4832                quote_ident(&table.primary_key),
4833                index
4834                    .columns
4835                    .iter()
4836                    .map(|column| format!("{} = ?", quote_ident(column)))
4837                    .collect::<Vec<_>>()
4838                    .join(" AND ")
4839            );
4840            let mut statement = self
4841                .conn
4842                .prepare_cached(&sql)
4843                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4844            let records = statement
4845                .query_map(rusqlite::params_from_iter(params), |record| {
4846                    let values = table
4847                        .columns
4848                        .iter()
4849                        .enumerate()
4850                        .map(|(index, column)| {
4851                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4852                            json_to_column_value(column, Some(&value)).map_err(|message| {
4853                                rusqlite::Error::ToSqlConversionFailure(message.into())
4854                            })
4855                        })
4856                        .collect::<Result<Row, _>>()?;
4857                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4858                })
4859                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4860            for record in records {
4861                let (row, version) =
4862                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4863                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4864                conflicts.push(RetainedUniqueConflict {
4865                    index: index.name.clone(),
4866                    columns: index.columns.clone(),
4867                    row_id,
4868                    server_row: table
4869                        .columns
4870                        .iter()
4871                        .zip(&row)
4872                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4873                        .collect(),
4874                    server_version: version,
4875                });
4876            }
4877        }
4878        Ok(conflicts)
4879    }
4880
4881    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4882        if let Some((commit, initial)) = self
4883            .failed_commits
4884            .iter()
4885            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4886        {
4887            let mut rows = Vec::new();
4888            for operation in &commit.ops {
4889                // A table the current schema removed has no rows to retain.
4890                let Some(table) = self.schema.table(&operation.table) else {
4891                    continue;
4892                };
4893                let key =
4894                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4895                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4896                let base = self.stored_row(table, &operation.row_id, true)?;
4897                rows.push(RetainedCommitRow {
4898                    table: operation.table.clone(),
4899                    row_id: operation.row_id.clone(),
4900                    unique_conflicts: if operation.upsert {
4901                        self.unique_conflicts(
4902                            table,
4903                            initial
4904                                .get(&key)
4905                                .and_then(Value::as_object)
4906                                .or(operation.values.as_ref())
4907                                .ok_or("sync.local_corrupt")?,
4908                            true,
4909                        )?
4910                    } else {
4911                        Vec::new()
4912                    },
4913                    local_row: if operation.upsert {
4914                        initial
4915                            .get(&key)
4916                            .and_then(Value::as_object)
4917                            .cloned()
4918                            .or_else(|| operation.values.clone())
4919                    } else {
4920                        None
4921                    },
4922                    server_row: base.as_ref().map(|(values, _)| {
4923                        table
4924                            .columns
4925                            .iter()
4926                            .zip(values)
4927                            .map(|(column, value)| {
4928                                (column.name.clone(), column_value_to_json(value))
4929                            })
4930                            .collect()
4931                    }),
4932                    server_version: base.map(|(_, version)| version),
4933                });
4934            }
4935            outcome.retained_rows = Some(rows);
4936        }
4937        Ok(outcome)
4938    }
4939
4940    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
4941        let row = self
4942            .conn
4943            .query_row(
4944                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
4945                        resolution, resolved_at_ms, replacement_client_commit_id
4946                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
4947                rusqlite::params![client_commit_id],
4948                |row| {
4949                    Ok(StoredCommitOutcomeRow {
4950                        sequence: row.get(0)?,
4951                        client_commit_id: row.get(1)?,
4952                        status: row.get(2)?,
4953                        recorded_at_ms: row.get(3)?,
4954                        results_json: row.get(4)?,
4955                        operations_json: row.get(5)?,
4956                        resolution: row.get(6)?,
4957                        resolved_at_ms: row.get(7)?,
4958                        replacement_client_commit_id: row.get(8)?,
4959                    })
4960                },
4961            )
4962            .optional()
4963            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4964        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
4965            .transpose()
4966    }
4967
4968    /// Internal host step. The host must run its idle drive before finishing.
4969    #[doc(hidden)]
4970    pub fn prepare_current_base(
4971        &self,
4972        client_commit_id: &str,
4973        sequence: i64,
4974    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
4975        let outcome = self
4976            .commit_outcome(client_commit_id)
4977            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
4978        let valid = outcome.as_ref().is_some_and(|outcome| {
4979            outcome.sequence == sequence
4980                && outcome.resolution == CommitOutcomeResolution::Active
4981                && matches!(
4982                    outcome.status,
4983                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
4984                )
4985                && outcome
4986                    .operations
4987                    .as_ref()
4988                    .zip(outcome.retained_rows.as_ref())
4989                    .is_some_and(|(operations, rows)| {
4990                        !rows.is_empty() && operations.len() == rows.len()
4991                    })
4992        });
4993        if !valid {
4994            return Err(CurrentBaseProof::failure(
4995                CurrentBaseProofReason::OutcomeUnavailable,
4996            ));
4997        }
4998        if !self.transport_enabled {
4999            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5000        }
5001        Ok(CurrentBasePreparation {
5002            client_commit_id: client_commit_id.to_owned(),
5003            sequence,
5004            units: load_units(&self.conn)
5005                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5006            transport_epoch: self.transport_epoch,
5007            registration_epoch: self.registration_epoch,
5008            security_epoch: self.security_epoch,
5009        })
5010    }
5011
5012    /// Internal host step after the completed post-rejection idle drive.
5013    #[doc(hidden)]
5014    pub fn finish_current_base(
5015        &mut self,
5016        prepared: CurrentBasePreparation,
5017        drive: SyncOutcome,
5018    ) -> CurrentBaseProof {
5019        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5020            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5021        }
5022        if self.registration_epoch != prepared.registration_epoch {
5023            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5024        }
5025        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5026            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5027        }
5028        let report = match drive {
5029            SyncOutcome::Ok(report) => report,
5030            SyncOutcome::BudgetExhausted(_) => {
5031                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5032            }
5033            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5034        };
5035        if report.schema_floor.is_some() {
5036            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5037        }
5038        if self
5039            .conn
5040            .execute_batch("SAVEPOINT syncular_base_proof")
5041            .is_err()
5042        {
5043            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5044        }
5045        let proof = (|| {
5046            let snapshot = snapshot_read_connection(
5047                &self.conn,
5048                &SnapshotReadRequest {
5049                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5050                    commit_ids: vec![prepared.client_commit_id.clone()],
5051                    ..Default::default()
5052                },
5053            )
5054            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5055            let current = self
5056                .commit_outcome(&prepared.client_commit_id)
5057                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5058            let unavailable = if report.revoked.is_empty() {
5059                CurrentBaseProofReason::OutcomeUnavailable
5060            } else {
5061                CurrentBaseProofReason::Revoked
5062            };
5063            let Some(outcome) = current else {
5064                return Err(unavailable);
5065            };
5066            if !matches!(
5067                snapshot.deliveries.first(),
5068                Some(CommitDelivery::Known { .. })
5069            ) || outcome.sequence != prepared.sequence
5070                || outcome.resolution != CommitOutcomeResolution::Active
5071                || !matches!(
5072                    outcome.status,
5073                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5074                )
5075            {
5076                return Err(unavailable);
5077            }
5078            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5079                return Err(unavailable);
5080            };
5081            if rows.is_empty() || operations.len() != rows.len() {
5082                return Err(unavailable);
5083            }
5084            let after =
5085                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5086            let complete: HashSet<&str> = snapshot
5087                .units
5088                .iter()
5089                .filter_map(|unit| match unit {
5090                    UnitCatchup::Known {
5091                        id,
5092                        bootstrap_complete: true,
5093                        ..
5094                    } => Some(id.as_str()),
5095                    _ => None,
5096                })
5097                .collect();
5098            // A unit id names one scope map (§4.1), so the same id before and
5099            // after the drive is the same registration.
5100            let candidates: Vec<&UnitRecord> = after
5101                .iter()
5102                .filter(|unit| {
5103                    complete.contains(unit.id.as_str())
5104                        && prepared.units.iter().any(|before| before.id == unit.id)
5105                })
5106                .collect();
5107            let covered = |table_name: &str, row: &Map<String, Value>| {
5108                let Some(table) = self.schema.table(table_name) else {
5109                    return false;
5110                };
5111                candidates.iter().any(|unit| {
5112                    covers(table, &unit.scopes)
5113                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5114                            |scopes| {
5115                                scopes.iter().all(|(variable, allowed)| {
5116                                    table
5117                                        .scope_column(variable)
5118                                        .and_then(|column| match row.get(column) {
5119                                            Some(Value::String(value)) => Some(value.clone()),
5120                                            Some(Value::Bool(value)) => Some(value.to_string()),
5121                                            Some(Value::Number(value)) => {
5122                                                value.as_f64().map(|number| {
5123                                                    ryu_js::Buffer::new()
5124                                                        .format_finite(number)
5125                                                        .to_owned()
5126                                                })
5127                                            }
5128                                            _ => None,
5129                                        })
5130                                        .is_some_and(|value| {
5131                                            !value.is_empty() && allowed.contains(&value)
5132                                        })
5133                                })
5134                            },
5135                        )
5136                })
5137            };
5138            for row in &rows {
5139                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5140                    return Err(if report.revoked.is_empty() {
5141                        CurrentBaseProofReason::MissingCoverage
5142                    } else {
5143                        CurrentBaseProofReason::Revoked
5144                    });
5145                };
5146                if !covered(&row.table, base)
5147                    || row
5148                        .unique_conflicts
5149                        .iter()
5150                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5151                {
5152                    return Err(if report.revoked.is_empty() {
5153                        CurrentBaseProofReason::MissingCoverage
5154                    } else {
5155                        CurrentBaseProofReason::Revoked
5156                    });
5157                }
5158            }
5159            Ok(rows)
5160        })();
5161        if self
5162            .conn
5163            .execute_batch("RELEASE syncular_base_proof")
5164            .is_err()
5165        {
5166            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5167        }
5168        match proof {
5169            Ok(rows) => CurrentBaseProof {
5170                proven: true,
5171                rows: Some(rows),
5172                reason: None,
5173            },
5174            Err(reason) => CurrentBaseProof::failure(reason),
5175        }
5176    }
5177
5178    pub fn prove_current_base(
5179        &mut self,
5180        transport: &mut dyn Transport,
5181        client_commit_id: &str,
5182        sequence: i64,
5183        max_rounds: Option<u32>,
5184    ) -> CurrentBaseProof {
5185        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5186            Ok(prepared) => prepared,
5187            Err(proof) => return proof,
5188        };
5189        let drive = self.sync_until_idle(transport, max_rounds);
5190        self.finish_current_base(prepared, drive)
5191    }
5192
5193    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5194        if query.limit == Some(0) {
5195            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5196        }
5197        let mut sql = String::from(
5198            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5199                    resolution, resolved_at_ms, replacement_client_commit_id
5200               FROM _syncular_commit_outcomes",
5201        );
5202        if query.active_only {
5203            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5204        }
5205        sql.push_str(" ORDER BY seq DESC");
5206        if let Some(limit) = query.limit {
5207            sql.push_str(&format!(" LIMIT {limit}"));
5208        }
5209        let mut stmt = self
5210            .conn
5211            .prepare(&sql)
5212            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5213        let rows = stmt
5214            .query_map([], |row| {
5215                Ok(StoredCommitOutcomeRow {
5216                    sequence: row.get(0)?,
5217                    client_commit_id: row.get(1)?,
5218                    status: row.get(2)?,
5219                    recorded_at_ms: row.get(3)?,
5220                    results_json: row.get(4)?,
5221                    operations_json: row.get(5)?,
5222                    resolution: row.get(6)?,
5223                    resolved_at_ms: row.get(7)?,
5224                    replacement_client_commit_id: row.get(8)?,
5225                })
5226            })
5227            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5228        let mut outcomes = Vec::new();
5229        for row in rows {
5230            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5231                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5232            )?)?);
5233        }
5234        Ok(outcomes)
5235    }
5236
5237    fn prune_commit_outcomes(&self) -> Result<(), String> {
5238        #[cfg(test)]
5239        self.outcome_prune_count
5240            .set(self.outcome_prune_count.get() + 1);
5241        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5242        let count = self
5243            .conn
5244            .query_row(
5245                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5246                [],
5247                |row| row.get::<_, i64>(0),
5248            )
5249            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5250            as usize;
5251        let excess = count.saturating_sub(max_entries);
5252        if excess == 0 {
5253            return Ok(());
5254        }
5255        let mut stmt = self
5256            .conn
5257            .prepare(
5258                "SELECT seq FROM _syncular_commit_outcomes
5259                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5260                  ORDER BY seq ASC LIMIT ?1",
5261            )
5262            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5263        let rows = stmt
5264            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5265            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5266        let sequences = rows
5267            .collect::<Result<Vec<_>, _>>()
5268            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5269        drop(stmt);
5270        for sequence in sequences {
5271            self.conn
5272                .execute(
5273                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5274                    rusqlite::params![sequence],
5275                )
5276                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5277        }
5278        Ok(())
5279    }
5280
5281    // -- units (§4.1) ------------------------------------------------------------
5282
5283    /// Every registered unit in registration order (§4.1).
5284    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5285        self.stored(load_units(&self.conn))
5286    }
5287
5288    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5289        self.stored(get_unit(&self.conn, id))
5290    }
5291
5292    /// Register one unit (§4.1) and return its id.
5293    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5294        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5295    }
5296
5297    /// Register several units in one local transaction and return their ids:
5298    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5299    /// cold start with hundreds of units pays it once). An error rolls back
5300    /// every registration of the call. Registering an active unit again
5301    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5302    /// re-arms a unit raises sync-needed with an interactive intent.
5303    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5304        self.begin_observation("syncular_subscribe")?;
5305        let mut changed = false;
5306        let registered = (|| {
5307            let mut ids = Vec::with_capacity(units.len());
5308            for scopes in units {
5309                let (id, held) = self.hold_unit(scopes, true)?;
5310                changed |= held;
5311                ids.push(id);
5312            }
5313            self.conn
5314                .execute_batch("RELEASE syncular_subscribe")
5315                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5316            Ok(ids)
5317        })();
5318        let ids = match registered {
5319            Ok(ids) => ids,
5320            Err(error) => {
5321                self.rollback_observation("syncular_subscribe");
5322                return Err(error);
5323            }
5324        };
5325        if changed {
5326            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5327            self.set_sync_needed(true, true);
5328        }
5329        Ok(ids)
5330    }
5331
5332    /// Release the explicit reference to a unit. The unit leaves the
5333    /// registration list when no window base holds it (§4.1); its rows stay.
5334    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5335        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5336            return Ok(());
5337        };
5338        if !unit.explicit {
5339            return Ok(());
5340        }
5341        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5342            return self.stored(save_unit(
5343                &self.conn,
5344                &UnitRecord {
5345                    explicit: false,
5346                    ..unit
5347                },
5348            ));
5349        }
5350        self.conn
5351            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5352            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5353        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5354        if unit.status == UnitStatus::Active {
5355            self.set_sync_needed(true, true);
5356        }
5357        Ok(())
5358    }
5359
5360    /// Take one reference on a unit (§4.1): validate the scope map, register a
5361    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5362    /// Returns the id and whether the registration list changed.
5363    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5364        if scopes.is_empty()
5365            || scopes.values().any(|value| value == "*")
5366            || !self
5367                .schema
5368                .tables
5369                .iter()
5370                .any(|table| covers(table, &scopes))
5371        {
5372            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5373        }
5374        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5375        let existing = self.stored(get_unit(&self.conn, &id))?;
5376        if let Some(existing) = existing
5377            .as_ref()
5378            .filter(|unit| unit.status != UnitStatus::Revoked)
5379        {
5380            if explicit && !existing.explicit {
5381                self.stored(save_unit(
5382                    &self.conn,
5383                    &UnitRecord {
5384                        explicit,
5385                        ..existing.clone()
5386                    },
5387                ))?;
5388            }
5389            return Ok((id, false));
5390        }
5391        self.delete_pending_evict(&id)?;
5392        self.stored(save_unit(
5393            &self.conn,
5394            &UnitRecord {
5395                id: id.clone(),
5396                scopes,
5397                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5398                status: UnitStatus::Active,
5399                reason_code: None,
5400                cursor: Some(-1),
5401                bootstrap_state: None,
5402                seeded: Vec::new(),
5403                echo: None,
5404            },
5405        ))?;
5406        Ok((id, true))
5407    }
5408
5409    // -- windows (§4.8) -----------------------------------------------------------
5410
5411    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5412    /// values take a reference on their unit (a new unit bootstraps on the
5413    /// next sync); removed values release theirs, and a unit no reference
5414    /// holds leaves the registration list fused with its first eviction chunk
5415    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5416    pub fn set_window(
5417        &mut self,
5418        base: &WindowBase,
5419        units: &[String],
5420    ) -> Result<CommandEffects, String> {
5421        let base_key = window_base_key(base);
5422        let live = self.load_window_units(&base_key)?;
5423        let mut changed = false;
5424        for value in units {
5425            if live.iter().any(|(held, _)| held == value) {
5426                continue;
5427            }
5428            let rollback = self.capture();
5429            self.begin_observation("syncular_window")?;
5430            let widened = (|| {
5431                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5432                self.sync_needed |= registered;
5433                self.conn
5434                    .execute(
5435                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5436                        rusqlite::params![base_key, value, id],
5437                    )
5438                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5439                let mut batch = ChangeAccumulator::default();
5440                batch.window(&base_key, value);
5441                batch.status = true;
5442                self.finish_observation("syncular_window", batch)
5443            })();
5444            if let Err(error) = widened {
5445                self.rollback_observation("syncular_window");
5446                self.restore(rollback);
5447                return Err(error);
5448            }
5449            changed = true;
5450            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5451        }
5452        for (value, id) in live {
5453            if units.contains(&value) {
5454                continue;
5455            }
5456            self.rebuild_overlay_if_dirty()?;
5457            let rollback = self.capture();
5458            self.begin_observation("syncular_window")?;
5459            let remaining = self.release_window_unit(&base_key, &value, &id);
5460            let remaining = match remaining {
5461                Ok(remaining) => remaining,
5462                Err(error) => {
5463                    self.rollback_observation("syncular_window");
5464                    self.restore(rollback);
5465                    return Err(error);
5466                }
5467            };
5468            changed = true;
5469            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5470            if remaining {
5471                self.drain_pending_evictions()?;
5472            }
5473        }
5474        Ok(if changed && self.transport_enabled {
5475            CommandEffects::interactive()
5476        } else {
5477            CommandEffects::none()
5478        })
5479    }
5480
5481    /// §4.8 completeness oracle: the windowed-in values of a base and those
5482    /// whose unit is not complete. A value is complete once its unit is active
5483    /// and no longer bootstrapping; registration alone is not completeness.
5484    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5485        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5486    }
5487
5488    /// Registry rows for a base, ordered by value: (value, unit id).
5489    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5490        let mut statement = self
5491            .conn
5492            .prepare_cached(
5493                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5494            )
5495            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5496        let rows = statement
5497            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5498            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5499        rows.collect::<Result<_, _>>()
5500            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5501    }
5502
5503    /// Every window registry row: (base key, value).
5504    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5505        let mut statement = self
5506            .conn
5507            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5508            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5509        let rows = statement
5510            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5511            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5512        rows.collect::<Result<_, _>>()
5513            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5514    }
5515
5516    /// §4.8: release a window value's reference inside the caller's
5517    /// observation. When no reference holds the unit, unregister it fused with
5518    /// its first eviction chunk (E3): delete the rows of every covered table
5519    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5520    /// owned by another unit (§3.3), and park the per-table scopes of what
5521    /// remains in the pending-eviction registry. Fails closed without a local
5522    /// scope-column mapping. Returns whether unpinned rows remain.
5523    fn release_window_unit(
5524        &mut self,
5525        base_key: &str,
5526        value: &str,
5527        id: &str,
5528    ) -> Result<bool, String> {
5529        self.conn
5530            .execute(
5531                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5532                rusqlite::params![base_key, value],
5533            )
5534            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5535        let mut batch = ChangeAccumulator::default();
5536        batch.window(base_key, value);
5537        batch.status = true;
5538        let unit = self.stored(get_unit(&self.conn, id))?;
5539        let mut remaining = false;
5540        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5541            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5542                let units = self.stored(load_units(&self.conn))?;
5543                self.conn
5544                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5545                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5546                self.sync_needed = true;
5547                for table in self.schema.tables.clone() {
5548                    let Some(effective) = unit
5549                        .echo
5550                        .as_deref()
5551                        .filter(|_| covers(&table, &unit.scopes))
5552                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5553                    else {
5554                        continue;
5555                    };
5556                    let pinned = self.pinned_row_ids(&table.name);
5557                    let owners = unit_owners(&units, &table, id);
5558                    let (more, deferred) =
5559                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5560                    remaining |= more;
5561                    if deferred {
5562                        self.save_pending_evict(id, &table.name, &effective)?;
5563                    }
5564                    self.record_scope_map(&mut batch, &table.name, &effective);
5565                }
5566            }
5567        }
5568        self.finish_observation("syncular_window", batch)?;
5569        Ok(remaining)
5570    }
5571
5572    /// §4.8 E1: delete base rows owned by the effective scopes
5573    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5574    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5575    /// fails without deleting rows.
5576    fn evict_scope_rows(
5577        &self,
5578        table_name: &str,
5579        effective: &[(String, Vec<String>)],
5580        owners: &[Vec<(String, Vec<String>)>],
5581        pinned: &std::collections::HashSet<String>,
5582    ) -> Result<(bool, bool), String> {
5583        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5584        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5585            .map_err(|()| "sync.scope_revoked".to_owned())?
5586        else {
5587            return Ok((false, false));
5588        };
5589        let scope_params = params.len();
5590        let pk = quote_ident(&table.primary_key);
5591        // Pinned ids are rowId text; the text form matches every key type.
5592        let predicate = if pinned.is_empty() {
5593            scope.clone()
5594        } else {
5595            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5596            let holes = vec!["?"; pinned.len()].join(", ");
5597            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5598        };
5599        let base = base_table(table_name);
5600        let visible = visible_table(table_name);
5601        let mut ids: Vec<SqlValue> = Vec::new();
5602        for source in [&base, &visible] {
5603            let mut stmt = self
5604                .conn
5605                .prepare_cached(&format!(
5606                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5607                ))
5608                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5609            ids = stmt
5610                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5611                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5612                .collect::<Result<_, _>>()
5613                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5614            if !ids.is_empty() {
5615                break;
5616            }
5617        }
5618        if !ids.is_empty() {
5619            for id in &ids {
5620                self.conn
5621                    .execute(
5622                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5623                        rusqlite::params![table_name, id],
5624                    )
5625                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5626            }
5627            let holes = vec!["?"; ids.len()].join(", ");
5628            for source in [&base, &visible] {
5629                self.conn
5630                    .execute(
5631                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5632                        rusqlite::params_from_iter(&ids),
5633                    )
5634                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5635            }
5636        }
5637        let mut remaining = false;
5638        let mut deferred = false;
5639        for source in [&base, &visible] {
5640            remaining |= self
5641                .conn
5642                .query_row(
5643                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5644                    rusqlite::params_from_iter(&params),
5645                    |row| row.get::<_, bool>(0),
5646                )
5647                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5648            deferred |= self
5649                .conn
5650                .query_row(
5651                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5652                    rusqlite::params_from_iter(&params[..scope_params]),
5653                    |row| row.get::<_, bool>(0),
5654                )
5655                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5656        }
5657        Ok((remaining, deferred))
5658    }
5659
5660    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5661    /// chunks, retaining outbox pins and their durable marker. A unit that
5662    /// re-entered the window has no pending entry left.
5663    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5664        let pending = self.load_pending_evictions()?;
5665        if pending.is_empty() {
5666            return Ok(());
5667        }
5668        self.rebuild_overlay_if_dirty()?;
5669        for (unit_id, table_name, effective) in pending {
5670            let Some(table) = self.schema.table(&table_name).cloned() else {
5671                self.delete_pending_evict_table(&unit_id, &table_name)?;
5672                continue;
5673            };
5674            loop {
5675                std::thread::yield_now();
5676                self.begin_observation("syncular_evict")?;
5677                let chunk = (|| {
5678                    let pinned = self.pinned_row_ids(&table_name);
5679                    let owners =
5680                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5681                    let (remaining, deferred) =
5682                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5683                    if !deferred {
5684                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5685                    }
5686                    let mut batch = ChangeAccumulator::default();
5687                    self.record_scope_map(&mut batch, &table_name, &effective);
5688                    self.finish_observation("syncular_evict", batch)?;
5689                    Ok(remaining)
5690                })();
5691                match chunk {
5692                    Ok(true) => {}
5693                    Ok(false) => break,
5694                    Err(error) => {
5695                        self.rollback_observation("syncular_evict");
5696                        return Err(error);
5697                    }
5698                }
5699            }
5700        }
5701        Ok(())
5702    }
5703
5704    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5705    /// commit — rows that MUST NOT be evicted until the commit drains.
5706    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5707        let mut pinned = std::collections::HashSet::new();
5708        for commit in self
5709            .outbox
5710            .iter()
5711            .map(|commit| &**commit)
5712            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5713        {
5714            for op in &commit.ops {
5715                if op.table == table {
5716                    pinned.insert(op.row_id.clone());
5717                }
5718            }
5719        }
5720        pinned
5721    }
5722
5723    fn save_pending_evict(
5724        &self,
5725        unit_id: &str,
5726        table: &str,
5727        effective: &[(String, Vec<String>)],
5728    ) -> Result<(), String> {
5729        self.conn.execute(
5730            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5731            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5732        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5733    }
5734
5735    /// Re-entry cancels every pending eviction of the unit.
5736    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5737        self.conn
5738            .execute(
5739                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5740                [unit_id],
5741            )
5742            .map(|_| ())
5743            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5744    }
5745
5746    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5747        self.conn
5748            .execute(
5749                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5750                [unit_id, table],
5751            )
5752            .map(|_| ())
5753            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5754    }
5755
5756    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5757        let mut stmt = self
5758            .conn
5759            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5760            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5761        let rows = stmt
5762            .query_map([], |row| {
5763                Ok((
5764                    row.get::<_, String>(0)?,
5765                    row.get::<_, String>(1)?,
5766                    row.get::<_, String>(2)?,
5767                ))
5768            })
5769            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5770        rows.map(|row| {
5771            let (unit_id, table, json) =
5772                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5773            let value: Value = serde_json::from_str(&json)
5774                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5775            let effective = json_to_scope_map(&value)
5776                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5777            Ok((unit_id, table, effective))
5778        })
5779        .collect()
5780    }
5781
5782    /// Select failed-intent retention; reload durable retained aggregates.
5783    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5784        self.retain_failed_commits = enabled;
5785        for (commit, _) in self.failed_commits.iter() {
5786            for operation in &commit.ops {
5787                self.overlay_dirty.table(&operation.table);
5788            }
5789        }
5790        self.failed_commits = self.load_failed_commits()?.into();
5791        for (commit, _) in self.failed_commits.iter() {
5792            for operation in &commit.ops {
5793                self.overlay_dirty.table(&operation.table);
5794            }
5795        }
5796        Ok(())
5797    }
5798
5799    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5800        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5801            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5802        let rows = statement
5803            .query_map([], |row| {
5804                Ok((
5805                    row.get::<_, String>(0)?,
5806                    row.get::<_, String>(1)?,
5807                    row.get::<_, String>(2)?,
5808                ))
5809            })
5810            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5811        rows.map(|row| {
5812            let (id, operations, initial) =
5813                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5814            let operations: Vec<CommitOperation> =
5815                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5816            let initial =
5817                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5818            Ok((
5819                OutboxCommit {
5820                    client_commit_id: id,
5821                    ops: operations
5822                        .into_iter()
5823                        .map(|op| OutboxOp {
5824                            upsert: op.op == "upsert",
5825                            table: op.table,
5826                            row_id: op.row_id,
5827                            base_version: op.base_version,
5828                            values: op.values,
5829                            insert: op.insert,
5830                        })
5831                        .collect(),
5832                },
5833                initial,
5834            ))
5835        })
5836        .collect()
5837    }
5838
5839    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5840        let mut initial = Map::new();
5841        for operation in operations {
5842            // A table the current schema removed has no rows to retain.
5843            let Some(table) = self.schema.table(&operation.table) else {
5844                continue;
5845            };
5846            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5847                Some(row) => Some(row),
5848                None => self.visible_row(table, &operation.row_id)?,
5849            };
5850            if let Some((values, _)) = initial_row {
5851                let mut intended: Map<String, Value> = table
5852                    .columns
5853                    .iter()
5854                    .zip(values.iter())
5855                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5856                    .collect();
5857                if let Some(patch) = &operation.values {
5858                    intended.extend(patch.clone());
5859                }
5860                initial.insert(
5861                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5862                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5863                    Value::Object(intended),
5864                );
5865            }
5866        }
5867        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5868            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5869            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5870            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5871        self.failed_commits.push((
5872            OutboxCommit {
5873                client_commit_id: id.to_owned(),
5874                ops: operations.to_vec(),
5875            },
5876            initial,
5877        ));
5878        Ok(())
5879    }
5880
5881    fn drop_failed_commits(
5882        &mut self,
5883        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5884    ) -> Result<Vec<OutboxCommit>, String> {
5885        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5886        for (commit, initial) in self.failed_commits.iter() {
5887            for operation in &commit.ops {
5888                let key =
5889                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5890                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5891                if initial
5892                    .get(&key)
5893                    .and_then(Value::as_object)
5894                    .is_some_and(|values| matches(&operation.table, values))
5895                {
5896                    doomed.push(commit.clone());
5897                    break;
5898                }
5899            }
5900        }
5901        for commit in &doomed {
5902            self.conn
5903                .execute(
5904                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5905                    rusqlite::params![commit.client_commit_id],
5906                )
5907                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5908        }
5909        for commit in &doomed {
5910            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5911        }
5912        self.conflicts.retain(|record| {
5913            !doomed
5914                .iter()
5915                .any(|commit| commit.client_commit_id == record.client_commit_id)
5916        });
5917        self.rejections.retain(|record| {
5918            !doomed
5919                .iter()
5920                .any(|commit| commit.client_commit_id == record.client_commit_id)
5921        });
5922        self.failed_commits.retain(|(commit, _)| {
5923            !doomed
5924                .iter()
5925                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
5926        });
5927        for commit in &doomed {
5928            for operation in &commit.ops {
5929                self.overlay_dirty.table(&operation.table);
5930            }
5931        }
5932        Ok(doomed)
5933    }
5934
5935    /// Record one commit of local mutations. The returned commit ID is the
5936    /// §2.3 idempotency key. Failures are structured so callers read
5937    /// `code`/`message`/`details` instead of parsing a string prefix.
5938    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
5939        self.mutate_with_commit_id(mutations, None)
5940    }
5941
5942    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
5943    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
5944    /// one this replica still holds fails with `client.commit_id_reused`.
5945    pub fn mutate_with_commit_id(
5946        &mut self,
5947        mutations: Vec<Mutation>,
5948        client_commit_id: Option<String>,
5949    ) -> Result<String, ClientError> {
5950        let prior = self.storage_failure.borrow_mut().take();
5951        if let Some(id) = &client_commit_id {
5952            let held = if id.is_empty() {
5953                Err("sync.invalid_request: clientCommitId is empty".to_owned())
5954            } else {
5955                self.conn
5956                    .query_row(
5957                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
5958                        [id],
5959                        |row| row.get(0),
5960                    )
5961                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5962            };
5963            match held {
5964                Ok(false) => {}
5965                Ok(true) => {
5966                    return self.finish_authoring(
5967                        prior,
5968                        Err(ClientError {
5969                            code: "client.commit_id_reused".to_owned(),
5970                            message: "clientCommitId is held".to_owned(),
5971                            details: Some(json!({ "clientCommitId": id })),
5972                            retryable: false,
5973                        }),
5974                    )
5975                }
5976                Err(message) => return self.finish_authoring(prior, Err(message.into())),
5977            }
5978        }
5979        let outcome = (|| -> Result<String, ClientError> {
5980            if mutations.is_empty() {
5981                return Err(
5982                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
5983                        .to_owned()
5984                        .into(),
5985                );
5986            }
5987            let mut ops = Vec::with_capacity(mutations.len());
5988            for mutation in mutations {
5989                match mutation {
5990                    Mutation::Patch {
5991                        table,
5992                        values,
5993                        base_version,
5994                    } => {
5995                        let schema_table = self
5996                            .schema
5997                            .table(&table)
5998                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
5999                        let values = normalize_values_casing(schema_table, values)?;
6000                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6001                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
6002                    }
6003                    Mutation::Upsert {
6004                        table,
6005                        values,
6006                        base_version,
6007                    } => {
6008                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6009                            format!("sync.unknown_table: unknown table {table:?}")
6010                        })?;
6011                        // §5: value keys are accepted in snake_case AND the
6012                        // generated row types' camelCase; normalize to SQL truth
6013                        // before the pk lookup / codec see them.
6014                        let values = normalize_values_casing(schema_table, values)?;
6015                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6016                        // §6.7 `mutate` marks every column present: fill missing
6017                        // nullable columns with NULL before the payload is built.
6018                        let values = full_row_values(schema_table, values)?;
6019                        self.refuse_sealed_row(schema_table, &row_id)?;
6020                        // §5.11: validate the payload encodes with the current
6021                        // codec. A key-selection or unknown-key failure is a
6022                        // durable rejection at the push seam (§10.3), never an
6023                        // author-time error. Full-row `mutate` presents every
6024                        // column, so no stored fallback is needed.
6025                        self.validate_author_encode(schema_table, &row_id, &values)?;
6026                        // §7.1 insert intent: the row is absent from the
6027                        // visible replica at call time, before this batch
6028                        // applies anything.
6029                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6030                        ops.push(OutboxOp {
6031                            upsert: true,
6032                            table,
6033                            row_id,
6034                            base_version,
6035                            values: Some(values),
6036                            insert,
6037                        });
6038                    }
6039                    Mutation::Delete {
6040                        table,
6041                        row_id,
6042                        base_version,
6043                    } => {
6044                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6045                            format!("sync.unknown_table: unknown table {table:?}")
6046                        })?;
6047                        self.refuse_sealed_row(schema_table, &row_id)?;
6048                        // §2.4: the server rejects a rowId holding U+0000.
6049                        if row_id.contains('\0') {
6050                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6051                                .to_owned()
6052                                .into());
6053                        }
6054                        ops.push(OutboxOp {
6055                            upsert: false,
6056                            table,
6057                            row_id,
6058                            base_version,
6059                            values: None,
6060                            insert: false,
6061                        });
6062                    }
6063                }
6064            }
6065            Ok(self.record_outbox_commit(ops, client_commit_id)?)
6066        })();
6067        self.finish_authoring(prior, outcome)
6068    }
6069
6070    /// Scope the retained `storage_failure` diagnostic to this call so an
6071    /// earlier operation's failure cannot classify a new one, and surface the
6072    /// call's own classified failure with its details. A fresh classified
6073    /// failure replaces the retained diagnostic; otherwise the earlier one is
6074    /// kept for the next sync.
6075    fn finish_authoring(
6076        &self,
6077        prior: Option<QueryReadFailure>,
6078        outcome: Result<String, ClientError>,
6079    ) -> Result<String, ClientError> {
6080        let fresh = self.storage_failure.borrow_mut().take();
6081        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6082        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6083    }
6084
6085    fn record_outbox_commit(
6086        &mut self,
6087        ops: Vec<OutboxOp>,
6088        client_commit_id: Option<String>,
6089    ) -> Result<String, String> {
6090        let commit = OutboxCommit {
6091            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6092            ops,
6093        };
6094        self.begin_observation("syncular_mutation")?;
6095        let mut batch = ChangeAccumulator::default();
6096        for op in &commit.ops {
6097            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6098            if let Some(values) = &op.values {
6099                if let Some(table) = self.schema.table(&op.table) {
6100                    for scope in &table.scope_variables {
6101                        if let Some(Value::String(value)) = values.get(&scope.column) {
6102                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6103                            precise = true;
6104                        }
6105                    }
6106                }
6107            }
6108            if !precise {
6109                batch.table(&op.table);
6110            }
6111        }
6112        if let Err(error) = self.persist_outbox_insert(&commit) {
6113            self.rollback_observation("syncular_mutation");
6114            return Err(error);
6115        }
6116        // A clean overlay contains the FIFO fold of every previous commit, so
6117        // the new commit applies over it. §7.1: an operation that does not
6118        // apply (a secondary unique collision) fails the whole commit; the
6119        // savepoint rollback removes the outbox entry and every visible write.
6120        // The append undoes itself: a rollback capture would copy the outbox.
6121        let was_dirty = self.overlay_dirty.snapshot();
6122        let id = commit.client_commit_id.clone();
6123        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6124            commit
6125                .ops
6126                .iter()
6127                .try_for_each(|op| self.apply_outbox_op(op))?;
6128            self.outbox.push(Arc::new(commit));
6129            batch.status = true;
6130            self.finish_observation("syncular_mutation", batch)
6131                .inspect_err(|_| drop(self.outbox.pop()))
6132        });
6133        if let Err(error) = appended {
6134            self.rollback_observation("syncular_mutation");
6135            self.overlay_dirty.restore(was_dirty);
6136            return Err(error);
6137        }
6138        Ok(id)
6139    }
6140
6141    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6142    /// non-scope columns are present; every other column is absent and stays
6143    /// untouched on the server and in the local overlay. An absent local base
6144    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6145    /// Failures are structured so callers read `code`/`message`/`details`.
6146    pub fn patch(
6147        &mut self,
6148        table: &str,
6149        row_id: &str,
6150        partial: Map<String, Value>,
6151        base_version: Option<i64>,
6152    ) -> Result<String, ClientError> {
6153        let prior = self.storage_failure.borrow_mut().take();
6154        let outcome = self.perform_patch(table, row_id, partial, base_version);
6155        self.finish_authoring(prior, outcome)
6156    }
6157
6158    /// The unscoped patch body. Public authoring entry points scope
6159    /// `storage_failure` around the whole operation, including any pre-read.
6160    fn perform_patch(
6161        &mut self,
6162        table: &str,
6163        row_id: &str,
6164        partial: Map<String, Value>,
6165        base_version: Option<i64>,
6166    ) -> Result<String, ClientError> {
6167        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6168        Ok(self.record_outbox_commit(vec![operation], None)?)
6169    }
6170
6171    fn prepare_patch(
6172        &self,
6173        table: &str,
6174        row_id: &str,
6175        partial: Map<String, Value>,
6176        base_version: Option<i64>,
6177    ) -> Result<OutboxOp, ClientError> {
6178        let schema_table = self
6179            .schema
6180            .table(table)
6181            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6182        let partial = normalize_values_casing(schema_table, partial)?;
6183        if let Some(pk) = partial.get(&schema_table.primary_key) {
6184            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6185                return Err(invalid_authoring(
6186                    "patch cannot change the primary key",
6187                    json!({ "table": table }),
6188                ));
6189            }
6190        }
6191        if !self.visible_row_exists(schema_table, row_id)? {
6192            return Err("sync.row_missing: a sparse patch requires a local row"
6193                .to_owned()
6194                .into());
6195        }
6196        self.refuse_sealed_row(schema_table, row_id)?;
6197        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6198        // server accepts a present scope column whose value equals the
6199        // stored row and applies it as a no-op, so a decoded envelope
6200        // round-trips. The local row is the only value the client can prove
6201        // equality against; when the row is absent locally the client cannot
6202        // prove equality against the server's stored row and fails closed.
6203        // A primary key that is also a scope column needs no stored-row
6204        // read: the primary key in a sparse payload is by construction the
6205        // row id being patched (§6.1), so its value is proven equal already.
6206        for scope in &schema_table.scope_variables {
6207            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6208                continue;
6209            }
6210            let Some(column) = schema_table
6211                .columns
6212                .iter()
6213                .find(|column| column.name == scope.column)
6214            else {
6215                continue;
6216            };
6217            let stored = self
6218                .conn
6219                .query_row(
6220                    &format!(
6221                        "SELECT {} FROM {} WHERE {}",
6222                        quote_ident(&scope.column),
6223                        visible_table(&schema_table.name),
6224                        row_id_predicate(schema_table)
6225                    ),
6226                    rusqlite::params![row_id],
6227                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6228                )
6229                .optional()
6230                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6231            // Decode the supplied value through the same column-type path the
6232            // rest of the patch encoding uses, so a value that only equals
6233            // the stored one in another number representation still compares
6234            // equal.
6235            let incoming = json_to_column_value(column, partial.get(&scope.column))
6236                .ok()
6237                .map(|value| column_value_to_json(&value));
6238            // A stored blob never equals a patch value, matching the server's
6239            // scope-column comparison (§3.4): bytes are unstorable as scope
6240            // values.
6241            let equal = stored.as_ref().is_some_and(|stored| {
6242                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6243            });
6244            if !equal {
6245                return Err(invalid_authoring(
6246                    "patch cannot write a scope column (§3.4)",
6247                    json!({ "table": table, "column": scope.column }),
6248                ));
6249            }
6250        }
6251        let mut values = partial;
6252        // Presence-set semantics: a proven-equal scope column is dropped, so
6253        // the patch leaves the stored value untouched (§6.2).
6254        for scope in &schema_table.scope_variables {
6255            values.remove(&scope.column);
6256        }
6257        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6258            // §2.4: the rowId of an integer key is its canonical decimal form.
6259            ColumnType::Integer => match row_id.parse::<i64>() {
6260                Ok(v)
6261                    if v.to_string() == row_id
6262                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6263                            .contains(&v) =>
6264                {
6265                    Value::from(v)
6266                }
6267                _ => {
6268                    return Err(format!(
6269                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6270                )
6271                    .into())
6272                }
6273            },
6274            ColumnType::Boolean => match row_id {
6275                "true" => Value::Bool(true),
6276                "false" => Value::Bool(false),
6277                _ => {
6278                    return Err(format!(
6279                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6280                )
6281                    .into())
6282                }
6283            },
6284            _ => Value::from(row_id.to_owned()),
6285        };
6286        values.insert(schema_table.primary_key.clone(), pk_value);
6287        // §6.1: a patch cannot null a non-nullable column.
6288        if let Some(column) = schema_table
6289            .columns
6290            .iter()
6291            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6292        {
6293            return Err(invalid_authoring(
6294                "a non-nullable column is null (§6.1)",
6295                json!({ "table": table, "column": column.name }),
6296            ));
6297        }
6298        // §5.11: structural validation only; an absent key-id selector is
6299        // resolved at the push seam from the stored local row, and a
6300        // key-selection or unknown-key failure becomes a durable
6301        // `client.encrypt_failed` rejection there (§10.3), never an
6302        // author-time error.
6303        self.validate_author_encode(schema_table, row_id, &values)?;
6304        Ok(OutboxOp {
6305            upsert: true,
6306            table: table.to_owned(),
6307            row_id: row_id.to_owned(),
6308            base_version,
6309            values: Some(values),
6310            insert: false,
6311        })
6312    }
6313
6314    /// §5.11 key-id fallback: the stored local row's selector value as a
6315    /// positional row (only the selector slot filled). Returns None when no
6316    /// selector is configured, the row is locally absent, or the stored
6317    /// value is not a usable key id. A failed storage read is an error.
6318    fn stored_key_fallback(
6319        &self,
6320        table: &TableSchema,
6321        row_id: &str,
6322    ) -> Result<Option<Row>, String> {
6323        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6324            return Ok(None);
6325        };
6326        let Some(index) = table
6327            .columns
6328            .iter()
6329            .position(|column| &column.name == selector)
6330        else {
6331            return Ok(None);
6332        };
6333        let sql = format!(
6334            "SELECT {} FROM {} WHERE {}",
6335            quote_ident(selector),
6336            visible_table(&table.name),
6337            row_id_predicate(table)
6338        );
6339        let key = self
6340            .conn
6341            .query_row(&sql, rusqlite::params![row_id], |row| {
6342                row.get::<_, Option<String>>(0)
6343            })
6344            .optional()
6345            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6346            .flatten()
6347            .filter(|key| !key.is_empty());
6348        let Some(key) = key else {
6349            return Ok(None);
6350        };
6351        let mut fallback: Row = vec![None; table.columns.len()];
6352        fallback[index] = Some(ColumnValue::String(key));
6353        Ok(Some(fallback))
6354    }
6355
6356    /// §5.11/§10.3 author-time encode check: a structural failure (a
6357    /// non-nullable NULL, a value the codec rejects) is an author-time
6358    /// error; a key-selection or unknown-key failure is deferred to the push
6359    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6360    fn validate_author_encode(
6361        &self,
6362        table: &TableSchema,
6363        row_id: &str,
6364        values: &Map<String, Value>,
6365    ) -> Result<(), String> {
6366        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6367            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6368            result => result.map(|_| ()),
6369        }
6370    }
6371
6372    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6373    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6374    /// raise one durable rejection per commit. `sync()` never aborts for an
6375    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6376    /// undoing the dropped commit's purely-optimistic rows.
6377    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6378    /// (an envelope or a type an earlier version accepted). `with_encryption`
6379    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6380    /// the startup boundary leaves those to the send seam, where hosts configure
6381    /// keys after open.
6382    /// Recovery runs as one observation. An enclosing observation (the guarded
6383    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6384    /// domains join that change event instead of publishing a second one.
6385    fn drop_unencodable_outbox(
6386        &mut self,
6387        with_encryption: bool,
6388        batch: Option<&mut ChangeAccumulator>,
6389    ) -> Result<bool, String> {
6390        let mut failures = Vec::new();
6391        for commit in self.outbox.iter() {
6392            for op in &commit.ops {
6393                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6394                    continue;
6395                };
6396                // An insert lacking a non-nullable column is left to
6397                // `drop_incompatible_outbox` (§7.4.4).
6398                let Some(values) = op.presented_values(table) else {
6399                    continue;
6400                };
6401                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6402                let failure = match encode_sparse_row_json(
6403                    table,
6404                    &op.row_id,
6405                    &values,
6406                    &self.encryption,
6407                    fallback.as_ref(),
6408                ) {
6409                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6410                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6411                    }
6412                    // §7.1/§7.4.4: a value the current codec refuses was
6413                    // persisted by a version that accepted it; it leaves
6414                    // through the incompatible-outbox path, so the FIFO
6415                    // queue never wedges.
6416                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6417                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6418                        OUTBOX_INCOMPATIBLE_CODE,
6419                    )),
6420                    _ => None,
6421                };
6422                if let Some((error, code)) = failure {
6423                    failures.push((commit.clone(), error, code));
6424                    break;
6425                }
6426            }
6427        }
6428        if failures.is_empty() {
6429            return Ok(false);
6430        }
6431        // §7.4.4: one observation covers the durable outcome writes, the outbox
6432        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6433        // leaves the replica and the client unchanged, a successful recovery
6434        // publishes one batch with the rejection, outcome, and table domains, and
6435        // an ordinary open with nothing to recover publishes nothing.
6436        let rollback = self.capture();
6437        let owns_observation = batch.is_none();
6438        let owns_transaction = self.conn.is_autocommit();
6439        let mut domains = ChangeAccumulator::default();
6440        if owns_observation {
6441            self.begin_observation("syncular_value_recovery")?;
6442        }
6443        let dropped = (|| -> Result<(), String> {
6444            for (commit, message, code) in failures {
6445                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6446                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6447                }
6448                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6449                let rejection = RejectionRecord {
6450                    client_commit_id: commit.client_commit_id.clone(),
6451                    op_index: 0,
6452                    code: code.to_owned(),
6453                    message,
6454                    retryable: false,
6455                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6456                        field_paths: None,
6457                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6458                        required_action: None,
6459                        references: None,
6460                    }),
6461                    operation: representable
6462                        .then(|| commit.ops.first())
6463                        .flatten()
6464                        .map(CommitOperation::from),
6465                };
6466                self.persist_commit_outcome(
6467                    &commit.client_commit_id,
6468                    CommitOutcomeStatus::Rejected,
6469                    &[CommitOperationOutcome::Error {
6470                        rejection: rejection.clone(),
6471                    }],
6472                    representable.then_some(commit.ops.as_slice()),
6473                )?;
6474                self.delete_outbox_persisted(&commit.client_commit_id)?;
6475                self.outbox
6476                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6477                self.rejections.push(rejection);
6478                for operation in &commit.ops {
6479                    domains.table(operation.table.as_str());
6480                }
6481            }
6482            domains.status = true;
6483            domains.rejections = true;
6484            domains.outcomes = true;
6485            self.prune_commit_outcomes()?;
6486            // The visible projection is re-derived from the surviving base + outbox.
6487            self.overlay_dirty.set(true);
6488            self.rebuild_overlay()?;
6489            Ok(())
6490        })();
6491        let result = match batch {
6492            Some(batch) => {
6493                if dropped.is_ok() {
6494                    for table in domains.tables.keys() {
6495                        batch.table(table.as_str());
6496                    }
6497                    batch.status |= domains.status;
6498                    batch.rejections |= domains.rejections;
6499                    batch.outcomes |= domains.outcomes;
6500                }
6501                dropped
6502            }
6503            None => {
6504                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6505            }
6506        };
6507        if let Err(error) = result {
6508            if owns_observation {
6509                if owns_transaction {
6510                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6511                        let mut failure = self.storage_failure.borrow_mut();
6512                        let failure =
6513                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6514                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6515                    }
6516                } else {
6517                    self.rollback_observation("syncular_value_recovery");
6518                }
6519            }
6520            self.restore(rollback);
6521            return Err(error);
6522        }
6523        Ok(true)
6524    }
6525
6526    pub fn pending_commit_ids(&self) -> Vec<String> {
6527        self.outbox
6528            .iter()
6529            .map(|c| c.client_commit_id.clone())
6530            .collect()
6531    }
6532
6533    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6534    /// FIFO — exactly what the next push round would put on the wire. Used by
6535    /// conformance B.21(h) to pin cross-core byte identity.
6536    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6537        let mut payloads = Vec::new();
6538        for commit in self.outbox.iter() {
6539            for op in &commit.ops {
6540                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6541                    continue;
6542                };
6543                let Some(values) = op.presented_values(table) else {
6544                    continue;
6545                };
6546                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6547                if let Ok(payload) = encode_sparse_row_json(
6548                    table,
6549                    &op.row_id,
6550                    &values,
6551                    &self.encryption,
6552                    fallback.as_ref(),
6553                ) {
6554                    payloads.push(payload);
6555                }
6556            }
6557        }
6558        Ok(payloads)
6559    }
6560
6561    pub fn conflicts(&self) -> &[ConflictRecord] {
6562        &self.conflicts
6563    }
6564
6565    pub fn rejections(&self) -> &[RejectionRecord] {
6566        &self.rejections
6567    }
6568
6569    pub fn resolve_commit_outcome(
6570        &mut self,
6571        input: ResolveCommitOutcomeInput,
6572    ) -> Result<CommitOutcome, String> {
6573        let current = self
6574            .commit_outcome(&input.client_commit_id)?
6575            .ok_or_else(|| {
6576                format!(
6577                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6578                    input.client_commit_id
6579                )
6580            })?;
6581        if current.resolution != CommitOutcomeResolution::Active {
6582            return Ok(current);
6583        }
6584        if input.resolution == CommitOutcomeResolution::Active {
6585            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6586        }
6587        match input.resolution {
6588            CommitOutcomeResolution::Superseded => {
6589                let replacement = input
6590                    .replacement_client_commit_id
6591                    .as_deref()
6592                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6593                    .ok_or_else(|| {
6594                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6595                            .to_owned()
6596                    })?;
6597                let _ = replacement;
6598            }
6599            _ if input.replacement_client_commit_id.is_some() => {
6600                return Err(
6601                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6602                        .to_owned(),
6603                );
6604            }
6605            _ => {}
6606        }
6607        let allowed = match current.status {
6608            CommitOutcomeStatus::Conflict => matches!(
6609                input.resolution,
6610                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6611            ),
6612            CommitOutcomeStatus::Rejected => {
6613                matches!(
6614                    input.resolution,
6615                    CommitOutcomeResolution::Superseded
6616                        | CommitOutcomeResolution::ResolvedKeepServer
6617                )
6618            }
6619            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6620                input.resolution == CommitOutcomeResolution::Dismissed
6621            }
6622        };
6623        if !allowed {
6624            return Err(format!(
6625                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6626                input.resolution, current.status
6627            ));
6628        }
6629
6630        let rollback = self.capture();
6631        self.begin_observation("syncular_outcome_resolution")?;
6632        let result = (|| {
6633            self.conn
6634                .execute(
6635                    "UPDATE _syncular_commit_outcomes
6636                        SET resolution = ?1, resolved_at_ms = ?2,
6637                            replacement_client_commit_id = ?3
6638                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6639                    rusqlite::params![
6640                        Self::outcome_resolution_name(input.resolution),
6641                        self.clock_now_ms(),
6642                        input.replacement_client_commit_id,
6643                        input.client_commit_id
6644                    ],
6645                )
6646                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6647            let resolved = self
6648                .commit_outcome(&current.client_commit_id)?
6649                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6650            self.conn
6651                .execute(
6652                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6653                    rusqlite::params![input.client_commit_id],
6654                )
6655                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6656            for (commit, _) in self.failed_commits.iter() {
6657                if commit.client_commit_id == input.client_commit_id {
6658                    for operation in &commit.ops {
6659                        self.overlay_dirty.table(&operation.table);
6660                    }
6661                }
6662            }
6663            self.failed_commits
6664                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6665            self.prune_commit_outcomes()?;
6666            Ok(resolved)
6667        })();
6668        let resolved = match result {
6669            Ok(outcome) => outcome,
6670            Err(error) => {
6671                self.rollback_observation("syncular_outcome_resolution");
6672                self.restore(rollback);
6673                return Err(error);
6674            }
6675        };
6676        self.conflicts
6677            .retain(|record| record.client_commit_id != current.client_commit_id);
6678        self.rejections
6679            .retain(|record| record.client_commit_id != current.client_commit_id);
6680        let mut batch = ChangeAccumulator {
6681            conflicts: current.status == CommitOutcomeStatus::Conflict,
6682            rejections: current.status == CommitOutcomeStatus::Rejected,
6683            outcomes: true,
6684            ..ChangeAccumulator::default()
6685        };
6686        if let Some(operations) = &current.operations {
6687            for operation in operations {
6688                batch.table(&operation.table);
6689            }
6690        }
6691        if let Err(error) = self
6692            .rebuild_overlay_if_dirty()
6693            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6694        {
6695            self.rollback_observation("syncular_outcome_resolution");
6696            self.restore(rollback);
6697            return Err(error);
6698        }
6699        Ok(resolved)
6700    }
6701
6702    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6703        self.schema_floor.as_ref()
6704    }
6705
6706    /// §7.3.5: the client's opaque auth-lease state, if any.
6707    pub fn lease_state(&self) -> Option<&LeaseState> {
6708        self.lease_state.as_ref()
6709    }
6710
6711    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6712    /// the two lease codes set it; other errors leave leaseState untouched.
6713    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6714        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6715            return Ok(());
6716        }
6717        let mut next = self.lease_state.clone().unwrap_or_default();
6718        next.error_code = Some(code.to_owned());
6719        self.set_lease_state(Some(next))
6720    }
6721
6722    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6723        if self.lease_state == next {
6724            return Ok(());
6725        }
6726        self.begin_observation("syncular_lease")?;
6727        let rollback = self.capture();
6728        self.lease_state = next;
6729        let persisted = match &self.lease_state {
6730            Some(lease) => serde_json::to_string(lease)
6731                .map_err(|error| error.to_string())
6732                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6733            None => self.delete_meta(LEASE_STATE_KEY),
6734        };
6735        let batch = ChangeAccumulator {
6736            status: true,
6737            ..ChangeAccumulator::default()
6738        };
6739        if let Err(error) =
6740            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6741        {
6742            self.rollback_observation("syncular_lease");
6743            self.restore(rollback);
6744            return Err(error);
6745        }
6746        Ok(())
6747    }
6748
6749    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6750        if self.schema_floor == next {
6751            return Ok(());
6752        }
6753        self.begin_observation("syncular_schema_floor")?;
6754        let rollback = self.capture();
6755        self.stopped = next.is_some();
6756        self.schema_floor = next;
6757        let persisted = match &self.schema_floor {
6758            Some(floor) => serde_json::to_string(floor)
6759                .map_err(|error| error.to_string())
6760                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6761            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6762        };
6763        let batch = ChangeAccumulator {
6764            status: true,
6765            ..ChangeAccumulator::default()
6766        };
6767        if let Err(error) =
6768            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6769        {
6770            self.rollback_observation("syncular_schema_floor");
6771            self.restore(rollback);
6772            return Err(error);
6773        }
6774        Ok(())
6775    }
6776
6777    fn set_upgrading(&mut self, value: bool) {
6778        if self.upgrading == value {
6779            return;
6780        }
6781        if self.begin_observation("syncular_upgrading").is_err() {
6782            return;
6783        }
6784        self.upgrading = value;
6785        let batch = ChangeAccumulator {
6786            status: true,
6787            ..ChangeAccumulator::default()
6788        };
6789        if self
6790            .finish_observation("syncular_upgrading", batch)
6791            .is_err()
6792        {
6793            self.rollback_observation("syncular_upgrading");
6794        }
6795    }
6796
6797    pub fn sync_needed(&self) -> bool {
6798        self.sync_needed
6799    }
6800
6801    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6802        let schema_table = self
6803            .schema
6804            .table(table)
6805            .ok_or_else(|| format!("unknown table {table:?}"))?;
6806        let sql = format!(
6807            "SELECT * FROM {} ORDER BY {} ASC",
6808            visible_table(table),
6809            quote_ident(&schema_table.primary_key)
6810        );
6811        let mut stmt = self
6812            .conn
6813            .prepare(&sql)
6814            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6815        let mut rows = stmt
6816            .query([])
6817            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6818        let mut out = Vec::new();
6819        while let Some(row) = rows
6820            .next()
6821            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6822        {
6823            let mut values = Map::new();
6824            for (i, column) in schema_table.columns.iter().enumerate() {
6825                let value = row
6826                    .get_ref(i)
6827                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6828                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6829            }
6830            let version: i64 = row
6831                .get(schema_table.columns.len())
6832                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6833            let row_id = match values.get(&schema_table.primary_key) {
6834                Some(Value::String(s)) => s.clone(),
6835                Some(Value::Number(n)) => n.to_string(),
6836                Some(Value::Bool(b)) => b.to_string(),
6837                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6838            };
6839            let sealed_key = if schema_table.has_encrypted_columns() {
6840                row.get(schema_table.columns.len() + 1)
6841                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6842            } else {
6843                None
6844            };
6845            out.push(RowState {
6846                row_id,
6847                version,
6848                values,
6849                sealed_key,
6850            });
6851        }
6852        Ok(out)
6853    }
6854
6855    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6856    //
6857    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6858    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6859    // the op with `yrs`, re-encodes the whole doc state, and records a
6860    // baseVersion-less sparse upsert carrying only the primary key and the
6861    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6862    // the overlay re-materializes the edit immediately (optimistic apply,
6863    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6864    // Byte-compatible with `@syncular/crdt-yjs`.
6865
6866    /// The current stored value of a `crdt` column for one row — the visible
6867    /// (optimistic) bytes, or `None` when the row is absent or the column is
6868    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6869    /// `crdt` column (guards the app against a typo'd column name).
6870    #[cfg(feature = "crdt-yjs")]
6871    fn crdt_column_bytes(
6872        &self,
6873        table: &str,
6874        row_id: &str,
6875        column: &str,
6876    ) -> Result<Option<Vec<u8>>, String> {
6877        let schema_table = self
6878            .schema
6879            .table(table)
6880            .ok_or_else(|| format!("unknown table {table:?}"))?;
6881        let col = schema_table
6882            .columns
6883            .iter()
6884            .find(|c| c.name == column)
6885            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6886        if col.ty != ColumnType::Crdt {
6887            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6888        }
6889        let sql = format!(
6890            "SELECT {} FROM {} WHERE {}",
6891            quote_ident(column),
6892            visible_table(table),
6893            row_id_predicate(schema_table)
6894        );
6895        let bytes: Option<Vec<u8>> = self
6896            .conn
6897            .prepare_cached(&sql)
6898            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6899            .query_row(rusqlite::params![row_id], |row| {
6900                row.get::<_, Option<Vec<u8>>>(0)
6901            })
6902            .map_err(|e| match e {
6903                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6904                other => Self::sqlite_failure(&self.storage_failure, other),
6905            })?;
6906        Ok(bytes)
6907    }
6908
6909    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6910    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
6911    /// An absent row / NULL column is the empty document (empty string).
6912    #[cfg(feature = "crdt-yjs")]
6913    pub fn crdt_text(
6914        &self,
6915        table: &str,
6916        row_id: &str,
6917        column: &str,
6918        name: &str,
6919    ) -> Result<String, String> {
6920        let bytes = self
6921            .crdt_column_bytes(table, row_id, column)?
6922            .unwrap_or_default();
6923        crate::crdt::text(&bytes, name)
6924    }
6925
6926    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
6927    /// record the resulting crdt-only sparse operation. Returns the enqueued
6928    /// `clientCommitId`.
6929    #[cfg(feature = "crdt-yjs")]
6930    pub fn crdt_insert_text(
6931        &mut self,
6932        table: &str,
6933        row_id: &str,
6934        column: &str,
6935        name: &str,
6936        index: u32,
6937        value: &str,
6938    ) -> Result<String, ClientError> {
6939        let prior = self.storage_failure.borrow_mut().take();
6940        let outcome = (|| -> Result<String, ClientError> {
6941            let current = self
6942                .crdt_column_bytes(table, row_id, column)?
6943                .unwrap_or_default();
6944            let update = crate::crdt::insert_text(&current, name, index, value)?;
6945            self.perform_crdt_push_update(table, row_id, column, &update)
6946        })();
6947        self.finish_authoring(prior, outcome)
6948    }
6949
6950    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
6951    /// record the resulting crdt-only sparse operation. Returns the enqueued
6952    /// `clientCommitId`.
6953    #[cfg(feature = "crdt-yjs")]
6954    pub fn crdt_delete_text(
6955        &mut self,
6956        table: &str,
6957        row_id: &str,
6958        column: &str,
6959        name: &str,
6960        index: u32,
6961        len: u32,
6962    ) -> Result<String, ClientError> {
6963        let prior = self.storage_failure.borrow_mut().take();
6964        let outcome = (|| -> Result<String, ClientError> {
6965            let current = self
6966                .crdt_column_bytes(table, row_id, column)?
6967                .unwrap_or_default();
6968            let update = crate::crdt::delete_text(&current, name, index, len)?;
6969            self.perform_crdt_push_update(table, row_id, column, &update)
6970        })();
6971        self.finish_authoring(prior, outcome)
6972    }
6973
6974    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
6975    /// `crdt` column's current state and record the resulting crdt-only sparse
6976    /// operation. The app authored the update with its own `yrs` model.
6977    /// Returns the enqueued `clientCommitId`.
6978    #[cfg(feature = "crdt-yjs")]
6979    pub fn crdt_apply_update(
6980        &mut self,
6981        table: &str,
6982        row_id: &str,
6983        column: &str,
6984        update: &[u8],
6985    ) -> Result<String, ClientError> {
6986        let prior = self.storage_failure.borrow_mut().take();
6987        let outcome = (|| -> Result<String, ClientError> {
6988            let current = self
6989                .crdt_column_bytes(table, row_id, column)?
6990                .unwrap_or_default();
6991            let next = crate::crdt::apply_update(&current, update)?;
6992            self.perform_crdt_push_update(table, row_id, column, &next)
6993        })();
6994        self.finish_authoring(prior, outcome)
6995    }
6996
6997    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
6998    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
6999    /// enqueue it. The server merges the crdt column and leaves every other
7000    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7001    /// clobber a concurrent edit to another column. A locally absent row
7002    /// records the same partial operation; the server answers per §6.2.
7003    #[cfg(feature = "crdt-yjs")]
7004    fn perform_crdt_push_update(
7005        &mut self,
7006        table: &str,
7007        row_id: &str,
7008        column: &str,
7009        crdt_bytes: &[u8],
7010    ) -> Result<String, ClientError> {
7011        let mut bytes_obj = Map::new();
7012        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7013        let mut partial = Map::new();
7014        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7015        self.perform_patch(table, row_id, partial, None)
7016    }
7017
7018    /// Run an arbitrary read-only SQL query against the local database and
7019    /// return each row as a `column-name → JSON value` map. This is the seam
7020    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7021    /// SQL over the visible tables/views, not a fixed table read like
7022    /// [`read_rows`]).
7023    ///
7024    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7025    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7026    /// same envelope the command surface uses everywhere else. Output BLOB
7027    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7028    ///
7029    /// The result column typing is dynamic (SQLite's stored affinity), because
7030    /// arbitrary SQL can alias, join, and compute — there is no schema column
7031    /// to consult per output cell, unlike [`read_rows`].
7032    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7033        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7034    }
7035
7036    /// Repository benchmark and conformance access, present only with `bench-internals`.
7037    #[cfg(feature = "bench-internals")]
7038    #[doc(hidden)]
7039    pub fn benchmark_connection(&mut self) -> &mut Connection {
7040        &mut self.conn
7041    }
7042
7043    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7044    #[cfg(feature = "bench-internals")]
7045    #[doc(hidden)]
7046    pub fn benchmark_phases(
7047        &self,
7048        enabled: Option<bool>,
7049        reset: bool,
7050    ) -> Result<Option<Value>, String> {
7051        self.benchmark_phases.configure(enabled, reset)?;
7052        Ok(self.benchmark_phases.snapshot())
7053    }
7054
7055    /// One serialized, read-only authority observation, including durable coverage.
7056    pub fn authority_snapshot(&self) -> Result<Value, String> {
7057        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7058        if self.limits.authority_reads.is_empty() {
7059            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7060        }
7061        self.conn
7062            .execute_batch("SAVEPOINT authority_snapshot")
7063            .map_err(|e| e.to_string())?;
7064        let result = (|| {
7065            let revision = self
7066                .get_meta(LOCAL_REVISION_KEY)?
7067                .ok_or("sync.local_corrupt: authority revision is missing")?;
7068            let parsed_revision = revision
7069                .parse::<u64>()
7070                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7071            if parsed_revision.to_string() != revision {
7072                return Err("sync.local_corrupt: authority revision is invalid".into());
7073            }
7074            let mut tables = Vec::new();
7075            for read in &self.limits.authority_reads {
7076                let table = self
7077                    .schema
7078                    .table(&read.table)
7079                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7080                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7081                    let ty = table
7082                        .columns
7083                        .iter()
7084                        .find(|c| c.name == column)
7085                        .expect("validated authority column")
7086                        .ty;
7087                    if ty == ColumnType::Boolean {
7088                        if let Some(value) = value.as_bool() {
7089                            return Ok(value.to_string());
7090                        }
7091                        return match value.as_i64() {
7092                            Some(0) => Ok("false".into()),
7093                            Some(1) => Ok("true".into()),
7094                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7095                        };
7096                    }
7097                    match value {
7098                        Value::String(s) => Ok(s.clone()),
7099                        Value::Number(n) => Ok(n.to_string()),
7100                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7101                            .get("$bigint")
7102                            .and_then(Value::as_str)
7103                            .map(str::to_owned)
7104                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7105                        _ => {
7106                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7107                        }
7108                    }
7109                };
7110                let matches = |row: &QueryRow| -> Result<bool, String> {
7111                    for (variable, values) in &read.scopes {
7112                        let column = table
7113                            .scope_column(variable)
7114                            .expect("validated authority scope");
7115                        let value = row
7116                            .get(column)
7117                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7118                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7119                            return Ok(false);
7120                        }
7121                    }
7122                    Ok(true)
7123                };
7124                let selected = read
7125                    .columns
7126                    .iter()
7127                    .map(|c| quote_ident(c))
7128                    .collect::<Vec<_>>()
7129                    .join(",");
7130                let mut intent = BTreeSet::new();
7131                let mut relevant = BTreeSet::new();
7132                for row in query_connection(&self.conn,
7133                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7134                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7135                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7136                    intent.insert(id.clone());
7137                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7138                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7139                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7140                    }
7141                }
7142                let mut rows = BTreeMap::new();
7143                for full in [base_table(&read.table), visible_table(&read.table)] {
7144                    for mut values in query_connection(
7145                        &self.conn,
7146                        &format!(
7147                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7148                        ),
7149                        &[],
7150                    )
7151                    .map_err(|e| e.to_string())?
7152                    {
7153                        if !matches(&values)? {
7154                            continue;
7155                        }
7156                        let id = scalar(
7157                            &table.primary_key,
7158                            values
7159                                .get(&table.primary_key)
7160                                .expect("selected primary key"),
7161                        )?;
7162                        relevant.insert(id.clone());
7163                        if full == base_table(&read.table) {
7164                            let version = values
7165                                .remove("authority_version")
7166                                .expect("selected version");
7167                            let version = version
7168                                .as_i64()
7169                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7170                            if version >= 0 {
7171                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7172                            }
7173                        }
7174                    }
7175                }
7176                intent.retain(|id| relevant.contains(id));
7177                let rows = rows.into_values().collect::<Vec<_>>();
7178                // Every unit covering the table, as its (unit, table) pair (§4.4).
7179                let replica = self.stored(replica_cursor(&self.conn))?;
7180                let persisted = self
7181                    .stored(load_units(&self.conn))?
7182                    .into_iter()
7183                    .filter(|unit| covers(table, &unit.scopes))
7184                    .map(|unit| {
7185                        let effective = unit.echo.as_deref().and_then(|echo| {
7186                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7187                        });
7188                        let requested = read
7189                            .scopes
7190                            .keys()
7191                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7192                            .collect::<Map<_, _>>();
7193                        let complete = effective
7194                            .as_ref()
7195                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7196                            && unit.status == UnitStatus::Active
7197                            && !unit.bootstrapping();
7198                        let exposed = effective.map(|scopes| {
7199                            read.scopes
7200                                .keys()
7201                                .map(|v| {
7202                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7203                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7204                                })
7205                                .collect::<Map<_, _>>()
7206                        });
7207                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7208                    })
7209                    .collect::<Vec<_>>();
7210                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7211                    match selectors.split_first() {
7212                        None => !candidates.is_empty(),
7213                        Some(((variable, values), rest)) => values.iter().all(|value| {
7214                            covered(
7215                                rest,
7216                                &candidates
7217                                    .iter()
7218                                    .copied()
7219                                    .filter(|p| {
7220                                        p["effectiveScopes"][variable.as_str()]
7221                                            .as_array()
7222                                            .is_some_and(|held| {
7223                                                held.contains(&Value::String(value.clone()))
7224                                                    || held.contains(&Value::String("*".into()))
7225                                            })
7226                                    })
7227                                    .collect::<Vec<_>>(),
7228                            )
7229                        }),
7230                    }
7231                }
7232                let coverage = if covered(
7233                    &read.scopes.iter().collect::<Vec<_>>(),
7234                    &persisted
7235                        .iter()
7236                        .filter(|p| p["complete"] == true)
7237                        .collect::<Vec<_>>(),
7238                ) {
7239                    "complete"
7240                } else if persisted.iter().any(|p| {
7241                    p["status"] == "active"
7242                        && read.scopes.iter().all(|(v, values)| {
7243                            values.iter().any(|value| {
7244                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7245                                    held.contains(&Value::String(value.clone()))
7246                                })
7247                            })
7248                        })
7249                }) {
7250                    "pending"
7251                } else {
7252                    "missing"
7253                };
7254                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7255            }
7256            Ok(
7257                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7258            )
7259        })();
7260        match result {
7261            Ok(snapshot) => {
7262                self.conn
7263                    .execute_batch("RELEASE authority_snapshot")
7264                    .map_err(|e| e.to_string())?;
7265                Ok(snapshot)
7266            }
7267            Err(error) => {
7268                self.conn
7269                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7270                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7271                Err(error)
7272            }
7273        }
7274    }
7275
7276    /// Rows, coverage, and local revision from one SQLite read snapshot.
7277    /// §7.5 atomic snapshot read. An `owner` records a failure in
7278    /// `diagnostics_snapshot().query_failures` until its next successful read.
7279    pub fn query_snapshot(
7280        &mut self,
7281        sql: &str,
7282        params: &[QueryValue],
7283        coverage: &[WindowCoverage],
7284        owner: Option<QueryOwner<'_>>,
7285    ) -> Result<QuerySnapshot, String> {
7286        let result = snapshot_connection(&self.conn, sql, params, coverage);
7287        if let Some(owner) = owner {
7288            self.record_query_read(owner, result.as_ref().err());
7289        }
7290        result.map_err(|failure| failure.to_string())
7291    }
7292
7293    /// §7.6: record the outcome of one owned snapshot read, including reads a
7294    /// host ran on a separate connection. A success removes the owner's entry;
7295    /// an identical failure leaves it unchanged; a changed failure moves last.
7296    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7297        let position = self
7298            .query_failures
7299            .iter()
7300            .position(|entry| entry.id == owner.id);
7301        let Some(failure) = failure else {
7302            if let Some(index) = position {
7303                self.query_failures.remove(index);
7304            }
7305            return;
7306        };
7307        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7308            .into_iter()
7309            .map(str::to_owned)
7310            .collect();
7311        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7312        if position.is_some_and(|index| {
7313            let entry = &self.query_failures[index];
7314            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7315        }) {
7316            return;
7317        }
7318        if let Some(index) = position {
7319            self.query_failures.remove(index);
7320        }
7321        self.query_failures.push(DiagnosticQueryFailure {
7322            id: owner.id.to_owned(),
7323            tables,
7324            code,
7325            sqlite_code: failure.sqlite_code,
7326            at_ms: self.clock_now_ms(),
7327        });
7328        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7329            self.query_failures.remove(0);
7330        }
7331    }
7332
7333    // -- request building ---------------------------------------------------------
7334
7335    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7336        #[cfg(feature = "bench-internals")]
7337        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7338        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7339            Box::new(SyncOutcome::Failed {
7340                error_code: "storage.failed".into(),
7341                message,
7342                details: None,
7343                recommended_action: None,
7344            })
7345        })?;
7346        let max_commits = self.limits.max_push_commits_per_request;
7347        let max_ops = self
7348            .limits
7349            .max_push_operations_per_request
7350            .unwrap_or(PUSH_OPS_PER_REQUEST);
7351        let byte_cap = self.limits.max_push_request_bytes;
7352        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7353        // bare URL — capability negotiation, decided per transport.
7354        let accept = self.limits.accept.unwrap_or(if url_capable {
7355            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7356        } else {
7357            DEFAULT_ACCEPT
7358        });
7359        let storage_failed = |message: String| {
7360            Box::new(SyncOutcome::Failed {
7361                error_code: "storage.failed".into(),
7362                message,
7363                details: None,
7364                recommended_action: None,
7365            })
7366        };
7367        let header = Frame::ReqHeader {
7368            client_id: self.client_id.clone(),
7369            schema_version: self.schema.version,
7370            log_epoch: log_epoch.clone(),
7371            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7372        };
7373        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7374        // which stays ahead of every push, pull, unit and progress frame; the
7375        // byte-capacity probes below measure this exact sequence.
7376        let mut lead = vec![header];
7377        lead.extend(
7378            self.round_failure
7379                .iter()
7380                .map(|code| Frame::RoundFailure { code: code.clone() }),
7381        );
7382        let registered: Vec<UnitRecord> = self
7383            .stored(load_units(&self.conn))
7384            .map_err(storage_failed)?
7385            .into_iter()
7386            .filter(|unit| unit.status == UnitStatus::Active)
7387            .collect();
7388        let registration_digest = ssp2::registration::registration_digest(
7389            registered
7390                .iter()
7391                .map(|unit| scope_pairs(&unit.scopes))
7392                .collect::<Vec<_>>()
7393                .iter()
7394                .map(Vec::as_slice),
7395        );
7396        let echo_digest = self
7397            .get_meta(ECHO_DIGEST_KEY)
7398            .map_err(storage_failed)?
7399            .filter(|hex| !hex.is_empty())
7400            .map(|hex| {
7401                crate::values::hex_to_bytes(&hex)
7402                    .ok()
7403                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7404                    .ok_or_else(|| {
7405                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7406                    })
7407            })
7408            .transpose()
7409            .map_err(storage_failed)?;
7410        let pull = Frame::PullHeader {
7411            limit_commits: self.limits.limit_commits.unwrap_or(0),
7412            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7413            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7414            accept,
7415            cursor: self
7416                .stored(replica_cursor(&self.conn))
7417                .map_err(storage_failed)?,
7418            registration_digest,
7419            echo_digest,
7420        };
7421        let registration_digest = bytes_to_hex(&registration_digest);
7422        // §4.1: the list travels when it changed since the server stored it,
7423        // after `unregistered` and after a reset (both clear the stored digest).
7424        let travels = self
7425            .get_meta(REGISTERED_DIGEST_KEY)
7426            .map_err(storage_failed)?
7427            .as_deref()
7428            != Some(registration_digest.as_str());
7429        let mut sub_frames: Vec<Frame> = Vec::new();
7430        if travels {
7431            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7432                scopes: scope_pairs(&unit.scopes),
7433            }));
7434        }
7435        // §4.3: one PROGRESS per catching-up unit, in registration order.
7436        sub_frames.extend(registered.iter().filter_map(|unit| {
7437            Some(Frame::Progress {
7438                id: unit.id.clone(),
7439                cursor: unit.cursor?,
7440                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7441            })
7442        }));
7443        // The fixed frames (order-independent size) and the header+pull probe
7444        // both encode canonically, so the byte budget is the exact wire size.
7445        let fixed_bytes = match byte_cap {
7446            Some(_) => {
7447                let mut frames = lead.clone();
7448                frames.push(pull.clone());
7449                frames.extend(sub_frames.iter().cloned());
7450                encode_message(&Message {
7451                    wire_version: WIRE_VERSION,
7452                    msg_kind: MsgKind::Request,
7453                    frames,
7454                })
7455                .len()
7456            }
7457            None => 0,
7458        };
7459        if let Some(cap) = byte_cap {
7460            if fixed_bytes > cap {
7461                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7462            }
7463        }
7464        let base_bytes = match byte_cap {
7465            Some(_) => {
7466                let mut frames = lead.clone();
7467                frames.push(pull.clone());
7468                encode_message(&Message {
7469                    wire_version: WIRE_VERSION,
7470                    msg_kind: MsgKind::Request,
7471                    frames,
7472                })
7473                .len()
7474            }
7475            None => 0,
7476        };
7477        let total_outbox = if log_epoch.is_some() {
7478            self.outbox.len()
7479        } else {
7480            0
7481        };
7482        let mut push_frames: Vec<Frame> = Vec::new();
7483        let mut pushed_ids = Vec::new();
7484        let mut ops_in_request = 0usize;
7485        let mut bytes_in_request = 0usize;
7486        let mut deferred_commits = 0usize;
7487        #[cfg(feature = "bench-internals")]
7488        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7489        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7490            // §6.1/§7.1: the commit-count and operation caps are hard. A
7491            // first commit over the operation cap is a typed capacity error
7492            // and stays queued; a later one defers with the complete suffix.
7493            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7494                deferred_commits = total_outbox - index;
7495                break;
7496            }
7497            if ops_in_request + commit.ops.len() > max_ops {
7498                if push_frames.is_empty() {
7499                    return Err(Self::push_capacity_outcome(
7500                        "operations",
7501                        max_ops,
7502                        commit.ops.len(),
7503                        Some(commit.client_commit_id.as_str()),
7504                    ));
7505                }
7506                deferred_commits = total_outbox - index;
7507                break;
7508            }
7509            // §7.1 chained base: the commit waits for the change of the commit
7510            // it chains on, which supplies its rebased version.
7511            if self.outbox[..index]
7512                .iter()
7513                .flat_map(|earlier| &earlier.ops)
7514                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7515            {
7516                deferred_commits = total_outbox - index;
7517                break;
7518            }
7519            let operations = commit
7520                .ops
7521                .iter()
7522                .map(|op| {
7523                    let payload = match self
7524                        .schema
7525                        .table(&op.table)
7526                        .and_then(|table| Some((table, op.presented_values(table)?)))
7527                    {
7528                        Some((table, values)) => {
7529                            // §0: outbox entries encode at send time with the
7530                            // current codec (validated at mutate()). §6.1: the
7531                            // values map's keys are the sparse presence set; an
7532                            // insert-intent operation presents every column (§7.1).
7533                            // §5.11: encrypted columns are encrypted here, with
7534                            // an absent key-id selector read from the stored row.
7535                            // `sync_inner` runs `drop_unencodable_outbox` first,
7536                            // so an encrypt failure cannot reach this encode; the
7537                            // author seam validated every other failure.
7538                            let fallback =
7539                                self.stored_key_fallback(table, &op.row_id)
7540                                    .map_err(|message| {
7541                                        Box::new(SyncOutcome::Failed {
7542                                            error_code: "storage.failed".into(),
7543                                            message,
7544                                            details: None,
7545                                            recommended_action: None,
7546                                        })
7547                                    })?;
7548                            encode_sparse_row_json(
7549                                table,
7550                                &op.row_id,
7551                                &values,
7552                                &self.encryption,
7553                                fallback.as_ref(),
7554                            )
7555                            .ok()
7556                        }
7557                        None => None,
7558                    };
7559                    Ok(ssp2::model::Operation {
7560                        table: op.table.clone(),
7561                        row_id: op.row_id.clone(),
7562                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7563                        base_version: op.base_version,
7564                        payload,
7565                    })
7566                })
7567                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7568            // §6.3.2, §7.1: the commit's outbox sequence.
7569            let client_seq = self
7570                .conn
7571                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7572                .and_then(|mut statement| {
7573                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7574                })
7575                .map_err(|error| {
7576                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7577                })?;
7578            let frame = Frame::PushCommit {
7579                client_commit_id: commit.client_commit_id.clone(),
7580                client_seq,
7581                operations,
7582            };
7583            let frame_bytes = match byte_cap {
7584                Some(_) => {
7585                    let mut frames = lead.clone();
7586                    frames.push(frame.clone());
7587                    frames.push(pull.clone());
7588                    encode_message(&Message {
7589                        wire_version: WIRE_VERSION,
7590                        msg_kind: MsgKind::Request,
7591                        frames,
7592                    })
7593                    .len()
7594                        - base_bytes
7595                }
7596                None => 0,
7597            };
7598            // The head is always encoded so the caller can report the full
7599            // projected request size; a later commit defers once it no longer
7600            // fits.
7601            if let Some(cap) = byte_cap {
7602                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7603                    deferred_commits = total_outbox - index;
7604                    break;
7605                }
7606            }
7607            push_frames.push(frame);
7608            pushed_ids.push(commit.client_commit_id.clone());
7609            ops_in_request += commit.ops.len();
7610            bytes_in_request += frame_bytes;
7611        }
7612        #[cfg(feature = "bench-internals")]
7613        drop(outbox_phase);
7614        if let Some(cap) = byte_cap {
7615            if fixed_bytes + bytes_in_request > cap {
7616                // The head alone does not fit this budget; the commit stays
7617                // queued and no transport round is attempted.
7618                let blocked = if push_frames.is_empty() {
7619                    None
7620                } else {
7621                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7622                };
7623                return Err(Self::push_capacity_outcome(
7624                    "bytes",
7625                    cap,
7626                    fixed_bytes + bytes_in_request,
7627                    blocked,
7628                ));
7629            }
7630        }
7631        let mut frames = lead;
7632        frames.extend(push_frames);
7633        frames.push(pull);
7634        frames.extend(sub_frames);
7635        let message = Message {
7636            wire_version: WIRE_VERSION,
7637            msg_kind: MsgKind::Request,
7638            frames,
7639        };
7640        Ok((
7641            message,
7642            RequestMeta {
7643                pushed_ids,
7644                registration_digest,
7645                accept,
7646                deferred_commits,
7647                applied: BTreeMap::new(),
7648                deferred: BTreeSet::new(),
7649            },
7650        ))
7651    }
7652
7653    /// §7.1: a configured push capacity limit cannot carry this request.
7654    fn push_capacity_outcome(
7655        kind: &str,
7656        limit: usize,
7657        size: usize,
7658        client_commit_id: Option<&str>,
7659    ) -> Box<SyncOutcome> {
7660        let mut details = Map::new();
7661        details.insert("kind".to_owned(), Value::from(kind));
7662        details.insert("limit".to_owned(), Value::from(limit as u64));
7663        details.insert("size".to_owned(), Value::from(size as u64));
7664        if let Some(id) = client_commit_id {
7665            details.insert("clientCommitId".to_owned(), Value::from(id));
7666        }
7667        Box::new(SyncOutcome::Failed {
7668            error_code: "client.push_request_too_large".into(),
7669            message: "the push request exceeds a configured capacity limit".into(),
7670            details: Some(Value::Object(details)),
7671            recommended_action: None,
7672        })
7673    }
7674
7675    // -- sync -------------------------------------------------------------------
7676
7677    /// Cloneable observation handle; no client lock is needed to read it.
7678    pub fn progress(&self) -> ProgressObserver {
7679        self.progress.clone()
7680    }
7681
7682    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7683        let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7684            Ok(prepared) => prepared,
7685            Err(outcome) => return *outcome,
7686        };
7687        // The round's network deadline rides `PreparedSyncRound` and is scoped
7688        // to each exchange, so uploads, continuations, the main request, and
7689        // every segment fetch share one monotonic anchor without leaving
7690        // deadline state on a reused transport.
7691        let mut next = prepared;
7692        loop {
7693            match self.apply_sync_round(next.exchange(transport)) {
7694                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7695                crate::AppliedSyncRound::Complete {
7696                    outcome, controls, ..
7697                } => {
7698                    if self.transport_enabled {
7699                        for text in controls {
7700                            let _ = transport.realtime_send(&text);
7701                        }
7702                    } else {
7703                        self.disconnect_realtime(transport);
7704                    }
7705                    return outcome;
7706                }
7707            }
7708        }
7709    }
7710
7711    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7712        if let SyncOutcome::Failed {
7713            error_code,
7714            message,
7715            details,
7716            ..
7717        } = &mut outcome
7718        {
7719            if let Some((failure, code)) = self
7720                .storage_failure
7721                .borrow()
7722                .as_ref()
7723                .and_then(|failure| failure.code.map(|code| (failure, code)))
7724            {
7725                *error_code = code.to_owned();
7726                *message = match code {
7727                    "client.storage_busy" => "local SQLite storage is busy",
7728                    "client.storage_full" => "local SQLite storage is full",
7729                    "client.storage_io" => "local SQLite storage I/O failed",
7730                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7731                    _ => unreachable!("unrecognized classified storage failure"),
7732                }
7733                .to_owned();
7734                *details = failure.details();
7735            }
7736        }
7737        let details = match &outcome {
7738            SyncOutcome::Failed { details, .. } => details.as_ref(),
7739            _ => None,
7740        };
7741        let cause_kind = details
7742            .and_then(|details| details.get("causeKind"))
7743            .and_then(Value::as_str)
7744            .and_then(|cause| match cause {
7745                "network" => Some(crate::TransportCauseKind::Network),
7746                "connect" => Some(crate::TransportCauseKind::Connect),
7747                "tls" => Some(crate::TransportCauseKind::Tls),
7748                "io" => Some(crate::TransportCauseKind::Io),
7749                "request" => Some(crate::TransportCauseKind::Request),
7750                "protocol" => Some(crate::TransportCauseKind::Protocol),
7751                "timeout" => Some(crate::TransportCauseKind::Timeout),
7752                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7753                "aborted" => Some(crate::TransportCauseKind::Aborted),
7754                "body" => Some(crate::TransportCauseKind::Body),
7755                "status" => Some(crate::TransportCauseKind::Status),
7756                "redirect" => Some(crate::TransportCauseKind::Redirect),
7757                "unknown" => Some(crate::TransportCauseKind::Unknown),
7758                _ => None,
7759            });
7760        let http_status = if matches!(
7761            cause_kind,
7762            Some(
7763                crate::TransportCauseKind::Status
7764                    | crate::TransportCauseKind::Body
7765                    | crate::TransportCauseKind::Redirect
7766            )
7767        ) {
7768            details
7769                .and_then(|details| details.get("httpStatus"))
7770                .and_then(Value::as_f64)
7771                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7772                .map(|status| status as u16)
7773        } else {
7774            None
7775        };
7776        let retry_delay_ms = self.round_retry_delay_ms;
7777        self.progress.update(|p| match &outcome {
7778            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7779                if report.failed.is_empty() =>
7780            {
7781                p.state = ProgressState::Complete
7782            }
7783            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7784                p.state = ProgressState::Failed;
7785                p.error_code = Some("sync.scope_revoked".into());
7786            }
7787            SyncOutcome::Failed { error_code, .. } => {
7788                p.state = ProgressState::Failed;
7789                p.error_code = Some(Self::diagnostic_code(error_code));
7790                p.cause_kind = cause_kind;
7791                p.http_status = http_status;
7792                p.retry_delay_ms = retry_delay_ms;
7793            }
7794            SyncOutcome::RealtimeUnavailable { .. } => {
7795                p.state = ProgressState::Failed;
7796                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7797                p.retry_delay_ms = retry_delay_ms;
7798            }
7799        });
7800        let completed_at_ms = self.clock_now_ms();
7801        self.last_round = Some(match &outcome {
7802            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7803                status: "succeeded".to_owned(),
7804                started_at_ms,
7805                completed_at_ms,
7806                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7807                counters: Some(DiagnosticRoundCounters {
7808                    pushed: report.pushed,
7809                    applied: report.applied.len(),
7810                    rejected: report.rejected.len(),
7811                    retryable: report.retryable.len(),
7812                    conflicts: report.conflicts,
7813                    commits_applied: report.commits_applied,
7814                    segment_rows_applied: report.segment_rows_applied,
7815                    bootstrapping: report.bootstrapping.len(),
7816                    resets: report.resets.len(),
7817                    revoked: report.revoked.len(),
7818                    failed: report.failed.len(),
7819                    deferred_commits: report.deferred_commits,
7820                }),
7821                error_code: None,
7822                cause_kind: None,
7823                http_status: None,
7824            },
7825            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7826                status: "failed".to_owned(),
7827                started_at_ms,
7828                completed_at_ms,
7829                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7830                counters: None,
7831                error_code: Some(Self::diagnostic_code(error_code)),
7832                cause_kind,
7833                http_status,
7834            },
7835            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7836                status: "failed".to_owned(),
7837                started_at_ms,
7838                completed_at_ms,
7839                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7840                counters: None,
7841                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7842                cause_kind: None,
7843                http_status: None,
7844            },
7845        });
7846        outcome
7847    }
7848
7849    /// Capture one request on the client owner. Mutations made after this
7850    /// call remain in the outbox for a later round; no network I/O occurs here.
7851    pub fn prepare_sync_round(
7852        &mut self,
7853        url_capable: bool,
7854    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7855        if !self.transport_enabled {
7856            return Err(Box::new(SyncOutcome::Failed {
7857                error_code: "sync.offline".into(),
7858                message: "transport gate is closed".into(),
7859                details: None,
7860                recommended_action: None,
7861            }));
7862        }
7863        if self.active_round.is_some() {
7864            return Err(Box::new(SyncOutcome::Failed {
7865                error_code: "client.round_in_progress".into(),
7866                message: "a sync round is already in flight".into(),
7867                details: None,
7868                recommended_action: None,
7869            }));
7870        }
7871        self.storage_failure.borrow_mut().take();
7872        self.progress.start();
7873        self.round_retry_delay_ms = None;
7874        let started_at_ms = self.clock_now_ms();
7875        let prepared = (|| {
7876            if self.stopped {
7877                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7878                    schema_floor: self.schema_floor.clone(),
7879                    ..SyncReport::default()
7880                })));
7881            }
7882            if self.security_preflight {
7883                return Err(Box::new(SyncOutcome::Failed {
7884                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7885                    message: "security preflight must complete before syncing".into(),
7886                    details: None,
7887                    recommended_action: None,
7888                }));
7889            }
7890            self.set_sync_needed(false, false);
7891            self.drain_pending_evictions()
7892                .and_then(|_| self.drop_incompatible_outbox())
7893                .and_then(|_| self.drop_unencodable_outbox(true, None))
7894                .map_err(|message| {
7895                    Box::new(SyncOutcome::Failed {
7896                        error_code: "storage.failed".into(),
7897                        message,
7898                        details: None,
7899                        recommended_action: None,
7900                    })
7901                })?;
7902            // §5.11: reveal rows sealed under a key the keyring gained since.
7903            self.reveal_sealed_rows().map_err(|error| {
7904                Box::new(SyncOutcome::Failed {
7905                    error_code: error.code,
7906                    message: error.message,
7907                    details: error.details,
7908                    recommended_action: None,
7909                })
7910            })?;
7911            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7912                Box::new(SyncOutcome::Failed {
7913                    error_code: "storage.failed".into(),
7914                    message,
7915                    details: None,
7916                    recommended_action: None,
7917                })
7918            })?;
7919            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
7920                self.pending_blob_uploads().map_err(|e| {
7921                    Box::new(SyncOutcome::Failed {
7922                        error_code: e.code,
7923                        message: e.message,
7924                        details: e.details,
7925                        recommended_action: None,
7926                    })
7927                })?
7928            } else {
7929                Vec::new()
7930            };
7931            if self.realtime_state != RealtimeState::Connected
7932                && self.realtime_policy == RealtimePolicy::Required
7933            {
7934                let retry_delay_ms = self.schedule_background_retry();
7935                self.realtime_retry_delay_ms = Some(retry_delay_ms);
7936                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
7937                    state: self.realtime_state,
7938                    reason_code: self.realtime_reason_code.clone(),
7939                    retry_delay_ms,
7940                }));
7941            }
7942            let (message, mut meta) = self.build_request(url_capable)?;
7943            let units = self.stored(load_units(&self.conn)).map_err(|message| {
7944                Box::new(SyncOutcome::Failed {
7945                    error_code: "storage.failed".into(),
7946                    message,
7947                    details: None,
7948                    recommended_action: None,
7949                })
7950            })?;
7951            meta.applied = units
7952                .iter()
7953                .filter_map(|unit| {
7954                    let (cursor, token, ids) = self.applied_segments.get(&unit.id)?;
7955                    (*cursor == unit.cursor && *token == unit.bootstrap_state)
7956                        .then(|| (unit.id.clone(), ids.clone()))
7957                })
7958                .collect();
7959            let id = uuid::Uuid::new_v4();
7960            self.active_round = Some(id);
7961            let phase = if uploads.is_empty() {
7962                crate::round::RoundPhase::Sync
7963            } else {
7964                crate::round::RoundPhase::UploadGrant
7965            };
7966            Ok(crate::PreparedSyncRound {
7967                id,
7968                started_at_ms,
7969                message,
7970                meta,
7971                uploads: uploads.into(),
7972                phase,
7973                encoded: None,
7974                continuation: None,
7975                completed: None,
7976                realtime: self.realtime_state == RealtimeState::Connected,
7977                // §5.5: the owning unit's scope map, each value a one-element list.
7978                scopes: units
7979                    .into_iter()
7980                    .map(|unit| {
7981                        (
7982                            unit.id,
7983                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
7984                        )
7985                    })
7986                    .collect(),
7987                #[cfg(feature = "bench-internals")]
7988                benchmark_phases: self.benchmark_phases.clone(),
7989                progress: self.progress.clone(),
7990                fixed_now: self.now_ms,
7991                round_deadline_at: None,
7992            })
7993        })();
7994        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
7995    }
7996
7997    /// Discard the reply of an old authorization or registration context. Its
7998    /// submitted commits stay durable and can be retried with the same ids.
7999    pub fn cancel_sync_round(&mut self) {
8000        self.active_round = None;
8001    }
8002
8003    /// Apply captured network results on the owner, preserving all normal
8004    /// version, schema, content-address and cursor checks. No network I/O.
8005    pub fn apply_sync_round(
8006        &mut self,
8007        completed: crate::CompletedSyncRound,
8008    ) -> crate::AppliedSyncRound {
8009        let mut prepared = completed.prepared;
8010        if self.active_round != Some(prepared.id) || self.security_preflight {
8011            return crate::AppliedSyncRound::Complete {
8012                outcome: SyncOutcome::Failed {
8013                    error_code: "client.round_cancelled".into(),
8014                    message: "sync round belongs to a released client context".into(),
8015                    details: None,
8016                    recommended_action: None,
8017                },
8018                controls: Vec::new(),
8019                more: false,
8020                bootstrap_advanced: false,
8021            };
8022        }
8023        self.storage_failure.borrow_mut().take();
8024        let (response, mut downloads, transport_failed) = match completed.exchange {
8025            crate::round::ExchangeResult::Upload { id, result } => {
8026                let result = match result {
8027                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8028                    // is the body's final outcome. The commits naming it take
8029                    // the incompatible-commit path with the server's code and
8030                    // leave this round's request; the round continues.
8031                    Err(error)
8032                        if error
8033                            .details
8034                            .as_ref()
8035                            .and_then(|details| details.get("retryable"))
8036                            == Some(&Value::Bool(false)) =>
8037                    {
8038                        self.conn
8039                            .prepare(
8040                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8041                            )
8042                            .and_then(|mut statement| {
8043                                statement
8044                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8045                                    .collect::<Result<HashSet<_>, _>>()
8046                            })
8047                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8048                            .and_then(|refused| {
8049                                let commits = self
8050                                    .outbox
8051                                    .iter()
8052                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8053                                    .cloned()
8054                                    .collect::<Vec<_>>();
8055                                if !commits.is_empty() {
8056                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8057                                }
8058                                prepared.message.frames.retain(|frame| {
8059                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8060                                        if refused.contains(client_commit_id))
8061                                });
8062                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8063                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8064                                Ok(())
8065                            })
8066                            .map_err(|message| TransportError::new("client.failed", message))
8067                    }
8068                    result => result,
8069                };
8070                let result = result.and_then(|()| {
8071                    self.conn
8072                        .execute(
8073                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8074                            rusqlite::params![id],
8075                        )
8076                        .map(|_| ())
8077                        .map_err(|error| {
8078                            TransportError::new(
8079                                "client.failed",
8080                                Self::sqlite_failure(&self.storage_failure, error),
8081                            )
8082                        })
8083                });
8084                match result {
8085                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8086                    Err(TransportError {
8087                        code,
8088                        message,
8089                        details,
8090                    }) => {
8091                        self.active_round = None;
8092                        if Self::unreachable_transport(&details) {
8093                            // The round's work is still owed; the host's
8094                            // connectivity resume runs it.
8095                            self.set_sync_needed(true, false);
8096                        } else if Self::retryable_failure_code(&code) {
8097                            self.schedule_background_retry();
8098                        }
8099                        let recommended_action = recommended_action_of(&details);
8100                        let outcome = self.finish_sync_round(
8101                            prepared.started_at_ms,
8102                            SyncOutcome::Failed {
8103                                error_code: code,
8104                                message,
8105                                details,
8106                                recommended_action,
8107                            },
8108                        );
8109                        return crate::AppliedSyncRound::Complete {
8110                            outcome,
8111                            controls: Vec::new(),
8112                            more: false,
8113                            bootstrap_advanced: false,
8114                        };
8115                    }
8116                }
8117            }
8118            crate::round::ExchangeResult::Reply {
8119                response,
8120                downloads,
8121                transport_failed,
8122            } => (response, downloads, transport_failed),
8123        };
8124        self.active_round = None;
8125        let before = self.resume_tokens();
8126        let outcome = (|| {
8127            let response = match response {
8128                Ok(response) => {
8129                    // §1.5: a complete, decodable response ends the report, an
8130                    // in-band ERROR included (the server already recorded it).
8131                    self.round_failure = None;
8132                    response
8133                }
8134                Err(TransportError {
8135                    code,
8136                    message,
8137                    details,
8138                }) => {
8139                    // §1.5: the server cannot see a round whose transport
8140                    // failed or whose response never decoded, so its code rides
8141                    // the following requests. A code the frame grammar rejects
8142                    // is not reportable, and the previous code stays.
8143                    if ssp2::model::valid_round_failure_code(&code) {
8144                        self.round_failure = Some(code.clone());
8145                    }
8146                    if transport_failed {
8147                        if prepared.realtime {
8148                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8149                        }
8150                        if let Err(message) = self.record_lease_error(&code) {
8151                            return SyncOutcome::Failed {
8152                                error_code: "storage.failed".to_owned(),
8153                                message,
8154                                details: None,
8155                                recommended_action: None,
8156                            };
8157                        }
8158                    }
8159                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8160                    {
8161                        let delay = self.schedule_background_retry();
8162                        if prepared.realtime && transport_failed {
8163                            self.realtime_retry_delay_ms = Some(delay);
8164                        }
8165                    }
8166                    let recommended_action = recommended_action_of(&details);
8167                    return SyncOutcome::Failed {
8168                        error_code: code,
8169                        message,
8170                        details,
8171                        recommended_action,
8172                    };
8173                }
8174            };
8175            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8176            if let SyncOutcome::Ok(report) = &mut outcome {
8177                report.deferred_commits = prepared.meta.deferred_commits;
8178            }
8179            match &outcome {
8180                SyncOutcome::Ok(_) => self.reset_background_retry(),
8181                SyncOutcome::Failed {
8182                    error_code,
8183                    details,
8184                    ..
8185                } if Self::retryable_failure_code(error_code)
8186                    && !Self::unreachable_transport(details) =>
8187                {
8188                    self.schedule_background_retry();
8189                }
8190                _ => {}
8191            }
8192            if prepared.meta.deferred_commits > 0 {
8193                self.set_sync_needed(true, true);
8194            }
8195            self.previous_version_lifetime_check();
8196            outcome
8197        })();
8198        match &outcome {
8199            // §7.1: a unit still catching up needs another round, also when the
8200            // drive that started it ends early.
8201            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8202                self.set_sync_needed(true, false)
8203            }
8204            // The round's work is still owed; the host's connectivity resume
8205            // runs it.
8206            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8207                self.set_sync_needed(true, false)
8208            }
8209            _ => {}
8210        }
8211        let more = self.transport_enabled
8212            && matches!(&outcome, SyncOutcome::Ok(report)
8213                if report.schema_floor.is_none()
8214                    && (!report.bootstrapping.is_empty()
8215                        || report.commits_applied > 0
8216                        || report.segment_rows_applied > 0
8217                        || !report.resets.is_empty()
8218                        || report.deferred_commits > 0
8219                        || self.sync_needed));
8220        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8221            && before != self.resume_tokens();
8222        crate::AppliedSyncRound::Complete {
8223            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8224            controls: if self.transport_enabled {
8225                downloads.controls
8226            } else {
8227                Vec::new()
8228            },
8229            more,
8230            bootstrap_advanced,
8231        }
8232    }
8233
8234    /// Every unit's resume token and applied-segment count: a bootstrap that
8235    /// advanced changes one (§4.7), also when it applied images of a section
8236    /// it deferred (§5.3, §5.6).
8237    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, usize)>> {
8238        load_units(&self.conn).ok().map(|units| {
8239            units
8240                .into_iter()
8241                .map(|unit| {
8242                    let applied = self
8243                        .applied_segments
8244                        .get(&unit.id)
8245                        .map_or(0, |(_, _, ids)| ids.len());
8246                    (unit.id, unit.bootstrap_state, applied)
8247                })
8248                .collect()
8249        })
8250    }
8251
8252    /// §7.8: one idle drive's default total deadline, the TS client's value.
8253    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8254
8255    pub fn sync_until_idle(
8256        &mut self,
8257        transport: &mut dyn Transport,
8258        max_rounds: Option<u32>,
8259    ) -> SyncOutcome {
8260        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8261    }
8262
8263    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8264    /// milliseconds (`None` disables it). The deadline is checked between
8265    /// rounds; one blocking transport call is bounded by the transport's own
8266    /// whole-round deadline (§1.8).
8267    pub fn sync_until_idle_with_deadline(
8268        &mut self,
8269        transport: &mut dyn Transport,
8270        max_rounds: Option<u32>,
8271        timeout_ms: Option<u64>,
8272    ) -> SyncOutcome {
8273        // §7.8: an explicit budget is a strict positive round count. Zero is a
8274        // caller error, not an exhaustion.
8275        if max_rounds == Some(0) {
8276            return SyncOutcome::Failed {
8277                details: None,
8278                error_code: "sync.invalid_request".into(),
8279                message: "maxRounds must be a positive integer".into(),
8280                recommended_action: None,
8281            };
8282        }
8283        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8284            return SyncOutcome::Failed {
8285                details: None,
8286                error_code: "sync.invalid_request".into(),
8287                message: "timeoutMs must be a positive integer".into(),
8288                recommended_action: None,
8289            };
8290        }
8291        let started_at_ms = self.clock_now_ms();
8292        let deadline_ms = timeout_ms.map(|ms| started_at_ms.saturating_add(ms as i64));
8293        self.sync_until_idle_bounded(transport, max_rounds, started_at_ms, deadline_ms)
8294    }
8295
8296    fn sync_until_idle_bounded(
8297        &mut self,
8298        transport: &mut dyn Transport,
8299        max_rounds: Option<u32>,
8300        started_at_ms: i64,
8301        deadline_ms: Option<i64>,
8302    ) -> SyncOutcome {
8303        let rounds = max_rounds.unwrap_or(20);
8304        let mut aggregate = SyncReport::default();
8305        let mut spent = 0;
8306        while spent < rounds {
8307            if let Some(deadline_ms) = deadline_ms {
8308                if self.clock_now_ms() >= deadline_ms {
8309                    return SyncOutcome::Failed {
8310                        error_code: "sync.idle_timeout".into(),
8311                        message: "sync_until_idle exceeded its deadline".into(),
8312                        details: Some(serde_json::json!({
8313                            "timeoutMs": deadline_ms
8314                                .saturating_sub(started_at_ms) as u64,
8315                            "rounds": spent,
8316                        })),
8317                        recommended_action: Some("retryLater".into()),
8318                    };
8319                }
8320            }
8321            spent += 1;
8322            let before = self.resume_tokens();
8323            match self.sync(transport) {
8324                SyncOutcome::Failed {
8325                    error_code,
8326                    message,
8327                    details,
8328                    recommended_action,
8329                } => {
8330                    return SyncOutcome::Failed {
8331                        details,
8332                        error_code,
8333                        message,
8334                        recommended_action,
8335                    };
8336                }
8337                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8338                SyncOutcome::BudgetExhausted(_) => {
8339                    unreachable!("sync() never exhausts a round budget")
8340                }
8341                SyncOutcome::Ok(report) => {
8342                    if max_rounds.is_none()
8343                        && report.segment_rows_applied > 0
8344                        && !report.bootstrapping.is_empty()
8345                        && before != self.resume_tokens()
8346                    {
8347                        spent = 0;
8348                    }
8349                    // TS parity: a disabled transport or a schema floor ends
8350                    // the loop. The latest round still merges first.
8351                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8352                    aggregate.merge(&report);
8353                    if stop {
8354                        return SyncOutcome::Ok(aggregate);
8355                    }
8356                    // §4.5: pull again whenever the response contained
8357                    // commits or segments; resets re-bootstrap; a pending
8358                    // resume token continues paging (§4.7); deferred outbox
8359                    // commits and a raised sync-needed signal push on the next
8360                    // round.
8361                    let more = !report.bootstrapping.is_empty()
8362                        || report.commits_applied > 0
8363                        || report.segment_rows_applied > 0
8364                        || !report.resets.is_empty()
8365                        || report.deferred_commits > 0
8366                        || self.sync_needed;
8367                    if !more {
8368                        return SyncOutcome::Ok(aggregate);
8369                    }
8370                }
8371            }
8372        }
8373        // §7.8: partial success. The aggregate report is retained and the run
8374        // is explicitly not idle.
8375        SyncOutcome::BudgetExhausted(aggregate)
8376    }
8377
8378    fn process_response(
8379        &mut self,
8380        transport: &mut dyn Transport,
8381        response: Message,
8382        meta: &RequestMeta,
8383    ) -> SyncOutcome {
8384        #[cfg(feature = "bench-internals")]
8385        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8386        let mut report = SyncReport {
8387            pushed: meta.pushed_ids.len() as u32,
8388            ..SyncReport::default()
8389        };
8390        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8391            HashMap::new();
8392        let pushed_ids = meta
8393            .pushed_ids
8394            .iter()
8395            .map(String::as_str)
8396            .collect::<HashSet<_>>();
8397        let mut last_final_push_result_id: Option<String> = None;
8398        for frame in &response.frames {
8399            match frame {
8400                Frame::PushResultDetails {
8401                    client_commit_id,
8402                    entries,
8403                } => {
8404                    let details = rejection_details_by_commit
8405                        .entry(client_commit_id.clone())
8406                        .or_default();
8407                    for entry in entries {
8408                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8409                            Ok(value) => value,
8410                            Err(message) => {
8411                                return SyncOutcome::Failed {
8412                                    details: None,
8413                                    error_code: "sync.invalid_request".to_owned(),
8414                                    message,
8415                                    recommended_action: None,
8416                                };
8417                            }
8418                        };
8419                        details.insert(entry.op_index, parsed);
8420                    }
8421                }
8422                Frame::PushResult {
8423                    client_commit_id,
8424                    status,
8425                    results,
8426                    ..
8427                } if pushed_ids.contains(client_commit_id.as_str())
8428                    && Self::push_result_is_final(*status, results) =>
8429                {
8430                    last_final_push_result_id = Some(client_commit_id.clone());
8431                }
8432                _ => {}
8433            }
8434        }
8435        let mut frames = response.frames.into_iter().peekable();
8436        match frames.next() {
8437            Some(Frame::RespHeader {
8438                required_schema_version,
8439                latest_schema_version,
8440                log_epoch,
8441                reset_required,
8442            }) => {
8443                if let Some(required) = required_schema_version {
8444                    // §1.6 schema-floor response: nothing else is processed;
8445                    // stop syncing and surface the upgrade requirement.
8446                    let floor = SchemaFloor {
8447                        required_schema_version: Some(required),
8448                        latest_schema_version,
8449                    };
8450                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8451                        return SyncOutcome::Failed {
8452                            details: None,
8453                            error_code: "storage.failed".to_owned(),
8454                            message,
8455                            recommended_action: None,
8456                        };
8457                    }
8458                    report.schema_floor = Some(floor);
8459                    return SyncOutcome::Ok(report);
8460                }
8461                if reset_required {
8462                    if frames.next().is_some() {
8463                        return SyncOutcome::Failed {
8464                            details: None,
8465                            error_code: "client.invalid_host_response".to_owned(),
8466                            message: "log-epoch reset response contains body frames".to_owned(),
8467                            recommended_action: None,
8468                        };
8469                    }
8470                    match self.run_log_epoch_reset(&log_epoch) {
8471                        Ok(resets) => {
8472                            report.resets = resets;
8473                            return SyncOutcome::Ok(report);
8474                        }
8475                        Err(message) => {
8476                            return SyncOutcome::Failed {
8477                                details: None,
8478                                error_code: "sync.local_corrupt".to_owned(),
8479                                message,
8480                                recommended_action: None,
8481                            };
8482                        }
8483                    }
8484                }
8485                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8486                    Ok(stored) => stored,
8487                    Err(message) => {
8488                        return SyncOutcome::Failed {
8489                            details: None,
8490                            error_code: "storage.failed".to_owned(),
8491                            message,
8492                            recommended_action: None,
8493                        };
8494                    }
8495                };
8496                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8497                    return SyncOutcome::Failed {
8498                        details: None,
8499                        error_code: "client.invalid_host_response".to_owned(),
8500                        message: "server changed logEpoch without requiring a reset".to_owned(),
8501                        recommended_action: None,
8502                    };
8503                }
8504            }
8505            _ => {
8506                return SyncOutcome::Failed {
8507                    details: None,
8508                    error_code: "sync.invalid_request".to_owned(),
8509                    message: "response does not start with RESP_HEADER".to_owned(),
8510                    recommended_action: None,
8511                };
8512            }
8513        }
8514
8515        if let Err(message) = self.begin_coalesced_apply() {
8516            return SyncOutcome::Failed {
8517                details: None,
8518                error_code: "storage.failed".into(),
8519                message,
8520                recommended_action: None,
8521            };
8522        }
8523        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8524        // §4.5 client application: the units whose persisted echo a live
8525        // change must match, fixed at LIVE_START after every SUB_END persisted.
8526        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8527        let mut live_ended = false;
8528        while let Some(frame) = frames.next() {
8529            match frame {
8530                frame @ Frame::PushResult { .. } => {
8531                    let successful = matches!(
8532                        &frame,
8533                        Frame::PushResult {
8534                            status: PushStatus::Applied | PushStatus::Cached,
8535                            ..
8536                        }
8537                    );
8538                    let mut results = vec![frame];
8539                    if successful {
8540                        while matches!(
8541                            frames.peek(),
8542                            Some(Frame::PushResult {
8543                                status: PushStatus::Applied | PushStatus::Cached,
8544                                ..
8545                            })
8546                        ) {
8547                            if let Some(next) = frames.next() {
8548                                results.push(next);
8549                            }
8550                        }
8551                    }
8552                    if self
8553                        .handle_push_results(
8554                            &results,
8555                            &pushed_ids,
8556                            &rejection_details_by_commit,
8557                            &mut report,
8558                            last_final_push_result_id.as_deref(),
8559                        )
8560                        .is_err()
8561                    {
8562                        failure = Some((
8563                            "client.outcome_persistence_failed".to_owned(),
8564                            "local commit outcome could not be persisted".to_owned(),
8565                            None,
8566                            None,
8567                        ));
8568                        break;
8569                    }
8570                }
8571                Frame::PushResultDetails { .. } => {}
8572                Frame::SubStart {
8573                    id,
8574                    status,
8575                    reason_code,
8576                    bootstrap,
8577                    fenced,
8578                } => {
8579                    // §1.4 rule 5: an ERROR closes the section; its body still
8580                    // applied, its SUB_END values are never persisted.
8581                    let mut body = Vec::new();
8582                    let mut sub_end: Option<(i64, Option<String>)> = None;
8583                    let mut error: Option<Frame> = None;
8584                    for inner in frames.by_ref() {
8585                        match inner {
8586                            Frame::SubEnd {
8587                                next_cursor,
8588                                bootstrap_state,
8589                            } => {
8590                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8591                                break;
8592                            }
8593                            inner @ Frame::Error { .. } => {
8594                                error = Some(inner);
8595                                break;
8596                            }
8597                            Frame::Unknown { .. } => {}
8598                            other => body.push(other),
8599                        }
8600                    }
8601                    let ended = sub_end.is_some();
8602                    let section = SectionStart {
8603                        id,
8604                        status,
8605                        reason_code,
8606                        bootstrap,
8607                        fenced,
8608                    };
8609                    match self.process_section(
8610                        transport,
8611                        &section,
8612                        body,
8613                        sub_end,
8614                        meta,
8615                        &mut report,
8616                    ) {
8617                        Err(SectionError::Abort(code, message)) => {
8618                            failure = Some((code, message, None, None));
8619                            break;
8620                        }
8621                        Err(SectionError::Transport(error)) => {
8622                            failure = Some((error.code, error.message, None, error.details));
8623                            break;
8624                        }
8625                        _ => {}
8626                    }
8627                    if ended {
8628                        // The counters stay: `complete` and `failed` report them.
8629                        self.progress.update(|p| {
8630                            p.unit_id = None;
8631                            p.table = None;
8632                            p.segment_id = None;
8633                        });
8634                    }
8635                    match error {
8636                        Some(Frame::Error {
8637                            code,
8638                            message,
8639                            recommended_action,
8640                            details,
8641                            ..
8642                        }) => {
8643                            failure = Some(Self::error_frame_failure(
8644                                code,
8645                                message,
8646                                recommended_action,
8647                                details,
8648                            ));
8649                            break;
8650                        }
8651                        Some(_) => unreachable!("only an ERROR closes a section early"),
8652                        None if !ended => {
8653                            failure = Some((
8654                                "sync.invalid_request".to_owned(),
8655                                "unit section without SUB_END".to_owned(),
8656                                None,
8657                                None,
8658                            ));
8659                            break;
8660                        }
8661                        None => {}
8662                    }
8663                }
8664                Frame::LiveStart {
8665                    status,
8666                    reason_code,
8667                } => {
8668                    let receivers = match self.stored(load_units(&self.conn)) {
8669                        Ok(units) => units
8670                            .into_iter()
8671                            .filter(|unit| {
8672                                unit.status == UnitStatus::Active
8673                                    && unit.echo.is_some()
8674                                    && !unit.bootstrapping()
8675                            })
8676                            .collect(),
8677                        Err(message) => {
8678                            failure = Some(("storage.failed".to_owned(), message, None, None));
8679                            break;
8680                        }
8681                    };
8682                    live = Some((status, reason_code, receivers));
8683                }
8684                Frame::Commit {
8685                    commit_seq,
8686                    tables,
8687                    changes,
8688                    ..
8689                } => {
8690                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8691                        continue;
8692                    };
8693                    let receivers = receivers.clone();
8694                    match self.apply_live_commit(
8695                        &receivers,
8696                        &tables,
8697                        changes,
8698                        commit_seq,
8699                        AppliedServerCommitSource::Pull,
8700                    ) {
8701                        Ok(applied) => report.commits_applied += u32::from(applied),
8702                        Err(error) => {
8703                            failure = Some((error.code, error.message, None, error.details));
8704                            break;
8705                        }
8706                    }
8707                }
8708                Frame::LiveEnd {
8709                    through_seq,
8710                    echo_digest,
8711                } => {
8712                    let Some((status, reason_code, _)) = live.take() else {
8713                        continue;
8714                    };
8715                    live_ended = true;
8716                    live = Some((status, String::new(), Vec::new()));
8717                    if let Err(message) = self.finish_live_section(
8718                        status,
8719                        &reason_code,
8720                        through_seq,
8721                        echo_digest,
8722                        &meta.registration_digest,
8723                        &mut report,
8724                    ) {
8725                        failure = Some(("storage.failed".to_owned(), message, None, None));
8726                        break;
8727                    }
8728                }
8729                Frame::Lease {
8730                    lease_id,
8731                    expires_at_ms,
8732                } => {
8733                    // §7.3.5: persist the opaque lease; a fresh lease clears
8734                    // any prior lease error (the outage/revocation is over).
8735                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8736                        lease_id: Some(lease_id),
8737                        expires_at_ms: Some(expires_at_ms),
8738                        error_code: None,
8739                    })) {
8740                        failure = Some(("storage.failed".to_owned(), message, None, None));
8741                        break;
8742                    }
8743                }
8744                Frame::Error {
8745                    code,
8746                    message,
8747                    recommended_action,
8748                    details,
8749                    ..
8750                } => {
8751                    // §1.6: the whole request failed; already-completed
8752                    // units keep their applied data and cursors.
8753                    failure = Some(Self::error_frame_failure(
8754                        code,
8755                        message,
8756                        recommended_action,
8757                        details,
8758                    ));
8759                    break;
8760                }
8761                Frame::Unknown { .. } => {}
8762                _ => {
8763                    failure = Some((
8764                        "sync.invalid_request".to_owned(),
8765                        "unexpected frame in response".to_owned(),
8766                        None,
8767                        None,
8768                    ));
8769                    break;
8770                }
8771            }
8772        }
8773
8774        let applied = match failure {
8775            Some(failure) => Err(failure),
8776            // Local apply boundaries already reconciled their affected overlay.
8777            // Refuse round success if any remaining dirty work fails to
8778            // reconcile. §4.8 E1: the push half may have drained commits that
8779            // pinned rows of a shrunk window unit, so retry deferred evictions.
8780            None => self
8781                .rebuild_overlay_if_dirty()
8782                .and_then(|()| self.drain_pending_evictions())
8783                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8784        };
8785        // §1.4 rule 4: frames completed before a failure stay committed; a
8786        // failed COMMIT of the coalesced transaction fails the round.
8787        if let Err(message) = self.finish_coalesced_apply() {
8788            return SyncOutcome::Failed {
8789                details: None,
8790                error_code: "storage.failed".into(),
8791                message,
8792                recommended_action: None,
8793            };
8794        }
8795        if let Err((error_code, message, recommended_action, details)) = applied {
8796            return SyncOutcome::Failed {
8797                details,
8798                error_code,
8799                message,
8800                recommended_action,
8801            };
8802        }
8803        let catching_up = match self.stored(load_units(&self.conn)) {
8804            Ok(units) => units
8805                .into_iter()
8806                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8807                .map(|unit| unit.id)
8808                .collect::<Vec<_>>(),
8809            Err(message) => {
8810                return SyncOutcome::Failed {
8811                    details: None,
8812                    error_code: "storage.failed".into(),
8813                    message,
8814                    recommended_action: None,
8815                }
8816            }
8817        };
8818        // §7.1: a response without LIVE_END, one answered `unregistered` or
8819        // `reset`, or a unit still catching up leaves sync-needed raised;
8820        // after `unregistered` the next round starts at once.
8821        let status = live.map(|(status, _, _)| status);
8822        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8823            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8824        }
8825        self.ack_after_pull(transport);
8826        // §7.4.5: the reset is over once the first post-reset pull round
8827        // leaves no unit catching up: the tables are rebuilt and current.
8828        if self.upgrading && catching_up.is_empty() {
8829            self.set_upgrading(false);
8830        }
8831        report.bootstrapping = catching_up;
8832        SyncOutcome::Ok(report)
8833    }
8834
8835    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8836    fn error_frame_failure(
8837        code: String,
8838        message: String,
8839        recommended_action: String,
8840        details: Option<RawJson>,
8841    ) -> (String, String, Option<String>, Option<Value>) {
8842        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8843        let details = details
8844            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8845            .filter(Value::is_object);
8846        (code, message, recommended_action, details)
8847    }
8848
8849    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8850    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8851    /// replica cursor, the echo digest and the digest of the list the server
8852    /// now stores.
8853    fn finish_live_section(
8854        &mut self,
8855        status: LiveStatus,
8856        reason_code: &str,
8857        through_seq: i64,
8858        echo_digest: Option<[u8; 32]>,
8859        registration_digest: &str,
8860        report: &mut SyncReport,
8861    ) -> Result<(), String> {
8862        let rollback = self.capture();
8863        let resets = report.resets.len();
8864        self.begin_observation("syncular_live_end")?;
8865        let finished = (|| {
8866            let mut batch = ChangeAccumulator::default();
8867            let before = self.stored(replica_cursor(&self.conn))?;
8868            let mut cursor = if status == LiveStatus::Unregistered {
8869                before
8870            } else {
8871                -1
8872            };
8873            let joined = before.max(through_seq);
8874            for unit in self.stored(load_units(&self.conn))? {
8875                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8876                    continue;
8877                }
8878                match status {
8879                    // §4.6 replica reset: every live unit bootstraps again;
8880                    // post-bootstrap units keep their cursors.
8881                    LiveStatus::Reset if unit.cursor.is_none() => {
8882                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8883                            batch.window(&base_key, &value);
8884                        }
8885                        report.resets.push(unit.id.clone());
8886                        self.stored(save_unit(
8887                            &self.conn,
8888                            &UnitRecord {
8889                                reason_code: Some(reason_code.to_owned()),
8890                                cursor: Some(-1),
8891                                seeded: Vec::new(),
8892                                ..unit
8893                            },
8894                        ))?;
8895                    }
8896                    // §4.5 join rule J.
8897                    LiveStatus::Active => {
8898                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8899                        if next >= joined {
8900                            cursor = joined;
8901                        }
8902                        if unit.cursor.is_some() {
8903                            self.stored(save_unit(
8904                                &self.conn,
8905                                &UnitRecord {
8906                                    cursor: (next < joined).then_some(next),
8907                                    ..unit
8908                                },
8909                            ))?;
8910                        }
8911                    }
8912                    _ => {}
8913                }
8914            }
8915            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
8916            self.set_meta(
8917                ECHO_DIGEST_KEY,
8918                &echo_digest
8919                    .map(|digest| bytes_to_hex(&digest))
8920                    .unwrap_or_default(),
8921            )?;
8922            if status == LiveStatus::Unregistered {
8923                self.delete_meta(REGISTERED_DIGEST_KEY)?;
8924            } else {
8925                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
8926            }
8927            self.finish_observation("syncular_live_end", batch)
8928        })();
8929        if finished.is_err() {
8930            self.rollback_observation("syncular_live_end");
8931            self.restore(rollback);
8932            report.resets.truncate(resets);
8933        }
8934        finished
8935    }
8936
8937    // -- push results (§6.3, §7.2) ------------------------------------------------
8938
8939    fn handle_push_results(
8940        &mut self,
8941        frames: &[Frame],
8942        pushed_ids: &HashSet<&str>,
8943        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
8944        report: &mut SyncReport,
8945        last_final_push_result_id: Option<&str>,
8946    ) -> Result<(), String> {
8947        self.begin_observation("syncular_push_results")?;
8948        let rollback = self.capture();
8949        let conflict_count = self.conflicts.len();
8950        let applied_count = report.applied.len();
8951        let rejected_count = report.rejected.len();
8952        let retryable_count = report.retryable.len();
8953        let mut drained = Vec::new();
8954        let mut batch = ChangeAccumulator::default();
8955        let persisted = (|| {
8956            for frame in frames {
8957                let Frame::PushResult {
8958                    client_commit_id,
8959                    status,
8960                    results,
8961                    commit_seq,
8962                } = frame
8963                else {
8964                    return Err("unexpected acknowledgement frame".to_owned());
8965                };
8966                if !pushed_ids.contains(client_commit_id.as_str()) {
8967                    continue;
8968                }
8969                let Some(index) = self
8970                    .outbox
8971                    .iter()
8972                    .position(|commit| commit.client_commit_id == *client_commit_id)
8973                else {
8974                    continue;
8975                };
8976                let status = *status;
8977                let rejection_details = rejection_details_by_commit.get(client_commit_id);
8978                let operations = self.outbox[index].ops.clone();
8979                match status {
8980                    PushStatus::Applied | PushStatus::Cached => {
8981                        // §7.2: a lost ack replays as `cached` — proceed as if the
8982                        // ack had arrived.
8983                        let sequence =
8984                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
8985                        let affected: HashSet<_> = operations
8986                            .iter()
8987                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
8988                            .collect();
8989                        let prior = self.acknowledged_operations(Some(&affected), None)?;
8990                        for (op_index, operation) in operations.iter().enumerate() {
8991                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
8992                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
8993                            if delivered {
8994                                continue;
8995                            }
8996                            let table = self
8997                                .schema
8998                                .table(&operation.table)
8999                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9000                            let mut intended: Map<String, Value> = self
9001                                .stored_row(table, &operation.row_id, true)?
9002                                .map(|(row, _)| {
9003                                    table
9004                                        .columns
9005                                        .iter()
9006                                        .zip(&row)
9007                                        .map(|(column, value)| {
9008                                            (column.name.clone(), column_value_to_json(value))
9009                                        })
9010                                        .collect()
9011                                })
9012                                .unwrap_or_default();
9013                            let mut scope_image = intended.clone();
9014                            for op in prior
9015                                .iter()
9016                                .chain(
9017                                    self.failed_commits
9018                                        .iter()
9019                                        .flat_map(|(commit, _)| &commit.ops),
9020                                )
9021                                .chain(
9022                                    self.outbox
9023                                        .iter()
9024                                        .take(index)
9025                                        .flat_map(|commit| &commit.ops),
9026                                )
9027                                .chain(operations.iter().take(op_index + 1))
9028                            {
9029                                if op.table != operation.table || op.row_id != operation.row_id {
9030                                    continue;
9031                                }
9032                                if let Some(values) = &op.values {
9033                                    intended.extend(values.clone());
9034                                } else {
9035                                    scope_image = intended.clone();
9036                                    intended.clear();
9037                                }
9038                            }
9039                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9040                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9041                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9042                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9043                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9044                        }
9045                        let journal_results = results
9046                            .iter()
9047                            .map(|result| {
9048                                let op_index = match result {
9049                                    OpResult::Applied { op_index }
9050                                    | OpResult::Conflict { op_index, .. }
9051                                    | OpResult::Error { op_index, .. } => *op_index,
9052                                };
9053                                CommitOperationOutcome::Applied { op_index }
9054                            })
9055                            .collect::<Vec<_>>();
9056                        let outcome_status = if status == PushStatus::Applied {
9057                            CommitOutcomeStatus::Applied
9058                        } else {
9059                            CommitOutcomeStatus::Cached
9060                        };
9061                        let persisted = self
9062                            .persist_commit_outcome(
9063                                client_commit_id,
9064                                outcome_status,
9065                                &journal_results,
9066                                None,
9067                            )
9068                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9069                        persisted?;
9070                        self.sync_needed = true;
9071                        batch.status = true;
9072                        batch.outcomes = true;
9073                    }
9074                    PushStatus::Rejected => {
9075                        if results.iter().any(|result| {
9076                            matches!(
9077                                result,
9078                                OpResult::Error {
9079                                    code,
9080                                    retryable: true,
9081                                    ..
9082                                } if code == "sync.idempotency_cache_miss"
9083                            )
9084                        }) {
9085                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9086                            // exact commit queued for an identical retry.
9087                            report.retryable.push(client_commit_id.to_owned());
9088                            continue;
9089                        }
9090
9091                        let mut journal_results = Vec::with_capacity(results.len());
9092                        let mut conflicts = Vec::new();
9093                        let mut rejections = Vec::new();
9094                        for result in results {
9095                            match result {
9096                                OpResult::Applied { op_index } => {
9097                                    journal_results.push(CommitOperationOutcome::Applied {
9098                                        op_index: *op_index,
9099                                    });
9100                                }
9101                                OpResult::Conflict {
9102                                    op_index,
9103                                    code,
9104                                    message,
9105                                    server_version,
9106                                    server_row,
9107                                    conflict_columns: conflict_columns_bitmap,
9108                                } => {
9109                                    let operation = operations
9110                                        .get(*op_index as usize)
9111                                        .map(CommitOperation::from);
9112                                    let (table, row_id) = operation
9113                                        .as_ref()
9114                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9115                                        .unwrap_or_default();
9116                                    let server_row_json = self
9117                                        .schema
9118                                        .table(&table)
9119                                        .and_then(|t| {
9120                                            decode_row_bytes(t, server_row, &self.encryption)
9121                                                .ok()
9122                                                .map(|(row, _)| (t, row))
9123                                        })
9124                                        .map(|(t, row)| {
9125                                            let mut map = Map::new();
9126                                            for (i, column) in t.columns.iter().enumerate() {
9127                                                map.insert(
9128                                                    column.name.clone(),
9129                                                    column_value_to_json(
9130                                                        row.get(i).unwrap_or(&None),
9131                                                    ),
9132                                                );
9133                                            }
9134                                            map
9135                                        })
9136                                        .unwrap_or_default();
9137                                    // §6.3: decode the presence-layout bitmap
9138                                    // into the marked column names.
9139                                    let conflict_columns = self
9140                                        .schema
9141                                        .table(&table)
9142                                        .map(|t| {
9143                                            t.columns
9144                                                .iter()
9145                                                .enumerate()
9146                                                .filter(|(index, _)| {
9147                                                    conflict_columns_bitmap
9148                                                        .get(index / 8)
9149                                                        .is_some_and(|byte| {
9150                                                            byte & (1 << (index % 8)) != 0
9151                                                        })
9152                                                })
9153                                                .map(|(_, column)| column.name.clone())
9154                                                .collect::<Vec<_>>()
9155                                        })
9156                                        .unwrap_or_default();
9157                                    let conflict = ConflictRecord {
9158                                        client_commit_id: client_commit_id.to_owned(),
9159                                        op_index: *op_index,
9160                                        table,
9161                                        row_id,
9162                                        code: code.clone(),
9163                                        message: message.clone(),
9164                                        server_version: *server_version,
9165                                        server_row: server_row_json,
9166                                        conflict_columns,
9167                                        operation,
9168                                    };
9169                                    journal_results.push(CommitOperationOutcome::Conflict {
9170                                        conflict: conflict.clone(),
9171                                    });
9172                                    conflicts.push(conflict);
9173                                }
9174                                OpResult::Error {
9175                                    op_index,
9176                                    code,
9177                                    message,
9178                                    retryable,
9179                                } => {
9180                                    let rejection = RejectionRecord {
9181                                        client_commit_id: client_commit_id.to_owned(),
9182                                        op_index: *op_index,
9183                                        code: code.clone(),
9184                                        message: message.clone(),
9185                                        retryable: *retryable,
9186                                        details: rejection_details
9187                                            .and_then(|details| details.get(op_index))
9188                                            .cloned(),
9189                                        operation: operations
9190                                            .get(*op_index as usize)
9191                                            .map(CommitOperation::from),
9192                                    };
9193                                    journal_results.push(CommitOperationOutcome::Error {
9194                                        rejection: rejection.clone(),
9195                                    });
9196                                    rejections.push(rejection);
9197                                }
9198                            }
9199                        }
9200                        if self.retain_failed_commits {
9201                            self.retain_failed_commit(client_commit_id, &operations)?;
9202                        }
9203                        let outcome_status = if conflicts.is_empty() {
9204                            CommitOutcomeStatus::Rejected
9205                        } else {
9206                            CommitOutcomeStatus::Conflict
9207                        };
9208                        let persisted = self
9209                            .persist_commit_outcome(
9210                                client_commit_id,
9211                                outcome_status,
9212                                &journal_results,
9213                                Some(&operations),
9214                            )
9215                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9216                        persisted?;
9217                        batch.conflicts = !conflicts.is_empty();
9218                        batch.rejections = !rejections.is_empty();
9219                        batch.status = true;
9220                        batch.outcomes = true;
9221                        self.conflicts.extend(conflicts);
9222                        self.rejections.extend(rejections);
9223                    }
9224                }
9225                drained.push(self.outbox.remove(index));
9226                if batch.status {
9227                    for operation in &operations {
9228                        if !self.record_row_scopes(
9229                            &mut batch,
9230                            &operation.table,
9231                            &operation.row_id,
9232                            false,
9233                        ) {
9234                            batch.table(&operation.table);
9235                        }
9236                    }
9237                }
9238                match status {
9239                    PushStatus::Applied | PushStatus::Cached => {
9240                        report.applied.push(client_commit_id.to_owned())
9241                    }
9242                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9243                }
9244            }
9245            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9246                self.prune_commit_outcomes()?;
9247            }
9248            self.reconcile_rows(&drained)?;
9249            self.rebuild_overlay_if_dirty()?;
9250            self.finish_observation("syncular_push_results", batch)
9251        })();
9252        if let Err(error) = persisted {
9253            self.rollback_observation("syncular_push_results");
9254            self.restore(rollback);
9255            report.applied.truncate(applied_count);
9256            report.rejected.truncate(rejected_count);
9257            report.retryable.truncate(retryable_count);
9258            return Err(error);
9259        }
9260        if !rollback.sync_needed && self.sync_needed {
9261            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9262        }
9263        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9264        Ok(())
9265    }
9266
9267    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9268        status != PushStatus::Rejected
9269            || !results.iter().any(|result| {
9270                matches!(
9271                    result,
9272                    OpResult::Error {
9273                        code,
9274                        retryable: true,
9275                        ..
9276                    } if code == "sync.idempotency_cache_miss"
9277                )
9278            })
9279    }
9280
9281    // -- unit sections and the live section ------------------------------------------
9282
9283    /// One unit section (§4.4): apply its segments, then persist its
9284    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9285    /// resume token). A section of a unit the app released while the round was
9286    /// in flight is skipped; a fail-closed first page fails the unit and skips
9287    /// the rest of its section (§5.6).
9288    fn process_section(
9289        &mut self,
9290        transport: &mut dyn Transport,
9291        start: &SectionStart,
9292        body: Vec<Frame>,
9293        sub_end: Option<(i64, Option<String>)>,
9294        meta: &RequestMeta,
9295        report: &mut SyncReport,
9296    ) -> Result<(), SectionError> {
9297        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9298        let unit = self
9299            .stored(get_unit(&self.conn, &start.id))
9300            .map_err(storage)?
9301            .filter(|unit| unit.status == UnitStatus::Active);
9302        // §7.7: a section boundary ends the previous payload.
9303        self.progress.update(|p| {
9304            p.phase = ProgressPhase::Import;
9305            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9306            p.table = None;
9307            p.segment_id = None;
9308            p.bytes_received = 0;
9309            p.bytes_total = None;
9310            p.rows_processed = 0;
9311            p.rows_total = None;
9312            p.import_write_ms = None;
9313            p.import_commit_ms = None;
9314        });
9315        let Some(unit) = unit else {
9316            return Ok(());
9317        };
9318        let echo: Vec<EchoRecord> = start
9319            .fenced
9320            .iter()
9321            .map(|record| EchoRecord {
9322                table: record.table.clone(),
9323                status: record.status.name().to_owned(),
9324                scopes: record.scopes.iter().cloned().collect(),
9325            })
9326            .collect();
9327        // §5.6: the applied-segment record ends when the presented cursor or
9328        // token changes.
9329        if self
9330            .applied_segments
9331            .get(&unit.id)
9332            .is_some_and(|(cursor, token, _)| {
9333                (*cursor, token) != (unit.cursor, &unit.bootstrap_state)
9334            })
9335        {
9336            self.applied_segments.remove(&unit.id);
9337        }
9338        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9339        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9340            BTreeSet::new()
9341        } else {
9342            unit.seeded.iter().cloned().collect()
9343        };
9344        let applied = (|| {
9345            for frame in body {
9346                match frame {
9347                    Frame::SegmentInline { payload } => {
9348                        let segment = decode_rows_segment(&payload).map_err(|e| {
9349                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9350                        })?;
9351                        self.progress.update(|p| {
9352                            p.phase = ProgressPhase::Import;
9353                            p.unit_id = Some(unit.id.clone());
9354                            p.table = Some(segment.table.clone());
9355                            p.segment_id = None;
9356                            p.bytes_received = payload.len() as u64;
9357                            p.bytes_total = Some(payload.len() as u64);
9358                            p.rows_processed = 0;
9359                            p.rows_total =
9360                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9361                            p.import_write_ms = None;
9362                            p.import_commit_ms = None;
9363                        });
9364                        let (pair, clear, effective) =
9365                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9366                        report.segment_rows_applied +=
9367                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9368                        seeded.insert(segment.table);
9369                    }
9370                    Frame::SegmentRef {
9371                        segment_id,
9372                        byte_length,
9373                        media_type,
9374                        table,
9375                        row_count,
9376                        as_of_commit_seq,
9377                        scope_digest,
9378                        row_cursor,
9379                        next_row_cursor,
9380                        url,
9381                        url_expires_at_ms,
9382                        ..
9383                    } => {
9384                        // §4.2: reject a descriptor whose mediaType was not
9385                        // advertised — never skip or guess.
9386                        let advertised = match media_type {
9387                            MediaType::Rows => {
9388                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9389                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9390                            }
9391                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9392                        };
9393                        if !advertised {
9394                            return Err(SectionError::Abort(
9395                                "sync.invalid_request".to_owned(),
9396                                format!(
9397                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9398                                    media_type.name()
9399                                ),
9400                            ));
9401                        }
9402                        // §5.6 applied-segment rule.
9403                        if meta
9404                            .applied
9405                            .get(&unit.id)
9406                            .is_some_and(|ids| ids.contains(&segment_id))
9407                        {
9408                            seeded.insert(table);
9409                            continue;
9410                        }
9411                        if meta
9412                            .deferred
9413                            .contains(&(unit.id.clone(), segment_id.clone()))
9414                        {
9415                            return Err(SectionError::Deferred);
9416                        }
9417                        self.progress.update(|p| {
9418                            p.phase = ProgressPhase::Download;
9419                            p.unit_id = Some(unit.id.clone());
9420                            p.table = Some(table.clone());
9421                            p.segment_id = Some(segment_id.clone());
9422                            p.bytes_received = 0;
9423                            p.bytes_total = u64::try_from(byte_length).ok();
9424                            p.rows_processed = 0;
9425                            p.rows_total = u64::try_from(row_count).ok();
9426                            p.import_write_ms = None;
9427                            p.import_commit_ms = None;
9428                        });
9429                        let observer = self.progress.clone();
9430                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9431                        let bytes = if let Some(url) = url {
9432                            // §5.4: a url-carrying descriptor MUST be fetched
9433                            // from that URL; failure invalidates the whole
9434                            // descriptor (no fall-through to §5.5 — re-pull
9435                            // recovers, §1.4 rule 5).
9436                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9437                                return Err(SectionError::Abort(
9438                                    "sync.invalid_request".to_owned(),
9439                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9440                                        .to_owned(),
9441                                ));
9442                            }
9443                            // §5.4: MUST NOT start a fetch at/past expiry.
9444                            if url_expires_at_ms.is_some_and(|exp| {
9445                                exp <= transport
9446                                    .segment_fetch_started_at(&url)
9447                                    .unwrap_or_else(|| self.clock_now_ms())
9448                            }) {
9449                                return Err(SectionError::Abort(
9450                                    "sync.segment_expired".to_owned(),
9451                                    format!(
9452                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9453                                    ),
9454                                ));
9455                            }
9456                            transport
9457                                .fetch_url(
9458                                    &url,
9459                                    u64::try_from(byte_length).unwrap_or(0),
9460                                    &mut on_progress,
9461                                )
9462                                .map_err(|e| self.segment_fetch_failed(e))?
9463                        } else {
9464                            // §5.5: the owning unit's scope map, each value a one-element list.
9465                            let requested_scopes_json =
9466                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9467                            transport
9468                                .download_segment(
9469                                    &SegmentRequest {
9470                                        segment_id: segment_id.clone(),
9471                                        table: table.clone(),
9472                                        requested_scopes_json,
9473                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9474                                    },
9475                                    &mut on_progress,
9476                                )
9477                                .map_err(|e| {
9478                                    // §5.5: access changed since the response;
9479                                    // the next round's SUB_START settles the unit.
9480                                    if e.code == "sync.forbidden" {
9481                                        SectionError::Refused
9482                                    } else {
9483                                        self.segment_fetch_failed(e)
9484                                    }
9485                                })?
9486                        };
9487                        // §5.4: the descriptor's byteLength caps the body before
9488                        // §5.1 verifies the content address; the client discards
9489                        // the segment and re-pulls on either failure.
9490                        if bytes.len() as i64 > byte_length {
9491                            self.schedule_background_retry();
9492                            return Err(SectionError::Abort(
9493                                "sync.invalid_request".to_owned(),
9494                                "segment exceeds its byteLength".to_owned(),
9495                            ));
9496                        }
9497                        on_progress(bytes.len() as u64);
9498                        let digest = Sha256::digest(&bytes);
9499                        let expected = segment_id
9500                            .strip_prefix("sha256:")
9501                            .unwrap_or(segment_id.as_str());
9502                        if bytes_to_hex(&digest) != expected {
9503                            // §5.1: the client discards the segment and re-pulls.
9504                            self.schedule_background_retry();
9505                            return Err(SectionError::Abort(
9506                                "sync.invalid_request".to_owned(),
9507                                "segment bytes do not match the content address (§5.1)".to_owned(),
9508                            ));
9509                        }
9510                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9511                        if media_type == MediaType::Sqlite {
9512                            // §5.3: a range image keeps its position in the resume
9513                            // token; a descriptor row cursor is invalid.
9514                            if row_cursor.is_some() || next_row_cursor.is_some() {
9515                                return Err(SectionError::Abort(
9516                                    "sync.invalid_request".to_owned(),
9517                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9518                                        .to_owned(),
9519                                ));
9520                            }
9521                            let (pair, clear, effective) =
9522                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9523                            report.segment_rows_applied += self.apply_sqlite_segment(
9524                                &pair,
9525                                &bytes,
9526                                clear,
9527                                &effective,
9528                                &unit.id,
9529                                row_count,
9530                                as_of_commit_seq,
9531                                &scope_digest,
9532                            )?;
9533                            seeded.insert(table);
9534                        } else {
9535                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9536                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9537                            })?;
9538                            let (pair, clear, effective) =
9539                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9540                            report.segment_rows_applied +=
9541                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9542                            seeded.insert(segment.table);
9543                        }
9544                        self.applied_segments
9545                            .entry(unit.id.clone())
9546                            .or_insert_with(|| {
9547                                (unit.cursor, unit.bootstrap_state.clone(), BTreeSet::new())
9548                            })
9549                            .2
9550                            .insert(segment_id);
9551                    }
9552                    _ => {
9553                        return Err(SectionError::Abort(
9554                            "sync.invalid_request".to_owned(),
9555                            "unexpected frame inside a unit section".to_owned(),
9556                        ));
9557                    }
9558                }
9559            }
9560            Ok(())
9561        })();
9562        match applied {
9563            Ok(()) => {}
9564            // The unit keeps catching up with its cursor unpersisted.
9565            Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9566            Err(SectionError::FailClosed) => {
9567                // §5.6: the failure is unit-local; the rest of the response
9568                // still applies and the failed unit leaves the list.
9569                self.begin_observation("syncular_section_failure")
9570                    .map_err(storage)?;
9571                let failed = (|| {
9572                    let mut batch = ChangeAccumulator::default();
9573                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9574                        batch.window(&base_key, &value);
9575                    }
9576                    batch.status = true;
9577                    self.stored(save_unit(
9578                        &self.conn,
9579                        &UnitRecord {
9580                            status: UnitStatus::Failed,
9581                            reason_code: Some("sync.scope_revoked".to_owned()),
9582                            ..unit.clone()
9583                        },
9584                    ))?;
9585                    self.finish_observation("syncular_section_failure", batch)
9586                })();
9587                if let Err(message) = failed {
9588                    self.rollback_observation("syncular_section_failure");
9589                    return Err(storage(message));
9590                }
9591                report.failed.push(unit.id);
9592                return Ok(());
9593            }
9594            Err(error) => return Err(error),
9595        }
9596        let Some((next_cursor, bootstrap_state)) = sub_end else {
9597            return Ok(());
9598        };
9599        let rollback = self.capture();
9600        let counts = (
9601            report.resets.len(),
9602            report.revoked.len(),
9603            report.failed.len(),
9604        );
9605        self.begin_observation("syncular_sub_end")
9606            .map_err(storage)?;
9607        let finished = self
9608            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9609            .and_then(|batch| {
9610                self.rebuild_overlay_if_dirty()?;
9611                self.finish_observation("syncular_sub_end", batch)
9612            });
9613        if let Err(message) = finished {
9614            self.rollback_observation("syncular_sub_end");
9615            self.restore(rollback);
9616            report.resets.truncate(counts.0);
9617            report.revoked.truncate(counts.1);
9618            report.failed.truncate(counts.2);
9619            return Err(storage(message));
9620        }
9621        self.applied_segments.remove(&unit.id);
9622        Ok(())
9623    }
9624
9625    /// §5.6: the table, first-page flag and effective scopes of one segment
9626    /// of `unit`'s section.
9627    #[allow(clippy::type_complexity)]
9628    fn segment_pair(
9629        &self,
9630        unit: &UnitRecord,
9631        echo: &[EchoRecord],
9632        seeded: &BTreeSet<String>,
9633        name: &str,
9634    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9635        let table = self.schema.table(name).cloned().ok_or_else(|| {
9636            SectionError::Abort(
9637                "sync.schema_mismatch".to_owned(),
9638                format!("segment table {name:?} is not in the client schema"),
9639            )
9640        })?;
9641        let effective = covers(&table, &unit.scopes)
9642            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9643            .flatten()
9644            .ok_or_else(|| {
9645                SectionError::Abort(
9646                    "sync.invalid_request".to_owned(),
9647                    "a segment names no active pair of its unit".to_owned(),
9648                )
9649            })?;
9650        Ok((table, !seeded.contains(name), effective))
9651    }
9652
9653    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9654    /// caller's observation: a reset discards cursor, resume token and
9655    /// first-page record and keeps the echo; a revoked pair purges by its last
9656    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9657    /// fenced record that turns active again re-bootstraps the unit, and a
9658    /// bootstrap section persists its pin, resume token and seeded tables.
9659    fn finish_section(
9660        &mut self,
9661        start: &SectionStart,
9662        echo: Vec<EchoRecord>,
9663        next_cursor: i64,
9664        bootstrap_state: Option<String>,
9665        seeded: BTreeSet<String>,
9666        report: &mut SyncReport,
9667    ) -> Result<ChangeAccumulator, String> {
9668        #[cfg(feature = "bench-internals")]
9669        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9670        let mut batch = ChangeAccumulator::default();
9671        // §4.4: SUB_END writes only into a unit that is still registered.
9672        let Some(unit) = self
9673            .stored(get_unit(&self.conn, &start.id))?
9674            .filter(|unit| unit.status == UnitStatus::Active)
9675        else {
9676            return Ok(batch);
9677        };
9678        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9679            batch.window(&base_key, &value);
9680        }
9681        if start.status == SubStatus::Reset {
9682            // §4.6: staleness, never a purge; the rows stay until the
9683            // bootstrap replaces them.
9684            report.resets.push(unit.id.clone());
9685            self.stored(save_unit(
9686                &self.conn,
9687                &UnitRecord {
9688                    reason_code: Some(start.reason_code.clone()),
9689                    cursor: Some(-1),
9690                    bootstrap_state: None,
9691                    seeded: Vec::new(),
9692                    ..unit
9693                },
9694            ))?;
9695            return Ok(batch);
9696        }
9697        let revoked = start.status == SubStatus::Revoked;
9698        let mut units = None;
9699        let mut failed = false;
9700        let mut rearmed = false;
9701        for table in self.schema.tables.clone() {
9702            if !covers(&table, &unit.scopes) {
9703                continue;
9704            }
9705            let last = unit
9706                .echo
9707                .as_deref()
9708                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9709            let next = (!revoked)
9710                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9711                .flatten();
9712            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9713            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9714            // §3.3: purge a pair the response revokes, keyed on the echo the
9715            // client persisted while the pair was active.
9716            if let (Some(last), None) = (&last, &next) {
9717                let units = match units {
9718                    Some(ref units) => units,
9719                    None => units.insert(self.stored(load_units(&self.conn))?),
9720                };
9721                let owners = unit_owners(units, &table, &unit.id);
9722                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9723            }
9724        }
9725        if revoked || failed {
9726            if revoked {
9727                report.revoked.push(unit.id.clone());
9728            }
9729            if failed {
9730                report.failed.push(unit.id.clone());
9731            }
9732            batch.status = true;
9733            self.stored(save_unit(
9734                &self.conn,
9735                &UnitRecord {
9736                    status: if failed {
9737                        UnitStatus::Failed
9738                    } else {
9739                        UnitStatus::Revoked
9740                    },
9741                    reason_code: Some(if failed {
9742                        "sync.scope_revoked".to_owned()
9743                    } else {
9744                        start.reason_code.clone()
9745                    }),
9746                    cursor: Some(-1),
9747                    bootstrap_state: None,
9748                    seeded: Vec::new(),
9749                    echo: None,
9750                    ..unit
9751                },
9752            ))?;
9753            return Ok(batch);
9754        }
9755        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9756        let (cursor, bootstrap_state, seeded) = if rearmed {
9757            (Some(-1), None, Vec::new())
9758        } else if start.bootstrap {
9759            let seeded = if bootstrap_state.is_none() {
9760                Vec::new()
9761            } else {
9762                seeded.into_iter().collect()
9763            };
9764            (Some(next_cursor), bootstrap_state, seeded)
9765        } else {
9766            (
9767                unit.cursor,
9768                unit.bootstrap_state.clone(),
9769                unit.seeded.clone(),
9770            )
9771        };
9772        self.stored(save_unit(
9773            &self.conn,
9774            &UnitRecord {
9775                reason_code: None,
9776                echo: Some(echo.clone()),
9777                cursor,
9778                bootstrap_state,
9779                seeded,
9780                ..unit.clone()
9781            },
9782        ))?;
9783        if !completed {
9784            return Ok(batch);
9785        }
9786        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9787        // at or below its pin.
9788        for table in self.schema.tables.clone() {
9789            let Some(effective) = covers(&table, &unit.scopes)
9790                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9791                .flatten()
9792            else {
9793                continue;
9794            };
9795            let mut statement = self
9796                .conn
9797                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9798                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9799            let rows = statement
9800                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9801                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9802                })
9803                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9804            let mut covered = HashSet::new();
9805            for row in rows {
9806                let (id, intent) =
9807                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9808                let values: Map<String, Value> =
9809                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9810                if owns_local_row(&table, &effective, &[], &values) {
9811                    covered.insert(id);
9812                }
9813            }
9814            drop(statement);
9815            for id in covered {
9816                self.conn
9817                    .execute(
9818                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9819                        rusqlite::params![table.name, id, next_cursor],
9820                    )
9821                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9822                self.overlay_dirty.table(&table.name);
9823                batch.table(&table.name);
9824            }
9825        }
9826        Ok(batch)
9827    }
9828
9829    /// §3.3: purge one revoked pair inside the caller's observation: delete
9830    /// the rows matching its last effective scopes that no owner holds, drop
9831    /// the retained intent and every pending commit that provably writes into
9832    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9833    /// table has no local scope-column mapping (fail closed: nothing deleted).
9834    fn purge_pair(
9835        &mut self,
9836        batch: &mut ChangeAccumulator,
9837        table: &TableSchema,
9838        last: &[(String, Vec<String>)],
9839        owners: &[Vec<(String, Vec<String>)>],
9840    ) -> Result<bool, String> {
9841        if local_scope_predicate(table, last, owners).is_err() {
9842            return Ok(false);
9843        }
9844        for commit in self.drop_failed_commits(|name, values| {
9845            name == table.name && owns_local_row(table, last, owners, values)
9846        })? {
9847            batch.conflicts = true;
9848            batch.rejections = true;
9849            batch.outcomes = true;
9850            for operation in commit.ops {
9851                batch.table(&operation.table);
9852            }
9853        }
9854        if !self.purge_scope_rows(&table.name, last, owners)? {
9855            return Err("storage.failed: the revoked scope purge failed".to_owned());
9856        }
9857        self.record_scope_map(batch, &table.name, last);
9858        if self.drop_doomed_outbox(table, last, owners)? {
9859            batch.status = true;
9860            batch.rejections = true;
9861            batch.outcomes = true;
9862        }
9863        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9864        self.delete_unreferenced_cached_blobs()?;
9865        Ok(true)
9866    }
9867
9868    /// §4.5 client application: apply one live-section or delta `COMMIT`
9869    /// with only the changes that match the persisted echo of a receiving
9870    /// unit; every applied change then goes through the version guard.
9871    /// Returns whether any change of the frame applied.
9872    fn apply_live_commit(
9873        &mut self,
9874        receivers: &[UnitRecord],
9875        tables: &[String],
9876        changes: Vec<ssp2::model::Change>,
9877        commit_seq: i64,
9878        source: AppliedServerCommitSource,
9879    ) -> Result<bool, TransportError> {
9880        let changes: Vec<ssp2::model::Change> = changes
9881            .into_iter()
9882            .filter(|change| {
9883                // An unknown table fails loud in the apply below.
9884                let Some(table) = tables
9885                    .get(change.table_index as usize)
9886                    .and_then(|name| self.schema.table(name))
9887                else {
9888                    return true;
9889                };
9890                receivers.iter().any(|unit| {
9891                    covers(table, &unit.scopes)
9892                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9893                            |scopes| {
9894                                scopes.iter().all(|(variable, values)| {
9895                                    change
9896                                        .scopes
9897                                        .iter()
9898                                        .find(|(key, _)| key == variable)
9899                                        .is_some_and(|(_, value)| values.contains(value))
9900                                })
9901                            },
9902                        )
9903                })
9904            })
9905            .collect();
9906        if changes.is_empty() {
9907            return Ok(false);
9908        }
9909        let affected_tables = changes
9910            .iter()
9911            .filter_map(|change| tables.get(change.table_index as usize))
9912            .cloned()
9913            .collect::<BTreeSet<_>>();
9914        let previous_batches = self.change_queue.len();
9915        self.apply_commit_frame(tables, &changes, commit_seq)?;
9916        if self.change_queue.len() > previous_batches {
9917            if let Some(batch) = self.change_queue.back_mut() {
9918                batch
9919                    .applied_commits
9920                    .extend(
9921                        affected_tables
9922                            .into_iter()
9923                            .map(|table| AppliedServerCommit {
9924                                table,
9925                                commit_seq,
9926                                source,
9927                            }),
9928                    );
9929            }
9930        }
9931        Ok(true)
9932    }
9933
9934    // Both pull and realtime frames use the same durable observation boundary.
9935    fn apply_commit_frame(
9936        &mut self,
9937        tables: &[String],
9938        changes: &[ssp2::model::Change],
9939        sequence: i64,
9940    ) -> Result<(), TransportError> {
9941        #[cfg(feature = "bench-internals")]
9942        let _phase = self.benchmark_phases.start(Phase::CommitApply);
9943        let rollback = self.capture();
9944        self.coalesce_unit(changes.len())
9945            .and_then(|()| self.begin_observation("syncular_commit"))
9946            .map_err(|message| TransportError::new("storage.failed", message))?;
9947        let applied = (|| {
9948            let mut batch = ChangeAccumulator::default();
9949            self.record_commit_changes(&mut batch, tables, changes);
9950            // §7.1 chained base: a change names the version an acknowledged own
9951            // commit wrote; pending upserts chained on it move to that version.
9952            for change in changes {
9953                let Some(table) = tables.get(change.table_index as usize) else {
9954                    continue;
9955                };
9956                let acknowledged = self
9957                    .conn
9958                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
9959                    .and_then(|mut statement| {
9960                        statement
9961                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
9962                            .collect::<Result<Vec<_>, _>>()
9963                    })
9964                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
9965                    .iter()
9966                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
9967                    .collect::<Result<Vec<_>, _>>()
9968                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
9969                for index in 0..self.outbox.len() {
9970                    if !self.outbox[index]
9971                        .ops
9972                        .iter()
9973                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
9974                    {
9975                        continue;
9976                    }
9977                    let commit = Arc::make_mut(&mut self.outbox[index]);
9978                    for op in &mut commit.ops {
9979                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
9980                            op.base_version = Some(change.row_version);
9981                        }
9982                    }
9983                    let ops = serde_json::to_string(
9984                        &commit
9985                            .ops
9986                            .iter()
9987                            .map(CommitOperation::from)
9988                            .collect::<Vec<_>>(),
9989                    )
9990                    .map_err(|_| {
9991                        TransportError::new(
9992                            "storage.failed",
9993                            "sync.local_corrupt: unencodable outbox".to_owned(),
9994                        )
9995                    })?;
9996                    let id = commit.client_commit_id.clone();
9997                    self.conn
9998                        .execute(
9999                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10000                            rusqlite::params![ops, id],
10001                        )
10002                        .map_err(|error| {
10003                            TransportError::new(
10004                                "storage.failed",
10005                                Self::sqlite_failure(&self.storage_failure, error),
10006                            )
10007                        })?;
10008                }
10009            }
10010            {
10011                let storage_failed = |error| {
10012                    TransportError::new(
10013                        "storage.failed",
10014                        Self::sqlite_failure(&self.storage_failure, error),
10015                    )
10016                };
10017                let pinned: HashSet<(&str, &str)> = self
10018                    .outbox
10019                    .iter()
10020                    .flat_map(|commit| &commit.ops)
10021                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10022                    .collect();
10023                for change in changes {
10024                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10025                        TransportError::new(
10026                            "sync.invalid_request",
10027                            "change tableIndex out of range".to_owned(),
10028                        )
10029                    })?;
10030                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10031                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10032                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10033                            .map_err(storage_failed)?;
10034                    }
10035                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10036                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10037                        .map_err(storage_failed)?;
10038                }
10039            }
10040            self.apply_commit_changes(tables, changes)?;
10041            self.rebuild_overlay_if_dirty()
10042                .map_err(|message| TransportError::new("storage.failed", message))?;
10043            self.finish_observation("syncular_commit", batch)
10044                .map_err(|message| TransportError::new("storage.failed", message))
10045        })();
10046        if applied.is_err() {
10047            self.rollback_observation("syncular_commit");
10048            self.restore(rollback);
10049        }
10050        applied
10051    }
10052
10053    fn apply_commit_changes(
10054        &mut self,
10055        tables: &[String],
10056        changes: &[ssp2::model::Change],
10057    ) -> Result<(), TransportError> {
10058        // A clean overlay can reconcile changed primary keys independently
10059        // when their tables have no secondary unique constraints. Such a
10060        // constraint can change whether another row's write succeeds, so a
10061        // table with one and any local intent takes the table rebuild.
10062        let changed_tables: BTreeSet<&str> = changes
10063            .iter()
10064            .filter_map(|change| tables.get(change.table_index as usize))
10065            .map(String::as_str)
10066            .collect();
10067        let mut mirror_visible = !self.overlay_dirty.get();
10068        for name in &changed_tables {
10069            if mirror_visible
10070                && self
10071                    .schema
10072                    .table(name)
10073                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10074            {
10075                let (pending, protected) = self
10076                    .table_intent(name)
10077                    .map_err(|message| TransportError::new("storage.failed", message))?;
10078                mirror_visible = !pending && !protected;
10079            }
10080        }
10081        for change in changes {
10082            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10083                TransportError::new(
10084                    "sync.invalid_request",
10085                    "change tableIndex out of range".to_owned(),
10086                )
10087            })?;
10088            let table = self.schema.table(table_name).ok_or_else(|| {
10089                TransportError::new(
10090                    "sync.schema_mismatch",
10091                    format!("change targets unknown table {table_name:?}"),
10092                )
10093            })?;
10094            let landed = match change.op {
10095                Op::Upsert => {
10096                    let payload = change.row.as_ref().ok_or_else(|| {
10097                        TransportError::new(
10098                            "sync.invalid_request",
10099                            "upsert change without row payload".to_owned(),
10100                        )
10101                    })?;
10102                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10103                    let (row, sealed) = {
10104                        #[cfg(feature = "bench-internals")]
10105                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10106                        // §5.11: a decrypt failure keeps its own identity.
10107                        decode_row_bytes(table, payload, &self.encryption)?
10108                    };
10109                    let table_name = table.name.clone();
10110                    // §4.5 version guard: only a newer upsert lands.
10111                    let landed = self
10112                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10113                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10114                    if mirror_visible && landed {
10115                        self.write_row(
10116                            &visible_table(&table_name),
10117                            &table_name,
10118                            &row,
10119                            change.row_version,
10120                            sealed.as_ref(),
10121                        )
10122                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10123                    }
10124                    landed
10125                }
10126                // §4.5 version guard: a delete at or below the local version
10127                // is skipped, a delete of an absent row is a no-op.
10128                Op::Delete => self
10129                    .delete_base_row(table_name, &change.row_id, change.row_version)
10130                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10131            };
10132            // The change dropped the row's acknowledged intent either way
10133            // (§7.1), so the overlay of a skipped change rebuilds from the
10134            // unchanged base.
10135            self.overlay_dirty.table(table_name);
10136            if mirror_visible && !(landed && change.op == Op::Upsert) {
10137                let predicate = row_id_predicate(table);
10138                let visible = visible_table(table_name);
10139                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10140                if !landed {
10141                    statements.push(format!(
10142                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10143                        base_table(table_name)
10144                    ));
10145                }
10146                for statement in &statements {
10147                    self.conn
10148                        .prepare_cached(statement)
10149                        .and_then(|mut statement| {
10150                            statement.execute(rusqlite::params![change.row_id])
10151                        })
10152                        .map_err(|error| {
10153                            TransportError::new("sync.invalid_request", error.to_string())
10154                        })?;
10155                }
10156            }
10157        }
10158        if mirror_visible {
10159            let changed_rows: HashSet<(&str, &str)> = changes
10160                .iter()
10161                .map(|change| {
10162                    (
10163                        tables[change.table_index as usize].as_str(),
10164                        change.row_id.as_str(),
10165                    )
10166                })
10167                .collect();
10168            self.replay_overlay(&changed_tables, Some(&changed_rows))
10169                .map_err(|message| TransportError::new("storage.failed", message))?;
10170            self.overlay_dirty.set(false);
10171        }
10172        Ok(())
10173    }
10174
10175    /// §5.6 segment application: validate against the generated schema,
10176    /// clear the pair's scope on the table's first page (fail closed
10177    /// without a mapping), then upsert each row with its segment-carried
10178    /// server version under the §4.5 version guard (§5.2).
10179    fn apply_segment(
10180        &mut self,
10181        table: &TableSchema,
10182        segment: &RowsSegment,
10183        first_page: bool,
10184        effective: &[(String, Vec<String>)],
10185        unit_id: &str,
10186    ) -> Result<u32, SectionError> {
10187        // §5.2: the column table validates against the generated schema —
10188        // order, names, types, nullability; mismatch is fatal. §5.11: the
10189        // server sends the WIRE types (bytes for an encrypted column), so
10190        // validate against wire_columns.
10191        let column = if segment.columns.len() != table.wire_columns.len() {
10192            Some(Value::from("count"))
10193        } else {
10194            segment
10195                .columns
10196                .iter()
10197                .zip(table.wire_columns.iter())
10198                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10199                .map(Value::from)
10200        };
10201        if segment.table != table.name
10202            || segment.schema_version != self.schema.version
10203            || column.is_some()
10204        {
10205            let mut details = Map::new();
10206            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10207            if let Some(column) = column {
10208                details.insert("column".to_owned(), column);
10209            }
10210            return Err(SectionError::Transport(TransportError {
10211                code: "sync.schema_mismatch".to_owned(),
10212                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10213                details: Some(Value::Object(details)),
10214            }));
10215        }
10216        let mut applied = 0u32;
10217        let empty = Vec::new();
10218        let blocks = if segment.blocks.is_empty() {
10219            std::slice::from_ref(&empty)
10220        } else {
10221            &segment.blocks
10222        };
10223        for (index, block) in blocks.iter().enumerate() {
10224            let clear = first_page && index == 0;
10225            if !clear && block.is_empty() {
10226                continue;
10227            }
10228            let rollback = self.capture();
10229            let (pending, protected) = self
10230                .table_intent(&table.name)
10231                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10232            let reconcile_pending =
10233                !protected && pending && table.indexes.iter().any(|index| index.unique);
10234            let mirror_visible = !protected && !reconcile_pending;
10235            self.coalesce_unit(block.len())
10236                .and_then(|()| self.begin_observation("syncular_segment_block"))
10237                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10238            let outcome = (|| {
10239                self.rebuild_overlay_if_dirty()
10240                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10241                let mut changed_keys = Vec::new();
10242                let mut batch = ChangeAccumulator::default();
10243                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10244                    batch.table(&table.name);
10245                }
10246                for (row_index, row) in block.iter().enumerate() {
10247                    let decrypted;
10248                    let mut sealed = None;
10249                    let values = if table.has_encrypted_columns() {
10250                        let mut values = row.values.clone();
10251                        sealed = crate::values::decrypt_segment_row(
10252                            table,
10253                            &mut values,
10254                            &self.encryption,
10255                        )
10256                        .map_err(SectionError::Transport)?;
10257                        decrypted = values;
10258                        &decrypted
10259                    } else {
10260                        &row.values
10261                    };
10262                    if !self
10263                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10264                        .map_err(|message| {
10265                            SectionError::Abort("sync.invalid_request".into(), message)
10266                        })?
10267                    {
10268                        continue;
10269                    }
10270                    if reconcile_pending {
10271                        changed_keys.push(
10272                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10273                                |message| SectionError::Abort("storage.failed".into(), message),
10274                            )?,
10275                        );
10276                    }
10277                    if mirror_visible {
10278                        self.write_row(
10279                            &visible_table(&table.name),
10280                            &table.name,
10281                            values,
10282                            row.server_version,
10283                            sealed.as_ref(),
10284                        )
10285                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10286                    }
10287                    let processed = u64::from(applied) + row_index as u64 + 1;
10288                    if processed.is_multiple_of(1024) {
10289                        self.progress.update(|p| p.rows_processed = processed);
10290                    }
10291                }
10292                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10293                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10294                self.finish_observation("syncular_segment_block", batch)
10295                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10296            })();
10297            if let Err(error) = outcome {
10298                self.rollback_observation("syncular_segment_block");
10299                self.restore(rollback);
10300                return Err(error);
10301            }
10302            applied += block.len() as u32;
10303            self.progress
10304                .update(|p| p.rows_processed = u64::from(applied));
10305            std::thread::yield_now();
10306        }
10307        self.progress
10308            .update(|p| p.rows_processed = u64::from(applied));
10309        Ok(applied)
10310    }
10311
10312    /// §5.3 sqlite-image application: validate the in-file metadata
10313    /// against the descriptor, validate column names/order against the
10314    /// generated schema, run the §5.6 first-page clear when fresh, then
10315    /// replace-or-upsert every image row with its `_syncular_version`.
10316    /// Mechanics: the image lands in a temp file read through a second
10317    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10318    /// Each image chunk commits its writes and revision before readers continue.
10319    #[allow(clippy::too_many_arguments)]
10320    fn apply_sqlite_segment(
10321        &mut self,
10322        table: &TableSchema,
10323        bytes: &[u8],
10324        first_page: bool,
10325        effective: &[(String, Vec<String>)],
10326        unit_id: &str,
10327        row_count: i64,
10328        as_of_commit_seq: i64,
10329        scope_digest: &str,
10330    ) -> Result<u32, SectionError> {
10331        let invalid = |detail: &str| {
10332            SectionError::Abort(
10333                "sync.invalid_request".to_owned(),
10334                format!("sqlite segment rejected: {detail} (§5.3)"),
10335            )
10336        };
10337        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10338        // one for a table with encrypted columns.
10339        if table.has_encrypted_columns() {
10340            return Err(invalid("table has encrypted columns"));
10341        }
10342
10343        // The image lands beside the replica so it shares the replica's
10344        // directory permissions and volume; an in-memory replica has no
10345        // directory and uses the system temp dir. The file is owner-only.
10346        let directory = self
10347            .previous_version_replica_path()
10348            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10349            .unwrap_or_else(std::env::temp_dir);
10350        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10351        let outcome = (|| {
10352            let mut options = std::fs::OpenOptions::new();
10353            options.write(true).create_new(true);
10354            #[cfg(unix)]
10355            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10356            std::io::Write::write_all(
10357                &mut options
10358                    .open(&path)
10359                    .map_err(|_| invalid("image temp file write failed"))?,
10360                bytes,
10361            )
10362            .map_err(|_| invalid("image temp file write failed"))?;
10363            let img = rusqlite::Connection::open_with_flags(
10364                &path,
10365                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10366            )
10367            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10368            self.apply_sqlite_image(
10369                &img,
10370                table,
10371                first_page,
10372                effective,
10373                unit_id,
10374                row_count,
10375                as_of_commit_seq,
10376                scope_digest,
10377            )
10378        })();
10379        let _ = std::fs::remove_file(&path);
10380        outcome
10381    }
10382
10383    #[allow(clippy::too_many_arguments)]
10384    fn apply_sqlite_image(
10385        &mut self,
10386        img: &rusqlite::Connection,
10387        table: &crate::schema::TableSchema,
10388        first_fresh_page: bool,
10389        effective: &[(String, Vec<String>)],
10390        unit_id: &str,
10391        row_count: i64,
10392        as_of_commit_seq: i64,
10393        scope_digest: &str,
10394    ) -> Result<u32, SectionError> {
10395        let invalid = |detail: String| {
10396            SectionError::Abort(
10397                "sync.invalid_request".to_owned(),
10398                format!("sqlite segment rejected: {detail} (§5.3)"),
10399            )
10400        };
10401
10402        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10403        //    one row).
10404        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10405        let meta: MetaRow = img
10406            .query_row(
10407                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10408                        \"scopeDigest\", \"rowCount\",
10409                        (SELECT count(*) FROM _syncular_segment)
10410                 FROM _syncular_segment",
10411                [],
10412                |row| {
10413                    Ok((
10414                        row.get(0)?,
10415                        row.get(1)?,
10416                        row.get(2)?,
10417                        row.get(3)?,
10418                        row.get(4)?,
10419                        row.get(5)?,
10420                        row.get(6)?,
10421                    ))
10422                },
10423            )
10424            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10425        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10426        if meta_count != 1 {
10427            return Err(invalid(format!(
10428                "_syncular_segment must contain exactly one row, found {meta_count}"
10429            )));
10430        }
10431        if format != 1 {
10432            return Err(invalid(format!("format {format}")));
10433        }
10434        if meta_table != table.name {
10435            return Err(invalid(format!("image table {meta_table:?}")));
10436        }
10437        if schema_version != i64::from(self.schema.version) {
10438            return Err(invalid(format!("schemaVersion {schema_version}")));
10439        }
10440        if pin != as_of_commit_seq {
10441            return Err(invalid(format!("asOfCommitSeq {pin}")));
10442        }
10443        if digest != scope_digest {
10444            return Err(invalid("scopeDigest mismatch".to_owned()));
10445        }
10446        if meta_rows != row_count {
10447            return Err(invalid(format!("rowCount {meta_rows}")));
10448        }
10449
10450        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10451        let mut names: Vec<String> = Vec::new();
10452        let mut primary_keys: Vec<String> = Vec::new();
10453        {
10454            let mut stmt = img
10455                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10456                .map_err(|_| invalid("image data table missing".to_owned()))?;
10457            let mut rows = stmt
10458                .query([])
10459                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10460            while let Some(row) = rows
10461                .next()
10462                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10463            {
10464                if row
10465                    .get::<_, i64>(5)
10466                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10467                    > 0
10468                {
10469                    primary_keys.push(row.get(1).map_err(|error| {
10470                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10471                    })?);
10472                }
10473                names.push(
10474                    row.get::<_, String>(1)
10475                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10476                );
10477            }
10478        }
10479        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10480        expected.push("_syncular_version");
10481        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10482            return Err(SectionError::Transport(TransportError {
10483                code: "sync.schema_mismatch".to_owned(),
10484                message: "sqlite segment columns do not match the generated schema (§5.3)"
10485                    .to_owned(),
10486                details: Some(serde_json::json!({ "table": table.name })),
10487            }));
10488        }
10489
10490        if primary_keys != vec![table.primary_key.clone()] {
10491            return Err(invalid(
10492                "image primary key does not match generated schema".into(),
10493            ));
10494        }
10495        let actual_count: i64 = img
10496            .query_row(
10497                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10498                [],
10499                |row| row.get(0),
10500            )
10501            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10502        if actual_count != row_count {
10503            return Err(invalid("image row count does not match descriptor".into()));
10504        }
10505        let insert = self.insert_row_sql(&base_table(&table.name), table);
10506        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10507        let column_list = names
10508            .iter()
10509            .map(|n| quote_ident(n))
10510            .collect::<Vec<_>>()
10511            .join(", ");
10512        let mut stmt = img
10513            .prepare(&format!(
10514                "SELECT {column_list} FROM {} ORDER BY {}",
10515                quote_ident(&table.name),
10516                quote_ident(&table.primary_key)
10517            ))
10518            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10519        let mut rows = stmt
10520            .query([])
10521            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10522        let mut applied = 0u32;
10523        let mut limit = if first_fresh_page {
10524            1024
10525        } else {
10526            self.image_chunk_rows
10527                .get(&table.name)
10528                .copied()
10529                .unwrap_or(1024)
10530        };
10531        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10532        loop {
10533            let clear = applied == 0 && first_fresh_page;
10534            let rollback = self.capture();
10535            let (pending, protected) = self
10536                .table_intent(&table.name)
10537                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10538            let reconcile_pending =
10539                !protected && pending && table.indexes.iter().any(|index| index.unique);
10540            let mirror_visible = !protected && !reconcile_pending;
10541            // §5.3: a table's first chunk holds 1,024 rows; a later range image
10542            // starts at the size its previous range reached. A later chunk holds at
10543            // least 1,024 rows, at most twice the rows of the one before, at
10544            // most 65,536, and at most the rows the one before applied per
10545            // 250 ms from its start to its COMMIT; a chunk that replays local
10546            // writes holds at most 1,024.
10547            let chunk = if pending || protected {
10548                limit.min(1024)
10549            } else {
10550                limit
10551            };
10552            self.coalesce_unit(chunk as usize)
10553                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10554                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10555            let started = std::time::Instant::now();
10556            let result = (|| {
10557                self.rebuild_overlay_if_dirty()
10558                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10559                let mut changed_keys = Vec::new();
10560                let mut batch = ChangeAccumulator::default();
10561                if clear && self.clear_scope(table, effective, unit_id)? {
10562                    batch.table(&table.name);
10563                }
10564                let mut processed = 0u32;
10565                {
10566                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10567                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10568                    })?;
10569                    let mut mirror = if mirror_visible {
10570                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10571                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10572                        })?)
10573                    } else {
10574                        None
10575                    };
10576                    for _ in 0..chunk {
10577                        let Some(row) = rows.next().map_err(|error| {
10578                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10579                        })?
10580                        else {
10581                            break;
10582                        };
10583                        for (i, column) in table.columns.iter().enumerate() {
10584                            let cell = row.get_ref(i).map_err(|error| {
10585                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10586                            })?;
10587                            let param = image_cell_param(column, cell).map_err(invalid)?;
10588                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10589                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10590                            })?;
10591                            if let Some(mirror) = &mut mirror {
10592                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10593                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10594                                })?;
10595                            }
10596                        }
10597                        let version_index = table.columns.len();
10598                        let version: i64 = row.get(version_index).map_err(|error| {
10599                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10600                        })?;
10601                        if version < 1 {
10602                            return Err(invalid("image row version must be positive".into()));
10603                        }
10604                        ins.raw_bind_parameter(version_index + 1, version)
10605                            .map_err(|error| {
10606                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10607                            })?;
10608                        // §4.5 version guard: the base upsert decides.
10609                        let landed = ins.raw_execute().map_err(|error| {
10610                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10611                        })? > 0;
10612                        processed += 1;
10613                        if processed.is_multiple_of(1024) && processed < chunk {
10614                            self.progress
10615                                .update(|p| p.rows_processed = u64::from(applied + processed));
10616                        }
10617                        if !landed {
10618                            continue;
10619                        }
10620                        if let Some(mirror) = &mut mirror {
10621                            mirror
10622                                .raw_bind_parameter(version_index + 1, version)
10623                                .map_err(|error| {
10624                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10625                                })?;
10626                            mirror.raw_execute().map_err(|error| {
10627                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10628                            })?;
10629                        }
10630                        if reconcile_pending {
10631                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10632                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10633                            )?);
10634                        }
10635                    }
10636                }
10637                if processed > 0 {
10638                    batch.table(&table.name);
10639                    self.overlay_dirty.table(&table.name);
10640                }
10641                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10642                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10643                let written = std::time::Instant::now();
10644                self.finish_observation("syncular_image_chunk", batch)
10645                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10646                Ok((processed, written))
10647            })();
10648            let (processed, written) = match result {
10649                Ok(done) => done,
10650                Err(error) => {
10651                    self.rollback_observation("syncular_image_chunk");
10652                    self.restore(rollback);
10653                    return Err(error);
10654                }
10655            };
10656            // The chunk commits before the yield, also inside a coalesced apply.
10657            let coalesced = self.coalesced.is_some();
10658            if coalesced {
10659                self.commit_coalesced_apply()
10660                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10661            }
10662            write += written - started;
10663            commit += written.elapsed();
10664            applied += processed;
10665            let last = processed < chunk || i64::from(applied) == actual_count;
10666            // A file replica's WAL keeps its high-water size until truncated.
10667            if last && self.conn.is_autocommit() {
10668                self.conn
10669                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10670                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10671            }
10672            if coalesced {
10673                self.begin_coalesced_apply()
10674                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10675            }
10676            self.progress.update(|p| {
10677                p.rows_processed = u64::from(applied);
10678                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10679                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10680            });
10681            let duration = started.elapsed().as_secs_f64() * 1000.0;
10682            #[cfg(test)]
10683            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10684            std::thread::yield_now();
10685            if last {
10686                break;
10687            }
10688            // A zero duration saturates the cast and leaves the doubling.
10689            limit = (processed * 2)
10690                .min(65_536)
10691                .min((f64::from(processed) * 250.0 / duration) as u32)
10692                .max(1024);
10693            self.image_chunk_rows.insert(table.name.clone(), limit);
10694        }
10695        if i64::from(applied) != actual_count {
10696            return Err(invalid(format!(
10697                "image yielded {applied} rows, expected {actual_count}"
10698            )));
10699        }
10700        Ok(applied)
10701    }
10702
10703    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10704    /// them from the base and replay their remaining intent. A table with a
10705    /// secondary unique index restores and replays every row with intent,
10706    /// since one row's value can admit or defer another row's write.
10707    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10708        let names: BTreeSet<&str> = drained
10709            .iter()
10710            .flat_map(|commit| &commit.ops)
10711            .map(|op| op.table.as_str())
10712            .collect();
10713        for name in names {
10714            let Some(table) = self.schema.table(name) else {
10715                continue;
10716            };
10717            let ops = drained.iter().flat_map(|commit| &commit.ops);
10718            let tables = BTreeSet::from([name]);
10719            let rows: HashSet<_> = ops
10720                .clone()
10721                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10722                .collect();
10723            let unique = table.indexes.iter().any(|index| index.unique);
10724            let acknowledged = if unique {
10725                self.acknowledged_operations(None, Some(&tables))?
10726            } else {
10727                Vec::new()
10728            };
10729            let peers = acknowledged
10730                .iter()
10731                .chain(
10732                    self.failed_commits
10733                        .iter()
10734                        .flat_map(|(commit, _)| &commit.ops),
10735                )
10736                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10737                .filter(|_| unique);
10738            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10739            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10740        }
10741        Ok(())
10742    }
10743
10744    /// Whether `table` has pending outbox intent, and whether it has protected
10745    /// (acknowledged or retained failed) intent.
10746    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10747        let touches = |op: &OutboxOp| op.table == table;
10748        Ok((
10749            self.outbox
10750                .iter()
10751                .flat_map(|commit| &commit.ops)
10752                .any(touches),
10753            self.failed_commits
10754                .iter()
10755                .flat_map(|(commit, _)| &commit.ops)
10756                .any(touches)
10757                || self
10758                    .conn
10759                    .query_row(
10760                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10761                        [table],
10762                        |row| row.get(0),
10763                    )
10764                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10765        ))
10766    }
10767
10768    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10769    /// only pending intent, so a table with protected intent rebuilds; a
10770    /// `restore` restores the imported `keys` and every pending row first.
10771    fn reconcile_import(
10772        &mut self,
10773        table: &TableSchema,
10774        keys: &mut Vec<SqlValue>,
10775        protected: bool,
10776        restore: bool,
10777    ) -> Result<(), String> {
10778        if protected {
10779            return self.rebuild_overlay_if_dirty();
10780        }
10781        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10782        if restore {
10783            self.restore_base_rows(table, keys, pending.clone())?;
10784        }
10785        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10786        self.overlay_dirty.set(false);
10787        Ok(())
10788    }
10789
10790    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10791    /// `table` from the base before the caller replays their intent. The
10792    /// caller owns the transaction and publishes its revision after replay.
10793    fn restore_base_rows<'a>(
10794        &self,
10795        table: &TableSchema,
10796        keys: &mut Vec<SqlValue>,
10797        ops: impl IntoIterator<Item = &'a OutboxOp>,
10798    ) -> Result<(), String> {
10799        let visible = visible_table(&table.name);
10800        let base = base_table(&table.name);
10801        let pk = quote_ident(&table.primary_key);
10802        let predicate = row_id_predicate(table);
10803        let mut lookup = self.conn.prepare_cached(&format!(
10804            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10805        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10806        let mut deleted = HashSet::new();
10807        for op in ops.into_iter().filter(|op| op.table == table.name) {
10808            if op.upsert {
10809                if let Some(values) = &op.values {
10810                    let key = json_to_column_value(
10811                        &table.columns[table.pk_index],
10812                        values.get(&table.primary_key),
10813                    )?;
10814                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10815                }
10816            } else if deleted.insert(&op.row_id) {
10817                keys.extend(
10818                    lookup
10819                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10820                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10821                        .collect::<Result<Vec<_>, _>>()
10822                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10823                );
10824            }
10825        }
10826        // Delete the complete affected set before restoring any base value, so
10827        // old optimistic unique values cannot collide with the new server rows.
10828        for chunk in keys.chunks(400) {
10829            let holes = vec!["?"; chunk.len()].join(",");
10830            self.conn
10831                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10832                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10833                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10834        }
10835        let updates = table
10836            .columns
10837            .iter()
10838            .map(|column| column.name.as_str())
10839            .chain(std::iter::once("_syncular_version"))
10840            .chain(table.sealed_columns().iter().copied())
10841            .filter(|name| *name != table.primary_key)
10842            .map(|name| {
10843                let name = quote_ident(name);
10844                format!("{name}=excluded.{name}")
10845            })
10846            .collect::<Vec<_>>()
10847            .join(",");
10848        for chunk in keys.chunks(400) {
10849            let holes = vec!["?"; chunk.len()].join(",");
10850            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10851                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10852                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10853        }
10854        Ok(())
10855    }
10856
10857    // -- application-authorized local purge ---------------------------------------
10858
10859    fn compile_local_data_purge(
10860        &self,
10861        input: &LocalDataPurgeInput,
10862    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10863        let invalid = |message: String| format!("sync.invalid_request: {message}");
10864        if input.purge_id.is_empty()
10865            || input.purge_id.len() > 128
10866            || !is_local_operation_code_like(&input.purge_id)
10867        {
10868            return Err(invalid(
10869                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10870            ));
10871        }
10872        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10873            return Err(invalid(format!(
10874                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10875            )));
10876        }
10877
10878        let mut deduplicated: BTreeMap<
10879            String,
10880            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10881        > = BTreeMap::new();
10882        for target in &input.targets {
10883            let table = self.schema.table(&target.table).ok_or_else(|| {
10884                invalid(format!(
10885                    "local purge names unknown table {:?}",
10886                    target.table
10887                ))
10888            })?;
10889            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10890                return Err(invalid(format!(
10891                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10892                    target.table
10893                )));
10894            }
10895            let mut selectors = Vec::with_capacity(target.selectors.len());
10896            let mut canonical_selectors = BTreeMap::new();
10897            for (column_name, raw_values) in &target.selectors {
10898                let Some((column_index, column)) = table
10899                    .columns
10900                    .iter()
10901                    .enumerate()
10902                    .find(|(_, column)| column.name == *column_name)
10903                else {
10904                    return Err(invalid(format!(
10905                        "local purge target {:?} names unknown column {:?}",
10906                        target.table, column_name
10907                    )));
10908                };
10909                let encrypted = table
10910                    .encrypted_columns
10911                    .iter()
10912                    .any(|candidate| candidate.index == column_index);
10913                if column.ty != ColumnType::String || encrypted {
10914                    return Err(invalid(format!(
10915                        "local purge selector {:?}.{:?} must be a plaintext string column",
10916                        target.table, column_name
10917                    )));
10918                }
10919                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
10920                    return Err(invalid(format!(
10921                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
10922                        target.table, column_name
10923                    )));
10924                }
10925                let mut values = raw_values.clone();
10926                values.sort();
10927                values.dedup();
10928                if values.iter().any(|value| {
10929                    value.is_empty()
10930                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
10931                        || !is_local_operation_code_like(value)
10932                }) {
10933                    return Err(invalid(format!(
10934                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
10935                    )));
10936                }
10937                selectors.push((column_name.clone(), values.clone()));
10938                canonical_selectors.insert(column_name.clone(), values);
10939            }
10940            let canonical = LocalDataPurgeTarget {
10941                table: target.table.clone(),
10942                selectors: canonical_selectors,
10943            };
10944            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10945            deduplicated.insert(
10946                key,
10947                (
10948                    CompiledLocalDataPurgeTarget {
10949                        table: target.table.clone(),
10950                        selectors,
10951                    },
10952                    canonical,
10953                ),
10954            );
10955        }
10956        let targets = deduplicated
10957            .values()
10958            .map(|(compiled, _)| compiled.clone())
10959            .collect::<Vec<_>>();
10960        let canonical = deduplicated
10961            .values()
10962            .map(|(_, target)| target.clone())
10963            .collect::<Vec<_>>();
10964        let canonical_plan =
10965            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10966        Ok((targets, canonical_plan))
10967    }
10968
10969    fn local_purge_base_row_ids(
10970        &self,
10971        targets: &[CompiledLocalDataPurgeTarget],
10972    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
10973        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
10974        for target in targets {
10975            let table = self
10976                .schema
10977                .table(&target.table)
10978                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
10979            let mut clauses = Vec::with_capacity(target.selectors.len());
10980            let mut params = Vec::new();
10981            for (column, values) in &target.selectors {
10982                clauses.push(format!(
10983                    "{} IN ({})",
10984                    quote_ident(column),
10985                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
10986                ));
10987                params.extend(values.iter().cloned().map(SqlValue::Text));
10988            }
10989            let sql = format!(
10990                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
10991                quote_ident(&table.primary_key),
10992                base_table(&target.table),
10993                clauses.join(" AND ")
10994            );
10995            let mut statement = self
10996                .conn
10997                .prepare(&sql)
10998                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10999            let rows = statement
11000                .query_map(rusqlite::params_from_iter(params), |row| {
11001                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11002                    render_row_id_json(Some(&value))
11003                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11004                })
11005                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11006            let ids = by_table.entry(target.table.clone()).or_default();
11007            for row in rows {
11008                ids.insert(
11009                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11010                );
11011            }
11012        }
11013        Ok(by_table)
11014    }
11015
11016    fn local_purge_values_match(
11017        target: &CompiledLocalDataPurgeTarget,
11018        values: &Map<String, Value>,
11019    ) -> bool {
11020        target.selectors.iter().all(|(column, allowed)| {
11021            values
11022                .get(column)
11023                .and_then(Value::as_str)
11024                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11025        })
11026    }
11027
11028    /// Apply one host-authorized local security purge. The host MUST gate the
11029    /// corresponding subscriptions first; this primitive owns local SQLite
11030    /// cleanup and never grants/revokes server authority by itself.
11031    pub fn purge_local_data(
11032        &mut self,
11033        input: &LocalDataPurgeInput,
11034    ) -> Result<LocalDataPurgeResult, String> {
11035        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11036        let meta_key = format!("localPurge:{}", input.purge_id);
11037        let applied_plan = self
11038            .conn
11039            .query_row(
11040                "SELECT value FROM _syncular_meta WHERE key = ?1",
11041                rusqlite::params![meta_key],
11042                |row| row.get::<_, String>(0),
11043            )
11044            .optional()
11045            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11046        if let Some(applied_plan) = applied_plan {
11047            if applied_plan != canonical_plan {
11048                return Err(format!(
11049                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11050                    input.purge_id
11051                ));
11052            }
11053            // A retry after a failed rewrite finishes it.
11054            self.erase_purged_pages()?;
11055            return Ok(LocalDataPurgeResult {
11056                already_applied: true,
11057                purged_rows: 0,
11058                dropped_commits: 0,
11059            });
11060        }
11061
11062        // RFC 0005 D7: the container file cannot be a purge target (its name is
11063        // not in the running schema), so purge drops it as a fixed step, before
11064        // any mirror row is touched and unconditionally.
11065        let _ = self.drop_previous_version();
11066
11067        let rollback = self.capture();
11068        self.begin_observation("syncular_local_purge")?;
11069        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11070            let row_ids = self.local_purge_base_row_ids(&targets)?;
11071            let retained_dropped = self.drop_failed_commits(|table, values| {
11072                targets.iter().any(|target| {
11073                    target.table == table && Self::local_purge_values_match(target, values)
11074                })
11075            })?;
11076            let doomed = self
11077                .outbox
11078                .iter()
11079                .filter(|commit| {
11080                    commit.ops.iter().any(|operation| {
11081                        let matching_targets = targets
11082                            .iter()
11083                            .filter(|target| target.table == operation.table)
11084                            .collect::<Vec<_>>();
11085                        if matching_targets.is_empty() {
11086                            return false;
11087                        }
11088                        if row_ids
11089                            .get(&operation.table)
11090                            .is_some_and(|ids| ids.contains(&operation.row_id))
11091                        {
11092                            return true;
11093                        }
11094                        operation.values.as_ref().is_some_and(|values| {
11095                            matching_targets
11096                                .iter()
11097                                .any(|target| Self::local_purge_values_match(target, values))
11098                        })
11099                    })
11100                })
11101                .cloned()
11102                .collect::<Vec<_>>();
11103            let doomed_ids = doomed
11104                .iter()
11105                .map(|commit| commit.client_commit_id.clone())
11106                .collect::<BTreeSet<_>>();
11107            let mut batch = ChangeAccumulator::default();
11108            for commit in &retained_dropped {
11109                batch.conflicts = true;
11110                batch.rejections = true;
11111                batch.outcomes = true;
11112                for operation in &commit.ops {
11113                    batch.table(&operation.table);
11114                }
11115            }
11116            let mut rejections = Vec::new();
11117            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11118            // codes, never its operations, so no purged value survives in it.
11119            for commit in &doomed {
11120                for operation in &commit.ops {
11121                    batch.table(&operation.table);
11122                    self.overlay_dirty.table(&operation.table);
11123                }
11124                let results = (0..commit.ops.len())
11125                    .map(|op_index| {
11126                        let rejection = RejectionRecord {
11127                            client_commit_id: commit.client_commit_id.clone(),
11128                            op_index: op_index as i32,
11129                            code: "client.local_data_purged".to_owned(),
11130                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11131                            retryable: false,
11132                            details: None,
11133                            operation: None,
11134                        };
11135                        rejections.push(rejection.clone());
11136                        CommitOperationOutcome::Error { rejection }
11137                    })
11138                    .collect::<Vec<_>>();
11139                self.persist_commit_outcome(
11140                    &commit.client_commit_id,
11141                    CommitOutcomeStatus::Rejected,
11142                    &results,
11143                    None,
11144                )?;
11145                self.delete_outbox_persisted(&commit.client_commit_id)?;
11146            }
11147            if !doomed.is_empty() {
11148                self.outbox
11149                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11150                self.rejections.extend(rejections);
11151                self.prune_commit_outcomes()?;
11152                batch.status = true;
11153                batch.rejections = true;
11154                batch.outcomes = true;
11155            }
11156
11157            let mut purged_rows = 0usize;
11158            for (table, ids) in &row_ids {
11159                if ids.is_empty() {
11160                    continue;
11161                }
11162                batch.table(table);
11163                self.overlay_dirty.table(table);
11164                for row_id in ids {
11165                    self.delete_base_row(table, row_id, i64::MAX)?;
11166                    purged_rows += 1;
11167                }
11168            }
11169            // §5.11 purge erasure: a journaled outcome with an operation or
11170            // conflict row the purge reaches keeps its identity, status, ids
11171            // and codes only.
11172            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11173                targets.iter().any(|target| {
11174                    target.table == table
11175                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11176                            || values.is_some_and(|values| {
11177                                Self::local_purge_values_match(target, values)
11178                            }))
11179                })
11180            };
11181            let entries = self
11182                .conn
11183                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11184                .and_then(|mut statement| {
11185                    statement
11186                        .query_map([], |row| {
11187                            Ok((
11188                                row.get::<_, String>(0)?,
11189                                row.get::<_, String>(1)?,
11190                                row.get::<_, Option<String>>(2)?,
11191                            ))
11192                        })?
11193                        .collect::<Result<Vec<_>, _>>()
11194                })
11195                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11196            for (client_commit_id, results_json, operations_json) in entries {
11197                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11198                let mut results: Vec<CommitOperationOutcome> =
11199                    serde_json::from_str(&results_json).map_err(corrupt)?;
11200                let operations: Vec<CommitOperation> = operations_json
11201                    .map(|json| serde_json::from_str(&json))
11202                    .transpose()
11203                    .map_err(corrupt)?
11204                    .unwrap_or_default();
11205                let named = operations
11206                    .iter()
11207                    .chain(results.iter().filter_map(|result| match result {
11208                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11209                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11210                        CommitOperationOutcome::Applied { .. } => None,
11211                    }))
11212                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11213                    || results.iter().any(|result| {
11214                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11215                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11216                    });
11217                if !named {
11218                    continue;
11219                }
11220                for result in &mut results {
11221                    match result {
11222                        CommitOperationOutcome::Conflict { conflict } => {
11223                            conflict.server_row = Map::new();
11224                            conflict.conflict_columns.clear();
11225                            conflict.operation = None;
11226                        }
11227                        CommitOperationOutcome::Error { rejection } => {
11228                            rejection.details = None;
11229                            rejection.operation = None;
11230                        }
11231                        CommitOperationOutcome::Applied { .. } => {}
11232                    }
11233                }
11234                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11235                self.conn
11236                    .execute(
11237                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11238                        rusqlite::params![results_json, client_commit_id],
11239                    )
11240                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11241                batch.outcomes = true;
11242            }
11243            self.rebuild_overlay_if_dirty()?;
11244            self.delete_unreferenced_cached_blobs()?;
11245            self.conn
11246                .execute(
11247                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11248                    rusqlite::params![meta_key, canonical_plan],
11249                )
11250                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11251            Ok((
11252                batch,
11253                LocalDataPurgeResult {
11254                    already_applied: false,
11255                    purged_rows,
11256                    dropped_commits: doomed.len(),
11257                },
11258            ))
11259        })()
11260        .and_then(|(batch, result)| {
11261            self.finish_observation("syncular_local_purge", batch)
11262                .map(|()| result)
11263        });
11264        let result = match applied {
11265            Ok(result) => result,
11266            Err(error) => {
11267                self.rollback_observation("syncular_local_purge");
11268                self.restore(rollback);
11269                return Err(error);
11270            }
11271        };
11272        self.cancel_sync_round();
11273        self.erase_purged_pages()?;
11274        Ok(result)
11275    }
11276
11277    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11278    /// database file, and the WAL keeps the frames written before the purge,
11279    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11280    /// rows, and the checkpoint truncates the WAL.
11281    fn erase_purged_pages(&self) -> Result<(), String> {
11282        let erased = self
11283            .conn
11284            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11285            .and_then(|()| {
11286                self.conn
11287                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11288            });
11289        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11290        erased
11291            .and(reset)
11292            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11293    }
11294
11295    /// Application-authorized recovery of the replicated projection. Unlike a
11296    /// local security purge, this retains the entire outbox, device identity,
11297    /// lease, outcomes, unit registrations, and protected bookkeeping.
11298    /// The projection reset, unit rewind, optimistic replay, and
11299    /// idempotency marker share one savepoint for interruption safety.
11300    pub fn rebootstrap_local_data(
11301        &mut self,
11302        input: &LocalDataRebootstrapInput,
11303    ) -> Result<LocalDataRebootstrapResult, String> {
11304        if input.rebootstrap_id.is_empty()
11305            || input.rebootstrap_id.len() > 128
11306            || !is_local_operation_code_like(&input.rebootstrap_id)
11307        {
11308            return Err(
11309                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11310                    .to_owned(),
11311            );
11312        }
11313        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11314        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11315            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11316        })? {
11317            let (retained_commits, reset_subscriptions) =
11318                decode_local_rebootstrap_receipt(&persisted)?;
11319            return Ok(LocalDataRebootstrapResult {
11320                already_applied: true,
11321                retained_commits,
11322                reset_subscriptions,
11323            });
11324        }
11325        if self.stopped || self.schema_floor.is_some() {
11326            return Err(
11327                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11328                    .to_owned(),
11329            );
11330        }
11331
11332        let retained_commits = self.outbox.len();
11333        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11334        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11335        let rollback = self.capture();
11336        self.begin_observation("syncular_local_rebootstrap")?;
11337        let mut batch = ChangeAccumulator::default();
11338        let applied = (|| -> Result<(), String> {
11339            self.run_schema_reset_observed(&mut batch, false)?;
11340            self.conn
11341                .execute(
11342                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11343                    rusqlite::params![meta_key, receipt],
11344                )
11345                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11346            self.sync_needed = true;
11347            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11348            batch.status = true;
11349            Ok(())
11350        })()
11351        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11352        if let Err(error) = applied {
11353            self.rollback_observation("syncular_local_rebootstrap");
11354            self.restore(rollback);
11355            return Err(error);
11356        }
11357
11358        Ok(LocalDataRebootstrapResult {
11359            already_applied: false,
11360            retained_commits,
11361            reset_subscriptions,
11362        })
11363    }
11364
11365    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11366
11367    /// Delete base and visible rows owned by the effective scopes
11368    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11369    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11370    fn purge_scope_rows(
11371        &mut self,
11372        table_name: &str,
11373        effective: &[(String, Vec<String>)],
11374        owners: &[Vec<(String, Vec<String>)>],
11375    ) -> Result<bool, String> {
11376        let Some(table) = self.schema.table(table_name) else {
11377            return Ok(false);
11378        };
11379        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11380            return Ok(false);
11381        };
11382        let Some((predicate, params)) = predicate else {
11383            return Ok(true);
11384        };
11385        self.overlay_dirty.table(table_name);
11386        for source in [base_table(table_name), visible_table(table_name)] {
11387            self.conn
11388                .execute(
11389                    &format!("DELETE FROM {source} WHERE {predicate}"),
11390                    rusqlite::params_from_iter(&params),
11391                )
11392                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11393        }
11394        Ok(true)
11395    }
11396
11397    /// §3.3: drop pending commits whose upserts provably land in the
11398    /// revoked effective scopes and no other owner covers (row ownership)
11399    /// — whole-commit, never per-operation.
11400    fn drop_doomed_outbox(
11401        &mut self,
11402        table: &TableSchema,
11403        effective: &[(String, Vec<String>)],
11404        owners: &[Vec<(String, Vec<String>)>],
11405    ) -> Result<bool, String> {
11406        let doomed: Vec<_> =
11407            self.outbox
11408                .iter()
11409                .filter(|commit| {
11410                    commit.ops.iter().any(|op| {
11411                        op.upsert
11412                            && op.table == table.name
11413                            && op.values.as_ref().is_some_and(|values| {
11414                                owns_local_row(table, effective, owners, values)
11415                            })
11416                    })
11417                })
11418                .cloned()
11419                .collect();
11420        if doomed.is_empty() {
11421            return Ok(false);
11422        }
11423        let mut rejections = Vec::new();
11424        for commit in &doomed {
11425            let results = commit
11426                .ops
11427                .iter()
11428                .enumerate()
11429                .map(|(op_index, operation)| {
11430                    self.overlay_dirty.table(&operation.table);
11431                    let rejection = RejectionRecord {
11432                        client_commit_id: commit.client_commit_id.clone(),
11433                        op_index: op_index as i32,
11434                        code: "sync.scope_revoked".to_owned(),
11435                        message: "the commit was dropped because its effective scope was revoked"
11436                            .to_owned(),
11437                        retryable: false,
11438                        details: None,
11439                        operation: Some(CommitOperation::from(operation)),
11440                    };
11441                    rejections.push(rejection.clone());
11442                    CommitOperationOutcome::Error { rejection }
11443                })
11444                .collect::<Vec<_>>();
11445            self.persist_commit_outcome(
11446                &commit.client_commit_id,
11447                CommitOutcomeStatus::Rejected,
11448                &results,
11449                Some(&commit.ops),
11450            )?;
11451            self.delete_outbox_persisted(&commit.client_commit_id)?;
11452        }
11453        self.prune_commit_outcomes()?;
11454        let doomed_ids = doomed
11455            .iter()
11456            .map(|commit| commit.client_commit_id.as_str())
11457            .collect::<BTreeSet<_>>();
11458        self.outbox
11459            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11460        self.rejections.extend(rejections);
11461        Ok(true)
11462    }
11463
11464    // -- blobs (§5.9) ----------------------------------------------------------------
11465
11466    /// §5.9.7: hash bytes into the content address, cache them, queue the
11467    /// upload (flushed before the next push, B4). Returns the canonical
11468    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11469    /// column value.
11470    pub fn upload_blob(
11471        &mut self,
11472        bytes: &[u8],
11473        media_type: Option<String>,
11474        name: Option<String>,
11475    ) -> Result<Value, String> {
11476        let blob_id = blob_id_for(bytes);
11477        let now = self.clock_now_ms();
11478        let tx = self
11479            .conn
11480            .transaction()
11481            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11482        tx.execute(
11483            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11484             ON CONFLICT(blob_id) DO NOTHING",
11485            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11486        )
11487        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11488        tx.execute(
11489            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11490            rusqlite::params![blob_id, media_type, now],
11491        )
11492        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11493        tx.commit()
11494            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11495        self.enforce_blob_cache_cap();
11496        let mut obj = Map::new();
11497        obj.insert("blobId".to_owned(), Value::from(blob_id));
11498        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11499        if let Some(mt) = media_type {
11500            obj.insert("mediaType".to_owned(), Value::from(mt));
11501        }
11502        if let Some(n) = name {
11503            obj.insert("name".to_owned(), Value::from(n));
11504        }
11505        Ok(Value::Object(obj))
11506    }
11507
11508    /// §5.9.7: resolve an owned blob body without driver encoding. A
11509    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11510    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11511    pub fn fetch_blob_bytes(
11512        &mut self,
11513        transport: &mut dyn Transport,
11514        blob_id_or_ref: &str,
11515    ) -> Result<FetchedBlob, (String, String)> {
11516        let simple = |m: String| ("client.failed".to_owned(), m);
11517        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11518        // blobId declares none.
11519        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11520            (blob_id_or_ref.to_owned(), None)
11521        } else {
11522            let value: Value = serde_json::from_str(blob_id_or_ref)
11523                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11524            (
11525                value
11526                    .get("blobId")
11527                    .and_then(Value::as_str)
11528                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11529                    .to_owned(),
11530                value.get("byteLength").and_then(Value::as_u64),
11531            )
11532        };
11533        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11534            return Ok(cached);
11535        }
11536        if !self.transport_enabled {
11537            return Err(("sync.offline".into(), "transport gate is closed".into()));
11538        }
11539        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11540        // blob.not_found) verbatim so the harness can assert on it. The
11541        // authorized endpoint serves bytes inline OR (always-issue, presign
11542        // configured) a signed url the client fetches directly — no host auth,
11543        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11544        #[cfg(feature = "bench-internals")]
11545        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11546        let bytes = match transport
11547            .blob_download(&blob_id, byte_length)
11548            .map_err(|e| (e.code, e.message))?
11549        {
11550            BlobDownload::Bytes(bytes) => bytes,
11551            BlobDownload::Url {
11552                url,
11553                url_expires_at_ms,
11554            } => {
11555                // §5.9.5: MUST NOT start a fetch at/past expiry.
11556                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11557                    return Err((
11558                        "sync.segment_expired".to_owned(),
11559                        format!(
11560                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11561                        ),
11562                    ));
11563                }
11564                transport
11565                    .fetch_blob_url(&url, byte_length)
11566                    .map_err(|e| (e.code, e.message))?
11567            }
11568        };
11569        #[cfg(feature = "bench-internals")]
11570        drop(download_phase);
11571        #[cfg(feature = "bench-internals")]
11572        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11573        // §5.9.5 inherits §5.1: the declared size caps the body before the
11574        // content address is verified.
11575        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11576            return Err((
11577                "sync.invalid_request".to_owned(),
11578                "blob exceeds its byteLength".to_owned(),
11579            ));
11580        }
11581        if blob_id_for(&bytes) != blob_id {
11582            return Err(simple(format!(
11583                "blob content address mismatch for {blob_id}"
11584            )));
11585        }
11586        #[cfg(feature = "bench-internals")]
11587        drop(validate_phase);
11588        #[cfg(feature = "bench-internals")]
11589        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11590        let now = self.clock_now_ms();
11591        self.conn
11592            .execute(
11593                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11594                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11595            )
11596            .map_err(|e| simple(e.to_string()))?;
11597        self.conn
11598            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11599            .map_err(|e| simple(e.to_string()))?;
11600        self.enforce_blob_cache_cap();
11601        #[cfg(feature = "bench-internals")]
11602        drop(insert_phase);
11603        self.get_cached_blob_bytes(&blob_id)
11604            .map_err(simple)?
11605            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11606    }
11607
11608    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11609        #[cfg(feature = "bench-internals")]
11610        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11611        let mut stmt = self
11612            .conn
11613            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11614            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11615        let mut rows = stmt
11616            .query(rusqlite::params![blob_id])
11617            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11618        if let Some(row) = rows
11619            .next()
11620            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11621        {
11622            let bytes: Vec<u8> = row
11623                .get(0)
11624                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11625            let byte_length: i64 = row
11626                .get(1)
11627                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11628            let media_type: Option<String> = row
11629                .get(2)
11630                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11631            return Ok(Some(FetchedBlob {
11632                blob_id: blob_id.to_owned(),
11633                byte_length,
11634                bytes,
11635                media_type,
11636            }));
11637        }
11638        Ok(None)
11639    }
11640
11641    /// §5.9.7 B4: upload every queued blob before push.
11642    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11643        let pending: Vec<(String, Option<String>)> = {
11644            let mut stmt = self
11645                .conn
11646                .prepare(
11647                    "SELECT blob_id, media_type FROM (
11648                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11649                      UNION ALL
11650                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11651                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11652                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11653                    ) ORDER BY created_at_ms, blob_id",
11654                )
11655                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11656            let rows = stmt
11657                .query_map([], |row| {
11658                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11659                })
11660                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11661            rows.collect::<Result<_, _>>()
11662                .map_err(|error| match error {
11663                    rusqlite::Error::InvalidColumnType(..)
11664                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11665                        "sync.local_corrupt",
11666                        "Pending blob upload metadata is invalid",
11667                    ),
11668                    _ => TransportError::new("client.failed", error.to_string()),
11669                })?
11670        };
11671        let mut uploads = Vec::new();
11672        for (blob_id, media_type) in pending {
11673            let bytes = (|| {
11674                let stored: Option<(SqlValue, SqlValue)> = self
11675                    .conn
11676                    .query_row(
11677                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11678                        rusqlite::params![blob_id],
11679                        |row| Ok((row.get(0)?, row.get(1)?)),
11680                    )
11681                    .optional()
11682                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11683                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11684                    return Err(TransportError::new(
11685                        "sync.local_corrupt",
11686                        "Pending blob upload body is missing or corrupt",
11687                    ));
11688                };
11689                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11690                    return Err(TransportError::new(
11691                        "sync.local_corrupt",
11692                        "Pending blob upload body is missing or corrupt",
11693                    ));
11694                }
11695                Ok(bytes)
11696            })();
11697            let failed = bytes.is_err();
11698            uploads.push(crate::round::PendingUpload {
11699                id: blob_id,
11700                bytes,
11701                media_type,
11702            });
11703            if failed {
11704                break;
11705            }
11706        }
11707        Ok(uploads)
11708    }
11709
11710    fn visible_blob_ids_sql(&self) -> String {
11711        let mut selects = Vec::new();
11712        for table in &self.schema.tables {
11713            for column in table
11714                .columns
11715                .iter()
11716                .filter(|column| column.ty == ColumnType::BlobRef)
11717            {
11718                let identifier = quote_ident(&column.name);
11719                selects.push(format!(
11720                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11721                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11722                     AND json_type({identifier}, '$.blobId') = 'text'",
11723                    quote_ident(&table.name),
11724                ));
11725            }
11726        }
11727        if selects.is_empty() {
11728            "SELECT NULL AS blob_id WHERE 0".to_owned()
11729        } else {
11730            selects.join(" UNION ")
11731        }
11732    }
11733
11734    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11735    /// staged uploads pin bodies directly; no derived refcount is stored.
11736    fn enforce_blob_cache_cap(&self) {
11737        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11738            return;
11739        };
11740        let mut total: i64 = self
11741            .conn
11742            .query_row(
11743                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11744                [],
11745                |row| row.get(0),
11746            )
11747            .unwrap_or(0);
11748        if total <= max_bytes {
11749            return;
11750        }
11751        let query = format!(
11752            "SELECT blob_id, byte_length FROM _syncular_blobs
11753             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11754               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11755               AND blob_id NOT IN ({})
11756             ORDER BY created_at_ms ASC, blob_id ASC",
11757            self.visible_blob_ids_sql(),
11758        );
11759        let candidates: Vec<(String, i64)> = {
11760            let Ok(mut stmt) = self.conn.prepare(&query) else {
11761                return;
11762            };
11763            let Ok(rows) = stmt.query_map([], |row| {
11764                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11765            }) else {
11766                return;
11767            };
11768            rows.filter_map(Result::ok).collect()
11769        };
11770        for (blob_id, byte_length) in candidates {
11771            if total <= max_bytes {
11772                break;
11773            }
11774            let _ = self.conn.execute(
11775                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11776                rusqlite::params![blob_id],
11777            );
11778            total -= byte_length;
11779        }
11780    }
11781
11782    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11783        #[cfg(feature = "bench-internals")]
11784        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11785        if !self.schema_has_blobs() {
11786            return Ok(());
11787        }
11788        self.rebuild_overlay_if_dirty()?;
11789        self.conn
11790            .execute(
11791                &format!(
11792                    "DELETE FROM _syncular_blobs
11793                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11794                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11795                   AND blob_id NOT IN ({})",
11796                    self.visible_blob_ids_sql(),
11797                ),
11798                [],
11799            )
11800            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11801        Ok(())
11802    }
11803
11804    // -- local row storage ----------------------------------------------------------
11805
11806    /// The cached per-table primary-key upsert SQL for `full_table` (see
11807    /// the `insert_sql` field: built once, reused per row).
11808    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11809        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11810            return sql.clone();
11811        }
11812        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11813        columns.push(quote_ident("_syncular_version"));
11814        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11815        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11816        let primary_key = quote_ident(&table.primary_key);
11817        let updates = columns
11818            .iter()
11819            .filter(|column| **column != primary_key)
11820            .map(|column| format!("{column}=excluded.{column}"))
11821            .collect::<Vec<_>>()
11822            .join(", ");
11823        // §4.5 version guard: server state lands in the base table only above
11824        // the stored version; the visible overlay takes every write.
11825        let guard = if full_table == base_table(&table.name) {
11826            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11827        } else {
11828            String::new()
11829        };
11830        let sql = format!(
11831            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11832            columns.join(", "),
11833            placeholders.join(", ")
11834        );
11835        self.insert_sql
11836            .borrow_mut()
11837            .insert(full_table.to_owned(), sql.clone());
11838        sql
11839    }
11840
11841    /// §4.5: write a server row under the version guard; returns whether it
11842    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
11843    fn write_base_row(
11844        &self,
11845        table_name: &str,
11846        row: &Row,
11847        version: i64,
11848        sealed: Option<&SealedRow>,
11849    ) -> Result<bool, String> {
11850        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11851        if landed {
11852            self.overlay_dirty.table(table_name);
11853        }
11854        Ok(landed)
11855    }
11856
11857    fn write_row(
11858        &self,
11859        full_table: &str,
11860        table_name: &str,
11861        row: &Row,
11862        version: i64,
11863        sealed: Option<&SealedRow>,
11864    ) -> Result<bool, String> {
11865        #[cfg(feature = "bench-internals")]
11866        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11867        let table = self
11868            .schema
11869            .table(table_name)
11870            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11871        let sql = self.insert_row_sql(full_table, table);
11872        let mut stmt = self
11873            .conn
11874            .prepare_cached(&sql)
11875            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11876        let params = row
11877            .iter()
11878            .map(RowParam::Cell)
11879            .chain(std::iter::once(RowParam::Version(version)))
11880            .chain(
11881                [
11882                    sealed.map(|sealed| sealed.key_id.as_str()),
11883                    sealed.map(|sealed| sealed.envelopes.as_str()),
11884                ]
11885                .into_iter()
11886                .take(table.sealed_columns().len())
11887                .map(RowParam::Text),
11888            );
11889        stmt.execute(rusqlite::params_from_iter(params))
11890            .map(|changed| changed > 0)
11891            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11892    }
11893
11894    /// §4.5: delete a server row whose stored version lies below `version`;
11895    /// returns whether a row left.
11896    fn delete_base_row(
11897        &self,
11898        table_name: &str,
11899        row_id: &str,
11900        version: i64,
11901    ) -> Result<bool, String> {
11902        let table = self
11903            .schema
11904            .table(table_name)
11905            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11906        let sql = format!(
11907            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
11908            base_table(table_name),
11909            row_id_predicate(table)
11910        );
11911        let mut stmt = self
11912            .conn
11913            .prepare_cached(&sql)
11914            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11915        let landed = stmt
11916            .execute(rusqlite::params![row_id, version])
11917            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11918            > 0;
11919        if landed {
11920            self.overlay_dirty.table(table_name);
11921        }
11922        Ok(landed)
11923    }
11924
11925    fn has_acknowledged_rows(&self) -> Result<bool, String> {
11926        self.conn
11927            .query_row(
11928                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
11929                [],
11930                |row| row.get(0),
11931            )
11932            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11933    }
11934
11935    fn acknowledged_operations(
11936        &self,
11937        rows: Option<&HashSet<(&str, &str)>>,
11938        tables: Option<&BTreeSet<&str>>,
11939    ) -> Result<Vec<OutboxOp>, String> {
11940        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
11941        let mut statement = self
11942            .conn
11943            .prepare_cached(if keys.is_some() {
11944                ACKED_ROWS_BY_ROW
11945            } else {
11946                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
11947            })
11948            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11949        let rows = statement
11950            .query_map(
11951                rusqlite::params![
11952                    keys,
11953                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
11954                ],
11955                |row| {
11956                    #[cfg(test)]
11957                    self.acknowledged_replay_count
11958                        .set(self.acknowledged_replay_count.get() + 1);
11959                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11960                },
11961            )
11962            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11963        rows.map(|row| {
11964            let (op, intent) =
11965                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11966            let op: CommitOperation =
11967                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
11968            let table = self
11969                .schema
11970                .table(&op.table)
11971                .ok_or_else(|| "sync.unknown_table".to_owned())?;
11972            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
11973                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
11974            } else {
11975                op.values.clone()
11976            };
11977            Ok(OutboxOp {
11978                values,
11979                ..op.into()
11980            })
11981        })
11982        .collect()
11983    }
11984
11985    fn drop_acknowledged_rows(
11986        &self,
11987        matches: impl Fn(&str, &Map<String, Value>) -> bool,
11988    ) -> Result<Vec<OutboxCommit>, String> {
11989        let mut statement = self
11990            .conn
11991            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
11992            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11993        let rows = statement
11994            .query_map([], |row| {
11995                Ok((
11996                    row.get::<_, String>(0)?,
11997                    row.get::<_, String>(1)?,
11998                    row.get::<_, String>(2)?,
11999                    row.get::<_, String>(3)?,
12000                ))
12001            })
12002            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12003        let mut doomed = HashSet::new();
12004        for row in rows {
12005            let (id, table, intent, row_id) =
12006                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12007            let values =
12008                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12009            let base_matches = self
12010                .schema
12011                .table(&table)
12012                .map(|schema| self.stored_row(schema, &row_id, true))
12013                .transpose()?
12014                .flatten()
12015                .is_some_and(|(base, _)| {
12016                    let schema = self.schema.table(&table).expect("acknowledged table");
12017                    let values = schema
12018                        .columns
12019                        .iter()
12020                        .zip(base.iter())
12021                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12022                        .collect();
12023                    matches(&table, &values)
12024                });
12025            if matches(&table, &values) || base_matches {
12026                doomed.insert(id);
12027            }
12028        }
12029        let mut dropped = Vec::new();
12030        for id in doomed {
12031            let mut statement = self
12032                .conn
12033                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12034                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12035            let operations = statement
12036                .query_map([&id], |row| row.get::<_, String>(0))
12037                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12038            let mut ops = Vec::new();
12039            for operation in operations {
12040                let json = operation
12041                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12042                let op: CommitOperation =
12043                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12044                self.overlay_dirty.table(&op.table);
12045                ops.push(OutboxOp {
12046                    table: op.table,
12047                    row_id: op.row_id,
12048                    upsert: op.op == "upsert",
12049                    base_version: op.base_version,
12050                    values: op.values,
12051                    insert: op.insert,
12052                });
12053            }
12054            dropped.push(OutboxCommit {
12055                client_commit_id: id.clone(),
12056                ops,
12057            });
12058            self.conn
12059                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12060                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12061        }
12062        Ok(dropped)
12063    }
12064
12065    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12066    /// the outbox changed since the last rebuild (the no-op sync round).
12067    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12068        if self.overlay_dirty.get() {
12069            self.rebuild_overlay()?;
12070        }
12071        Ok(())
12072    }
12073
12074    /// §7.1: local reads see outbox state applied optimistically — rebuild
12075    /// every visible table as (base server state) + (pending outbox replay
12076    /// on top). Optimistic rows carry version `-1`.
12077    fn rebuild_overlay(&mut self) -> Result<(), String> {
12078        #[cfg(feature = "bench-internals")]
12079        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12080        #[cfg(test)]
12081        self.overlay_rebuild_count
12082            .set(self.overlay_rebuild_count.get() + 1);
12083        let dirty = self.overlay_dirty.snapshot();
12084        let tables: Vec<_> = self
12085            .schema
12086            .tables
12087            .iter()
12088            .filter(|table| {
12089                dirty
12090                    .as_ref()
12091                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12092            })
12093            .cloned()
12094            .collect();
12095        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12096        self.begin_observation("syncular_overlay")?;
12097        let rebuilt = (|| {
12098            for table in &tables {
12099                for index in &table.fts_indexes {
12100                    self.drop_fts_triggers(index)?;
12101                }
12102                let visible = visible_table(&table.name);
12103                let base = base_table(&table.name);
12104                self.conn
12105                    .execute_batch(&format!(
12106                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12107                    ))
12108                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12109            }
12110            self.replay_overlay(&names, None)?;
12111            for table in &tables {
12112                for index in &table.fts_indexes {
12113                    self.rebuild_fts_projection(table, index)?;
12114                    self.create_fts_triggers(table, index)?;
12115                }
12116            }
12117            self.conn
12118                .execute_batch("RELEASE syncular_overlay")
12119                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12120            Ok(())
12121        })();
12122        if rebuilt.is_err() {
12123            self.conn
12124                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12125                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12126        } else {
12127            self.overlay_dirty.set(false);
12128        }
12129        rebuilt
12130    }
12131
12132    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12133    /// that order, for the operations of `tables` on `rows` (every row for
12134    /// `None`) over their current visible state.
12135    fn replay_overlay(
12136        &self,
12137        tables: &BTreeSet<&str>,
12138        rows: Option<&HashSet<(&str, &str)>>,
12139    ) -> Result<(), String> {
12140        let replays = |op: &&OutboxOp| {
12141            tables.contains(op.table.as_str())
12142                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12143        };
12144        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12145        for (commit, initial) in self.failed_commits.iter() {
12146            for operation in commit.ops.iter().filter(replays) {
12147                let Some(table) = self.schema.table(&operation.table) else {
12148                    continue;
12149                };
12150                if operation.upsert
12151                    && self.visible_row(table, &operation.row_id)?.is_none()
12152                    && operation.presented_values(table).is_some_and(|values| {
12153                        table
12154                            .columns
12155                            .iter()
12156                            .all(|column| values.contains_key(&column.name))
12157                    })
12158                {
12159                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12160                        .expect("row key serialization");
12161                    if let Some(Value::Object(values)) = initial.get(&key) {
12162                        self.apply_outbox_ops(&[OutboxOp {
12163                            values: Some(values.clone()),
12164                            ..operation.clone()
12165                        }])?;
12166                    }
12167                }
12168            }
12169            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12170        }
12171        self.apply_outbox_ops(
12172            self.outbox
12173                .iter()
12174                .flat_map(|commit| &commit.ops)
12175                .filter(replays),
12176        )
12177    }
12178
12179    /// §7.1 replay: every operation is applied over the current visible
12180    /// state. An operation that no longer applies (a server row now holds its
12181    /// unique value) stays pending and invisible until a later replay admits
12182    /// it or the server answers its push.
12183    fn apply_outbox_ops<'a>(
12184        &self,
12185        ops: impl IntoIterator<Item = &'a OutboxOp>,
12186    ) -> Result<(), String> {
12187        #[cfg(feature = "bench-internals")]
12188        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12189        for op in ops {
12190            #[cfg(feature = "bench-internals")]
12191            if let Some(phase) = &mut phase {
12192                phase.unit();
12193            }
12194            if op.upsert {
12195                if let Some((table, values)) = self
12196                    .schema
12197                    .table(&op.table)
12198                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12199                    .and_then(|table| Some((table, op.presented_values(table)?)))
12200                {
12201                    let mut intended: Map<String, Value> = self
12202                        .visible_row(table, &op.row_id)?
12203                        .map(|(row, _)| {
12204                            table
12205                                .columns
12206                                .iter()
12207                                .zip(&row)
12208                                .map(|(column, value)| {
12209                                    (column.name.clone(), column_value_to_json(value))
12210                                })
12211                                .collect()
12212                        })
12213                        .unwrap_or_default();
12214                    intended.extend(values.into_owned());
12215                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12216                        continue;
12217                    }
12218                }
12219            }
12220            self.apply_outbox_op(op)?;
12221        }
12222        Ok(())
12223    }
12224
12225    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12226        let Some(table) = self.schema.table(&op.table) else {
12227            return Ok(());
12228        };
12229        let visible = visible_table(&table.name);
12230        if !op.upsert {
12231            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12232            self.conn
12233                .prepare_cached(&sql)
12234                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12235                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12236            return Ok(());
12237        }
12238        // §7.1: an insert-intent operation that lacks a non-nullable column
12239        // cannot encode; it stays invisible until the send-time prepass drops
12240        // it as sync.outbox_incompatible (§7.4.4).
12241        let Some(values) = op.presented_values(table) else {
12242            return Ok(());
12243        };
12244        // §7.1: the overlay applies the operation's PRESENT columns
12245        // over the current local row. A partial operation over an
12246        // absent local row leaves it absent — the server answers it
12247        // with sync.row_deleted or sync.row_missing (§6.2). An
12248        // insert-intent operation presents every column.
12249        let full = table
12250            .columns
12251            .iter()
12252            .all(|column| values.contains_key(&column.name));
12253        let local = self.visible_row(table, &op.row_id)?;
12254        if local.is_none() && !full {
12255            return Ok(());
12256        }
12257        let mut row: Row = Vec::with_capacity(table.columns.len());
12258        for (index, column) in table.columns.iter().enumerate() {
12259            let incoming = match values.get(&column.name) {
12260                Some(raw) => json_to_column_value(
12261                    if raw.get("$bytes").is_some()
12262                        && table
12263                            .encrypted_columns
12264                            .iter()
12265                            .any(|encrypted| encrypted.index == index)
12266                    {
12267                        &table.wire_columns[index]
12268                    } else {
12269                        column
12270                    },
12271                    Some(raw),
12272                ),
12273                None => Ok(local
12274                    .as_ref()
12275                    .and_then(|(values, _)| values.get(index).cloned())
12276                    .flatten()),
12277            };
12278            row.push(incoming.map_err(|_| {
12279                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12280            })?);
12281        }
12282        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12283        self.write_row(&visible, &table.name, &row, version, None)
12284            .map(|_| ())
12285            .map_err(|error| {
12286                if error.starts_with("UNIQUE constraint failed:") {
12287                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12288                } else {
12289                    error
12290                }
12291            })
12292    }
12293
12294    /// Whether the row is present in the overlay. Existence reads only the
12295    /// key: locked encrypted values stay opaque, and decoding the complete row
12296    /// would reject a valid plain write.
12297    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12298        self.conn
12299            .query_row(
12300                &format!(
12301                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12302                    visible_table(&table.name),
12303                    row_id_predicate(table)
12304                ),
12305                rusqlite::params![row_id],
12306                |row| row.get(0),
12307            )
12308            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12309    }
12310
12311    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12312    /// over ciphertext this device cannot see.
12313    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12314        if !table.has_encrypted_columns() {
12315            return Ok(());
12316        }
12317        let key_id: Option<String> = self
12318            .conn
12319            .query_row(
12320                &format!(
12321                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12322                    visible_table(&table.name),
12323                    row_id_predicate(table)
12324                ),
12325                rusqlite::params![row_id],
12326                |row| row.get(0),
12327            )
12328            .optional()
12329            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12330            .flatten();
12331        match key_id {
12332            None => Ok(()),
12333            Some(key_id) => Err(ClientError {
12334                code: "client.encryption_key_missing".to_owned(),
12335                message: "the row is sealed under a key this device does not hold".to_owned(),
12336                details: Some(json!({ "table": table.name, "keyId": key_id })),
12337                retryable: false,
12338            }),
12339        }
12340    }
12341
12342    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12343    /// keyring now holds into its columns and clear both reserved columns,
12344    /// one change batch per table. `client.decrypt_failed` fails the caller.
12345    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12346        let storage = |message: String| TransportError::new("storage.failed", message);
12347        let tables: Vec<TableSchema> = self
12348            .schema
12349            .tables
12350            .iter()
12351            .filter(|table| table.has_encrypted_columns())
12352            .cloned()
12353            .collect();
12354        for table in &tables {
12355            let base = base_table(&table.name);
12356            let pk = quote_ident(&table.primary_key);
12357            let sealed: Vec<(SqlValue, i64, String)> = self
12358                .conn
12359                .prepare_cached(&format!(
12360                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12361                ))
12362                .and_then(|mut statement| {
12363                    statement
12364                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12365                        .collect()
12366                })
12367                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12368            let mut revealed = Vec::new();
12369            for (key, version, envelopes) in sealed {
12370                if let Some(row) =
12371                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12372                {
12373                    revealed.push((key, version, row));
12374                }
12375            }
12376            if revealed.is_empty() {
12377                continue;
12378            }
12379            self.begin_observation("syncular_reveal").map_err(storage)?;
12380            let outcome = (|| {
12381                let assignments = table
12382                    .encrypted_columns
12383                    .iter()
12384                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12385                    .collect::<Vec<_>>()
12386                    .join(",");
12387                let mut statement = self
12388                    .conn
12389                    .prepare_cached(&format!(
12390                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12391                    ))
12392                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12393                for (key, version, row) in &revealed {
12394                    let mut params = table
12395                        .encrypted_columns
12396                        .iter()
12397                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12398                        .collect::<Result<Vec<_>, _>>()?;
12399                    params.push(key.clone());
12400                    params.push(SqlValue::Integer(*version));
12401                    statement
12402                        .execute(rusqlite::params_from_iter(params))
12403                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12404                }
12405                drop(statement);
12406                self.overlay_dirty.table(&table.name);
12407                self.rebuild_overlay()?;
12408                let mut batch = ChangeAccumulator::default();
12409                batch.table(&table.name);
12410                self.finish_observation("syncular_reveal", batch)
12411            })();
12412            if outcome.is_err() {
12413                self.rollback_observation("syncular_reveal");
12414            }
12415            outcome.map_err(storage)?;
12416        }
12417        Ok(())
12418    }
12419
12420    /// The visible (optimistic) row's values plus version, or `None` when the
12421    /// row is absent from the overlay.
12422    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12423        self.stored_row(table, row_id, false)
12424    }
12425
12426    fn stored_row(
12427        &self,
12428        table: &TableSchema,
12429        row_id: &str,
12430        base: bool,
12431    ) -> Result<Option<(Row, i64)>, String> {
12432        let sql = format!(
12433            "SELECT * FROM {} WHERE {}",
12434            if base {
12435                base_table(&table.name)
12436            } else {
12437                visible_table(&table.name)
12438            },
12439            row_id_predicate(table)
12440        );
12441        let mut stmt = self
12442            .conn
12443            .prepare_cached(&sql)
12444            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12445        let mut rows = stmt
12446            .query(rusqlite::params![row_id])
12447            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12448        let Some(row) = rows
12449            .next()
12450            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12451        else {
12452            return Ok(None);
12453        };
12454        let mut values = Vec::with_capacity(table.columns.len());
12455        for (index, column) in table.columns.iter().enumerate() {
12456            let raw = row
12457                .get_ref(index)
12458                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12459            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12460                if table
12461                    .encrypted_columns
12462                    .iter()
12463                    .any(|encrypted| encrypted.index == index)
12464                {
12465                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12466                    continue;
12467                }
12468            }
12469            let json = sql_ref_to_json(column, raw);
12470            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12471                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12472            })?);
12473        }
12474        let version: i64 = row
12475            .get(table.columns.len())
12476            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12477        Ok(Some((values, version)))
12478    }
12479
12480    // -- realtime (§8) ---------------------------------------------------------------
12481
12482    /// §8.8: the configured realtime policy.
12483    #[must_use]
12484    pub fn realtime_policy(&self) -> RealtimePolicy {
12485        self.realtime_policy
12486    }
12487
12488    /// §8.8: the explicit realtime availability state.
12489    #[must_use]
12490    pub fn realtime_state(&self) -> RealtimeState {
12491        self.realtime_state
12492    }
12493
12494    /// §8.8: set the realtime policy. Refused while the socket is connected:
12495    /// a policy change never silently re-routes an ownership state.
12496    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12497        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12498            return Err(
12499                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12500                    .to_owned(),
12501            );
12502        }
12503        self.realtime_policy = policy;
12504        match policy {
12505            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12506            RealtimePolicy::Required | RealtimePolicy::Optional
12507                if self.realtime_state == RealtimeState::Disabled =>
12508            {
12509                self.set_realtime_state(RealtimeState::Disconnected, None);
12510            }
12511            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12512        }
12513        Ok(())
12514    }
12515
12516    /// Record an availability transition, keeping the reason code bounded and
12517    /// code-like (§7.6) and clearing it where it cannot apply.
12518    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12519        self.realtime_state = state;
12520        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12521        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12522            self.realtime_reason_code = None;
12523            self.realtime_retry_delay_ms = None;
12524        }
12525    }
12526
12527    #[must_use]
12528    fn realtime_connected(&self) -> bool {
12529        self.realtime_state == RealtimeState::Connected
12530    }
12531
12532    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12533        if !self.transport_enabled {
12534            return Err("sync.offline: transport gate is closed".into());
12535        }
12536        if self.realtime_policy == RealtimePolicy::Off {
12537            return Err(
12538                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12539            );
12540        }
12541        if self.realtime_connected() {
12542            return Ok(());
12543        }
12544        self.realtime_state = RealtimeState::Connecting;
12545        match transport.realtime_connect_for_client(&self.client_id) {
12546            Ok(()) => {
12547                self.set_realtime_state(RealtimeState::Connected, None);
12548                Ok(())
12549            }
12550            Err(error) => {
12551                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12552                Err(format!("{}: {}", error.code, error.message))
12553            }
12554        }
12555    }
12556
12557    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12558        self.cancel_sync_round();
12559        // A deliberately disconnected or policy-disabled client has nothing
12560        // to release; a `lost` socket still does (the host owns the handle).
12561        if self.realtime_state == RealtimeState::Disconnected
12562            || self.realtime_state == RealtimeState::Disabled
12563        {
12564            return;
12565        }
12566        let _ = transport.realtime_close();
12567        self.set_realtime_state(
12568            if self.realtime_policy == RealtimePolicy::Off {
12569                RealtimeState::Disabled
12570            } else {
12571                RealtimeState::Disconnected
12572            },
12573            None,
12574        );
12575        self.presence.clear(); // §8.6.1: presence is per-connection
12576    }
12577
12578    /// §8.8: the host observed the socket end without disconnecting it (a
12579    /// peer close, an I/O error, a failed send; the native transport reports
12580    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12581    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12582    /// one background retry intent so the host reconnects and catches up.
12583    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12584        if self.realtime_state != RealtimeState::Connected {
12585            return;
12586        }
12587        self.cancel_sync_round();
12588        let _ = transport.realtime_close();
12589        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12590        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12591        self.presence.clear();
12592    }
12593
12594    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12595    /// document for `scope_key`. Requires a live socket; the document is
12596    /// ephemeral (lost on disconnect). Authorization is the connection's
12597    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12598    /// server with `presence.forbidden`.
12599    pub fn set_presence(
12600        &mut self,
12601        transport: &mut dyn Transport,
12602        scope_key: &str,
12603        doc: Option<&Value>,
12604    ) -> Result<(), String> {
12605        if !self.transport_enabled {
12606            return Err("sync.offline: transport gate is closed".into());
12607        }
12608        if !self.realtime_connected() {
12609            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12610        }
12611        let text = encode_presence_publish(scope_key, doc);
12612        transport
12613            .realtime_send(&text)
12614            .map_err(|e| format!("{}: {}", e.code, e.message))
12615    }
12616
12617    /// §8.6: the peers currently present on a scope key (ephemeral).
12618    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12619        self.presence
12620            .get(scope_key)
12621            .map(|peers| peers.values().cloned().collect())
12622            .unwrap_or_default()
12623    }
12624
12625    /// §8.6 apply an inbound presence fanout to the local map.
12626    fn apply_presence(
12627        &mut self,
12628        scope_key: String,
12629        kind: Option<PresenceKind>,
12630        actor_id: Option<String>,
12631        client_id: Option<String>,
12632        doc: Option<Value>,
12633        error: Option<String>,
12634    ) {
12635        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12636        // it as a `presence` event carrying the server's `error` code.
12637        if error.is_some() {
12638            return;
12639        }
12640        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12641            return;
12642        };
12643        let peer_key = format!("{actor_id} {client_id}");
12644        match kind {
12645            PresenceKind::Leave => {
12646                if let Some(peers) = self.presence.get_mut(&scope_key) {
12647                    peers.remove(&peer_key);
12648                    if peers.is_empty() {
12649                        self.presence.remove(&scope_key);
12650                    }
12651                }
12652            }
12653            _ => {
12654                let doc = match doc {
12655                    Some(Value::Object(_)) => doc.unwrap(),
12656                    _ => return,
12657                };
12658                self.presence.entry(scope_key).or_default().insert(
12659                    peer_key,
12660                    PresencePeer {
12661                        actor_id,
12662                        client_id,
12663                        doc,
12664                    },
12665                );
12666            }
12667        }
12668    }
12669
12670    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12671    pub fn on_realtime_text(&mut self, text: &str) {
12672        if self.stopped || !self.transport_enabled || self.security_preflight {
12673            return;
12674        }
12675        match parse_control(text) {
12676            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12677                if requires_sync {
12678                    // §8.1: pull before trusting the socket for continuity.
12679                    self.set_sync_needed(true, true);
12680                }
12681            }
12682            Ok(ControlMessage::Presence {
12683                scope_key,
12684                kind,
12685                actor_id,
12686                client_id,
12687                doc,
12688                error,
12689                ..
12690            }) => {
12691                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12692            }
12693            Ok(ControlMessage::Wake { .. }) => {
12694                // §8.3: any wake-up means "run a pull soon", never data.
12695                self.set_sync_needed(true, true);
12696            }
12697            _ => {}
12698        }
12699    }
12700
12701    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12702    /// a pull response per section; an unapplied delta is a wake-up.
12703    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12704    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12705        let mut transport = crate::round::DownloadResults::default();
12706        self.on_realtime_binary(&mut transport, bytes);
12707        transport.controls
12708    }
12709
12710    /// §8.2: a delta applies exactly like a live section and sets the replica
12711    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12712    /// duplicate. A delta the client does not apply (no live unit, a status
12713    /// other than `active`, a decode or apply failure) is a drop and raises
12714    /// sync-needed.
12715    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12716        if self.stopped || !self.transport_enabled || self.security_preflight {
12717            return;
12718        }
12719        #[cfg(feature = "bench-internals")]
12720        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12721        let message = match decode_message(bytes) {
12722            Ok(m) if m.msg_kind == MsgKind::Response => m,
12723            _ => {
12724                self.set_sync_needed(true, true);
12725                return;
12726            }
12727        };
12728        #[cfg(feature = "bench-internals")]
12729        drop(decode_phase);
12730        // §8.2: a delta's header means what a pull response's header means. A
12731        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12732        if let Some(Frame::RespHeader {
12733            required_schema_version: Some(required),
12734            latest_schema_version,
12735            ..
12736        }) = message.frames.first()
12737        {
12738            let floor = SchemaFloor {
12739                required_schema_version: Some(*required),
12740                latest_schema_version: *latest_schema_version,
12741            };
12742            if self.set_schema_floor(Some(floor)).is_err() {
12743                self.set_sync_needed(true, true);
12744            }
12745            return;
12746        }
12747        let through = message.frames.iter().find_map(|frame| match frame {
12748            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12749            _ => None,
12750        });
12751        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12752        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12753                reset_required: false,
12754                log_epoch,
12755                ..
12756            }) if Some(log_epoch) == epoch.as_ref());
12757        let mut applied: Option<i64> = None;
12758        dropped |= self.begin_coalesced_apply().is_err();
12759        let mut receivers: Option<Vec<UnitRecord>> = None;
12760        for frame in message.frames {
12761            if dropped {
12762                break;
12763            }
12764            match frame {
12765                Frame::LiveStart { status, .. } => {
12766                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12767                        dropped = true;
12768                        break;
12769                    };
12770                    if status != LiveStatus::Active || cursor < 0 {
12771                        dropped = true;
12772                    } else if !through.is_some_and(|through| through <= cursor) {
12773                        // A catching-up unit receives no delta (§8.2).
12774                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12775                            units
12776                                .into_iter()
12777                                .filter(|unit| {
12778                                    unit.status == UnitStatus::Active
12779                                        && unit.echo.is_some()
12780                                        && unit.cursor.is_none()
12781                                })
12782                                .collect()
12783                        });
12784                        dropped = receivers.is_none();
12785                    }
12786                }
12787                Frame::Commit {
12788                    commit_seq,
12789                    tables,
12790                    changes,
12791                    ..
12792                } => {
12793                    let Some(receivers) = receivers.clone() else {
12794                        continue;
12795                    };
12796                    dropped = self
12797                        .apply_live_commit(
12798                            &receivers,
12799                            &tables,
12800                            changes,
12801                            commit_seq,
12802                            AppliedServerCommitSource::Delta,
12803                        )
12804                        .is_err();
12805                }
12806                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12807                    dropped = self
12808                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12809                        .is_err();
12810                    applied = Some(through_seq);
12811                }
12812                Frame::Error { .. } => dropped = true,
12813                _ => {}
12814            }
12815        }
12816        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12817        // the replica cursor, so the delta counts as a drop.
12818        if self.finish_coalesced_apply().is_err() || dropped {
12819            applied = None;
12820            dropped = true;
12821        }
12822        if let Some(cursor) = applied {
12823            // §8.2 ack point: the delta's throughSeq.
12824            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12825            let _ = transport.realtime_send(&ack);
12826        }
12827        if dropped {
12828            self.set_sync_needed(true, true);
12829        }
12830    }
12831
12832    /// §8.2 ack point after a pull round on a live connection: the replica
12833    /// cursor while it is at least `0`.
12834    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12835        if !self.realtime_connected() {
12836            return;
12837        }
12838        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12839            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12840            let _ = transport.realtime_send(&ack);
12841        }
12842    }
12843}
12844
12845#[cfg(test)]
12846mod previous_version_wiring_tests;
12847
12848#[cfg(test)]
12849mod identity_tests;
12850
12851#[cfg(test)]
12852mod insert_intent_tests;
12853
12854#[cfg(test)]
12855mod robustness_tests;
12856
12857#[cfg(test)]
12858mod range_image_tests;