Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use std::cell::{Cell, RefCell};
12use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
13
14use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
15use rusqlite::{Connection, OpenFlags, OptionalExtension};
16use serde::{Deserialize, Serialize};
17use serde_json::{Map, Value};
18use sha2::{Digest, Sha256};
19use ssp2::decode::WIRE_VERSION;
20use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
21use ssp2::primitives::RawJson;
22use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
23use ssp2::{
24    decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
25    PresenceKind,
26};
27
28use crate::api::{
29    ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
30    ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
31    ClientDiagnosticsStorage, ClientLimits, CommandEffects, CommitOperation,
32    CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution,
33    CommitOutcomeStatus, ConflictRecord, CoverageSnapshot, DiagnosticLastChange,
34    DiagnosticLastRound, DiagnosticRoundCounters, DiagnosticSubscription, FetchedBlob, LeaseState,
35    LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget, LocalDataRebootstrapInput,
36    LocalDataRebootstrapResult, Mutation, PresencePeer, QueryRow, QuerySnapshot, QueryValue,
37    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
38    SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
39    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
40    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
41};
42#[cfg(feature = "bench-internals")]
43use crate::bench::{Phase, Recorder};
44use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
45use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
46use crate::values::{
47    bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
48    encode_sparse_row_json, full_row_values, json_to_column_value, json_to_scope_map,
49    normalize_values_casing, render_row_id_json, scope_map_to_json, sort_scope_map,
50};
51
52/// §4.2 default: rows and SQLite images, both applied in committed chunks.
53/// Bit 3 (signed URLs, §5.4) is
54/// added per transport capability at request-build time.
55const DEFAULT_ACCEPT: u8 = 0b0111;
56const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
57const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
58const ACCEPT_SQLITE: u8 = 1 << 2;
59const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
60const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
61
62/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
63const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
64const LOCAL_REVISION_KEY: &str = "localRevision";
65const CLIENT_ID_KEY: &str = "clientId";
66const LEASE_STATE_KEY: &str = "leaseState";
67const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
68const LOG_EPOCH_KEY: &str = "logEpoch";
69/// Persisted fail-closed quarantine marker: present while a security preflight
70/// is pending and `activateSecurity` has yet to run. Storing it in the replica
71/// keeps the gate with the data it protects, so a rebuilt host handle reopening
72/// the same file returns in preflight even though its in-memory state is fresh.
73const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
74const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
75const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
76/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
77/// the new schema after a bump. Never a wire code (§10.3).
78const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
79/// §5.11 client-local code: an encode at the push seam resolved no usable key
80/// id or named an unknown key. Never a wire code (§10.3).
81const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
82
83#[derive(Debug, Serialize, Deserialize)]
84#[serde(rename_all = "camelCase", deny_unknown_fields)]
85struct PersistedLocalDataRebootstrapReceipt {
86    version: u8,
87    retained_commits: u64,
88    reset_subscriptions: u64,
89}
90
91fn invalid_local_rebootstrap_receipt() -> String {
92    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
93}
94
95fn encode_local_rebootstrap_receipt(
96    retained_commits: usize,
97    reset_subscriptions: usize,
98) -> Result<String, String> {
99    let retained_commits =
100        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
101    let reset_subscriptions =
102        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
103    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
104        return Err(invalid_local_rebootstrap_receipt());
105    }
106    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
107        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
108        retained_commits,
109        reset_subscriptions,
110    })
111    .map_err(|_| invalid_local_rebootstrap_receipt())
112}
113
114fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
115    // Pre-0.15.36 markers proved application but did not retain the receipt.
116    if value == "v1" {
117        return Ok((0, 0));
118    }
119    let receipt: PersistedLocalDataRebootstrapReceipt =
120        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
121    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
122        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
123        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
124    {
125        return Err(invalid_local_rebootstrap_receipt());
126    }
127    Ok((
128        usize::try_from(receipt.retained_commits)
129            .map_err(|_| invalid_local_rebootstrap_receipt())?,
130        usize::try_from(receipt.reset_subscriptions)
131            .map_err(|_| invalid_local_rebootstrap_receipt())?,
132    ))
133}
134
135#[derive(Debug, Clone, Copy, PartialEq, Eq)]
136enum SubState {
137    Active,
138    Revoked,
139    Failed,
140}
141
142#[cfg(test)]
143mod observation_tests {
144    use super::*;
145    use crate::native_transport::HostTransport;
146    use crate::values::encode_row_json;
147    use serde_json::json;
148
149    fn client() -> SyncClient {
150        SyncClient::new(
151            "retry-test".to_owned(),
152            &json!({
153                "version": 1,
154                "tables": [{
155                    "name": "tasks",
156                    "primaryKey": "id",
157                    "columns": [
158                        { "name": "id", "type": "string", "nullable": false },
159                        { "name": "project_id", "type": "string", "nullable": false }
160                    ],
161                    "scopes": [{ "pattern": "project:{project_id}" }]
162                }]
163            }),
164            ClientLimits::default(),
165        )
166        .expect("test client")
167    }
168
169    #[cfg(feature = "e2ee")]
170    #[test]
171    fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
172        let schema = json!({
173            "version": 1,
174            "tables": [{
175                "name": "secrets",
176                "primaryKey": "id",
177                "columns": [
178                    { "name": "id", "type": "string", "nullable": false },
179                    { "name": "encryption_key_id", "type": "string", "nullable": true },
180                    { "name": "note", "type": "bytes", "nullable": true,
181                      "encrypted": true, "declaredType": "string" }
182                ],
183                "scopes": []
184            }]
185        });
186        let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
187            .expect("test client");
188        let mut config = crate::values::EncryptionConfig::default();
189        config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
190        config
191            .key_id_columns
192            .insert("secrets".to_owned(), "encryption_key_id".to_owned());
193        client.set_encryption(config);
194        // A patch on a locally absent row presents an encrypted column with no
195        // stored key id to fall back to. Authoring succeeds...
196        let commit_id = client
197            .patch(
198                "secrets",
199                "ghost",
200                Map::from_iter([("note".to_owned(), json!("x"))]),
201                None,
202            )
203            .expect("patch records without an author-time encrypt failure");
204        assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
205        // ...and the push seam raises a durable local rejection, not a throw.
206        assert!(client.drop_unencodable_outbox().unwrap());
207        assert!(client.pending_commit_ids().is_empty());
208        let rejections = client.rejections();
209        assert_eq!(rejections.len(), 1);
210        assert_eq!(rejections[0].code, "client.encrypt_failed");
211        assert_eq!(rejections[0].client_commit_id, commit_id);
212        assert_eq!(
213            client
214                .commit_outcome(&commit_id)
215                .unwrap()
216                .map(|outcome| outcome.status),
217            Some(CommitOutcomeStatus::Rejected)
218        );
219    }
220
221    #[cfg(feature = "e2ee")]
222    #[test]
223    fn stored_key_fallback_resolves_an_integer_primary_key() {
224        let schema = json!({
225            "version": 1,
226            "tables": [{
227                "name": "patients",
228                "primaryKey": "id",
229                "columns": [
230                    { "name": "id", "type": "integer", "nullable": false },
231                    { "name": "encryption_key_id", "type": "string", "nullable": true },
232                    { "name": "note", "type": "bytes", "nullable": true,
233                      "encrypted": true, "declaredType": "string" }
234                ],
235                "scopes": []
236            }]
237        });
238        let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
239            .expect("test client");
240        client.create_synced_tables().unwrap();
241        client
242            .conn
243            .execute(
244                "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
245                 VALUES (5, 'k1', NULL, 1)",
246                [],
247            )
248            .unwrap();
249        let mut config = crate::values::EncryptionConfig::default();
250        config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
251        config
252            .key_id_columns
253            .insert("patients".to_owned(), "encryption_key_id".to_owned());
254        client.set_encryption(config);
255        let table = client.schema.table("patients").unwrap();
256        let fallback = client
257            .stored_key_fallback(table, "5")
258            .expect("the stored integer key resolves the fallback");
259        assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
260    }
261
262    #[test]
263    fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
264        let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
265            {"name":"id","type":"string","nullable":false},
266            {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
267        for state in ["absent", "cached", "pinned"] {
268            let path = std::env::temp_dir().join(format!(
269                "syncular-blob-stage-{}.sqlite",
270                uuid::Uuid::new_v4()
271            ));
272            let mut client = SyncClient::open_path(
273                "stage-test".into(),
274                &schema,
275                Default::default(),
276                path.to_str().unwrap(),
277            )
278            .unwrap();
279            let bytes = b"atomic stage";
280            if state != "absent" {
281                client
282                    .upload_blob(bytes, Some("text/plain".into()), None)
283                    .unwrap();
284                if state == "cached" {
285                    client
286                        .conn
287                        .execute("DELETE FROM _syncular_blob_uploads", [])
288                        .unwrap();
289                }
290            }
291            let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
292            client
293                .conn
294                .execute_batch(
295                    "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
296                   BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
297                 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
298                   BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
299                )
300                .unwrap();
301            let error = client
302                .upload_blob(bytes, Some("application/octet-stream".into()), None)
303                .unwrap_err();
304            assert!(error.contains("injected stage failure"), "{state}: {error}");
305            assert!(client.conn.is_autocommit());
306            let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
307            drop(client);
308            let mut reopened = SyncClient::open_path(
309                "stage-test".into(),
310                &schema,
311                Default::default(),
312                path.to_str().unwrap(),
313            )
314            .unwrap();
315            let durable = reopened
316                .query("SELECT * FROM _syncular_blobs", &[])
317                .unwrap();
318            reopened
319                .conn
320                .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
321                .unwrap();
322            let retried = reopened.upload_blob(bytes, None, None).unwrap();
323            assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
324            assert_eq!(
325                reopened
326                    .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
327                    .unwrap(),
328                vec![Map::from_iter([(
329                    "blob_id".into(),
330                    Value::from(blob_id_for(bytes)),
331                )])]
332            );
333            drop(reopened);
334            std::fs::remove_file(&path).unwrap();
335            assert_eq!(durable, bodies, "{state}: reopened body state");
336            assert_eq!(immediate, bodies, "{state}: immediate body state");
337        }
338    }
339
340    #[test]
341    fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
342        let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
343            {"name":"id","type":"string","nullable":false},
344            {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
345        for fault in [
346            "missing",
347            "body-type",
348            "length",
349            "hash",
350            "pin-metadata",
351            "read",
352            "pin-delete",
353        ] {
354            let path = std::env::temp_dir().join(format!(
355                "syncular-blob-upload-{}.sqlite",
356                uuid::Uuid::new_v4()
357            ));
358            let mut client = SyncClient::open_path(
359                "upload-test".into(),
360                &schema,
361                Default::default(),
362                path.to_str().unwrap(),
363            )
364            .unwrap();
365            let bytes = b"durable upload";
366            let ref_value = client
367                .upload_blob(bytes, Some("text/plain".into()), None)
368                .unwrap();
369            let commit = client
370                .mutate(vec![Mutation::Upsert {
371                    table: "attachments".into(),
372                    values: Map::from_iter([
373                        ("id".into(), Value::from("pending")),
374                        (
375                            "file".into(),
376                            Value::from(format!(
377                                "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
378                                blob_id_for(bytes),
379                                bytes.len()
380                            )),
381                        ),
382                    ]),
383                    base_version: None,
384                }])
385                .unwrap();
386            client
387                .conn
388                .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
389                .unwrap();
390            client.conn.execute_batch(match fault {
391                "missing" => "DELETE FROM _syncular_blobs",
392                "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
393                "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
394                "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
395                "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
396                "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
397                _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
398            }).unwrap();
399            let state = if fault == "read" {
400                Vec::new()
401            } else {
402                client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
403            };
404            let mut transport = CountingRealtimeTransport::default();
405            let error = client.flush_blob_uploads(&mut transport).unwrap_err();
406            assert_eq!(
407                error.code,
408                if matches!(fault, "read" | "pin-delete") {
409                    "client.failed"
410                } else {
411                    "sync.local_corrupt"
412                },
413                "{fault}"
414            );
415            assert_eq!(
416                transport.blob_uploads.len(),
417                usize::from(fault == "pin-delete"),
418                "{fault}: corrupt bytes must never be uploaded"
419            );
420            assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
421            if fault != "read" {
422                assert_eq!(
423                    client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
424                    state
425                );
426            } else {
427                client
428                    .conn
429                    .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
430                    .unwrap();
431            }
432            drop(client);
433            let mut reopened = SyncClient::open_path(
434                "upload-test".into(),
435                &schema,
436                Default::default(),
437                path.to_str().unwrap(),
438            )
439            .unwrap();
440            assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
441            if fault != "read" {
442                assert_eq!(
443                    reopened
444                        .query("SELECT * FROM _syncular_blobs", &[])
445                        .unwrap(),
446                    state
447                );
448            }
449            if fault == "pin-delete" {
450                reopened
451                    .conn
452                    .execute_batch("DROP TRIGGER fail_pin_delete")
453                    .unwrap();
454            }
455            if fault == "pin-metadata" {
456                reopened
457                    .conn
458                    .execute(
459                        "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
460                        [],
461                    )
462                    .unwrap();
463            }
464            reopened
465                .conn
466                .execute_batch(
467                    "DELETE FROM _syncular_blobs;
468                     INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
469                )
470                .unwrap();
471            transport.blob_uploads.clear();
472            reopened.flush_blob_uploads(&mut transport).unwrap();
473            assert_eq!(
474                transport.blob_uploads,
475                [(blob_id_for(bytes), bytes.to_vec())]
476            );
477            assert_eq!(
478                reopened
479                    .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
480                    .unwrap(),
481                Vec::<Map<String, Value>>::new()
482            );
483            assert_eq!(reopened.pending_commit_ids(), [commit]);
484            assert_eq!(
485                reopened
486                    .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
487                    .unwrap()
488                    .unwrap()
489                    .bytes,
490                bytes
491            );
492            drop(reopened);
493            std::fs::remove_file(path).unwrap();
494        }
495    }
496
497    #[test]
498    fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
499        let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
500            {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
501            "scopes":[{"pattern":"project:{project_id}"}]}]});
502        for realtime in [false, true] {
503            for fault in [
504                "none",
505                "row",
506                "revision",
507                "commit",
508                "cursor",
509                "error",
510                "missing-end",
511            ] {
512                if realtime && fault == "missing-end" {
513                    continue;
514                }
515                let path = std::env::temp_dir()
516                    .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
517                let mut client = SyncClient::open_path(
518                    "frame-test".into(),
519                    &schema,
520                    Default::default(),
521                    path.to_str().unwrap(),
522                )
523                .unwrap();
524                client.set_meta(LOG_EPOCH_KEY, "epoch-1");
525                client
526                    .subscribe(
527                        "tasks".into(),
528                        "tasks".into(),
529                        vec![("project_id".into(), vec!["p1".into()])],
530                        None,
531                    )
532                    .unwrap();
533                client.subs[0].cursor = 0;
534                client.subs[0].synced_once = true;
535                client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
536                client.persist_sub(&client.subs[0]).unwrap();
537                client.realtime_connected = true;
538                client.drain_change_batches();
539                let mut transport = CountingRealtimeTransport::default();
540                let (_, meta) = client.build_request(false);
541                let mut response = Message {
542                    wire_version: WIRE_VERSION,
543                    msg_kind: MsgKind::Response,
544                    frames: vec![
545                        Frame::RespHeader {
546                            required_schema_version: None,
547                            latest_schema_version: None,
548                            log_epoch: Some("epoch-1".into()),
549                            reset_required: Some(false),
550                        },
551                        Frame::SubStart {
552                            id: "tasks".into(),
553                            status: SubStatus::Active,
554                            reason_code: "".into(),
555                            effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
556                            bootstrap: false,
557                        },
558                    ],
559                };
560                for (index, id) in ["first", "second"].iter().enumerate() {
561                    let values = Map::from_iter([
562                        ("id".into(), Value::from(*id)),
563                        ("project_id".into(), Value::from("p1")),
564                    ]);
565                    response.frames.push(Frame::Commit {
566                        commit_seq: index as i64 + 1,
567                        actor_id: "writer".into(),
568                        created_at_ms: 1,
569                        tables: vec!["tasks".into()],
570                        changes: vec![ssp2::model::Change {
571                            table_index: 0,
572                            row_id: (*id).into(),
573                            op: Op::Upsert,
574                            row_version: Some(index as i64 + 1),
575                            scopes: vec![("project_id".into(), "p1".into())],
576                            row: Some(
577                                encode_row_json(
578                                    client.schema.table("tasks").unwrap(),
579                                    id,
580                                    &values,
581                                    &client.encryption,
582                                )
583                                .unwrap(),
584                            ),
585                        }],
586                    });
587                }
588                response.frames.push(Frame::SubEnd {
589                    next_cursor: 2,
590                    bootstrap_state: None,
591                });
592                let valid = response.clone();
593                match fault {
594                    "row" => {
595                        let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
596                        let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
597                    },
598                    "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
599                    "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
600                    "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
601                    "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
602                    "missing-end" => {response.frames.truncate(3);},
603                    _ => {}
604                }
605                if realtime {
606                    client.on_realtime_binary(&mut transport, &encode_message(&response));
607                } else {
608                    assert_eq!(
609                        matches!(
610                            client.process_response(&mut transport, response, &meta),
611                            SyncOutcome::Ok(_)
612                        ),
613                        fault == "none",
614                        "{fault}"
615                    );
616                }
617                assert!(
618                    client.conn.is_autocommit(),
619                    "no open transaction after {realtime}/{fault}"
620                );
621                let count = if matches!(fault, "none" | "cursor") {
622                    2
623                } else {
624                    1
625                };
626                let expected_revision = count + u64::from(realtime && fault != "none");
627                assert_eq!(
628                    client.local_revision(),
629                    expected_revision,
630                    "{realtime}/{fault}"
631                );
632                let batches = client.drain_change_batches();
633                assert_eq!(
634                    batches.len(),
635                    expected_revision as usize,
636                    "{realtime}/{fault}"
637                );
638                let row_batches: Vec<_> = batches
639                    .iter()
640                    .filter(|batch| !batch.tables.is_empty())
641                    .collect();
642                assert_eq!(row_batches.len(), count as usize);
643                assert_eq!(
644                    row_batches
645                        .iter()
646                        .map(|b| b.revision.clone())
647                        .collect::<Vec<_>>(),
648                    (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
649                );
650                for table in ["tasks", "_syncular_base_tasks"] {
651                    assert_eq!(
652                        client
653                            .conn
654                            .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
655                                .get::<_, u64>(0))
656                            .unwrap(),
657                        count,
658                        "{realtime}/{fault}/{table}"
659                    );
660                }
661                assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
662                assert_eq!(
663                    transport
664                        .messages
665                        .iter()
666                        .filter(|m| m.contains("\"ack\""))
667                        .count(),
668                    usize::from(fault == "none")
669                );
670                if matches!(fault, "revision" | "commit" | "cursor") {
671                    client
672                        .conn
673                        .execute_batch("DROP TRIGGER fail_frame")
674                        .unwrap();
675                }
676                drop(client);
677                let mut reopened = SyncClient::open_path(
678                    "frame-test".into(),
679                    &schema,
680                    Default::default(),
681                    path.to_str().unwrap(),
682                )
683                .unwrap();
684                assert_eq!(reopened.local_revision(), expected_revision);
685                assert_eq!(
686                    reopened
687                        .query("SELECT id FROM tasks ORDER BY id", &[])
688                        .unwrap()
689                        .len(),
690                    count as usize
691                );
692                assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
693                let (_, meta) = reopened.build_request(false);
694                assert!(matches!(
695                    reopened.process_response(&mut transport, valid, &meta),
696                    SyncOutcome::Ok(_)
697                ));
698                assert_eq!(
699                    reopened
700                        .query("SELECT id FROM tasks ORDER BY id", &[])
701                        .unwrap()
702                        .len(),
703                    2
704                );
705                assert_eq!(reopened.subs[0].cursor, 2);
706                drop(reopened);
707                std::fs::remove_file(path).unwrap();
708            }
709        }
710    }
711
712    #[test]
713    fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
714        for status in [SubStatus::Reset, SubStatus::Revoked] {
715            for fault in ["subscription", "revision"] {
716                let mut client = client();
717                client
718                    .subscribe(
719                        "tasks".into(),
720                        "tasks".into(),
721                        vec![("project_id".into(), vec!["p1".into()])],
722                        None,
723                    )
724                    .unwrap();
725                client.subs[0].cursor = 41;
726                client.subs[0].synced_once = true;
727                client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
728                client.persist_sub(&client.subs[0]).unwrap();
729                client
730                    .mutate(vec![Mutation::Upsert {
731                        table: "tasks".into(),
732                        values: Map::from_iter([
733                            ("id".into(), Value::from("pending")),
734                            ("project_id".into(), Value::from("p1")),
735                        ]),
736                        base_version: None,
737                    }])
738                    .unwrap();
739                let ids = client.pending_commit_ids();
740                let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
741                let revision = client.local_revision();
742                client.drain_change_batches();
743                let (_, meta) = client.build_request(false);
744                client.conn.execute_batch(if fault == "subscription" {
745                    "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
746                } else {
747                    "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
748                }).unwrap();
749                let mut transport = CountingRealtimeTransport::default();
750                let mut report = SyncReport::default();
751                // Reset changes window completeness and therefore needs a revision.
752                if status == SubStatus::Reset {
753                    client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
754                }
755                let result = client.process_section(
756                    &mut transport,
757                    "tasks",
758                    status,
759                    "sync.scope_revoked",
760                    vec![],
761                    vec![],
762                    Some((42, None)),
763                    &meta,
764                    &mut report,
765                );
766                assert!(result.is_err(), "{status:?}/{fault}");
767                assert!(client.conn.is_autocommit());
768                assert_eq!(client.subs[0].cursor, 41);
769                assert_eq!(client.subs[0].state, SubState::Active);
770                assert_eq!(client.pending_commit_ids(), ids);
771                assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
772                assert!(client.rejections.is_empty());
773                assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
774                assert_eq!(client.local_revision(), revision);
775                assert!(client.drain_change_batches().is_empty());
776                assert!(report.revoked.is_empty() && report.resets.is_empty());
777            }
778        }
779    }
780
781    #[test]
782    fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
783        let mut client = client();
784        client
785            .subscribe(
786                "tasks".into(),
787                "tasks".into(),
788                vec![("project_id".into(), vec!["p1".into()])],
789                None,
790            )
791            .unwrap();
792        client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
793        client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
794            CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
795        let row = |id: &str| ssp2::segment::SegmentRow {
796            server_version: 1,
797            values: vec![
798                Some(ColumnValue::String(id.into())),
799                Some(ColumnValue::String("p1".into())),
800            ],
801        };
802        let segment = RowsSegment {
803            table: "tasks".into(),
804            schema_version: 1,
805            columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
806            blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
807        };
808        assert!(client.apply_segment(0, &segment, true).is_err());
809        assert!(client.conn.is_autocommit());
810        assert_eq!(
811            client
812                .query("SELECT id FROM tasks ORDER BY id", &[])
813                .unwrap(),
814            vec![Map::from_iter([("id".into(), Value::from("first"))])]
815        );
816        assert_eq!(client.local_revision(), 1);
817        assert_eq!(client.drain_change_batches().len(), 1);
818        assert_eq!(client.subs[0].cursor, -1);
819        client
820            .conn
821            .execute_batch("DROP TRIGGER fail_block")
822            .unwrap();
823        assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
824        assert_eq!(client.local_revision(), 3);
825        assert_eq!(client.drain_change_batches().len(), 2);
826    }
827
828    #[test]
829    fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
830        assert_eq!(
831            client().diagnostics_storage().blob_cache_bytes_approx,
832            Some(0)
833        );
834        let client = SyncClient::new(
835            "diagnostic-storage".to_owned(),
836            &json!({
837                "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
838                    "columns": [{"name": "id", "type": "string", "nullable": false},
839                        {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
840            }),
841            ClientLimits {
842                blob_cache_max_bytes: Some(5),
843                ..Default::default()
844            },
845        )
846        .unwrap();
847        let empty = client.diagnostics_storage();
848        assert_eq!(empty.status, "healthy");
849        assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
850        assert_eq!(empty.retained_outcome_entries, Some(0));
851        assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
852        assert_eq!(empty.blob_cache_bytes_approx, Some(0));
853        client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
854            INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
855                VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
856            INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
857                VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
858        let written = client.diagnostics_storage();
859        assert_eq!(written.status, "pressure");
860        assert_eq!(written.pending_outbox_bytes_approx, Some(2));
861        assert_eq!(written.retained_outcome_entries, Some(2));
862        assert_eq!(written.retained_outcome_bytes_approx, Some(6));
863        assert_eq!(written.blob_cache_bytes_approx, Some(8));
864        assert_eq!(
865            written.pressure_reason_code.as_deref(),
866            Some("client.blob_cache_over_limit")
867        );
868        client
869            .conn
870            .execute_batch(
871                "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
872            DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
873            )
874            .unwrap();
875        let staged = client.diagnostics_storage();
876        assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
877        assert_eq!(staged.retained_outcome_entries, Some(0));
878        assert_eq!(staged.blob_cache_bytes_approx, Some(0));
879        assert_eq!(staged.status, "healthy");
880        client.conn.execute_batch("ROLLBACK").unwrap();
881        assert_eq!(client.diagnostics_storage(), written);
882        client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
883        let unreadable = client.diagnostics_storage();
884        assert_eq!(unreadable.status, "unreadable");
885        assert!(unreadable.database_bytes_approx.is_none());
886        assert!(unreadable.pending_outbox_bytes_approx.is_none());
887        assert!(unreadable.retained_outcome_entries.is_none());
888        assert!(unreadable.retained_outcome_bytes_approx.is_none());
889        assert!(unreadable.blob_cache_bytes_approx.is_none());
890        client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
891        let restored = client.diagnostics_storage();
892        assert_eq!(restored.status, written.status);
893        assert_eq!(
894            restored.retained_outcome_bytes_approx,
895            written.retained_outcome_bytes_approx
896        );
897        client.conn.set_prepared_statement_cache_capacity(0);
898        assert_eq!(client.diagnostics_storage(), restored);
899    }
900
901    #[test]
902    fn background_retry_deadlines_back_off_and_reset() {
903        let mut client = client();
904        client.schedule_background_retry();
905        assert!(matches!(
906            client.drain_sync_intents().as_slice(),
907            [SyncIntent::Background { delay_ms: 250 }]
908        ));
909        client.schedule_background_retry();
910        assert!(matches!(
911            client.drain_sync_intents().as_slice(),
912            [SyncIntent::Background { delay_ms: 500 }]
913        ));
914        client.reset_background_retry();
915        client.schedule_background_retry();
916        assert!(matches!(
917            client.drain_sync_intents().as_slice(),
918            [SyncIntent::Background { delay_ms: 250 }]
919        ));
920    }
921
922    #[test]
923    fn log_epoch_reset_requests_an_immediate_follow_up_round() {
924        let mut client = client();
925        client
926            .subscribe(
927                "epoch-tasks".to_owned(),
928                "tasks".to_owned(),
929                vec![("project_id".to_owned(), vec!["p1".to_owned()])],
930                None,
931            )
932            .expect("subscribe");
933        client.drain_sync_intents();
934
935        client
936            .run_log_epoch_reset("epoch-2")
937            .expect("reset partition log epoch");
938
939        assert!(matches!(
940            client.drain_sync_intents().as_slice(),
941            [SyncIntent::Interactive]
942        ));
943    }
944
945    #[derive(Default)]
946    struct CountingRealtimeTransport {
947        connects: usize,
948        closes: usize,
949        messages: Vec<String>,
950        blob_uploads: Vec<(String, Vec<u8>)>,
951        blob_download: Option<Vec<u8>>,
952    }
953
954    impl Transport for CountingRealtimeTransport {
955        fn blob_upload(
956            &mut self,
957            blob_id: &str,
958            bytes: &[u8],
959            _media_type: Option<&str>,
960        ) -> Result<(), TransportError> {
961            self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
962            Ok(())
963        }
964
965        fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
966            self.blob_download
967                .clone()
968                .map(BlobDownload::Bytes)
969                .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
970        }
971
972        fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
973            Err(TransportError::new("sync.transport_failed", "offline"))
974        }
975
976        fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
977            Err(TransportError::new("sync.transport_failed", "offline"))
978        }
979
980        fn download_segment(
981            &mut self,
982            _request: &SegmentRequest,
983            _on_progress: &mut dyn FnMut(u64),
984        ) -> Result<Vec<u8>, TransportError> {
985            Err(TransportError::new("sync.transport_failed", "offline"))
986        }
987
988        fn realtime_connect(&mut self) -> Result<(), TransportError> {
989            self.connects += 1;
990            Ok(())
991        }
992
993        fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
994            self.messages.push(text.to_owned());
995            Ok(())
996        }
997
998        fn realtime_close(&mut self) -> Result<(), TransportError> {
999            self.closes += 1;
1000            Ok(())
1001        }
1002    }
1003
1004    #[test]
1005    fn blob_results_own_downloaded_bytes() {
1006        let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
1007            {"name":"id","type":"string","nullable":false},
1008            {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
1009        let bytes = b"owned typed result".to_vec();
1010        let blob_id = blob_id_for(&bytes);
1011        let mut transport = CountingRealtimeTransport {
1012            blob_download: Some(bytes.clone()),
1013            ..Default::default()
1014        };
1015        let result = {
1016            let mut client =
1017                SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
1018                    .unwrap();
1019            let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
1020            assert_eq!(result.blob_id, blob_id);
1021            assert_eq!(result.byte_length, bytes.len() as i64);
1022            assert_eq!(result.bytes, bytes);
1023            assert_eq!(result.media_type, None);
1024            assert_eq!(
1025                client
1026                    .conn
1027                    .query_row(
1028                        "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
1029                        rusqlite::params![blob_id],
1030                        |row| row.get::<_, i64>(0),
1031                    )
1032                    .unwrap(),
1033                0
1034            );
1035            result
1036        };
1037        assert_eq!(result.bytes, bytes);
1038
1039        let mut client =
1040            SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
1041        assert_eq!(
1042            client
1043                .fetch_blob_bytes(&mut transport, "not json")
1044                .unwrap_err(),
1045            (
1046                "client.failed".to_owned(),
1047                "blob ref is not JSON".to_owned()
1048            )
1049        );
1050    }
1051
1052    #[test]
1053    fn visible_blob_references_survive_cache_trimming() {
1054        let schema = json!({"version":1,"tables":[
1055            {"name":"attachments","primaryKey":"id","columns":[
1056                {"name":"id","type":"string","nullable":false},
1057                {"name":"body","type":"blob_ref","nullable":true},
1058                {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
1059            {"name":"avatars","primaryKey":"id","columns":[
1060                {"name":"id","type":"string","nullable":false},
1061                {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
1062        ]});
1063        let bytes = b"referenced above cap".to_vec();
1064        let blob_id = blob_id_for(&bytes);
1065        let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
1066        let mut client = SyncClient::new(
1067            "visible-blob-refs".to_owned(),
1068            &schema,
1069            ClientLimits {
1070                blob_cache_max_bytes: Some(1),
1071                ..Default::default()
1072            },
1073        )
1074        .unwrap();
1075        client.conn.execute(
1076            "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
1077            rusqlite::params![reference, reference, reference, reference],
1078        ).unwrap();
1079        client
1080            .conn
1081            .execute(
1082                "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
1083                rusqlite::params![reference],
1084            )
1085            .unwrap();
1086        client.conn.execute(
1087            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
1088            [],
1089        ).unwrap();
1090        let mut transport = CountingRealtimeTransport {
1091            blob_download: Some(bytes.clone()),
1092            ..Default::default()
1093        };
1094
1095        assert_eq!(
1096            client
1097                .fetch_blob_bytes(&mut transport, &blob_id)
1098                .unwrap()
1099                .bytes,
1100            bytes
1101        );
1102        let rows = client
1103            .conn
1104            .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
1105            .unwrap()
1106            .query_map([], |row| row.get::<_, String>(0))
1107            .unwrap()
1108            .collect::<Result<Vec<_>, _>>()
1109            .unwrap();
1110        assert_eq!(rows, vec![blob_id]);
1111    }
1112
1113    #[test]
1114    fn realtime_connection_ownership_is_idempotent() {
1115        let mut client = client();
1116        let mut transport = CountingRealtimeTransport {
1117            connects: 0,
1118            closes: 0,
1119            ..Default::default()
1120        };
1121        client
1122            .connect_realtime(&mut transport)
1123            .expect("first connect");
1124        client
1125            .connect_realtime(&mut transport)
1126            .expect("idempotent connect");
1127        assert_eq!(transport.connects, 1);
1128        client.disconnect_realtime(&mut transport);
1129        client.disconnect_realtime(&mut transport);
1130        assert_eq!(transport.closes, 1);
1131        client
1132            .connect_realtime(&mut transport)
1133            .expect("deliberate reconnect");
1134        assert_eq!(transport.connects, 2);
1135    }
1136
1137    #[test]
1138    fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
1139        let mut client = client();
1140        client
1141            .subscribe(
1142                "repair-tasks".to_owned(),
1143                "tasks".to_owned(),
1144                vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1145                None,
1146            )
1147            .expect("subscribe");
1148        {
1149            let sub = client
1150                .subs
1151                .iter_mut()
1152                .find(|sub| sub.id == "repair-tasks")
1153                .expect("subscription");
1154            sub.cursor = 42;
1155            sub.synced_once = true;
1156            let persisted = sub.clone();
1157            client.persist_sub(&persisted).unwrap();
1158        }
1159        for table in ["_syncular_base_tasks", "tasks"] {
1160            client
1161                .conn
1162                .execute(
1163                    &format!(
1164                        "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
1165                    ),
1166                    rusqlite::params!["server-row", "p1"],
1167                )
1168                .expect("seed server row");
1169        }
1170        let pending = client
1171            .mutate(vec![Mutation::Upsert {
1172                table: "tasks".to_owned(),
1173                values: Map::from_iter([
1174                    ("id".to_owned(), Value::from("offline-row")),
1175                    ("project_id".to_owned(), Value::from("p1")),
1176                ]),
1177                base_version: None,
1178            }])
1179            .expect("queue offline work");
1180        client.drain_change_batches();
1181        client.drain_sync_intents();
1182
1183        assert_eq!(
1184            client
1185                .rebootstrap_local_data(&LocalDataRebootstrapInput {
1186                    rebootstrap_id: "support-case-001".to_owned(),
1187                })
1188                .expect("rebootstrap"),
1189            LocalDataRebootstrapResult {
1190                already_applied: false,
1191                retained_commits: 1,
1192                reset_subscriptions: 1,
1193            }
1194        );
1195        let visible_ids = client
1196            .conn
1197            .prepare("SELECT id FROM tasks ORDER BY id")
1198            .expect("prepare visible ids")
1199            .query_map([], |row| row.get::<_, String>(0))
1200            .expect("query visible ids")
1201            .collect::<Result<Vec<_>, _>>()
1202            .expect("collect visible ids");
1203        assert_eq!(visible_ids, vec!["offline-row"]);
1204        assert_eq!(client.pending_commit_ids(), vec![pending]);
1205        assert_eq!(
1206            client
1207                .subscription_state("repair-tasks")
1208                .expect("subscription")
1209                .cursor,
1210            -1
1211        );
1212        assert!(client.upgrading());
1213        assert!(client.sync_needed());
1214        assert!(matches!(
1215            client.drain_sync_intents().as_slice(),
1216            [SyncIntent::Interactive]
1217        ));
1218        assert_eq!(client.drain_change_batches().len(), 1);
1219
1220        assert_eq!(
1221            client
1222                .rebootstrap_local_data(&LocalDataRebootstrapInput {
1223                    rebootstrap_id: "support-case-001".to_owned(),
1224                })
1225                .expect("idempotent retry"),
1226            LocalDataRebootstrapResult {
1227                already_applied: true,
1228                retained_commits: 1,
1229                reset_subscriptions: 1,
1230            }
1231        );
1232        assert!(client.drain_change_batches().is_empty());
1233    }
1234
1235    #[test]
1236    fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
1237        let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
1238        assert_eq!(
1239            decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
1240            (3, 4)
1241        );
1242        assert_eq!(
1243            decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
1244            (0, 0)
1245        );
1246        for malformed in [
1247            "",
1248            "{}",
1249            r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
1250            r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
1251            r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
1252        ] {
1253            assert_eq!(
1254                decode_local_rebootstrap_receipt(malformed)
1255                    .expect_err("malformed receipt must fail"),
1256                "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
1257            );
1258        }
1259    }
1260
1261    #[test]
1262    fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
1263        let path = std::env::temp_dir().join(format!(
1264            "syncular-rebootstrap-receipt-{}.db",
1265            uuid::Uuid::new_v4()
1266        ));
1267        let schema = json!({
1268            "version": 1,
1269            "tables": [{
1270                "name": "tasks",
1271                "primaryKey": "id",
1272                "columns": [
1273                    { "name": "id", "type": "string", "nullable": false },
1274                    { "name": "project_id", "type": "string", "nullable": false }
1275                ],
1276                "scopes": [{ "pattern": "project:{project_id}" }]
1277            }]
1278        });
1279        let path_string = path.to_str().expect("UTF-8 temp path");
1280
1281        {
1282            let mut first = SyncClient::open_path(
1283                "repair-restart-client".to_owned(),
1284                &schema,
1285                ClientLimits::default(),
1286                path_string,
1287            )
1288            .expect("first open");
1289            first
1290                .subscribe(
1291                    "repair-tasks".to_owned(),
1292                    "tasks".to_owned(),
1293                    vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1294                    None,
1295                )
1296                .expect("subscribe");
1297            first
1298                .mutate(vec![Mutation::Upsert {
1299                    table: "tasks".to_owned(),
1300                    values: Map::from_iter([
1301                        ("id".to_owned(), Value::from("offline-row")),
1302                        ("project_id".to_owned(), Value::from("p1")),
1303                    ]),
1304                    base_version: None,
1305                }])
1306                .expect("queue offline work");
1307            assert_eq!(
1308                first
1309                    .rebootstrap_local_data(&LocalDataRebootstrapInput {
1310                        rebootstrap_id: "restart-receipt".to_owned(),
1311                    })
1312                    .expect("first rebootstrap"),
1313                LocalDataRebootstrapResult {
1314                    already_applied: false,
1315                    retained_commits: 1,
1316                    reset_subscriptions: 1,
1317                }
1318            );
1319        }
1320
1321        let mut reopened = SyncClient::open_path(
1322            "repair-restart-client".to_owned(),
1323            &schema,
1324            ClientLimits::default(),
1325            path_string,
1326        )
1327        .expect("reopen");
1328        reopened.drain_change_batches();
1329        reopened.drain_sync_intents();
1330        assert_eq!(
1331            reopened
1332                .rebootstrap_local_data(&LocalDataRebootstrapInput {
1333                    rebootstrap_id: "restart-receipt".to_owned(),
1334                })
1335                .expect("receipt replay"),
1336            LocalDataRebootstrapResult {
1337                already_applied: true,
1338                retained_commits: 1,
1339                reset_subscriptions: 1,
1340            }
1341        );
1342        assert!(reopened.drain_change_batches().is_empty());
1343        assert!(reopened.drain_sync_intents().is_empty());
1344        drop(reopened);
1345        std::fs::remove_file(path).expect("remove temp database");
1346    }
1347
1348    #[test]
1349    fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
1350        let mut malformed = client();
1351        malformed
1352            .subscribe(
1353                "repair-tasks".to_owned(),
1354                "tasks".to_owned(),
1355                vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1356                None,
1357            )
1358            .expect("subscribe");
1359        malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
1360        malformed.drain_change_batches();
1361        malformed.drain_sync_intents();
1362        let malformed_error = malformed
1363            .rebootstrap_local_data(&LocalDataRebootstrapInput {
1364                rebootstrap_id: "malformed".to_owned(),
1365            })
1366            .expect_err("malformed receipt must fail closed");
1367        assert_eq!(
1368            malformed_error,
1369            "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
1370        );
1371        assert!(!malformed.upgrading());
1372        assert_eq!(
1373            malformed
1374                .subscription_state("repair-tasks")
1375                .expect("unchanged subscription")
1376                .cursor,
1377            -1
1378        );
1379        assert!(malformed.drain_change_batches().is_empty());
1380        assert!(malformed.drain_sync_intents().is_empty());
1381
1382        let mut unreadable = client();
1383        unreadable
1384            .conn
1385            .execute(
1386                "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
1387                rusqlite::params!["server-row", "p1"],
1388            )
1389            .expect("seed visible row");
1390        unreadable
1391            .conn
1392            .execute("DROP TABLE _syncular_meta", [])
1393            .expect("break marker storage");
1394        let unreadable_error = unreadable
1395            .rebootstrap_local_data(&LocalDataRebootstrapInput {
1396                rebootstrap_id: "unreadable".to_owned(),
1397            })
1398            .expect_err("unreadable marker storage must fail closed");
1399        assert_eq!(
1400            unreadable_error,
1401            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
1402        );
1403        let visible_rows = unreadable
1404            .conn
1405            .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
1406            .expect("visible projection remains");
1407        assert_eq!(visible_rows, 1);
1408        assert!(!unreadable.upgrading());
1409        assert!(unreadable.drain_change_batches().is_empty());
1410        assert!(unreadable.drain_sync_intents().is_empty());
1411    }
1412
1413    #[test]
1414    fn local_rebootstrap_cannot_bypass_schema_floor() {
1415        let mut client = client();
1416        client.set_schema_floor(Some(SchemaFloor {
1417            required_schema_version: Some(2),
1418            latest_schema_version: Some(2),
1419        }));
1420        let error = client
1421            .rebootstrap_local_data(&LocalDataRebootstrapInput {
1422                rebootstrap_id: "blocked-floor".to_owned(),
1423            })
1424            .expect_err("schema floor must block repair");
1425        assert!(error.contains("cannot bypass an active schema-floor stop"));
1426    }
1427
1428    #[test]
1429    fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
1430        let mut client = client();
1431        client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1432        const COMMIT_COUNT: usize = 32;
1433
1434        for index in 0..COMMIT_COUNT {
1435            client
1436                .mutate(vec![Mutation::Upsert {
1437                    table: "tasks".to_owned(),
1438                    values: Map::from_iter([
1439                        ("id".to_owned(), Value::from(format!("task-{index}"))),
1440                        ("project_id".to_owned(), Value::from("project-1")),
1441                    ]),
1442                    base_version: None,
1443                }])
1444                .expect("queue commit");
1445        }
1446
1447        let (_, request_meta) = client.build_request(false);
1448        assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
1449        let mut frames = vec![Frame::RespHeader {
1450            required_schema_version: None,
1451            latest_schema_version: None,
1452            log_epoch: Some("epoch-1".to_owned()),
1453            reset_required: Some(false),
1454        }];
1455        frames.extend(request_meta.pushed_ids.iter().enumerate().map(
1456            |(index, client_commit_id)| Frame::PushResult {
1457                client_commit_id: client_commit_id.clone(),
1458                status: PushStatus::Applied,
1459                commit_seq: Some(index as i64 + 1),
1460                results: vec![OpResult::Applied { op_index: 0 }],
1461            },
1462        ));
1463        let response = Message {
1464            wire_version: WIRE_VERSION,
1465            msg_kind: MsgKind::Response,
1466            frames,
1467        };
1468        let mut transport =
1469            HostTransport::new_from_config(&json!({})).expect("no-network host transport");
1470
1471        client.overlay_rebuild_count.set(0);
1472        client.outcome_prune_count.set(0);
1473        let outcome = client.process_response(&mut transport, response, &request_meta);
1474        assert!(matches!(outcome, SyncOutcome::Ok(_)));
1475        assert!(client.pending_commit_ids().is_empty());
1476        assert_eq!(
1477            client.overlay_rebuild_count.get(),
1478            1,
1479            "one response must reconcile its acknowledged commits with one overlay rebuild"
1480        );
1481        assert_eq!(
1482            client.outcome_prune_count.get(),
1483            1,
1484            "one response must enforce outcome retention once"
1485        );
1486    }
1487
1488    #[test]
1489    fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
1490        let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
1491            "columns": [
1492                {"name":"id", "type":"string", "nullable":false},
1493                {"name":"project_id", "type":"string", "nullable":false},
1494                {"name":"title", "type":"string", "nullable":false}
1495            ], "scopes": [{"pattern":"project:{project_id}"}],
1496            "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
1497            "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
1498        }] });
1499        let mut incremental =
1500            SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
1501        let mut reference =
1502            SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
1503        let tables = vec!["tasks".to_owned()];
1504        for index in 0..32 {
1505            let id = format!("task-{}", index % 8);
1506            let values = Map::from_iter([
1507                ("id".to_owned(), Value::from(id.clone())),
1508                (
1509                    "project_id".to_owned(),
1510                    Value::from(format!("p{}", index % 2)),
1511                ),
1512                ("title".to_owned(), Value::from(format!("needle{index}"))),
1513            ]);
1514            let payload = encode_row_json(
1515                incremental.schema.table("tasks").unwrap(),
1516                &id,
1517                &values,
1518                &incremental.encryption,
1519            )
1520            .unwrap();
1521            let change = ssp2::model::Change {
1522                table_index: 0,
1523                row_id: id,
1524                op: if index % 5 == 0 {
1525                    Op::Delete
1526                } else {
1527                    Op::Upsert
1528                },
1529                row_version: Some(index + 1),
1530                scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
1531                row: Some(payload),
1532            };
1533            reference.overlay_dirty.set(true);
1534            for instance in [&mut incremental, &mut reference] {
1535                instance.begin_observation("mirror_test").unwrap();
1536                let mut batch = ChangeAccumulator::default();
1537                instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
1538                instance
1539                    .apply_commit_changes(&tables, std::slice::from_ref(&change))
1540                    .unwrap();
1541                instance.rebuild_overlay_if_dirty();
1542                instance.finish_observation("mirror_test", batch).unwrap();
1543            }
1544            for sql in [
1545                "SELECT * FROM tasks ORDER BY id",
1546                "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
1547            ] {
1548                assert_eq!(
1549                    incremental.query(sql, &[]).unwrap(),
1550                    reference.query(sql, &[]).unwrap()
1551                );
1552            }
1553            assert_eq!(incremental.local_revision(), reference.local_revision());
1554            assert_eq!(
1555                serde_json::to_value(incremental.drain_change_batches()).unwrap(),
1556                serde_json::to_value(reference.drain_change_batches()).unwrap()
1557            );
1558        }
1559        assert_eq!(incremental.overlay_rebuild_count.get(), 0);
1560        assert_eq!(reference.overlay_rebuild_count.get(), 32);
1561    }
1562
1563    #[test]
1564    fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
1565        let cases = [
1566            (
1567                "string",
1568                vec![
1569                    json!("1"),
1570                    json!("01"),
1571                    json!(""),
1572                    json!("nul\u{0}id"),
1573                    json!("é"),
1574                ],
1575            ),
1576            (
1577                "integer",
1578                vec![
1579                    json!(i64::MIN),
1580                    json!(-1),
1581                    json!(0),
1582                    json!(1),
1583                    json!(i64::MAX),
1584                ],
1585            ),
1586            ("boolean", vec![json!(false), json!(true)]),
1587            ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
1588            (
1589                "float",
1590                vec![
1591                    json!(1.0),
1592                    json!(1.0000000000000002),
1593                    json!(-1.0),
1594                    json!(1e20),
1595                ],
1596            ),
1597        ];
1598        for (kind, ids) in cases {
1599            let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
1600                "columns":[{"name":"id","type":kind,"nullable":false},
1601                    {"name":"project_id","type":"string","nullable":false}],
1602                "scopes":[{"pattern":"project:{project_id}"}]}]});
1603            let mut client =
1604                SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
1605            for (index, id) in ids.iter().enumerate() {
1606                let table = client.schema.table("tasks").unwrap();
1607                client
1608                    .write_base_row(
1609                        "tasks",
1610                        &vec![
1611                            json_to_column_value(&table.columns[0], Some(id)).unwrap(),
1612                            Some(ColumnValue::String(format!("p{index}"))),
1613                        ],
1614                        1,
1615                    )
1616                    .unwrap();
1617            }
1618            client.rebuild_overlay_if_dirty();
1619            for base in [true, false] {
1620                let full = if base {
1621                    base_table("tasks")
1622                } else {
1623                    visible_table("tasks")
1624                };
1625                let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
1626                for row_id in [
1627                    "1",
1628                    "01",
1629                    "1.0",
1630                    "1.0000000000000002",
1631                    "+1",
1632                    "1e0",
1633                    " 1",
1634                    "-1",
1635                    "0",
1636                    "9223372036854775807",
1637                    "-9223372036854775808",
1638                    "true",
1639                    "false",
1640                    "",
1641                    "nul\u{0}id",
1642                    "é",
1643                    "null",
1644                    "{\"a\":1}",
1645                ] {
1646                    let old_sql =
1647                        format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
1648                    let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
1649                    let old = client
1650                        .conn
1651                        .prepare(&old_sql)
1652                        .unwrap()
1653                        .query_map([row_id], |row| row.get::<_, String>(0))
1654                        .unwrap()
1655                        .collect::<Result<Vec<_>, _>>()
1656                        .unwrap();
1657                    let new = client
1658                        .conn
1659                        .prepare_cached(&new_sql)
1660                        .unwrap()
1661                        .query_map([row_id], |row| row.get::<_, String>(0))
1662                        .unwrap()
1663                        .collect::<Result<Vec<_>, _>>()
1664                        .unwrap();
1665                    assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
1666                    // The floating-point text collision is deliberate: an
1667                    // indexed numeric equality must not silently lose a match.
1668                    if kind == "float" && row_id == "1.0" {
1669                        assert_eq!(old.len(), 2);
1670                    }
1671                    let mut batch = ChangeAccumulator::default();
1672                    assert_eq!(
1673                        client.record_row_scopes(&mut batch, "tasks", row_id, base),
1674                        !old.is_empty()
1675                    );
1676                    let expected = old.first().map(|scope| {
1677                        BTreeMap::from([(
1678                            "tasks".to_owned(),
1679                            Some(BTreeSet::from([format!("project:{scope}")])),
1680                        )])
1681                    });
1682                    assert_eq!(batch.tables, expected.unwrap_or_default());
1683                    client
1684                        .conn
1685                        .execute_batch("SAVEPOINT lookup_delete")
1686                        .unwrap();
1687                    if base {
1688                        client.delete_base_row("tasks", row_id).unwrap();
1689                    } else {
1690                        client.apply_outbox_ops(&[OutboxOp {
1691                            upsert: false,
1692                            table: "tasks".into(),
1693                            row_id: row_id.into(),
1694                            base_version: None,
1695                            values: None,
1696                        }]);
1697                    }
1698                    let remaining: i64 = client
1699                        .conn
1700                        .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
1701                            row.get(0)
1702                        })
1703                        .unwrap();
1704                    assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
1705                    client
1706                        .conn
1707                        .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
1708                        .unwrap();
1709                    client.overlay_dirty.set(false);
1710                }
1711                for (predicate, expected) in [
1712                    ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
1713                    (predicate, if kind == "float" { "SCAN" } else { "SEARCH" }),
1714                ] {
1715                    let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
1716                    let details = client
1717                        .conn
1718                        .prepare(&sql)
1719                        .unwrap()
1720                        .query_map(["1"], |row| row.get::<_, String>(3))
1721                        .unwrap()
1722                        .collect::<Result<Vec<_>, _>>()
1723                        .unwrap();
1724                    assert!(
1725                        details.iter().any(|detail| detail.contains(expected)),
1726                        "{kind}: {details:?}"
1727                    );
1728                }
1729            }
1730        }
1731    }
1732
1733    #[test]
1734    fn pending_row_reconciliation_matches_full_fifo_replay() {
1735        for unique in [false, true] {
1736            let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
1737                "name": name, "primaryKey": "id", "columns": [
1738                    {"name":"id", "type":"string", "nullable":false},
1739                    {"name":"project_id", "type":"string", "nullable":false},
1740                    {"name":"title", "type":"string", "nullable":false}],
1741                "scopes": [{"pattern":"project:{project_id}"}],
1742                "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
1743                "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
1744            })))});
1745            let mut incremental =
1746                SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
1747            let mut reference =
1748                SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
1749            let tables = vec!["tasks".to_owned(), "notes".to_owned()];
1750            for instance in [&mut incremental, &mut reference] {
1751                for table in &tables {
1752                    for index in 0..12 {
1753                        instance
1754                            .write_base_row(
1755                                table,
1756                                &vec![
1757                                    Some(ColumnValue::String(format!("row{index}"))),
1758                                    Some(ColumnValue::String("p0".into())),
1759                                    Some(ColumnValue::String(format!("base{index}"))),
1760                                ],
1761                                1,
1762                            )
1763                            .unwrap();
1764                    }
1765                }
1766                instance.rebuild_overlay_if_dirty();
1767            }
1768            for index in 0..80 {
1769                let mutations = (0..3)
1770                    .map(|offset| {
1771                        let table = tables[(index + offset) % 2].clone();
1772                        let id = format!("row{}", (index + offset) % 16);
1773                        if (index + offset) % 7 == 0 {
1774                            Mutation::Delete {
1775                                table,
1776                                row_id: id,
1777                                base_version: None,
1778                            }
1779                        } else {
1780                            Mutation::Upsert {
1781                                table,
1782                                values: Map::from_iter([
1783                                    ("id".into(), Value::from(id)),
1784                                    ("project_id".into(), Value::from(format!("p{}", index % 3))),
1785                                    (
1786                                        "title".into(),
1787                                        Value::from(format!("needle{}", (index + offset) % 5)),
1788                                    ),
1789                                ]),
1790                                base_version: None,
1791                            }
1792                        }
1793                    })
1794                    .collect::<Vec<_>>();
1795                for instance in [&mut incremental, &mut reference] {
1796                    instance.mutate(mutations.clone()).unwrap();
1797                }
1798                // Most frames touch only the table without unique constraints.
1799                // Mixed frames must retain full replay for both tables.
1800                let changes = (0..3)
1801                    .map(|offset| {
1802                        let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
1803                        let id = format!("row{}", (index + offset) % 16);
1804                        let values = Map::from_iter([
1805                            ("id".into(), Value::from(id.clone())),
1806                            (
1807                                "project_id".into(),
1808                                Value::from(format!("p{}", (index + 1) % 3)),
1809                            ),
1810                            (
1811                                "title".into(),
1812                                Value::from(format!("remote{index}_{offset}")),
1813                            ),
1814                        ]);
1815                        ssp2::model::Change {
1816                            table_index: table_index as u16,
1817                            row_id: id.clone(),
1818                            op: if (index + offset) % 5 == 0 {
1819                                Op::Delete
1820                            } else {
1821                                Op::Upsert
1822                            },
1823                            row_version: Some(index as i64 + 2),
1824                            scopes: vec![],
1825                            row: Some(
1826                                encode_row_json(
1827                                    incremental.schema.table(&tables[table_index]).unwrap(),
1828                                    &id,
1829                                    &values,
1830                                    &incremental.encryption,
1831                                )
1832                                .unwrap(),
1833                            ),
1834                        }
1835                    })
1836                    .collect::<Vec<_>>();
1837                reference.overlay_dirty.set(true);
1838                incremental.apply_commit_frame(&tables, &changes).unwrap();
1839                reference.apply_commit_frame(&tables, &changes).unwrap();
1840                for table in &tables {
1841                    for sql in [format!("SELECT * FROM {table} ORDER BY id"),
1842                        format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
1843                        format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
1844                        format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
1845                        assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
1846                    }
1847                }
1848                assert_eq!(incremental.local_revision(), reference.local_revision());
1849                assert_eq!(
1850                    serde_json::to_value(incremental.drain_change_batches()).unwrap(),
1851                    serde_json::to_value(reference.drain_change_batches()).unwrap()
1852                );
1853            }
1854            assert_eq!(
1855                incremental.overlay_rebuild_count.get(),
1856                if unique { 81 } else { 21 }
1857            );
1858            assert_eq!(reference.overlay_rebuild_count.get(), 81);
1859        }
1860    }
1861
1862    #[test]
1863    fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
1864        let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
1865            "columns":[{"name":"id", "type":"string", "nullable":false},
1866                {"name":"title", "type":"string", "nullable":false}],
1867            "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
1868        let mut client =
1869            SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
1870        for (id, title) in [("a", "original"), ("b", "occupied")] {
1871            client
1872                .write_base_row(
1873                    "tasks",
1874                    &vec![
1875                        Some(ColumnValue::String(id.into())),
1876                        Some(ColumnValue::String(title.into())),
1877                    ],
1878                    1,
1879                )
1880                .unwrap();
1881        }
1882        client.rebuild_overlay_if_dirty();
1883        let pending = client
1884            .mutate(vec![Mutation::Upsert {
1885                table: "tasks".into(),
1886                values: Map::from_iter([
1887                    ("id".into(), Value::from("a")),
1888                    ("title".into(), Value::from("occupied")),
1889                ]),
1890                base_version: None,
1891            }])
1892            .unwrap();
1893        assert_eq!(
1894            client
1895                .conn
1896                .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
1897                    .get::<_, String>(
1898                    0
1899                ))
1900                .unwrap(),
1901            "original"
1902        );
1903        for (title, expected) in [("available", "occupied"), ("occupied", "original")] {
1904            let change = ssp2::model::Change {
1905                table_index: 0,
1906                row_id: "b".into(),
1907                op: Op::Upsert,
1908                row_version: Some(2),
1909                scopes: vec![],
1910                row: Some(
1911                    encode_row_json(
1912                        client.schema.table("tasks").unwrap(),
1913                        "b",
1914                        &Map::from_iter([
1915                            ("id".into(), Value::from("b")),
1916                            ("title".into(), Value::from(title)),
1917                        ]),
1918                        &client.encryption,
1919                    )
1920                    .unwrap(),
1921                ),
1922            };
1923            client
1924                .apply_commit_frame(&["tasks".into()], &[change])
1925                .unwrap();
1926            assert_eq!(
1927                client
1928                    .conn
1929                    .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
1930                        .get::<_, String>(
1931                        0
1932                    ))
1933                    .unwrap(),
1934                expected
1935            );
1936            assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
1937        }
1938        assert_eq!(client.overlay_rebuild_count.get(), 3);
1939    }
1940
1941    #[test]
1942    fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
1943        for failure in ["visible", "revision", "commit"] {
1944            let mut client = client();
1945            let id = client
1946                .mutate(vec![Mutation::Upsert {
1947                    table: "tasks".into(),
1948                    values: Map::from_iter([
1949                        ("id".into(), Value::from("t1")),
1950                        ("project_id".into(), Value::from("pending")),
1951                    ]),
1952                    base_version: None,
1953                }])
1954                .unwrap();
1955            client.drain_change_batches();
1956            let revision = client.local_revision();
1957            let before = client.query("SELECT * FROM tasks", &[]).unwrap();
1958            let trigger = match failure {
1959                "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
1960                "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
1961                _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
1962            };
1963            client.conn.execute_batch(&trigger).unwrap();
1964            let change = ssp2::model::Change {
1965                table_index: 0,
1966                row_id: "t1".into(),
1967                op: Op::Upsert,
1968                row_version: Some(2),
1969                scopes: vec![],
1970                row: Some(
1971                    encode_row_json(
1972                        client.schema.table("tasks").unwrap(),
1973                        "t1",
1974                        &Map::from_iter([
1975                            ("id".into(), Value::from("t1")),
1976                            ("project_id".into(), Value::from("remote")),
1977                        ]),
1978                        &client.encryption,
1979                    )
1980                    .unwrap(),
1981                ),
1982            };
1983            assert!(client
1984                .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change))
1985                .is_err());
1986            assert!(client.conn.is_autocommit());
1987            assert!(!client.overlay_dirty.get());
1988            assert_eq!(client.local_revision(), revision);
1989            assert!(client.drain_change_batches().is_empty());
1990            assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
1991            assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
1992            assert!(client
1993                .query("SELECT * FROM _syncular_base_tasks", &[])
1994                .unwrap()
1995                .is_empty());
1996            client
1997                .conn
1998                .execute_batch("DROP TRIGGER fail_pending")
1999                .unwrap();
2000            client
2001                .apply_commit_frame(&["tasks".into()], &[change])
2002                .unwrap();
2003            assert_eq!(client.local_revision(), revision + 1);
2004            assert_eq!(client.pending_commit_ids(), vec![id]);
2005            // The overlay keeps the pending write visible; the row now carries
2006            // the server row's version it is layered on (§7.1).
2007            let mut expected = before[0].clone();
2008            expected.insert("_syncular_version".to_owned(), Value::from(2));
2009            assert_eq!(
2010                client.query("SELECT * FROM tasks", &[]).unwrap(),
2011                vec![expected]
2012            );
2013            assert_eq!(
2014                client
2015                    .conn
2016                    .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
2017                        .get::<_, String>(0))
2018                    .unwrap(),
2019                "remote"
2020            );
2021        }
2022    }
2023
2024    #[test]
2025    fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
2026        let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
2027            "columns": [
2028                {"name":"id", "type":"string", "nullable":false},
2029                {"name":"project_id", "type":"string", "nullable":false},
2030                {"name":"title", "type":"string", "nullable":false}],
2031            "scopes": [{"pattern":"project:{project_id}"}],
2032            "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
2033            "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
2034        let mut incremental =
2035            SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
2036        let mut reference =
2037            SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
2038        for instance in [&mut incremental, &mut reference] {
2039            for index in 0..16 {
2040                instance
2041                    .write_base_row(
2042                        "tasks",
2043                        &vec![
2044                            Some(ColumnValue::String(format!("task{index}"))),
2045                            Some(ColumnValue::String("p0".to_owned())),
2046                            Some(ColumnValue::String(format!("base{index}"))),
2047                        ],
2048                        1,
2049                    )
2050                    .unwrap();
2051            }
2052            instance.rebuild_overlay_if_dirty();
2053            instance.overlay_rebuild_count.set(0);
2054        }
2055        for index in 0..100 {
2056            let mutations = (0..1 + index % 4)
2057                .map(|offset| {
2058                    let id = format!("task{}", (index + offset) % 24);
2059                    if (index + offset) % 7 == 0 {
2060                        Mutation::Delete {
2061                            table: "tasks".to_owned(),
2062                            row_id: id,
2063                            base_version: None,
2064                        }
2065                    } else {
2066                        Mutation::Upsert {
2067                            table: "tasks".to_owned(),
2068                            values: Map::from_iter([
2069                                ("id".to_owned(), Value::from(id)),
2070                                (
2071                                    "project_id".to_owned(),
2072                                    Value::from(format!("p{}", index % 2)),
2073                                ),
2074                                (
2075                                    "title".to_owned(),
2076                                    Value::from(format!("needle{}", (index + offset) % 5)),
2077                                ),
2078                            ]),
2079                            base_version: None,
2080                        }
2081                    }
2082                })
2083                .collect::<Vec<_>>();
2084            reference.overlay_dirty.set(true);
2085            incremental.mutate(mutations.clone()).unwrap();
2086            reference.mutate(mutations).unwrap();
2087            for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
2088                "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
2089                assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
2090            }
2091            assert_eq!(incremental.local_revision(), reference.local_revision());
2092            assert_eq!(
2093                serde_json::to_value(incremental.drain_change_batches()).unwrap(),
2094                serde_json::to_value(reference.drain_change_batches()).unwrap()
2095            );
2096        }
2097        assert_eq!(incremental.outbox.len(), 100);
2098        assert_eq!(incremental.overlay_rebuild_count.get(), 0);
2099        assert_eq!(reference.overlay_rebuild_count.get(), 100);
2100    }
2101
2102    #[test]
2103    fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
2104        for fail_at in ["outbox", "revision"] {
2105            let mut client = client();
2106            let values = Map::from_iter([
2107                ("id".to_owned(), Value::from("t1")),
2108                ("project_id".to_owned(), Value::from("p1")),
2109            ]);
2110            let mutation = Mutation::Upsert {
2111                table: "tasks".to_owned(),
2112                values,
2113                base_version: None,
2114            };
2115            let trigger = if fail_at == "outbox" {
2116                "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
2117            } else {
2118                format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
2119            };
2120            client.conn.execute_batch(&trigger).unwrap();
2121            let revision = client.local_revision();
2122            assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
2123            assert_eq!(client.local_revision(), revision);
2124            assert!(client.outbox.is_empty());
2125            assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
2126            assert_eq!(
2127                client
2128                    .conn
2129                    .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
2130                        .get::<_, i64>(0))
2131                    .unwrap(),
2132                0
2133            );
2134            assert!(client.drain_change_batches().is_empty());
2135            assert!(!client.overlay_dirty.get());
2136            client
2137                .conn
2138                .execute_batch("DROP TRIGGER fail_append")
2139                .unwrap();
2140            client.mutate(vec![mutation]).unwrap();
2141            assert_eq!(client.outbox.len(), 1);
2142            assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
2143        }
2144    }
2145
2146    #[test]
2147    fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
2148        let mut client = client();
2149        let tables = vec!["tasks".to_owned(), "missing".to_owned()];
2150        let values = Map::from_iter([
2151            ("id".to_owned(), Value::from("t1")),
2152            ("project_id".to_owned(), Value::from("p1")),
2153        ]);
2154        let payload = encode_row_json(
2155            client.schema.table("tasks").unwrap(),
2156            "t1",
2157            &values,
2158            &client.encryption,
2159        )
2160        .unwrap();
2161        let first = ssp2::model::Change {
2162            table_index: 0,
2163            row_id: "t1".to_owned(),
2164            op: Op::Upsert,
2165            row_version: Some(1),
2166            scopes: vec![("project_id".to_owned(), "p1".to_owned())],
2167            row: Some(payload),
2168        };
2169        let mut invalid = first.clone();
2170        invalid.table_index = 1;
2171        let revision = client.local_revision();
2172        client.begin_observation("failed_mirror").unwrap();
2173        assert!(client
2174            .apply_commit_changes(&tables, &[first, invalid])
2175            .is_err());
2176        client.rollback_observation("failed_mirror");
2177        client.rebuild_overlay_if_dirty();
2178        assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
2179        assert_eq!(
2180            client
2181                .conn
2182                .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
2183                    .get::<_, i64>(
2184                    0
2185                ))
2186                .unwrap(),
2187            0
2188        );
2189        assert_eq!(client.local_revision(), revision);
2190        assert!(client.drain_change_batches().is_empty());
2191    }
2192
2193    #[test]
2194    fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
2195        for failure in ["journal", "revision", "commit", "later-frame"] {
2196            let mut client = client();
2197            client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2198            for id in ["first", "second"] {
2199                client
2200                    .mutate(vec![Mutation::Upsert {
2201                        table: "tasks".to_owned(),
2202                        values: Map::from_iter([
2203                            ("id".to_owned(), Value::from(id)),
2204                            ("project_id".to_owned(), Value::from("p1")),
2205                        ]),
2206                        base_version: None,
2207                    }])
2208                    .unwrap();
2209            }
2210            let (_, meta) = client.build_request(false);
2211            // A malicious or stale response must not drain an unsent local commit.
2212            client
2213                .mutate(vec![Mutation::Upsert {
2214                    table: "tasks".to_owned(),
2215                    values: Map::from_iter([
2216                        ("id".to_owned(), Value::from("later")),
2217                        ("project_id".to_owned(), Value::from("p1")),
2218                    ]),
2219                    base_version: None,
2220                }])
2221                .unwrap();
2222            let ids = client.pending_commit_ids();
2223            let rows = client
2224                .query("SELECT * FROM tasks ORDER BY id", &[])
2225                .unwrap();
2226            let revision = client.local_revision();
2227            client.drain_change_batches();
2228            if failure != "later-frame" {
2229                client.conn.execute_batch(match failure {
2230                    "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
2231                    "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
2232                    _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
2233                }).unwrap();
2234            }
2235            let mut response = Message {
2236                wire_version: WIRE_VERSION,
2237                msg_kind: MsgKind::Response,
2238                frames: vec![Frame::RespHeader {
2239                    required_schema_version: None,
2240                    latest_schema_version: None,
2241                    log_epoch: Some("epoch-1".to_owned()),
2242                    reset_required: Some(false),
2243                }],
2244            };
2245            for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
2246                response.frames.push(Frame::PushResult {
2247                    client_commit_id: id.to_owned(),
2248                    status: PushStatus::Applied,
2249                    commit_seq: Some(1),
2250                    results: vec![OpResult::Applied { op_index: 0 }],
2251                });
2252            }
2253            if failure == "later-frame" {
2254                response.frames.push(Frame::Error {
2255                    code: "sync.invalid_request".to_owned(),
2256                    message: "later error".to_owned(),
2257                    category: "protocol".to_owned(),
2258                    retryable: false,
2259                    recommended_action: "retry".to_owned(),
2260                    details: None,
2261                });
2262            }
2263            let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2264            let outcome = client.process_response(&mut transport, response.clone(), &meta);
2265            assert!(
2266                matches!(outcome, SyncOutcome::Failed { error_code, .. } if error_code == if failure == "later-frame" { "sync.invalid_request" } else { "client.outcome_persistence_failed" })
2267            );
2268            if failure == "later-frame" {
2269                assert_eq!(client.pending_commit_ids(), ids[2..]);
2270                assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
2271                assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
2272                let batches = client.drain_change_batches();
2273                assert_eq!(batches.len(), 1);
2274                assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
2275                assert_eq!(client.local_revision(), revision + 1);
2276                response.frames.pop();
2277            } else {
2278                assert_eq!(client.pending_commit_ids(), ids);
2279                assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
2280                assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
2281                assert_eq!(client.local_revision(), revision);
2282                assert!(client.drain_change_batches().is_empty());
2283                assert_eq!(
2284                    client
2285                        .query("SELECT * FROM tasks ORDER BY id", &[])
2286                        .unwrap(),
2287                    rows
2288                );
2289                client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
2290            }
2291            for frame in &mut response.frames {
2292                if let Frame::PushResult { status, .. } = frame {
2293                    *status = PushStatus::Cached;
2294                }
2295            }
2296            let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
2297            else {
2298                panic!("retry failed");
2299            };
2300            assert_eq!(
2301                report.applied,
2302                if failure == "later-frame" {
2303                    vec![]
2304                } else {
2305                    ids[..2].to_vec()
2306                }
2307            );
2308            assert_eq!(client.pending_commit_ids(), ids[2..]);
2309            let batches = client.drain_change_batches();
2310            assert_eq!(batches.len(), usize::from(failure != "later-frame"));
2311            if let Some(batch) = batches.first() {
2312                assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
2313            }
2314            assert_eq!(client.local_revision(), revision + 1);
2315        }
2316    }
2317
2318    #[test]
2319    fn failed_push_result_preserves_memory_disk_and_retry_identity() {
2320        for failure in ["revision", "journal", "commit"] {
2321            for status in ["applied", "cached", "rejected", "conflict"] {
2322                let mut client = client();
2323                client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2324                for id in ["first", "later"] {
2325                    client
2326                        .mutate(vec![Mutation::Upsert {
2327                            table: "tasks".to_owned(),
2328                            values: Map::from_iter([
2329                                ("id".to_owned(), Value::from(id)),
2330                                ("project_id".to_owned(), Value::from("p1")),
2331                            ]),
2332                            base_version: None,
2333                        }])
2334                        .unwrap();
2335                }
2336                let (_, meta) = client.build_request(false);
2337                let ids = client.pending_commit_ids();
2338                let rows = client
2339                    .query("SELECT * FROM tasks ORDER BY id", &[])
2340                    .unwrap();
2341                let revision = client.local_revision();
2342                client.drain_change_batches();
2343                let (push_status, results) = match status {
2344                    "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
2345                    "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
2346                    "conflict" => (
2347                        PushStatus::Rejected,
2348                        vec![OpResult::Conflict {
2349                            op_index: 0,
2350                            code: "sync.version_conflict".to_owned(),
2351                            message: "conflict".to_owned(),
2352                            server_version: 1,
2353                            server_row: encode_row_json(
2354                                client.schema.table("tasks").unwrap(),
2355                                "first",
2356                                &Map::from_iter([
2357                                    ("id".to_owned(), Value::from("first")),
2358                                    ("project_id".to_owned(), Value::from("p1")),
2359                                ]),
2360                                &client.encryption,
2361                            )
2362                            .unwrap(),
2363                            conflict_columns: Vec::new(),
2364                        }],
2365                    ),
2366                    _ => (
2367                        PushStatus::Rejected,
2368                        vec![OpResult::Error {
2369                            op_index: 0,
2370                            code: "sync.validation_failed".to_owned(),
2371                            message: "rejected".to_owned(),
2372                            retryable: false,
2373                        }],
2374                    ),
2375                };
2376                let response = Message {
2377                    wire_version: WIRE_VERSION,
2378                    msg_kind: MsgKind::Response,
2379                    frames: vec![
2380                        Frame::RespHeader {
2381                            required_schema_version: None,
2382                            latest_schema_version: None,
2383                            log_epoch: Some("epoch-1".to_owned()),
2384                            reset_required: Some(false),
2385                        },
2386                        Frame::PushResult {
2387                            client_commit_id: ids[0].clone(),
2388                            status: push_status,
2389                            commit_seq: Some(1),
2390                            results,
2391                        },
2392                    ],
2393                };
2394                client.conn.execute_batch(match failure {
2395                    "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
2396                    "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
2397                    _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
2398                }).unwrap();
2399                let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2400                let outcome = client.process_response(&mut transport, response.clone(), &meta);
2401                assert_eq!(
2402                    client.pending_commit_ids(),
2403                    ids,
2404                    "{failure}/{status}: pending identity"
2405                );
2406                assert_eq!(
2407                    client.local_revision(),
2408                    revision,
2409                    "{failure}/{status}: revision"
2410                );
2411                assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
2412                assert!(client.conflicts().is_empty());
2413                assert!(client.rejections().is_empty());
2414                assert!(client.drain_change_batches().is_empty());
2415                assert_eq!(
2416                    client
2417                        .query("SELECT * FROM tasks ORDER BY id", &[])
2418                        .unwrap(),
2419                    rows
2420                );
2421                assert!(
2422                    matches!(outcome, SyncOutcome::Failed { error_code, .. } if error_code == "client.outcome_persistence_failed"),
2423                    "{failure}/{status}: failed round"
2424                );
2425                client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
2426                assert!(matches!(
2427                    client.process_response(&mut transport, response, &meta),
2428                    SyncOutcome::Ok(_)
2429                ));
2430                assert_eq!(client.pending_commit_ids(), ids[1..]);
2431                assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
2432                assert_eq!(client.drain_change_batches().len(), 1);
2433            }
2434        }
2435    }
2436
2437    #[test]
2438    fn mixed_push_results_reconcile_and_prune_once_per_response() {
2439        let mut client = client();
2440        client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2441        for index in 0..4 {
2442            client
2443                .mutate(vec![Mutation::Upsert {
2444                    table: "tasks".to_owned(),
2445                    values: Map::from_iter([
2446                        ("id".to_owned(), Value::from(format!("task-{index}"))),
2447                        ("project_id".to_owned(), Value::from("project-1")),
2448                    ]),
2449                    base_version: None,
2450                }])
2451                .expect("queue commit");
2452        }
2453
2454        let (_, request_meta) = client.build_request(false);
2455        let ids = &request_meta.pushed_ids;
2456        assert_eq!(ids.len(), 4);
2457        let response = Message {
2458            wire_version: WIRE_VERSION,
2459            msg_kind: MsgKind::Response,
2460            frames: vec![
2461                Frame::RespHeader {
2462                    required_schema_version: None,
2463                    latest_schema_version: None,
2464                    log_epoch: Some("epoch-1".to_owned()),
2465                    reset_required: Some(false),
2466                },
2467                Frame::PushResult {
2468                    client_commit_id: ids[0].clone(),
2469                    status: PushStatus::Applied,
2470                    commit_seq: Some(1),
2471                    results: vec![OpResult::Applied { op_index: 0 }],
2472                },
2473                Frame::PushResult {
2474                    client_commit_id: ids[1].clone(),
2475                    status: PushStatus::Cached,
2476                    commit_seq: Some(2),
2477                    results: vec![OpResult::Applied { op_index: 0 }],
2478                },
2479                Frame::PushResult {
2480                    client_commit_id: ids[2].clone(),
2481                    status: PushStatus::Rejected,
2482                    commit_seq: None,
2483                    results: vec![OpResult::Error {
2484                        op_index: 0,
2485                        code: "sync.validation_failed".to_owned(),
2486                        message: "rejected".to_owned(),
2487                        retryable: false,
2488                    }],
2489                },
2490                Frame::PushResult {
2491                    client_commit_id: ids[3].clone(),
2492                    status: PushStatus::Rejected,
2493                    commit_seq: None,
2494                    results: vec![OpResult::Error {
2495                        op_index: 0,
2496                        code: "sync.idempotency_cache_miss".to_owned(),
2497                        message: "retry".to_owned(),
2498                        retryable: true,
2499                    }],
2500                },
2501            ],
2502        };
2503        let mut transport =
2504            HostTransport::new_from_config(&json!({})).expect("no-network host transport");
2505
2506        client.overlay_rebuild_count.set(0);
2507        client.outcome_prune_count.set(0);
2508        let outcome = client.process_response(&mut transport, response, &request_meta);
2509        let SyncOutcome::Ok(report) = outcome else {
2510            panic!("mixed push-result response failed");
2511        };
2512
2513        assert_eq!(report.applied, ids[..2]);
2514        assert_eq!(report.rejected, ids[2..3]);
2515        assert_eq!(report.retryable, ids[3..4]);
2516        assert_eq!(client.pending_commit_ids(), ids[3..4]);
2517        assert_eq!(
2518            client
2519                .query("SELECT id FROM tasks ORDER BY id", &[])
2520                .expect("query visible overlay"),
2521            vec![Map::from_iter([("id".to_owned(), Value::from("task-3"))])]
2522        );
2523        assert_eq!(client.overlay_rebuild_count.get(), 1);
2524        assert_eq!(client.outcome_prune_count.get(), 1);
2525    }
2526
2527    #[test]
2528    fn secondary_unique_collision_preserves_existing_synced_row() {
2529        let client = SyncClient::new(
2530            "unique-upsert-test".to_owned(),
2531            &json!({
2532                "version": 1,
2533                "tables": [{
2534                    "name": "tasks",
2535                    "primaryKey": "id",
2536                    "columns": [
2537                        { "name": "id", "type": "string", "nullable": false },
2538                        { "name": "project_id", "type": "string", "nullable": false },
2539                        { "name": "title", "type": "string", "nullable": false }
2540                    ],
2541                    "scopes": [{ "pattern": "project:{project_id}" }],
2542                    "indexes": [{
2543                        "name": "tasks_by_project_title",
2544                        "columns": ["project_id", "title"],
2545                        "unique": true
2546                    }]
2547                }]
2548            }),
2549            ClientLimits::default(),
2550        )
2551        .expect("test client");
2552        let table = client.schema.table("tasks").expect("tasks table");
2553        let sql = client.insert_row_sql(&base_table("tasks"), table);
2554
2555        client
2556            .conn
2557            .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
2558            .expect("insert first row");
2559        client
2560            .conn
2561            .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
2562            .expect("update same primary key");
2563        client
2564            .conn
2565            .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
2566            .expect("insert second row");
2567        assert!(client
2568            .conn
2569            .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
2570            .is_err());
2571
2572        let rows = client
2573            .conn
2574            .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
2575            .expect("prepare rows")
2576            .query_map([], |row| {
2577                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2578            })
2579            .expect("query rows")
2580            .collect::<Result<Vec<_>, _>>()
2581            .expect("collect rows");
2582        assert_eq!(
2583            rows,
2584            vec![
2585                ("t1".to_owned(), "updated".to_owned()),
2586                ("t2".to_owned(), "original".to_owned())
2587            ]
2588        );
2589    }
2590
2591    #[test]
2592    fn reopening_active_subscriptions_emits_a_catch_up_intent() {
2593        let path = std::env::temp_dir().join(format!(
2594            "syncular-startup-intent-{}.db",
2595            uuid::Uuid::new_v4()
2596        ));
2597        let schema = json!({
2598            "version": 1,
2599            "tables": [{
2600                "name": "tasks",
2601                "primaryKey": "id",
2602                "columns": [
2603                    { "name": "id", "type": "string", "nullable": false },
2604                    { "name": "project_id", "type": "string", "nullable": false }
2605                ],
2606                "scopes": [{ "pattern": "project:{project_id}" }]
2607            }]
2608        });
2609
2610        {
2611            let mut first = SyncClient::open_path_with_identity(
2612                None,
2613                &schema,
2614                ClientLimits::default(),
2615                path.to_str().expect("UTF-8 temp path"),
2616            )
2617            .expect("first open");
2618            first
2619                .set_window(
2620                    &WindowBase {
2621                        table: "tasks".to_owned(),
2622                        variable: "project_id".to_owned(),
2623                        fixed_scopes: Vec::new(),
2624                        params: None,
2625                    },
2626                    &["persisted".to_owned()],
2627                )
2628                .expect("persist window");
2629        }
2630
2631        let mut reopened = SyncClient::open_path_with_identity(
2632            None,
2633            &schema,
2634            ClientLimits::default(),
2635            path.to_str().expect("UTF-8 temp path"),
2636        )
2637        .expect("reopen");
2638        assert!(reopened.sync_needed());
2639        assert!(matches!(
2640            reopened.drain_sync_intents().as_slice(),
2641            [SyncIntent::Interactive]
2642        ));
2643        drop(reopened);
2644        std::fs::remove_file(path).expect("remove temp database");
2645    }
2646
2647    #[test]
2648    fn reopening_preserves_immutable_subscription_identity_and_progress() {
2649        let path = std::env::temp_dir().join(format!(
2650            "syncular-subscription-identity-{}.db",
2651            uuid::Uuid::new_v4()
2652        ));
2653        let schema = json!({
2654            "version": 1,
2655            "tables": [
2656                {
2657                    "name": "tasks",
2658                    "primaryKey": "id",
2659                    "columns": [
2660                        { "name": "id", "type": "string", "nullable": false },
2661                        { "name": "project_id", "type": "string", "nullable": false }
2662                    ],
2663                    "scopes": [{ "pattern": "project:{project_id}" }]
2664                },
2665                {
2666                    "name": "docs",
2667                    "primaryKey": "id",
2668                    "columns": [
2669                        { "name": "id", "type": "string", "nullable": false },
2670                        { "name": "org_id", "type": "string", "nullable": false },
2671                        { "name": "project_id", "type": "string", "nullable": false }
2672                    ],
2673                    "scopes": [
2674                        { "pattern": "org:{org_id}" },
2675                        { "pattern": "project:{projectId}", "column": "project_id" }
2676                    ]
2677                }
2678            ]
2679        });
2680
2681        {
2682            let mut first = SyncClient::open_path_with_identity(
2683                None,
2684                &schema,
2685                ClientLimits::default(),
2686                path.to_str().expect("UTF-8 temp path"),
2687            )
2688            .expect("first open");
2689            first
2690                .subscribe(
2691                    "stable-subscription".to_owned(),
2692                    "tasks".to_owned(),
2693                    vec![(
2694                        "project_id".to_owned(),
2695                        vec!["p2".to_owned(), "p1".to_owned()],
2696                    )],
2697                    Some(r#"{"view":"v1"}"#.to_owned()),
2698                )
2699                .expect("persist subscription");
2700            let persisted = {
2701                let subscription = first
2702                    .subs
2703                    .iter_mut()
2704                    .find(|subscription| subscription.id == "stable-subscription")
2705                    .expect("subscription");
2706                subscription.cursor = 41;
2707                subscription.bootstrap_state = Some("resume-token".to_owned());
2708                subscription.effective = Some(vec![(
2709                    "project_id".to_owned(),
2710                    vec!["p1".to_owned(), "p2".to_owned()],
2711                )]);
2712                subscription.synced_once = true;
2713                subscription.clone()
2714            };
2715            first.persist_sub(&persisted).unwrap();
2716        }
2717
2718        let mut reopened = SyncClient::open_path_with_identity(
2719            None,
2720            &schema,
2721            ClientLimits::default(),
2722            path.to_str().expect("UTF-8 temp path"),
2723        )
2724        .expect("reopen");
2725        let progress = reopened
2726            .subscription_state("stable-subscription")
2727            .expect("persisted state");
2728        assert_eq!(progress.cursor, 41);
2729        assert!(progress.has_resume_token);
2730
2731        reopened
2732            .subscribe(
2733                "stable-subscription".to_owned(),
2734                "tasks".to_owned(),
2735                vec![(
2736                    "project_id".to_owned(),
2737                    vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
2738                )],
2739                Some(r#"{"view":"v1"}"#.to_owned()),
2740            )
2741            .expect("canonical intent is idempotent");
2742        assert_eq!(
2743            reopened
2744                .subscription_state("stable-subscription")
2745                .expect("unchanged state")
2746                .cursor,
2747            progress.cursor
2748        );
2749
2750        for (table, scopes, params) in [
2751            (
2752                "tasks",
2753                vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2754                Some(r#"{"view":"v1"}"#.to_owned()),
2755            ),
2756            (
2757                "tasks",
2758                vec![(
2759                    "project_id".to_owned(),
2760                    vec!["p2".to_owned(), "p1".to_owned()],
2761                )],
2762                Some(r#"{"view":"v2"}"#.to_owned()),
2763            ),
2764            (
2765                "docs",
2766                vec![
2767                    ("org_id".to_owned(), vec!["o1".to_owned()]),
2768                    ("projectId".to_owned(), vec!["p1".to_owned()]),
2769                ],
2770                Some(r#"{"view":"v1"}"#.to_owned()),
2771            ),
2772        ] {
2773            let error = reopened
2774                .subscribe(
2775                    "stable-subscription".to_owned(),
2776                    table.to_owned(),
2777                    scopes,
2778                    params,
2779                )
2780                .expect_err("identity rebind must fail");
2781            assert!(error.starts_with("client.subscription_intent_mismatch:"));
2782            assert_eq!(
2783                reopened
2784                    .subscription_state("stable-subscription")
2785                    .expect("unchanged state")
2786                    .cursor,
2787                progress.cursor
2788            );
2789        }
2790
2791        drop(reopened);
2792        std::fs::remove_file(path).expect("remove temp database");
2793    }
2794
2795    #[test]
2796    fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
2797        let path = std::env::temp_dir().join(format!(
2798            "syncular-satisfied-schema-floor-{}.db",
2799            uuid::Uuid::new_v4()
2800        ));
2801        let schema = json!({
2802            "version": 23,
2803            "tables": [{
2804                "name": "tasks",
2805                "primaryKey": "id",
2806                "columns": [
2807                    { "name": "id", "type": "string", "nullable": false },
2808                    { "name": "project_id", "type": "string", "nullable": false }
2809                ],
2810                "scopes": [{ "pattern": "project:{project_id}" }]
2811            }]
2812        });
2813
2814        {
2815            let mut first = SyncClient::open_path_with_identity(
2816                None,
2817                &schema,
2818                ClientLimits::default(),
2819                path.to_str().expect("UTF-8 temp path"),
2820            )
2821            .expect("first open");
2822            first
2823                .subscribe(
2824                    "tasks".to_owned(),
2825                    "tasks".to_owned(),
2826                    vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2827                    None,
2828                )
2829                .expect("persist subscription");
2830            first.set_schema_floor(Some(SchemaFloor {
2831                required_schema_version: Some(22),
2832                latest_schema_version: Some(22),
2833            }));
2834        }
2835
2836        let mut reopened = SyncClient::open_path_with_identity(
2837            None,
2838            &schema,
2839            ClientLimits::default(),
2840            path.to_str().expect("UTF-8 temp path"),
2841        )
2842        .expect("reopen");
2843        assert!(reopened.schema_floor().is_none());
2844        assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
2845        assert!(reopened.sync_needed());
2846        assert!(matches!(
2847            reopened.drain_sync_intents().as_slice(),
2848            [SyncIntent::Interactive]
2849        ));
2850        drop(reopened);
2851        std::fs::remove_file(path).expect("remove temp database");
2852    }
2853
2854    #[test]
2855    fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
2856        let path = std::env::temp_dir().join(format!(
2857            "syncular-unsatisfied-schema-floor-{}.db",
2858            uuid::Uuid::new_v4()
2859        ));
2860        let schema = json!({
2861            "version": 1,
2862            "tables": [{
2863                "name": "tasks",
2864                "primaryKey": "id",
2865                "columns": [
2866                    { "name": "id", "type": "string", "nullable": false },
2867                    { "name": "project_id", "type": "string", "nullable": false }
2868                ],
2869                "scopes": [{ "pattern": "project:{project_id}" }]
2870            }]
2871        });
2872
2873        {
2874            let mut first = SyncClient::open_path_with_identity(
2875                None,
2876                &schema,
2877                ClientLimits::default(),
2878                path.to_str().expect("UTF-8 temp path"),
2879            )
2880            .expect("first open");
2881            first
2882                .subscribe(
2883                    "tasks".to_owned(),
2884                    "tasks".to_owned(),
2885                    vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2886                    None,
2887                )
2888                .expect("persist subscription");
2889            first.set_schema_floor(Some(SchemaFloor {
2890                required_schema_version: Some(2),
2891                latest_schema_version: Some(2),
2892            }));
2893        }
2894
2895        let reopened = SyncClient::open_path_with_identity(
2896            None,
2897            &schema,
2898            ClientLimits::default(),
2899            path.to_str().expect("UTF-8 temp path"),
2900        )
2901        .expect("reopen");
2902        assert_eq!(
2903            reopened.schema_floor(),
2904            Some(&SchemaFloor {
2905                required_schema_version: Some(2),
2906                latest_schema_version: Some(2),
2907            })
2908        );
2909        assert!(!reopened.sync_needed());
2910        drop(reopened);
2911        std::fs::remove_file(path).expect("remove temp database");
2912    }
2913
2914    #[test]
2915    fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
2916        let path = std::env::temp_dir().join(format!(
2917            "syncular-indexed-column-bump-{}.db",
2918            uuid::Uuid::new_v4()
2919        ));
2920        let old_schema = json!({
2921            "version": 1,
2922            "tables": [
2923                {
2924                    "name": "tasks",
2925                    "primaryKey": "id",
2926                    "columns": [
2927                        { "name": "id", "type": "string", "nullable": false },
2928                        { "name": "project_id", "type": "string", "nullable": false }
2929                    ],
2930                    "scopes": [{ "pattern": "project:{project_id}" }]
2931                },
2932                {
2933                    "name": "legacy",
2934                    "primaryKey": "id",
2935                    "columns": [
2936                        { "name": "id", "type": "string", "nullable": false },
2937                        { "name": "project_id", "type": "string", "nullable": false }
2938                    ],
2939                    "scopes": [{ "pattern": "project:{project_id}" }]
2940                }
2941            ]
2942        });
2943        {
2944            let mut first = SyncClient::open_path_with_identity(
2945                None,
2946                &old_schema,
2947                ClientLimits::default(),
2948                path.to_str().expect("UTF-8 temp path"),
2949            )
2950            .expect("open old schema");
2951            first
2952                .subscribe(
2953                    "legacy-sub".to_owned(),
2954                    "legacy".to_owned(),
2955                    vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2956                    None,
2957                )
2958                .expect("persist legacy subscription");
2959        }
2960
2961        let new_schema = json!({
2962            "version": 2,
2963            "tables": [{
2964                "name": "tasks",
2965                "primaryKey": "id",
2966                "columns": [
2967                    { "name": "id", "type": "string", "nullable": false },
2968                    { "name": "project_id", "type": "string", "nullable": false },
2969                    { "name": "facility_membership_id", "type": "string", "nullable": true }
2970                ],
2971                "scopes": [{ "pattern": "project:{project_id}" }],
2972                "indexes": [{
2973                    "name": "tasks_by_membership",
2974                    "columns": ["project_id", "facility_membership_id"],
2975                    "unique": false
2976                }]
2977            }]
2978        });
2979        let upgraded = SyncClient::open_path_with_identity(
2980            None,
2981            &new_schema,
2982            ClientLimits::default(),
2983            path.to_str().expect("UTF-8 temp path"),
2984        )
2985        .expect("open upgraded schema");
2986        let columns = upgraded
2987            .conn
2988            .prepare("PRAGMA table_info(tasks)")
2989            .expect("prepare columns")
2990            .query_map([], |row| row.get::<_, String>(1))
2991            .expect("query columns")
2992            .collect::<Result<Vec<_>, _>>()
2993            .expect("collect columns");
2994        assert!(columns.contains(&"facility_membership_id".to_owned()));
2995        let index_count: i64 = upgraded
2996            .conn
2997            .query_row(
2998                "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
2999                [],
3000                |row| row.get(0),
3001            )
3002            .expect("query index");
3003        assert_eq!(index_count, 1);
3004        assert!(upgraded.subscription_state("legacy-sub").is_none());
3005        drop(upgraded);
3006        std::fs::remove_file(path).expect("remove temp database");
3007    }
3008
3009    #[test]
3010    fn migrates_pre_envelope_outcome_journal_additively() {
3011        let path = std::env::temp_dir().join(format!(
3012            "syncular-outcome-migration-{}.db",
3013            uuid::Uuid::new_v4()
3014        ));
3015        let conn = Connection::open(&path).expect("open legacy database");
3016        conn.execute_batch(
3017            "CREATE TABLE _syncular_commit_outcomes (
3018               seq INTEGER PRIMARY KEY AUTOINCREMENT,
3019               client_commit_id TEXT NOT NULL UNIQUE,
3020               status TEXT NOT NULL,
3021               recorded_at_ms INTEGER NOT NULL,
3022               results_json TEXT NOT NULL,
3023               resolution TEXT NOT NULL DEFAULT 'active',
3024               resolved_at_ms INTEGER,
3025               replacement_client_commit_id TEXT);",
3026        )
3027        .expect("create legacy outcome journal");
3028        drop(conn);
3029        let schema = json!({
3030            "version": 1,
3031            "tables": [{
3032                "name": "tasks",
3033                "primaryKey": "id",
3034                "columns": [
3035                    { "name": "id", "type": "string", "nullable": false },
3036                    { "name": "project_id", "type": "string", "nullable": false }
3037                ],
3038                "scopes": [{ "pattern": "project:{project_id}" }]
3039            }]
3040        });
3041        let client = SyncClient::open_path(
3042            "migration-native".to_owned(),
3043            &schema,
3044            ClientLimits::default(),
3045            path.to_str().expect("UTF-8 temp path"),
3046        )
3047        .expect("migrate database");
3048        let has_operations = client
3049            .conn
3050            .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
3051            .expect("prepare table info")
3052            .query_map([], |row| row.get::<_, String>(1))
3053            .expect("query table info")
3054            .filter_map(Result::ok)
3055            .any(|column| column == "operations_json");
3056        assert!(has_operations);
3057        drop(client);
3058        std::fs::remove_file(path).expect("remove temp database");
3059    }
3060
3061    #[test]
3062    fn durable_conflict_outcome_and_resolution_survive_reopen() {
3063        let path = std::env::temp_dir().join(format!(
3064            "syncular-durable-outcome-{}.db",
3065            uuid::Uuid::new_v4()
3066        ));
3067        let schema = json!({
3068            "version": 1,
3069            "tables": [{
3070                "name": "tasks",
3071                "primaryKey": "id",
3072                "columns": [
3073                    { "name": "id", "type": "string", "nullable": false },
3074                    { "name": "project_id", "type": "string", "nullable": false }
3075                ],
3076                "scopes": [{ "pattern": "project:{project_id}" }]
3077            }]
3078        });
3079        let conflict = ConflictRecord {
3080            client_commit_id: "losing-commit".to_owned(),
3081            op_index: 0,
3082            table: "tasks".to_owned(),
3083            row_id: "t1".to_owned(),
3084            code: "sync.version_conflict".to_owned(),
3085            message: "stale base version".to_owned(),
3086            server_version: 2,
3087            server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
3088            conflict_columns: vec!["project_id".to_owned()],
3089            operation: Some(CommitOperation {
3090                table: "tasks".to_owned(),
3091                row_id: "t1".to_owned(),
3092                op: "upsert".to_owned(),
3093                base_version: Some(1),
3094                values: None,
3095            }),
3096        };
3097        let failed_operations = vec![
3098            OutboxOp {
3099                upsert: true,
3100                table: "tasks".to_owned(),
3101                row_id: "t1".to_owned(),
3102                base_version: Some(1),
3103                values: None,
3104            },
3105            OutboxOp {
3106                upsert: true,
3107                table: "tasks".to_owned(),
3108                row_id: "status-event-1".to_owned(),
3109                base_version: Some(0),
3110                values: None,
3111            },
3112        ];
3113
3114        {
3115            let mut first = SyncClient::open_path(
3116                "durable-native".to_owned(),
3117                &schema,
3118                ClientLimits::default(),
3119                path.to_str().expect("UTF-8 temp path"),
3120            )
3121            .expect("first open");
3122            first
3123                .begin_observation("test_outcome")
3124                .expect("begin outcome");
3125            first
3126                .persist_commit_outcome(
3127                    "losing-commit",
3128                    CommitOutcomeStatus::Conflict,
3129                    &[CommitOperationOutcome::Conflict {
3130                        conflict: conflict.clone(),
3131                    }],
3132                    Some(&failed_operations),
3133                )
3134                .expect("persist outcome");
3135            first.conflicts.push(conflict);
3136            first
3137                .finish_observation(
3138                    "test_outcome",
3139                    ChangeAccumulator {
3140                        conflicts: true,
3141                        outcomes: true,
3142                        ..ChangeAccumulator::default()
3143                    },
3144                )
3145                .expect("commit outcome");
3146        }
3147
3148        {
3149            let mut reopened = SyncClient::open_path(
3150                "durable-native".to_owned(),
3151                &schema,
3152                ClientLimits::default(),
3153                path.to_str().expect("UTF-8 temp path"),
3154            )
3155            .expect("reopen");
3156            assert_eq!(reopened.conflicts().len(), 1);
3157            assert_eq!(
3158                reopened.conflicts()[0].conflict_columns,
3159                vec!["project_id".to_owned()],
3160                "conflictColumns survives the journal round trip"
3161            );
3162            let outcome = reopened
3163                .commit_outcome("losing-commit")
3164                .expect("read outcome")
3165                .expect("outcome");
3166            assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
3167            let operations = outcome.operations.expect("aggregate envelope");
3168            assert_eq!(operations.len(), 2);
3169            assert_eq!(operations[1].row_id, "status-event-1");
3170            let resolved = reopened
3171                .resolve_commit_outcome(ResolveCommitOutcomeInput {
3172                    client_commit_id: "losing-commit".to_owned(),
3173                    resolution: CommitOutcomeResolution::ResolvedKeepServer,
3174                    replacement_client_commit_id: None,
3175                })
3176                .expect("resolve");
3177            assert_eq!(
3178                resolved.resolution,
3179                CommitOutcomeResolution::ResolvedKeepServer
3180            );
3181            assert!(reopened.conflicts().is_empty());
3182        }
3183
3184        let reopened = SyncClient::open_path(
3185            "durable-native".to_owned(),
3186            &schema,
3187            ClientLimits::default(),
3188            path.to_str().expect("UTF-8 temp path"),
3189        )
3190        .expect("second reopen");
3191        assert!(reopened.conflicts().is_empty());
3192        assert_eq!(
3193            reopened
3194                .commit_outcome("losing-commit")
3195                .expect("read outcome")
3196                .expect("outcome")
3197                .resolution,
3198            CommitOutcomeResolution::ResolvedKeepServer
3199        );
3200        drop(reopened);
3201        std::fs::remove_file(path).expect("remove temp database");
3202    }
3203
3204    #[test]
3205    fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
3206        let path =
3207            std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
3208        let schema = json!({
3209            "version": 1,
3210            "tables": [{
3211                "name": "tasks",
3212                "primaryKey": "id",
3213                "columns": [
3214                    { "name": "id", "type": "string", "nullable": false },
3215                    { "name": "project_id", "type": "string", "nullable": false }
3216                ],
3217                "scopes": [{ "pattern": "project:{project_id}" }]
3218            }]
3219        });
3220        let mut client = SyncClient::open_path_with_identity(
3221            Some("sidecar-client".to_owned()),
3222            &schema,
3223            ClientLimits::default(),
3224            path.to_str().expect("UTF-8 temp path"),
3225        )
3226        .expect("open owner");
3227        let base = WindowBase {
3228            table: "tasks".to_owned(),
3229            variable: "project_id".to_owned(),
3230            fixed_scopes: Vec::new(),
3231            params: None,
3232        };
3233        client
3234            .set_window(&base, &["one".to_owned()])
3235            .expect("set window");
3236        client
3237            .mutate(vec![Mutation::Upsert {
3238                table: "tasks".to_owned(),
3239                values: Map::from_iter([
3240                    ("id".to_owned(), Value::from("t1")),
3241                    ("project_id".to_owned(), Value::from("one")),
3242                ]),
3243                base_version: None,
3244            }])
3245            .expect("local mutate");
3246
3247        let coverage = [WindowCoverage {
3248            base,
3249            units: vec!["one".to_owned(), "missing".to_owned()],
3250        }];
3251        let owner = client
3252            .query_snapshot(
3253                "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
3254                &[],
3255                &coverage,
3256            )
3257            .expect("owner snapshot");
3258        let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
3259        let sidecar = reader
3260            .query_snapshot(
3261                "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
3262                &[],
3263                &coverage,
3264            )
3265            .expect("sidecar snapshot");
3266
3267        assert_eq!(sidecar.revision, owner.revision);
3268        assert_eq!(sidecar.rows, owner.rows);
3269        assert_eq!(
3270            serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
3271            serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
3272        );
3273        assert_eq!(sidecar.revision, "2");
3274        assert_eq!(sidecar.rows[0]["id"], "t1");
3275        assert_eq!(sidecar.rows[0]["server_version"], -1);
3276        assert!(!sidecar.coverage.complete);
3277        assert_eq!(sidecar.coverage.pending.len(), 1);
3278        assert_eq!(sidecar.coverage.missing.len(), 1);
3279
3280        drop(reader);
3281        drop(client);
3282        std::fs::remove_file(path).expect("remove temp database");
3283    }
3284
3285    #[test]
3286    fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
3287        let mut client = client();
3288        let image = Connection::open_in_memory().unwrap();
3289        image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
3290            WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
3291            INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
3292            CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3293            INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
3294        let table = client.schema.table("tasks").unwrap().clone();
3295        let scopes = vec![("project_id".into(), vec!["p1".into()])];
3296        assert!(client
3297            .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
3298            .is_err());
3299        assert!(client.conn.is_autocommit());
3300        assert_eq!(
3301            client
3302                .query("SELECT count(*) AS n FROM tasks", &[])
3303                .unwrap()[0]["n"],
3304            1024
3305        );
3306        assert_eq!(client.drain_change_batches().len(), 1);
3307        image
3308            .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
3309            .unwrap();
3310        assert!(matches!(
3311            client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
3312            Ok(2050)
3313        ));
3314        assert_eq!(
3315            client
3316                .query("SELECT count(*) AS n FROM tasks", &[])
3317                .unwrap()[0]["n"],
3318            2050
3319        );
3320        assert_eq!(client.drain_change_batches().len(), 3);
3321    }
3322
3323    #[test]
3324    fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
3325        let mut client = client();
3326        let base = WindowBase {
3327            table: "tasks".into(),
3328            variable: "project_id".into(),
3329            fixed_scopes: Vec::new(),
3330            params: None,
3331        };
3332        client
3333            .set_window(&base, &["p1".into(), "p2".into()])
3334            .unwrap();
3335        client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
3336            INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
3337            INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
3338            INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
3339            CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
3340        client.drain_change_batches();
3341        assert!(client.set_window(&base, &["p2".into()]).is_err());
3342        assert!(client.conn.is_autocommit());
3343        assert_eq!(
3344            client
3345                .query("SELECT count(*) AS n FROM tasks", &[])
3346                .unwrap()[0]["n"],
3347            1027
3348        );
3349        assert_eq!(client.window_state(&base).units, vec!["p2"]);
3350        assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
3351        assert_eq!(client.drain_change_batches().len(), 1);
3352        client
3353            .conn
3354            .execute_batch("DROP TRIGGER interrupt_evict")
3355            .unwrap();
3356        client.drain_pending_evictions().unwrap();
3357        assert!(client.load_pending_evictions().unwrap().is_empty());
3358        assert_eq!(
3359            client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3360            "held"
3361        );
3362        assert_eq!(client.drain_change_batches().len(), 2);
3363    }
3364
3365    #[test]
3366    fn eviction_marker_failures_roll_back_window_changes() {
3367        let mut client = client();
3368        let base = WindowBase {
3369            table: "tasks".into(),
3370            variable: "project_id".into(),
3371            fixed_scopes: Vec::new(),
3372            params: None,
3373        };
3374        client.set_window(&base, &["p1".into()]).unwrap();
3375        client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
3376            INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
3377            INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
3378            CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
3379        let revision = client.local_revision();
3380        assert!(client.set_window(&base, &[]).is_err());
3381        assert!(client.conn.is_autocommit());
3382        assert_eq!(client.local_revision(), revision);
3383        assert_eq!(
3384            client
3385                .query("SELECT count(*) AS n FROM tasks", &[])
3386                .unwrap()[0]["n"],
3387            2050
3388        );
3389        assert_eq!(client.window_state(&base).units, vec!["p1"]);
3390        client.conn.execute_batch("DROP TRIGGER fail_marker;
3391            CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
3392        assert!(client.set_window(&base, &[]).is_err());
3393        client.conn.execute_batch("DROP TRIGGER interrupt_evict;
3394            CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
3395        assert!(client.drain_pending_evictions().is_err());
3396        assert!(client.conn.is_autocommit());
3397        assert_eq!(
3398            client
3399                .query("SELECT count(*) AS n FROM tasks", &[])
3400                .unwrap()[0]["n"],
3401            2
3402        );
3403        assert!(client.set_window(&base, &["p1".into()]).is_err());
3404        assert!(client.conn.is_autocommit());
3405        assert!(client.window_state(&base).units.is_empty());
3406        assert!(client.subs.is_empty());
3407        assert_eq!(
3408            client
3409                .query(
3410                    "SELECT count(*) AS n FROM _syncular_window_pending_evict",
3411                    &[]
3412                )
3413                .unwrap()[0]["n"],
3414            1
3415        );
3416        client
3417            .conn
3418            .execute_batch("DROP TRIGGER fail_marker")
3419            .unwrap();
3420        client.set_window(&base, &["p1".into()]).unwrap();
3421        assert_eq!(client.window_state(&base).units, vec!["p1"]);
3422    }
3423
3424    #[test]
3425    fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
3426        let mut client = client();
3427        client
3428            .conn
3429            .execute(
3430                "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
3431                [],
3432            )
3433            .unwrap();
3434        assert_eq!(
3435            client.drain_pending_evictions().unwrap_err(),
3436            "sync.local_corrupt: invalid pending eviction scopes"
3437        );
3438        assert!(client.conn.is_autocommit());
3439        assert!(client.enqueue_startup_sync_if_needed().is_err());
3440    }
3441
3442    #[test]
3443    fn image_chunks_reject_null_primary_keys() {
3444        let mut client = client();
3445        let image = Connection::open_in_memory().unwrap();
3446        image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3447            WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
3448            INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
3449            CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3450            INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
3451        let table = client.schema.table("tasks").unwrap().clone();
3452        assert!(client
3453            .apply_sqlite_image(
3454                &image,
3455                &table,
3456                true,
3457                &[("project_id".into(), vec!["p1".into()])],
3458                1025,
3459                7,
3460                "digest"
3461            )
3462            .is_err());
3463        assert!(client.conn.is_autocommit());
3464        assert_eq!(
3465            client
3466                .query("SELECT count(*) AS n FROM tasks", &[])
3467                .unwrap()[0]["n"],
3468            0
3469        );
3470    }
3471
3472    #[test]
3473    fn import_chunks_with_unique_constraints_match_full_pending_replay() {
3474        for images in [false, true] {
3475            let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
3476                "name":name,"primaryKey":"id","columns":[
3477                    {"name":"id","type":"string","nullable":false},
3478                    {"name":"project_id","type":"string","nullable":false},
3479                    {"name":"title","type":"string","nullable":false}],
3480                "scopes":[{"pattern":"project:{project_id}"}],
3481                "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
3482                "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
3483            })))});
3484            let mut actual =
3485                SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
3486            let mut reference =
3487                SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
3488            let scopes = vec![("project_id".into(), vec!["p1".into()])];
3489            for client in [&mut actual, &mut reference] {
3490                client
3491                    .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
3492                    .unwrap();
3493                client.subs[0].effective = Some(scopes.clone());
3494                for (table, id, title) in [
3495                    ("tasks", "a", "original"),
3496                    ("tasks", "b", "occupied"),
3497                    ("tasks", "c", "deleted"),
3498                    ("notes", "n", "untouched"),
3499                ] {
3500                    client
3501                        .write_base_row(
3502                            table,
3503                            &vec![
3504                                Some(ColumnValue::String(id.into())),
3505                                Some(ColumnValue::String("p1".into())),
3506                                Some(ColumnValue::String(title.into())),
3507                            ],
3508                            1,
3509                        )
3510                        .unwrap();
3511                }
3512                client.rebuild_overlay_if_dirty();
3513                client
3514                    .mutate(vec![
3515                        Mutation::Upsert {
3516                            table: "tasks".into(),
3517                            values: Map::from_iter([
3518                                ("id".into(), Value::from("a")),
3519                                ("project_id".into(), Value::from("p1")),
3520                                ("title".into(), Value::from("occupied")),
3521                            ]),
3522                            base_version: None,
3523                        },
3524                        Mutation::Delete {
3525                            table: "tasks".into(),
3526                            row_id: "c".into(),
3527                            base_version: None,
3528                        },
3529                        Mutation::Upsert {
3530                            table: "tasks".into(),
3531                            values: Map::from_iter([
3532                                ("id".into(), Value::from("local")),
3533                                ("project_id".into(), Value::from("p1")),
3534                                ("title".into(), Value::from("local value")),
3535                            ]),
3536                            base_version: None,
3537                        },
3538                        Mutation::Upsert {
3539                            table: "notes".into(),
3540                            values: Map::from_iter([
3541                                ("id".into(), Value::from("n")),
3542                                ("project_id".into(), Value::from("p1")),
3543                                ("title".into(), Value::from("pending note")),
3544                            ]),
3545                            base_version: None,
3546                        },
3547                    ])
3548                    .unwrap();
3549            }
3550            let pending = actual.pending_commit_ids();
3551            actual.overlay_rebuild_count.set(0);
3552            for (id, title) in [
3553                ("b", "available"),
3554                ("b", "occupied"),
3555                ("a", "server original"),
3556                ("d", "local value"),
3557                ("d", "released"),
3558                ("c", "redelivered"),
3559            ] {
3560                let table = actual.schema.table("tasks").unwrap().clone();
3561                let values = vec![
3562                    Some(ColumnValue::String(id.into())),
3563                    Some(ColumnValue::String("p1".into())),
3564                    Some(ColumnValue::String(title.into())),
3565                ];
3566                if images {
3567                    let image = Connection::open_in_memory().unwrap();
3568                    image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3569                        CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3570                        INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
3571                    image
3572                        .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
3573                        .unwrap();
3574                    assert!(matches!(
3575                        actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
3576                        Ok(1)
3577                    ));
3578                } else {
3579                    let segment = RowsSegment {
3580                        table: "tasks".into(),
3581                        schema_version: 1,
3582                        columns: table.wire_columns.clone(),
3583                        blocks: vec![vec![ssp2::segment::SegmentRow {
3584                            server_version: 2,
3585                            values: values.clone(),
3586                        }]],
3587                    };
3588                    assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
3589                }
3590                reference.write_base_row("tasks", &values, 2).unwrap();
3591                reference.rebuild_overlay();
3592                for table in ["tasks", "notes"] {
3593                    for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
3594                        assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
3595                    }
3596                }
3597                assert_eq!(actual.pending_commit_ids(), pending);
3598            }
3599            assert_eq!(
3600                actual.overlay_rebuild_count.get(),
3601                0,
3602                "each chunk must leave unrelated tables alone"
3603            );
3604            let before = actual
3605                .query("SELECT * FROM tasks ORDER BY id", &[])
3606                .unwrap();
3607            let fts_before = actual
3608                .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
3609                .unwrap();
3610            let revision = actual.local_revision();
3611            actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
3612            let table = actual.schema.table("tasks").unwrap().clone();
3613            let segment = RowsSegment {
3614                table: "tasks".into(),
3615                schema_version: 1,
3616                columns: table.wire_columns.clone(),
3617                blocks: vec![vec![ssp2::segment::SegmentRow {
3618                    server_version: 3,
3619                    values: vec![
3620                        Some(ColumnValue::String("b".into())),
3621                        Some(ColumnValue::String("p1".into())),
3622                        Some(ColumnValue::String("fresh".into())),
3623                    ],
3624                }]],
3625            };
3626            assert!(actual.apply_segment(0, &segment, false).is_err());
3627            assert!(actual.conn.is_autocommit());
3628            assert_eq!(
3629                actual
3630                    .query("SELECT * FROM tasks ORDER BY id", &[])
3631                    .unwrap(),
3632                before
3633            );
3634            assert_eq!(
3635                actual
3636                    .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
3637                    .unwrap(),
3638                fts_before
3639            );
3640            assert_eq!(actual.local_revision(), revision);
3641            assert_eq!(actual.pending_commit_ids(), pending);
3642        }
3643    }
3644
3645    #[test]
3646    fn import_reconciliation_preserves_typed_pending_primary_keys() {
3647        for (kind, a, b) in [
3648            ("string", json!("a"), json!("b")),
3649            ("integer", json!(1), json!(2)),
3650            ("float", json!(1.5), json!(2.5)),
3651            ("boolean", json!(true), json!(false)),
3652            ("json", json!("1"), json!("2")),
3653        ] {
3654            let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
3655            for images in [false, true] {
3656                let mut client =
3657                    SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
3658                let table = client.schema.table("tasks").unwrap().clone();
3659                client
3660                    .subscribe("tasks".into(), "tasks".into(), vec![], None)
3661                    .unwrap();
3662                for (id, title) in [(&a, "original"), (&b, "occupied")] {
3663                    client
3664                        .write_base_row(
3665                            "tasks",
3666                            &vec![
3667                                json_to_column_value(&table.columns[0], Some(id)).unwrap(),
3668                                Some(ColumnValue::String(title.into())),
3669                            ],
3670                            1,
3671                        )
3672                        .unwrap();
3673                }
3674                client.rebuild_overlay_if_dirty();
3675                client
3676                    .mutate(vec![Mutation::Upsert {
3677                        table: "tasks".into(),
3678                        values: Map::from_iter([
3679                            ("id".into(), a.clone()),
3680                            ("title".into(), json!("occupied")),
3681                        ]),
3682                        base_version: None,
3683                    }])
3684                    .unwrap();
3685                client.overlay_rebuild_count.set(0);
3686                for (title, expected) in [("available", "occupied"), ("occupied", "original")] {
3687                    let values = vec![
3688                        json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
3689                        Some(ColumnValue::String(title.into())),
3690                    ];
3691                    if images {
3692                        let image = Connection::open_in_memory().unwrap();
3693                        image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3694                            CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3695                            INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
3696                        image
3697                            .execute(
3698                                "INSERT INTO tasks VALUES (?1,?2,2)",
3699                                rusqlite::params![RowParam::Cell(&values[0]), title],
3700                            )
3701                            .unwrap();
3702                        assert!(
3703                            matches!(
3704                                client.apply_sqlite_image(
3705                                    &image,
3706                                    &table,
3707                                    false,
3708                                    &[],
3709                                    1,
3710                                    7,
3711                                    "digest"
3712                                ),
3713                                Ok(1)
3714                            ),
3715                            "{kind}, images={images}"
3716                        );
3717                    } else {
3718                        let segment = RowsSegment {
3719                            table: "tasks".into(),
3720                            schema_version: 1,
3721                            columns: table.wire_columns.clone(),
3722                            blocks: vec![vec![ssp2::segment::SegmentRow {
3723                                server_version: 2,
3724                                values,
3725                            }]],
3726                        };
3727                        assert!(
3728                            matches!(client.apply_segment(0, &segment, false), Ok(1)),
3729                            "{kind}"
3730                        );
3731                    }
3732                    let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
3733                    let actual: String = client
3734                        .conn
3735                        .query_row(
3736                            "SELECT title FROM tasks WHERE id=?1",
3737                            [RowParam::Cell(&key)],
3738                            |row| row.get(0),
3739                        )
3740                        .unwrap();
3741                    assert_eq!(actual, expected, "{kind}, images={images}");
3742                }
3743                assert_eq!(client.overlay_rebuild_count.get(), 0);
3744            }
3745        }
3746    }
3747
3748    fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
3749        let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
3750            "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
3751            "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
3752            "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
3753        })))});
3754        let mut client =
3755            SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
3756        client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
3757        client.rebuild_overlay();
3758        client
3759            .mutate(vec![Mutation::Upsert {
3760                table: "tasks".into(),
3761                values: Map::from_iter([
3762                    ("id".into(), json!("offline")),
3763                    ("project_id".into(), json!("p1")),
3764                    ("title".into(), json!("pending edit")),
3765                ]),
3766                base_version: None,
3767            }])
3768            .unwrap();
3769        let image = Connection::open_in_memory().unwrap();
3770        image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3771            CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
3772        image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
3773        image
3774            .execute(
3775                "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
3776                [rows],
3777            )
3778            .unwrap();
3779        client.overlay_rebuild_count.set(0);
3780        let table = client.schema.table("tasks").unwrap().clone();
3781        (client, image, table)
3782    }
3783
3784    #[test]
3785    fn large_unique_import_copies_no_unrelated_tables() {
3786        let (mut client, image, table) = unique_import_fixture(10240);
3787        let pending = client.pending_commit_ids();
3788        client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
3789            CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
3790        assert!(matches!(
3791            client.apply_sqlite_image(
3792                &image,
3793                &table,
3794                true,
3795                &[("project_id".into(), vec!["p1".into()])],
3796                10240,
3797                7,
3798                "digest"
3799            ),
3800            Ok(10240)
3801        ));
3802        assert_eq!(client.overlay_rebuild_count.get(), 0);
3803        assert_eq!(client.pending_commit_ids(), pending);
3804        assert_eq!(
3805            client
3806                .query("SELECT count(*) AS n FROM tasks", &[])
3807                .unwrap()[0]["n"],
3808            10241
3809        );
3810        assert_eq!(
3811            client
3812                .query("SELECT count(*) AS n FROM tasks_fts", &[])
3813                .unwrap()[0]["n"],
3814            10241
3815        );
3816        assert_eq!(
3817            client
3818                .query("SELECT count(*) AS n FROM notes_fts", &[])
3819                .unwrap()[0]["n"],
3820            8192
3821        );
3822    }
3823
3824    #[test]
3825    #[ignore = "manual scaling benchmark; see bench/README.md"]
3826    fn benchmark_unique_image_import() {
3827        let mut samples = Vec::new();
3828        for rows in [10240, 51200, 102400] {
3829            for trial in 0..4 {
3830                let (mut client, image, table) = unique_import_fixture(rows);
3831                let start = std::time::Instant::now();
3832                assert!(
3833                    matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
3834                );
3835                let elapsed = start.elapsed().as_secs_f64() * 1000.0;
3836                assert_eq!(
3837                    client
3838                        .query("SELECT count(*) AS n FROM tasks", &[])
3839                        .unwrap()[0]["n"],
3840                    rows + 1
3841                );
3842                assert_eq!(
3843                    client
3844                        .query("SELECT count(*) AS n FROM tasks_fts", &[])
3845                        .unwrap()[0]["n"],
3846                    rows + 1
3847                );
3848                assert_eq!(
3849                    client
3850                        .query("SELECT count(*) AS n FROM notes_fts", &[])
3851                        .unwrap()[0]["n"],
3852                    8192
3853                );
3854                assert_eq!(client.pending_commit_ids().len(), 1);
3855                if trial > 0 {
3856                    samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
3857                }
3858            }
3859        }
3860        let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
3861        let output = serde_json::to_string_pretty(&artifact).unwrap();
3862        if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
3863            use std::io::Write;
3864            let mut file = std::fs::OpenOptions::new()
3865                .write(true)
3866                .create_new(true)
3867                .open(path)
3868                .unwrap();
3869            file.write_all(output.as_bytes()).unwrap();
3870        } else {
3871            println!("{output}");
3872        }
3873    }
3874
3875    #[test]
3876    fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
3877        let schema = json!({
3878            "version": 1,
3879            "tables": [{
3880                "name": "catalogue_codes",
3881                "primaryKey": "id",
3882                "columns": [
3883                    { "name": "id", "type": "string", "nullable": false },
3884                    { "name": "release_id", "type": "string", "nullable": false },
3885                    { "name": "code", "type": "string", "nullable": false },
3886                    { "name": "title", "type": "string", "nullable": false }
3887                ],
3888                "scopes": [{ "pattern": "release:{release_id}" }],
3889                "ftsIndexes": [{
3890                    "name": "catalogue_codes_fts",
3891                    "columns": ["code", "title"],
3892                    "tokenize": "unicode61 remove_diacritics 2"
3893                }]
3894            }]
3895        });
3896        let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
3897            .expect("FTS5 client");
3898        let insert_trigger: String = client
3899            .conn
3900            .query_row(
3901                "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
3902                [],
3903                |row| row.get(0),
3904            )
3905            .expect("insert trigger");
3906        let replace_guard: String = client
3907            .conn
3908            .query_row(
3909                "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
3910                [],
3911                |row| row.get(0),
3912            )
3913            .expect("replace guard");
3914        assert!(!insert_trigger.contains("DELETE FROM"));
3915        assert!(replace_guard.contains("BEFORE INSERT"));
3916        assert!(replace_guard.contains("WHEN EXISTS"));
3917        let search = |client: &SyncClient, query: &str| {
3918            client
3919                .query(
3920                    "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
3921                    &[Value::from(query)],
3922                )
3923                .expect("FTS query")
3924        };
3925
3926        client
3927            .mutate(vec![Mutation::Upsert {
3928                table: "catalogue_codes".to_owned(),
3929                values: Map::from_iter([
3930                    ("id".to_owned(), Value::from("c1")),
3931                    ("release_id".to_owned(), Value::from("r1")),
3932                    ("code".to_owned(), Value::from("A01")),
3933                    ("title".to_owned(), Value::from("Cholera")),
3934                ]),
3935                base_version: None,
3936            }])
3937            .expect("insert code");
3938        assert_eq!(search(&client, "cholera").len(), 1);
3939        let prior = client
3940            .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
3941            .unwrap();
3942        client
3943            .conn
3944            .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
3945            .unwrap();
3946        client.create_synced_tables().unwrap();
3947        assert_eq!(
3948            client
3949                .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
3950                .unwrap(),
3951            prior
3952        );
3953        let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
3954        let rendered = serde_json::to_string(&plan).unwrap();
3955        assert!(rendered.contains("COVERING INDEX"));
3956        assert!(rendered.contains("INDEX 0:="));
3957        client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
3958        assert_eq!(search(&client, "cholera").len(), 1);
3959
3960        client
3961            .mutate(vec![Mutation::Upsert {
3962                table: "catalogue_codes".to_owned(),
3963                values: Map::from_iter([
3964                    ("id".to_owned(), Value::from("c1")),
3965                    ("release_id".to_owned(), Value::from("r1")),
3966                    ("code".to_owned(), Value::from("A01")),
3967                    ("title".to_owned(), Value::from("Enteric infection")),
3968                ]),
3969                base_version: None,
3970            }])
3971            .expect("update code");
3972        assert!(search(&client, "cholera").is_empty());
3973        assert_eq!(search(&client, "enteric").len(), 1);
3974
3975        client
3976            .mutate(vec![Mutation::Delete {
3977                table: "catalogue_codes".to_owned(),
3978                row_id: "c1".to_owned(),
3979                base_version: None,
3980            }])
3981            .expect("delete code");
3982        assert!(search(&client, "enteric").is_empty());
3983    }
3984
3985    #[test]
3986    fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
3987        let schema = json!({
3988            "version": 1,
3989            "tables": [{
3990                "name": "patient_notes",
3991                "primaryKey": "id",
3992                "columns": [
3993                    { "name": "id", "type": "string", "nullable": false },
3994                    { "name": "practice_id", "type": "string", "nullable": false },
3995                    { "name": "encryption_key_id", "type": "string", "nullable": false },
3996                    { "name": "title", "type": "string", "nullable": false }
3997                ],
3998                "scopes": [{ "pattern": "practice:{practice_id}" }],
3999                "ftsIndexes": [{
4000                    "name": "patient_notes_fts",
4001                    "columns": ["title"],
4002                    "tokenize": "unicode61 remove_diacritics 2"
4003                }]
4004            }]
4005        });
4006        let mut client = SyncClient::new(
4007            "local-purge-test".to_owned(),
4008            &schema,
4009            ClientLimits::default(),
4010        )
4011        .expect("local purge client");
4012        client
4013            .conn
4014            .execute(
4015                "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
4016                 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
4017                 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
4018                [],
4019            )
4020            .expect("seed base rows");
4021        client.overlay_dirty.set(true);
4022        client.rebuild_overlay_if_dirty();
4023
4024        let note = |id: &str, key_id: &str, title: &str| {
4025            Map::from_iter([
4026                ("id".to_owned(), Value::from(id)),
4027                ("practice_id".to_owned(), Value::from("practice-1")),
4028                ("encryption_key_id".to_owned(), Value::from(key_id)),
4029                ("title".to_owned(), Value::from(title)),
4030            ])
4031        };
4032        let doomed = client
4033            .mutate(vec![
4034                Mutation::Upsert {
4035                    table: "patient_notes".to_owned(),
4036                    values: note("target", "key-revoked", "Target changed"),
4037                    base_version: None,
4038                },
4039                Mutation::Upsert {
4040                    table: "patient_notes".to_owned(),
4041                    values: note("unrelated", "key-held", "Unrelated changed"),
4042                    base_version: None,
4043                },
4044            ])
4045            .expect("doomed commit");
4046        let kept = client
4047            .mutate(vec![Mutation::Upsert {
4048                table: "patient_notes".to_owned(),
4049                values: note("kept", "key-held", "Kept optimistic"),
4050                base_version: None,
4051            }])
4052            .expect("kept commit");
4053        client.drain_change_batches();
4054
4055        let input = LocalDataPurgeInput {
4056            purge_id: "purge-001".to_owned(),
4057            targets: vec![LocalDataPurgeTarget {
4058                table: "patient_notes".to_owned(),
4059                selectors: BTreeMap::from([(
4060                    "encryption_key_id".to_owned(),
4061                    vec!["key-revoked".to_owned()],
4062                )]),
4063            }],
4064        };
4065        assert_eq!(
4066            client.purge_local_data(&input).expect("apply purge"),
4067            LocalDataPurgeResult {
4068                already_applied: false,
4069                purged_rows: 1,
4070                dropped_commits: 1,
4071            }
4072        );
4073        let rows = client
4074            .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
4075            .expect("visible rows");
4076        assert_eq!(rows.len(), 2);
4077        assert_eq!(rows[0]["id"], "kept");
4078        assert_eq!(rows[1]["id"], "unrelated");
4079        assert_eq!(rows[1]["title"], "Unrelated original");
4080        let fts = client
4081            .query(
4082                "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
4083                &[],
4084            )
4085            .expect("fts query");
4086        assert!(fts.is_empty());
4087        assert_eq!(client.outbox.len(), 1);
4088        assert_eq!(client.outbox[0].client_commit_id, kept);
4089        let outcome = client
4090            .commit_outcome(&doomed)
4091            .expect("read doomed outcome")
4092            .expect("doomed outcome");
4093        assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
4094        match &outcome.results[0] {
4095            CommitOperationOutcome::Error { rejection } => {
4096                assert_eq!(rejection.code, "client.local_data_purged");
4097                assert_eq!(rejection.client_commit_id, doomed);
4098            }
4099            other => panic!("expected local purge rejection, got {other:?}"),
4100        }
4101        assert_eq!(client.drain_change_batches().len(), 1);
4102        assert_eq!(
4103            client.purge_local_data(&input).expect("retry purge"),
4104            LocalDataPurgeResult {
4105                already_applied: true,
4106                purged_rows: 0,
4107                dropped_commits: 0,
4108            }
4109        );
4110        let conflicting = LocalDataPurgeInput {
4111            purge_id: input.purge_id.clone(),
4112            targets: vec![LocalDataPurgeTarget {
4113                table: "patient_notes".to_owned(),
4114                selectors: BTreeMap::from([(
4115                    "encryption_key_id".to_owned(),
4116                    vec!["key-held".to_owned()],
4117                )]),
4118            }],
4119        };
4120        assert!(client
4121            .purge_local_data(&conflicting)
4122            .expect_err("id collision must fail")
4123            .contains("already used with a different plan"));
4124    }
4125}
4126
4127impl SubState {
4128    fn name(self) -> &'static str {
4129        match self {
4130            SubState::Active => "active",
4131            SubState::Revoked => "revoked",
4132            SubState::Failed => "failed",
4133        }
4134    }
4135
4136    fn parse(value: &str) -> Self {
4137        match value {
4138            "revoked" => Self::Revoked,
4139            "failed" => Self::Failed,
4140            _ => Self::Active,
4141        }
4142    }
4143}
4144
4145#[derive(Debug, Clone)]
4146struct Subscription {
4147    id: String,
4148    table: String,
4149    requested: Vec<(String, Vec<String>)>,
4150    params: Option<String>,
4151    cursor: i64,
4152    /// §4.7 resume token, round-tripped opaquely.
4153    bootstrap_state: Option<String>,
4154    state: SubState,
4155    reason_code: Option<String>,
4156    /// Last effective scopes echoed while active (§3.3: persisted for the
4157    /// purge contract; each active echo replaces it).
4158    effective: Option<Vec<(String, Vec<String>)>>,
4159    synced_once: bool,
4160}
4161
4162#[derive(Debug, Clone)]
4163struct OutboxOp {
4164    upsert: bool,
4165    table: String,
4166    row_id: String,
4167    base_version: Option<i64>,
4168    /// Schema-agnostic local form (§0): driver JSON values, encoded with
4169    /// the current codec at send time. A key's presence IS the §6.1 sparse
4170    /// presence set — both the overlay and the wire payload derive from it.
4171    values: Option<Map<String, Value>>,
4172}
4173
4174impl From<&OutboxOp> for CommitOperation {
4175    fn from(operation: &OutboxOp) -> Self {
4176        Self {
4177            table: operation.table.clone(),
4178            row_id: operation.row_id.clone(),
4179            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
4180            base_version: operation.base_version,
4181            values: operation.values.clone(),
4182        }
4183    }
4184}
4185
4186#[derive(Debug, Clone)]
4187struct OutboxCommit {
4188    client_commit_id: String,
4189    ops: Vec<OutboxOp>,
4190}
4191
4192#[derive(Debug, Clone)]
4193struct CompiledLocalDataPurgeTarget {
4194    table: String,
4195    selectors: Vec<(String, Vec<String>)>,
4196}
4197
4198struct StoredCommitOutcomeRow {
4199    sequence: i64,
4200    client_commit_id: String,
4201    status: String,
4202    recorded_at_ms: i64,
4203    results_json: String,
4204    operations_json: Option<String>,
4205    resolution: String,
4206    resolved_at_ms: Option<i64>,
4207    replacement_client_commit_id: Option<String>,
4208}
4209
4210/// Section outcome distinguishing the §5.6 subscription-local fail-closed
4211/// path from a round-aborting failure (§1.4 rule 5).
4212enum SectionError {
4213    FailClosed,
4214    Abort(String, String),
4215}
4216
4217struct RequestMeta {
4218    pushed_ids: Vec<String>,
4219    /// Subscription id → the request carried `cursor < 0` and no resume
4220    /// token (§5.6 first-page detection: a *fresh* bootstrap).
4221    fresh: Vec<(String, bool)>,
4222    accept: u8,
4223    /// §6.1 splitBatch: outbox commits held back from THIS request because
4224    /// the running operation count reached the push cap — the next round
4225    /// pushes them (`sync_needed` stays set while any remain).
4226    deferred_commits: usize,
4227}
4228
4229#[derive(Default)]
4230struct ChangeAccumulator {
4231    /// None means table-wide; Some is the exact scoped domain.
4232    tables: BTreeMap<String, Option<BTreeSet<String>>>,
4233    windows: BTreeMap<(String, String), BTreeSet<String>>,
4234    status: bool,
4235    conflicts: bool,
4236    rejections: bool,
4237    outcomes: bool,
4238}
4239
4240impl ChangeAccumulator {
4241    fn table(&mut self, table: &str) {
4242        self.tables.insert(table.to_owned(), None);
4243    }
4244
4245    fn scope(&mut self, table: &str, key: String) {
4246        match self.tables.get_mut(table) {
4247            Some(None) => {}
4248            Some(Some(keys)) => {
4249                keys.insert(key);
4250            }
4251            None => {
4252                self.tables
4253                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
4254            }
4255        }
4256    }
4257
4258    fn window(&mut self, base_key: &str, table: &str, unit: &str) {
4259        self.windows
4260            .entry((base_key.to_owned(), table.to_owned()))
4261            .or_default()
4262            .insert(unit.to_owned());
4263    }
4264
4265    fn touched(&self) -> bool {
4266        !self.tables.is_empty()
4267            || !self.windows.is_empty()
4268            || self.status
4269            || self.conflicts
4270            || self.rejections
4271            || self.outcomes
4272    }
4273}
4274
4275/// §6.1: the server caps total operations per request (reference default
4276/// 500) and rejects the whole batch with `sync.too_many_operations`; the
4277/// client "splits and retries". Splitting happens at build time: commits are
4278/// included IN ORDER until the operation budget is spent, the rest wait for
4279/// the next round.
4280const PUSH_OPS_PER_REQUEST: usize = 500;
4281const MAX_LOCAL_PURGE_TARGETS: usize = 64;
4282const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
4283const MAX_LOCAL_PURGE_VALUES: usize = 128;
4284const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
4285pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
4286
4287pub struct SyncClient {
4288    #[cfg(feature = "bench-internals")]
4289    benchmark_phases: Recorder,
4290    conn: Connection,
4291    schema: ClientSchema,
4292    client_id: String,
4293    limits: ClientLimits,
4294    subs: Vec<Subscription>,
4295    outbox: Vec<OutboxCommit>,
4296    conflicts: Vec<ConflictRecord>,
4297    rejections: Vec<RejectionRecord>,
4298    schema_floor: Option<SchemaFloor>,
4299    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
4300    lease_state: Option<LeaseState>,
4301    /// §1.6: the schema-floor response stops syncing until an upgrade.
4302    stopped: bool,
4303    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
4304    upgrading: bool,
4305    /// §8.4 coalesced sync-needed signal.
4306    sync_needed: bool,
4307    realtime_connected: bool,
4308    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
4309    presence: HashMap<String, HashMap<String, PresencePeer>>,
4310    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check; the
4311    /// host may pin it (conformance runs on a virtual clock).
4312    now_ms: Option<i64>,
4313    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
4314    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
4315    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
4316    encryption: crate::values::EncryptionConfig,
4317    /// Fail-closed host bootstrap gate. While set, command hosts permit only
4318    /// status/lifecycle inspection and an exact authorized local purge.
4319    security_preflight: bool,
4320    /// Per-table primary-key upsert SQL, built once per (full table name) —
4321    /// the row write path runs per row during bootstrap (§5.6), so the SQL
4322    /// string (and, via `prepare_cached`, its compiled statement) is reused
4323    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
4324    /// schema reset (the column lists may have changed).
4325    insert_sql: RefCell<HashMap<String, String>>,
4326    /// §7.1 rebuild gate: true whenever the base tables or the outbox have
4327    /// diverged from the visible overlay since the last rebuild. Lets a
4328    /// no-op sync round skip the full base→visible copy.
4329    overlay_dirty: Cell<bool>,
4330    /// Test-only structural performance signal: response processing must not
4331    /// turn a batch of acknowledgements into one full overlay rebuild each.
4332    #[cfg(test)]
4333    overlay_rebuild_count: Cell<usize>,
4334    /// Test-only structural performance signal: outcome retention is enforced
4335    /// once per response rather than once per acknowledgement.
4336    #[cfg(test)]
4337    outcome_prune_count: Cell<usize>,
4338    /// Exact observer-transaction output drained by command/FFI hosts.
4339    progress: ProgressObserver,
4340    change_queue: VecDeque<ClientChangeBatch>,
4341    sync_intent_queue: VecDeque<SyncIntent>,
4342    /// Explicit exponential retry policy for transient transport failures.
4343    retry_delay_ms: u64,
4344    last_round: Option<DiagnosticLastRound>,
4345    last_change: Option<DiagnosticLastChange>,
4346}
4347
4348fn quote_ident(name: &str) -> String {
4349    format!("\"{}\"", name.replace('"', "\"\""))
4350}
4351
4352/// Seek by the existing key while retaining the exact legacy text match.
4353fn row_id_predicate(table: &TableSchema) -> String {
4354    let key = quote_ident(&table.primary_key);
4355    let text_match = format!("CAST({key} AS TEXT) = ?1");
4356    match table.columns[table.pk_index].ty {
4357        ColumnType::String | ColumnType::Json => {
4358            format!("{key} = ?1 AND {text_match}")
4359        }
4360        // Native columns have no declared SQLite affinity. Match the stored
4361        // integer bind type explicitly. Unary + removes expression affinity so
4362        // SQLite can seek the typeless key; the text check preserves exact IDs.
4363        ColumnType::Integer | ColumnType::Boolean => {
4364            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
4365        }
4366        // REAL-to-text conversion can round distinct values to the same text.
4367        // Narrowing those matches by numeric equality would change behavior.
4368        _ => text_match,
4369    }
4370}
4371
4372fn is_local_operation_code_like(value: &str) -> bool {
4373    let bytes = value.as_bytes();
4374    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
4375        && bytes
4376            .iter()
4377            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
4378}
4379
4380fn base_table(name: &str) -> String {
4381    quote_ident(&format!("_syncular_base_{name}"))
4382}
4383
4384/// §4.8: a stable, server-opaque key for a window base — table + variable +
4385/// canonical fixed scopes. Two `set_window` calls with the same base
4386/// address the same registry rows.
4387/// §4.8 deferred eviction record: (sub id, table, effective scope map).
4388type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
4389
4390fn window_base_key(base: &WindowBase) -> String {
4391    format!(
4392        "{}\0{}\0{}",
4393        base.table,
4394        base.variable,
4395        canonical_scope_json(&base.fixed_scopes)
4396    )
4397}
4398
4399/// §4.8: the full requested scope map for one unit (fixed scopes + unit).
4400fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
4401    let mut scopes = base.fixed_scopes.clone();
4402    scopes.retain(|(k, _)| k != &base.variable);
4403    scopes.push((base.variable.clone(), vec![unit.to_owned()]));
4404    scopes
4405}
4406
4407/// §4.1 guidance: `w:<table>:<sha256(canonical scope map)[0..16]>`. Ids are
4408/// echoed not interpreted by the server, so the exact hash is client
4409/// convention; SHA-256 matches the SPEC's worked example.
4410fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
4411    let canonical = canonical_scope_json(&unit_scopes(base, unit));
4412    let digest = Sha256::digest(canonical.as_bytes());
4413    let hex = bytes_to_hex(&digest);
4414    format!("w:{}:{}", base.table, &hex[..16])
4415}
4416
4417fn visible_table(name: &str) -> String {
4418    quote_ident(name)
4419}
4420
4421const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
4422
4423/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
4424/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
4425fn is_synced_table_name(name: &str) -> bool {
4426    if name.starts_with("sqlite_") {
4427        return false;
4428    }
4429    if name.starts_with("_syncular_base_") {
4430        return true; // the base half of a synced table pair
4431    }
4432    // Bookkeeping: outbox, subscriptions, meta, blob cache/uploads.
4433    !name.starts_with("_syncular_")
4434}
4435
4436/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
4437fn blob_id_for(bytes: &[u8]) -> String {
4438    let digest = Sha256::digest(bytes);
4439    format!("sha256:{}", bytes_to_hex(&digest))
4440}
4441
4442/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
4443/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
4444/// per row on the §5.6 bootstrap path), scalars bind owned.
4445enum RowParam<'a> {
4446    Cell(&'a Option<ColumnValue>),
4447    Version(i64),
4448}
4449
4450impl rusqlite::ToSql for RowParam<'_> {
4451    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
4452        Ok(match self {
4453            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
4454            RowParam::Cell(cell) => match cell {
4455                None => ToSqlOutput::Owned(SqlValue::Null),
4456                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
4457                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
4458                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
4459                Some(ColumnValue::Boolean(b)) => {
4460                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
4461                }
4462                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
4463                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
4464                }
4465                // §5.10: crdt bytes store as BLOB, like bytes.
4466                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
4467                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
4468                }
4469            },
4470        })
4471    }
4472}
4473
4474// Retain a typed primary key across the current import chunk without copying row payloads.
4475fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
4476    match param.to_sql().map_err(|error| error.to_string())? {
4477        ToSqlOutput::Owned(value) => Ok(value),
4478        ToSqlOutput::Borrowed(value) => Ok(value.into()),
4479        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
4480    }
4481}
4482
4483/// §5.3 image cell → bind parameter, strict per the declared column type
4484/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
4485/// nullable). Mismatches are image-producer violations. Returns the cell
4486/// borrowed when the stored representation already matches what the row
4487/// codec would write, or the normalized scalar (boolean → 0/1, float from
4488/// INTEGER → REAL) otherwise — the local write is byte-identical to the
4489/// old convert-then-insert path without allocating per cell.
4490fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
4491    use ssp2::segment::ColumnType;
4492    let mismatch = || {
4493        Err(format!(
4494            "image column {:?} holds a value of the wrong type",
4495            column.name
4496        ))
4497    };
4498    match value {
4499        ValueRef::Null => {
4500            if !column.nullable {
4501                return Err(format!(
4502                    "image column {:?} is NULL but not nullable",
4503                    column.name
4504                ));
4505            }
4506            Ok(ToSqlOutput::Owned(SqlValue::Null))
4507        }
4508        ValueRef::Integer(i) => match column.ty {
4509            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
4510            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
4511            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
4512            _ => mismatch(),
4513        },
4514        ValueRef::Real(_) => match column.ty {
4515            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
4516            _ => mismatch(),
4517        },
4518        ValueRef::Text(t) => {
4519            std::str::from_utf8(t)
4520                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
4521            match column.ty {
4522                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
4523                    Ok(ToSqlOutput::Borrowed(value))
4524                }
4525                _ => mismatch(),
4526            }
4527        }
4528        ValueRef::Blob(_) => match column.ty {
4529            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
4530            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
4531            _ => mismatch(),
4532        },
4533    }
4534}
4535
4536fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
4537    use rusqlite::types::ValueRef;
4538    match value {
4539        ValueRef::Null => Value::Null,
4540        ValueRef::Integer(i) => match column.ty {
4541            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
4542            ssp2::segment::ColumnType::Float => {
4543                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
4544            }
4545            _ => Value::from(i),
4546        },
4547        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
4548        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
4549        ValueRef::Blob(b) => {
4550            let mut map = Map::new();
4551            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
4552            Value::Object(map)
4553        }
4554    }
4555}
4556
4557/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
4558/// Objects are accepted in lossless `{"$bytes": hex}` and
4559/// `{"$bigint": decimal}` envelope forms.
4560fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
4561    Ok(match value {
4562        Value::Null => SqlValue::Null,
4563        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
4564        Value::Number(n) => {
4565            if let Some(i) = n.as_i64() {
4566                SqlValue::Integer(i)
4567            } else if let Some(f) = n.as_f64() {
4568                SqlValue::Real(f)
4569            } else {
4570                return Err(format!("query param number {n} is out of range"));
4571            }
4572        }
4573        Value::String(s) => SqlValue::Text(s.clone()),
4574        Value::Object(_) => {
4575            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
4576                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
4577            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
4578                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
4579                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
4580                })?)
4581            } else {
4582                return Err(
4583                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
4584                        .to_owned(),
4585                );
4586            }
4587        }
4588        Value::Array(_) => return Err("query array params are not supported".to_owned()),
4589    })
4590}
4591
4592/// Map a rusqlite value with no schema column to consult (arbitrary query
4593/// output): integers/reals/text pass through by stored affinity, blobs ride
4594/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
4595/// schema column type to recover booleans/floats/json.
4596fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
4597    use rusqlite::types::ValueRef;
4598    match value {
4599        ValueRef::Null => Value::Null,
4600        ValueRef::Integer(i) => {
4601            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
4602            // ordinary UI-sized integers ergonomic and envelope only values
4603            // beyond JavaScript's safe range.
4604            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
4605            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
4606                Value::from(i)
4607            } else {
4608                let mut map = Map::new();
4609                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
4610                Value::Object(map)
4611            }
4612        }
4613        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
4614        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
4615        ValueRef::Blob(b) => {
4616            let mut map = Map::new();
4617            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
4618            Value::Object(map)
4619        }
4620    }
4621}
4622
4623fn query_connection(
4624    conn: &Connection,
4625    sql: &str,
4626    params: &[QueryValue],
4627) -> Result<Vec<QueryRow>, String> {
4628    crate::query_guard::assert_read_only_query(sql)?;
4629    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
4630    let bound: Vec<SqlValue> = params
4631        .iter()
4632        .map(json_param_to_sql)
4633        .collect::<Result<_, _>>()?;
4634    let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
4635    let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
4636    let bound_refs: Vec<&dyn rusqlite::ToSql> =
4637        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
4638    let mut sql_rows = stmt
4639        .query(bound_refs.as_slice())
4640        .map_err(|e| e.to_string())?;
4641    let mut out = Vec::new();
4642    while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
4643        let mut record = Map::new();
4644        for (i, name) in column_names.iter().enumerate() {
4645            let value = row.get_ref(i).map_err(|e| e.to_string())?;
4646            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
4647        }
4648        out.push(record);
4649    }
4650    Ok(out)
4651}
4652
4653fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
4654    let mut stmt = conn
4655        .prepare(
4656            "SELECT windows.unit, subscriptions.state_json
4657               FROM _syncular_windows AS windows
4658               JOIN _syncular_subscriptions AS subscriptions
4659                 ON subscriptions.id = windows.sub_id
4660              WHERE windows.base = ?1
4661              ORDER BY windows.unit ASC",
4662        )
4663        .map_err(|error| error.to_string())?;
4664    let rows = stmt
4665        .query_map(rusqlite::params![window_base_key(base)], |row| {
4666            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
4667        })
4668        .map_err(|error| error.to_string())?;
4669    let mut units = Vec::new();
4670    let mut pending = Vec::new();
4671    for row in rows {
4672        let (unit, raw) = row.map_err(|error| error.to_string())?;
4673        let state: Value = serde_json::from_str(&raw)
4674            .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
4675        let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
4676            || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
4677            || state
4678                .get("bootstrapState")
4679                .is_some_and(|value| !value.is_null());
4680        if is_pending {
4681            pending.push(unit.clone());
4682        }
4683        units.push(unit);
4684    }
4685    Ok(WindowState { units, pending })
4686}
4687
4688fn snapshot_connection(
4689    conn: &Connection,
4690    sql: &str,
4691    params: &[Value],
4692    coverage: &[WindowCoverage],
4693) -> Result<QuerySnapshot, String> {
4694    conn.execute_batch("SAVEPOINT syncular_snapshot_read")
4695        .map_err(|error| error.to_string())?;
4696    let result = (|| {
4697        let revision = conn
4698            .query_row(
4699                "SELECT value FROM _syncular_meta WHERE key = ?1",
4700                rusqlite::params![LOCAL_REVISION_KEY],
4701                |row| row.get::<_, String>(0),
4702            )
4703            .ok()
4704            .and_then(|value| value.parse::<u64>().ok())
4705            .unwrap_or(0);
4706        let rows = query_connection(conn, sql, params)?;
4707        let mut pending = Vec::new();
4708        let mut missing = Vec::new();
4709        for requested in coverage {
4710            let base_key = window_base_key(&requested.base);
4711            let state = persisted_window_state(conn, &requested.base)?;
4712            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
4713                let reference = WindowUnitRef {
4714                    base_key: base_key.clone(),
4715                    unit: unit.clone(),
4716                };
4717                if !state.units.iter().any(|held| held == &unit) {
4718                    missing.push(reference);
4719                } else if state.pending.iter().any(|held| held == &unit) {
4720                    pending.push(reference);
4721                }
4722            }
4723        }
4724        Ok(QuerySnapshot {
4725            revision: revision.to_string(),
4726            rows,
4727            coverage: CoverageSnapshot {
4728                complete: pending.is_empty() && missing.is_empty(),
4729                pending,
4730                missing,
4731            },
4732        })
4733    })();
4734    match result {
4735        Ok(snapshot) => {
4736            conn.execute_batch("RELEASE syncular_snapshot_read")
4737                .map_err(|error| error.to_string())?;
4738            Ok(snapshot)
4739        }
4740        Err(error) => {
4741            let _ = conn.execute_batch(
4742                "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
4743            );
4744            Err(error)
4745        }
4746    }
4747}
4748
4749/// A long-lived read-only SQLite sidecar for latency-critical native views.
4750/// Network rounds stay serialized on the mutable core owner, while atomic
4751/// query snapshots use this independent connection and therefore never queue
4752/// behind HTTP/WebSocket latency.
4753pub struct FileQuerySnapshotReader {
4754    path: String,
4755    conn: Option<Connection>,
4756}
4757
4758impl FileQuerySnapshotReader {
4759    #[must_use]
4760    pub fn new(path: impl Into<String>) -> Self {
4761        Self {
4762            path: path.into(),
4763            conn: None,
4764        }
4765    }
4766
4767    fn connection(&mut self) -> Result<&Connection, String> {
4768        if self.conn.is_none() {
4769            let conn = Connection::open_with_flags(
4770                &self.path,
4771                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
4772            )
4773            .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
4774            conn.busy_timeout(std::time::Duration::from_millis(250))
4775                .map_err(|error| error.to_string())?;
4776            self.conn = Some(conn);
4777        }
4778        self.conn
4779            .as_ref()
4780            .ok_or_else(|| "read sidecar connection missing".to_owned())
4781    }
4782
4783    pub fn query_snapshot(
4784        &mut self,
4785        sql: &str,
4786        params: &[Value],
4787        coverage: &[WindowCoverage],
4788    ) -> Result<QuerySnapshot, String> {
4789        snapshot_connection(self.connection()?, sql, params, coverage)
4790    }
4791}
4792
4793impl SyncClient {
4794    pub fn new_with_identity(
4795        client_id: Option<String>,
4796        schema_json: &Value,
4797        limits: ClientLimits,
4798    ) -> Result<Self, String> {
4799        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
4800        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
4801        Self::with_connection(resolved, schema_json, limits, conn)
4802    }
4803
4804    pub fn new(
4805        client_id: String,
4806        schema_json: &Value,
4807        limits: ClientLimits,
4808    ) -> Result<Self, String> {
4809        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
4810        Self::with_connection(client_id, schema_json, limits, conn)
4811    }
4812
4813    /// Build a client backed by an on-disk SQLite database at `path` — the
4814    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
4815    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
4816    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
4817    /// of the command router's dependency set (the router only holds a path).
4818    pub fn open_path(
4819        client_id: String,
4820        schema_json: &Value,
4821        limits: ClientLimits,
4822        path: &str,
4823    ) -> Result<Self, String> {
4824        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
4825        Self::with_connection(client_id, schema_json, limits, conn)
4826    }
4827
4828    pub fn open_path_with_identity(
4829        client_id: Option<String>,
4830        schema_json: &Value,
4831        limits: ClientLimits,
4832        path: &str,
4833    ) -> Result<Self, String> {
4834        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
4835        // File-backed native clients use an independent read connection for
4836        // latency-critical snapshots. WAL is SQLite's intended reader/writer
4837        // concurrency mode: a view read never holds a rollback-journal lock
4838        // that delays the mutable client's next commit, and a short busy
4839        // timeout absorbs the tiny checkpoint/schema-lock windows.
4840        conn.busy_timeout(std::time::Duration::from_millis(250))
4841            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
4842        conn.pragma_update(None, "journal_mode", "WAL")
4843            .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
4844        let persisted = conn
4845            .query_row(
4846                "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
4847                [],
4848                |row| row.get::<_, String>(0),
4849            )
4850            .ok();
4851        let resolved = persisted
4852            .clone()
4853            .or(client_id.clone())
4854            .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
4855        if let (Some(existing), Some(requested)) = (persisted, client_id) {
4856            if existing != requested {
4857                return Err(format!(
4858                    "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
4859                ));
4860            }
4861        }
4862        Self::with_connection(resolved, schema_json, limits, conn)
4863    }
4864
4865    #[must_use]
4866    pub fn client_id(&self) -> &str {
4867        &self.client_id
4868    }
4869
4870    /// Build a client over a caller-supplied rusqlite connection — the seam a
4871    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
4872    /// with an on-disk database (`Connection::open(path)`) rather than the
4873    /// default `:memory:`. The connection MUST be fresh (no pre-existing
4874    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
4875    /// the same file across process restarts reuses the persisted rows.
4876    pub fn with_connection(
4877        client_id: String,
4878        schema_json: &Value,
4879        limits: ClientLimits,
4880        conn: Connection,
4881    ) -> Result<Self, String> {
4882        if limits.outcome_retention_max_entries == Some(0) {
4883            return Err(
4884                "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
4885            );
4886        }
4887        let schema = parse_schema_json(schema_json)?;
4888        let mut client = SyncClient {
4889            #[cfg(feature = "bench-internals")]
4890            benchmark_phases: Recorder::default(),
4891            conn,
4892            schema,
4893            client_id,
4894            limits,
4895            subs: Vec::new(),
4896            outbox: Vec::new(),
4897            conflicts: Vec::new(),
4898            rejections: Vec::new(),
4899            schema_floor: None,
4900            lease_state: None,
4901            stopped: false,
4902            upgrading: false,
4903            sync_needed: false,
4904            realtime_connected: false,
4905            presence: HashMap::new(),
4906            now_ms: None,
4907            encryption: crate::values::EncryptionConfig::default(),
4908            security_preflight: false,
4909            insert_sql: RefCell::new(HashMap::new()),
4910            overlay_dirty: Cell::new(false),
4911            #[cfg(test)]
4912            overlay_rebuild_count: Cell::new(0),
4913            #[cfg(test)]
4914            outcome_prune_count: Cell::new(0),
4915            progress: ProgressObserver::default(),
4916            change_queue: VecDeque::new(),
4917            sync_intent_queue: VecDeque::new(),
4918            retry_delay_ms: 250,
4919            last_round: None,
4920            last_change: None,
4921        };
4922        // The row write path leans on the prepared-statement cache (two
4923        // insert statements per synced table, plus the bookkeeping
4924        // statements); size it so a multi-table schema never thrashes.
4925        client
4926            .conn
4927            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
4928        // Protected bookkeeping must exist before inspecting the persisted
4929        // schema marker. New app indexes may reference columns that only
4930        // exist after a version-bump reset.
4931        client.create_bookkeeping_tables()?;
4932        match client.get_meta(CLIENT_ID_KEY) {
4933            Some(existing) if existing != client.client_id => {
4934                return Err(format!(
4935                    "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
4936                    client.client_id
4937                ));
4938            }
4939            None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
4940            _ => {}
4941        }
4942        client.restore_persisted_state()?;
4943        let marker = client
4944            .get_meta(LOCAL_SCHEMA_VERSION_KEY)
4945            .and_then(|value| value.parse::<i32>().ok());
4946        match marker {
4947            None => {
4948                client.create_synced_tables()?;
4949                client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
4950            }
4951            Some(version) if version == client.schema.version => {
4952                client.create_synced_tables()?;
4953            }
4954            Some(_) => client.run_schema_reset()?,
4955        }
4956        client.clear_satisfied_persisted_schema_floor();
4957        client.prune_unknown_subscriptions()?;
4958        if marker == Some(client.schema.version) && !client.outbox.is_empty() {
4959            // Reconstruct the visible optimistic overlay from the durable base
4960            // plus outbox instead of trusting a process-interrupted mirror.
4961            client.overlay_dirty.set(true);
4962            client.rebuild_overlay();
4963        }
4964        // Every persisted active subscription needs one catch-up pull on open:
4965        // realtime only covers changes after connection, while an idempotent
4966        // setWindow correctly creates no fresh command effect. Pending outbox
4967        // work has the same restart requirement. The core owns this intent so
4968        // native hosts never poll or require an application-issued sync().
4969        client.enqueue_startup_sync_if_needed()?;
4970        Ok(client)
4971    }
4972
4973    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
4974    /// against this instead of system time (conformance virtual clock).
4975    pub fn set_now_ms(&mut self, now_ms: i64) {
4976        self.now_ms = Some(now_ms);
4977    }
4978
4979    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
4980    /// The command router parses these from the `create` command's
4981    /// `encryption` config (keys as `{$bytes: hex}`).
4982    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
4983        self.encryption = encryption;
4984    }
4985
4986    #[must_use]
4987    pub fn security_lifecycle(&self) -> &'static str {
4988        if self.security_preflight {
4989            "preflight"
4990        } else {
4991            "active"
4992        }
4993    }
4994
4995    #[must_use]
4996    pub fn security_preflight(&self) -> bool {
4997        self.security_preflight
4998    }
4999
5000    /// Quarantine this replica: enter the fail-closed gate, release all
5001    /// core-owned key material, and record the gate in the database so
5002    /// reopening re-enters preflight until `activate_security` clears it.
5003    ///
5004    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
5005    /// which must NOT leave that durable mark.
5006    pub fn begin_security_preflight(&mut self) {
5007        self.seal_security_on_teardown();
5008        // Persist the quarantine so it survives handle teardown and restart:
5009        // reopening this replica re-enters preflight until activation clears it.
5010        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
5011    }
5012
5013    /// Release core-owned key material as a host tears the client down,
5014    /// WITHOUT recording a quarantine.
5015    ///
5016    /// Shutting an activated client down is not a quarantine event. Persisting
5017    /// the gate here marks every cleanly closed replica as pending, and the
5018    /// next plain `create` is then refused permanently: the reopen path
5019    /// restores the flag from the marker and the create guard rejects it. The
5020    /// in-memory flag still closes the gate for anything still holding this
5021    /// instance, which is all a teardown barrier needs.
5022    pub fn seal_security_on_teardown(&mut self) {
5023        self.security_preflight = true;
5024        self.encryption = crate::values::EncryptionConfig::default();
5025        self.sync_intent_queue.clear();
5026    }
5027
5028    /// Install the post-authentication keyring and release the host loop.
5029    pub fn activate_security(
5030        &mut self,
5031        encryption: crate::values::EncryptionConfig,
5032    ) -> Result<(), String> {
5033        if !self.security_preflight {
5034            return Err(
5035                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
5036            );
5037        }
5038        self.enqueue_startup_sync_if_needed()?;
5039        self.encryption = encryption;
5040        self.security_preflight = false;
5041        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
5042        Ok(())
5043    }
5044
5045    fn clock_now_ms(&self) -> i64 {
5046        self.now_ms.unwrap_or_else(|| {
5047            std::time::SystemTime::now()
5048                .duration_since(std::time::UNIX_EPOCH)
5049                .map(|d| d.as_millis() as i64)
5050                .unwrap_or(0)
5051        })
5052    }
5053
5054    fn create_bookkeeping_tables(&self) -> Result<(), String> {
5055        // Durable client bookkeeping (outbox + subscription + meta).
5056        self.conn
5057            .execute_batch(
5058                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
5059                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
5060                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
5061                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
5062                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
5063                   client_commit_id TEXT NOT NULL UNIQUE,
5064                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
5065                   recorded_at_ms INTEGER NOT NULL,
5066                   results_json TEXT NOT NULL,
5067                   operations_json TEXT,
5068                   resolution TEXT NOT NULL DEFAULT 'active'
5069                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
5070                   resolved_at_ms INTEGER,
5071                   replacement_client_commit_id TEXT);
5072                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
5073                   ON _syncular_commit_outcomes(resolution, seq);
5074                 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
5075                   id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
5076                 CREATE TABLE IF NOT EXISTS _syncular_meta (
5077                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
5078                 CREATE TABLE IF NOT EXISTS _syncular_windows (
5079                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
5080                   PRIMARY KEY (base, unit));
5081                 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
5082                   sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
5083                   effective_scopes TEXT NOT NULL);
5084                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
5085                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
5086                   PRIMARY KEY(commit_id, blob_id));
5087                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
5088                   ON _syncular_blob_commit_refs(blob_id);",
5089            )
5090            .map_err(|e| e.to_string())?;
5091        // Migrate an outcome journal created before failed aggregate
5092        // envelopes were retained. Historical rows intentionally stay NULL.
5093        let _ = self
5094            .conn
5095            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
5096        if self.get_meta(LOCAL_REVISION_KEY).is_none() {
5097            self.set_meta(LOCAL_REVISION_KEY, "0");
5098        }
5099        // §5.9.7 blob cache. Mutable upload state stays off the body row so
5100        // completing a large upload does not rewrite its BLOB record.
5101        if self.schema_has_blobs() {
5102            let exists = self
5103                .conn
5104                .query_row(
5105                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
5106                    [],
5107                    |row| row.get::<_, bool>(0),
5108                )
5109                .map_err(|e| e.to_string())?;
5110            if exists {
5111                let mut statement = self
5112                    .conn
5113                    .prepare("PRAGMA table_info(_syncular_blobs)")
5114                    .map_err(|e| e.to_string())?;
5115                let mut columns = statement
5116                    .query_map([], |row| row.get::<_, String>(1))
5117                    .map_err(|e| e.to_string())?
5118                    .collect::<Result<Vec<_>, _>>()
5119                    .map_err(|e| e.to_string())?;
5120                columns.sort();
5121                let mut expected = vec![
5122                    "blob_id",
5123                    "byte_length",
5124                    "bytes",
5125                    "created_at_ms",
5126                    "media_type",
5127                ];
5128                expected.sort_unstable();
5129                if columns != expected {
5130                    return Err(
5131                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
5132                    );
5133                }
5134            } else {
5135                self.conn
5136                    .execute_batch(
5137                        "CREATE TABLE _syncular_blobs(
5138                           blob_id TEXT PRIMARY KEY,
5139                           bytes BLOB NOT NULL,
5140                           byte_length INTEGER NOT NULL,
5141                           media_type TEXT,
5142                           created_at_ms INTEGER NOT NULL);",
5143                    )
5144                    .map_err(|e| e.to_string())?;
5145            }
5146            self.conn
5147                .execute_batch(
5148                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
5149                       blob_id TEXT PRIMARY KEY,
5150                       media_type TEXT,
5151                       created_at_ms INTEGER NOT NULL);",
5152                )
5153                .map_err(|e| e.to_string())?;
5154        }
5155        Ok(())
5156    }
5157
5158    /// True iff any synced table declares a `blob_ref` column (§5.9).
5159    fn schema_has_blobs(&self) -> bool {
5160        self.schema
5161            .tables
5162            .iter()
5163            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
5164    }
5165
5166    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
5167
5168    fn get_meta(&self, key: &str) -> Option<String> {
5169        self.conn
5170            .query_row(
5171                "SELECT value FROM _syncular_meta WHERE key = ?1",
5172                rusqlite::params![key],
5173                |row| row.get::<_, String>(0),
5174            )
5175            .ok()
5176    }
5177
5178    fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
5179        self.conn
5180            .query_row(
5181                "SELECT value FROM _syncular_meta WHERE key = ?1",
5182                rusqlite::params![key],
5183                |row| row.get::<_, String>(0),
5184            )
5185            .optional()
5186            .map_err(|_| {
5187                "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
5188            })
5189    }
5190
5191    fn set_meta(&self, key: &str, value: &str) {
5192        let _ = self.conn.execute(
5193            "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
5194            rusqlite::params![key, value],
5195        );
5196    }
5197
5198    fn delete_meta(&self, key: &str) {
5199        let _ = self.conn.execute(
5200            "DELETE FROM _syncular_meta WHERE key = ?1",
5201            rusqlite::params![key],
5202        );
5203    }
5204
5205    fn restore_persisted_state(&mut self) -> Result<(), String> {
5206        self.subs = {
5207            let mut stmt = self
5208                .conn
5209                .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
5210                .map_err(|error| error.to_string())?;
5211            let rows = stmt
5212                .query_map([], |row| {
5213                    Ok((
5214                        row.get::<_, String>(0)?,
5215                        row.get::<_, String>(1)?,
5216                        row.get::<_, String>(2)?,
5217                    ))
5218                })
5219                .map_err(|error| error.to_string())?;
5220            let mut subscriptions = Vec::new();
5221            for row in rows {
5222                let (id, table, raw) = row.map_err(|error| error.to_string())?;
5223                let state: Value = serde_json::from_str(&raw)
5224                    .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
5225                let requested = json_to_scope_map(
5226                    state.get("requested").unwrap_or(&Value::Object(Map::new())),
5227                )?;
5228                let effective = state
5229                    .get("effectiveScopes")
5230                    .filter(|value| !value.is_null())
5231                    .map(json_to_scope_map)
5232                    .transpose()?;
5233                subscriptions.push(Subscription {
5234                    id,
5235                    table,
5236                    requested,
5237                    params: state
5238                        .get("params")
5239                        .and_then(Value::as_str)
5240                        .map(str::to_owned),
5241                    cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
5242                    bootstrap_state: state
5243                        .get("bootstrapState")
5244                        .and_then(Value::as_str)
5245                        .map(str::to_owned),
5246                    state: SubState::parse(
5247                        state
5248                            .get("status")
5249                            .and_then(Value::as_str)
5250                            .unwrap_or("active"),
5251                    ),
5252                    reason_code: state
5253                        .get("reasonCode")
5254                        .and_then(Value::as_str)
5255                        .map(str::to_owned),
5256                    effective,
5257                    synced_once: state
5258                        .get("syncedOnce")
5259                        .and_then(Value::as_bool)
5260                        .unwrap_or(false),
5261                });
5262            }
5263            subscriptions
5264        };
5265
5266        self.outbox = {
5267            let mut stmt = self
5268                .conn
5269                .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
5270                .map_err(|error| error.to_string())?;
5271            let rows = stmt
5272                .query_map([], |row| {
5273                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
5274                })
5275                .map_err(|error| error.to_string())?;
5276            let mut commits = Vec::new();
5277            for row in rows {
5278                let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
5279                let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
5280                    format!("invalid persisted outbox {client_commit_id:?}: {error}")
5281                })?;
5282                let mut ops = Vec::with_capacity(entries.len());
5283                for entry in entries {
5284                    let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
5285                    ops.push(OutboxOp {
5286                        upsert: op == "upsert",
5287                        table: entry
5288                            .get("table")
5289                            .and_then(Value::as_str)
5290                            .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
5291                            .to_owned(),
5292                        row_id: entry
5293                            .get("rowId")
5294                            .and_then(Value::as_str)
5295                            .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
5296                            .to_owned(),
5297                        base_version: entry.get("baseVersion").and_then(Value::as_i64),
5298                        // A legacy entry's `changedFields` key is ignored:
5299                        // the values map is the presence set (§6.1).
5300                        values: entry.get("values").and_then(Value::as_object).cloned(),
5301                    });
5302                }
5303                commits.push(OutboxCommit {
5304                    client_commit_id,
5305                    ops,
5306                });
5307            }
5308            commits
5309        };
5310
5311        self.prune_commit_outcomes()?;
5312        let active = self.commit_outcomes(CommitOutcomeQuery {
5313            active_only: true,
5314            ..CommitOutcomeQuery::default()
5315        })?;
5316        self.conflicts = active
5317            .iter()
5318            .flat_map(|outcome| outcome.results.iter())
5319            .filter_map(|result| match result {
5320                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
5321                _ => None,
5322            })
5323            .collect();
5324        self.rejections = active
5325            .iter()
5326            .flat_map(|outcome| outcome.results.iter())
5327            .filter_map(|result| match result {
5328                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
5329                _ => None,
5330            })
5331            .collect();
5332
5333        self.lease_state = self
5334            .get_meta(LEASE_STATE_KEY)
5335            .map(|raw| serde_json::from_str(&raw))
5336            .transpose()
5337            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
5338        self.schema_floor = self
5339            .get_meta(SCHEMA_FLOOR_KEY)
5340            .map(|raw| serde_json::from_str(&raw))
5341            .transpose()
5342            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
5343        self.stopped = self.schema_floor.is_some();
5344        // A replica left in unactivated preflight reopens gated: the quarantine
5345        // decision persists with the data across handle teardown and restart.
5346        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
5347            self.security_preflight = true;
5348        }
5349        Ok(())
5350    }
5351
5352    /// Remove registrations for tables the running schema no longer knows.
5353    /// Otherwise the server rejects every pull with `sync.unknown_table`.
5354    fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
5355        let valid_tables: BTreeSet<String> = self
5356            .schema
5357            .tables
5358            .iter()
5359            .map(|table| table.name.clone())
5360            .collect();
5361        let stale_ids: Vec<String> = self
5362            .subs
5363            .iter()
5364            .filter(|sub| !valid_tables.contains(&sub.table))
5365            .map(|sub| sub.id.clone())
5366            .collect();
5367        for id in &stale_ids {
5368            self.conn
5369                .execute(
5370                    "DELETE FROM _syncular_windows WHERE sub_id = ?1",
5371                    rusqlite::params![id],
5372                )
5373                .map_err(|error| error.to_string())?;
5374            self.conn
5375                .execute(
5376                    "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
5377                    rusqlite::params![id],
5378                )
5379                .map_err(|error| error.to_string())?;
5380            self.conn
5381                .execute(
5382                    "DELETE FROM _syncular_subscriptions WHERE id = ?1",
5383                    rusqlite::params![id],
5384                )
5385                .map_err(|error| error.to_string())?;
5386        }
5387        self.subs.retain(|sub| valid_tables.contains(&sub.table));
5388        Ok(())
5389    }
5390
5391    #[must_use]
5392    pub fn local_revision(&self) -> u64 {
5393        self.get_meta(LOCAL_REVISION_KEY)
5394            .and_then(|value| value.parse().ok())
5395            .unwrap_or(0)
5396    }
5397
5398    #[must_use]
5399    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
5400        SyncStatusSnapshot {
5401            current_schema_version: self.schema.version,
5402            outbox: self.outbox.len(),
5403            upgrading: self.upgrading,
5404            lease_state: self.lease_state.clone(),
5405            schema_floor: self.schema_floor.clone(),
5406            sync_needed: self.sync_needed,
5407        }
5408    }
5409
5410    pub fn diagnostics_snapshot(
5411        &self,
5412        request: &ClientDiagnosticsRequest,
5413    ) -> Result<ClientDiagnosticsSnapshot, String> {
5414        if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
5415            return Err(format!(
5416                "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
5417            ));
5418        }
5419        let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
5420        for sub in &self.subs {
5421            let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
5422            let complete =
5423                sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
5424            let state = match sub.state {
5425                SubState::Revoked => "revoked",
5426                SubState::Failed => "failed",
5427                SubState::Active if reset => "reset",
5428                SubState::Active if complete => "complete",
5429                SubState::Active => "bootstrapping",
5430            };
5431            subscriptions.insert(
5432                sub.id.clone(),
5433                DiagnosticSubscription {
5434                    id: sub.id.clone(),
5435                    table: sub.table.clone(),
5436                    state: state.to_owned(),
5437                    complete,
5438                    cursor: Some(sub.cursor),
5439                    reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
5440                },
5441            );
5442        }
5443        for expected in &request.expected_subscriptions {
5444            if expected.id.is_empty() || expected.table.is_empty() {
5445                return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
5446            }
5447            if subscriptions
5448                .get(&expected.id)
5449                .is_some_and(|registered| registered.table != expected.table)
5450            {
5451                subscriptions.insert(
5452                    expected.id.clone(),
5453                    DiagnosticSubscription {
5454                        id: expected.id.clone(),
5455                        table: expected.table.clone(),
5456                        state: "failed".to_owned(),
5457                        complete: false,
5458                        cursor: None,
5459                        reason_code: Some("client.subscription_intent_mismatch".to_owned()),
5460                    },
5461                );
5462            } else {
5463                subscriptions.entry(expected.id.clone()).or_insert_with(|| {
5464                    DiagnosticSubscription {
5465                        id: expected.id.clone(),
5466                        table: expected.table.clone(),
5467                        state: "unregistered".to_owned(),
5468                        complete: false,
5469                        cursor: None,
5470                        reason_code: None,
5471                    }
5472                });
5473            }
5474        }
5475        let mut ordered_subscriptions = Vec::new();
5476        let mut included = BTreeSet::new();
5477        for expected in &request.expected_subscriptions {
5478            if included.insert(expected.id.clone()) {
5479                if let Some(subscription) = subscriptions.get(&expected.id) {
5480                    ordered_subscriptions.push(subscription.clone());
5481                }
5482            }
5483        }
5484        for (id, subscription) in &subscriptions {
5485            if included.insert(id.clone()) {
5486                ordered_subscriptions.push(subscription.clone());
5487            }
5488        }
5489        let subscriptions_truncated =
5490            ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
5491        ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
5492        let captured_at_ms = self.clock_now_ms();
5493        let lease = if let Some(error_code) = self
5494            .lease_state
5495            .as_ref()
5496            .and_then(|state| state.error_code.clone())
5497        {
5498            ClientDiagnosticsLease {
5499                state: "stopped".to_owned(),
5500                expires_at_ms: self
5501                    .lease_state
5502                    .as_ref()
5503                    .and_then(|state| state.expires_at_ms),
5504                error_code: Some(Self::diagnostic_code(&error_code)),
5505            }
5506        } else if let Some(expires_at_ms) = self
5507            .lease_state
5508            .as_ref()
5509            .and_then(|state| state.expires_at_ms)
5510        {
5511            ClientDiagnosticsLease {
5512                state: if expires_at_ms <= captured_at_ms {
5513                    "expired".to_owned()
5514                } else {
5515                    "active".to_owned()
5516                },
5517                expires_at_ms: Some(expires_at_ms),
5518                error_code: None,
5519            }
5520        } else {
5521            ClientDiagnosticsLease {
5522                state: "none".to_owned(),
5523                expires_at_ms: None,
5524                error_code: None,
5525            }
5526        };
5527        let connectivity = match self.last_round.as_ref() {
5528            Some(round) if round.status == "succeeded" => "online",
5529            Some(round)
5530                if round.status == "failed"
5531                    && round
5532                        .error_code
5533                        .as_deref()
5534                        .is_some_and(Self::retryable_transport_code) =>
5535            {
5536                "offline"
5537            }
5538            _ => "unknown",
5539        };
5540        Ok(ClientDiagnosticsSnapshot {
5541            version: CLIENT_DIAGNOSTICS_VERSION,
5542            captured_at_ms,
5543            host: ClientDiagnosticsHost {
5544                kind: "direct".to_owned(),
5545                role: "single".to_owned(),
5546                connectivity: connectivity.to_owned(),
5547                realtime: if self.realtime_connected {
5548                    "connected".to_owned()
5549                } else {
5550                    "disconnected".to_owned()
5551                },
5552            },
5553            security_lifecycle: self.security_lifecycle().to_owned(),
5554            schema: ClientDiagnosticsSchema {
5555                current_version: self.schema.version,
5556                upgrading: self.upgrading,
5557                required_version: self
5558                    .schema_floor
5559                    .as_ref()
5560                    .and_then(|floor| floor.required_schema_version),
5561                latest_version: self
5562                    .schema_floor
5563                    .as_ref()
5564                    .and_then(|floor| floor.latest_schema_version),
5565            },
5566            replica: ClientDiagnosticsReplica {
5567                local_revision: self.local_revision().to_string(),
5568                sync_needed: self.sync_needed,
5569                pending_outbox: self.outbox.len(),
5570            },
5571            lease,
5572            subscriptions: ordered_subscriptions,
5573            subscriptions_truncated,
5574            last_round: self.last_round.clone(),
5575            last_change: self.last_change.clone(),
5576            storage: self.diagnostics_storage(),
5577        })
5578    }
5579
5580    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
5581        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
5582            let page_count: i64 = self
5583                .conn
5584                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
5585            let page_size: i64 = self
5586                .conn
5587                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
5588            // Reuse compiled aggregates; every invocation still reads current storage.
5589            let outbox_bytes: i64 = self
5590                .conn
5591                .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
5592                .query_row([], |row| row.get(0))?;
5593            let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
5594                "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
5595            )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
5596            let blob_bytes = if self.schema_has_blobs() {
5597                self.conn
5598                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
5599                    .query_row([], |row| row.get(0))?
5600            } else {
5601                0
5602            };
5603            let pressure = self
5604                .limits
5605                .blob_cache_max_bytes
5606                .is_some_and(|limit| blob_bytes > limit);
5607            Ok(ClientDiagnosticsStorage {
5608                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
5609                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
5610                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
5611                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
5612                retained_outcome_entries: Some(outcome_entries.max(0)),
5613                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
5614                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
5615            })
5616        };
5617        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
5618            status: "unreadable".to_owned(),
5619            database_bytes_approx: None,
5620            pending_outbox_bytes_approx: None,
5621            retained_outcome_bytes_approx: None,
5622            retained_outcome_entries: None,
5623            blob_cache_bytes_approx: None,
5624            pressure_reason_code: None,
5625        })
5626    }
5627
5628    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
5629        self.change_queue.drain(..).collect()
5630    }
5631
5632    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
5633        self.sync_intent_queue.drain(..).collect()
5634    }
5635
5636    fn schedule_background_retry(&mut self) {
5637        self.sync_intent_queue.push_back(SyncIntent::Background {
5638            delay_ms: self.retry_delay_ms,
5639        });
5640        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
5641    }
5642
5643    fn reset_background_retry(&mut self) {
5644        self.retry_delay_ms = 250;
5645    }
5646
5647    fn retryable_transport_code(code: &str) -> bool {
5648        code == "transport.failed"
5649            || code == "transport.unavailable"
5650            || code == "sync.transport_failed"
5651    }
5652
5653    fn diagnostic_code(code: &str) -> String {
5654        let valid = !code.is_empty()
5655            && code.len() <= 96
5656            && code.contains('.')
5657            && code
5658                .bytes()
5659                .next()
5660                .is_some_and(|byte| byte.is_ascii_lowercase())
5661            && code.bytes().all(|byte| {
5662                byte.is_ascii_lowercase()
5663                    || byte.is_ascii_digit()
5664                    || matches!(byte, b'.' | b'_' | b'-')
5665            });
5666        if valid {
5667            code.to_owned()
5668        } else {
5669            "client.unknown_failure".to_owned()
5670        }
5671    }
5672
5673    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
5674        if self.sync_needed != value {
5675            if self.begin_observation("syncular_status").is_ok() {
5676                self.sync_needed = value;
5677                let batch = ChangeAccumulator {
5678                    status: true,
5679                    ..ChangeAccumulator::default()
5680                };
5681                if self.finish_observation("syncular_status", batch).is_err() {
5682                    self.rollback_observation("syncular_status");
5683                }
5684            } else {
5685                self.sync_needed = value;
5686            }
5687        }
5688        if value && interactive {
5689            self.sync_intent_queue.push_back(SyncIntent::Interactive);
5690        }
5691    }
5692
5693    fn begin_observation(&self, name: &str) -> Result<(), String> {
5694        self.conn
5695            .execute_batch(&format!("SAVEPOINT {name}"))
5696            .map_err(|error| error.to_string())
5697    }
5698
5699    fn rollback_observation(&self, name: &str) {
5700        let _ = self
5701            .conn
5702            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
5703    }
5704
5705    fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
5706        #[cfg(feature = "bench-internals")]
5707        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
5708        if !batch.touched() {
5709            self.conn
5710                .execute_batch(&format!("RELEASE {name}"))
5711                .map_err(|error| error.to_string())?;
5712            return Ok(());
5713        }
5714        let revision = self
5715            .local_revision()
5716            .checked_add(1)
5717            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
5718        self.conn
5719            .execute(
5720                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
5721                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
5722            )
5723            .map_err(|error| error.to_string())?;
5724        let status = batch.status.then(|| self.status_snapshot());
5725        let event = ClientChangeBatch {
5726            revision: revision.to_string(),
5727            tables: batch
5728                .tables
5729                .into_iter()
5730                .map(|(table, scope_keys)| TableChange {
5731                    table,
5732                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
5733                })
5734                .collect(),
5735            windows: batch
5736                .windows
5737                .into_iter()
5738                .map(|((base_key, table), units)| WindowChange {
5739                    base_key,
5740                    table,
5741                    units: units.into_iter().collect(),
5742                })
5743                .collect(),
5744            status,
5745            conflicts_changed: batch.conflicts,
5746            rejections_changed: batch.rejections,
5747            outcomes_changed: batch.outcomes,
5748        };
5749        self.conn
5750            .execute_batch(&format!("RELEASE {name}"))
5751            .map_err(|error| error.to_string())?;
5752        let mut diagnostic_tables = event
5753            .tables
5754            .iter()
5755            .map(|entry| entry.table.clone())
5756            .collect::<BTreeSet<_>>()
5757            .into_iter()
5758            .collect::<Vec<_>>();
5759        let mut diagnostic_windows = event
5760            .windows
5761            .iter()
5762            .map(|entry| entry.table.clone())
5763            .collect::<BTreeSet<_>>()
5764            .into_iter()
5765            .collect::<Vec<_>>();
5766        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
5767            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
5768        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
5769        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
5770        self.last_change = Some(DiagnosticLastChange {
5771            revision: event.revision.clone(),
5772            recorded_at_ms: self.clock_now_ms(),
5773            tables: diagnostic_tables,
5774            windows: diagnostic_windows,
5775            domains_truncated,
5776            status_changed: event.status.is_some(),
5777            conflicts_changed: event.conflicts_changed,
5778            rejections_changed: event.rejections_changed,
5779            outcomes_changed: event.outcomes_changed,
5780        });
5781        self.change_queue.push_back(event);
5782        Ok(())
5783    }
5784
5785    fn record_scope_map(
5786        &self,
5787        batch: &mut ChangeAccumulator,
5788        table_name: &str,
5789        scopes: &[(String, Vec<String>)],
5790    ) {
5791        let Some(table) = self.schema.table(table_name) else {
5792            return;
5793        };
5794        for (variable, values) in scopes {
5795            let Some(scope) = table
5796                .scope_variables
5797                .iter()
5798                .find(|scope| &scope.variable == variable)
5799            else {
5800                continue;
5801            };
5802            for value in values {
5803                batch.scope(table_name, format!("{}:{value}", scope.prefix));
5804            }
5805        }
5806    }
5807
5808    /// Record a row's current scope keys from the base or visible table.
5809    fn record_row_scopes(
5810        &self,
5811        batch: &mut ChangeAccumulator,
5812        table_name: &str,
5813        row_id: &str,
5814        base: bool,
5815    ) -> bool {
5816        let Some(table) = self.schema.table(table_name) else {
5817            return false;
5818        };
5819        if table.scope_variables.is_empty() {
5820            return false;
5821        }
5822        let columns = table
5823            .scope_variables
5824            .iter()
5825            .map(|scope| quote_ident(&scope.column))
5826            .collect::<Vec<_>>()
5827            .join(", ");
5828        let full_table = if base {
5829            base_table(table_name)
5830        } else {
5831            visible_table(table_name)
5832        };
5833        let sql = format!(
5834            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
5835            row_id_predicate(table)
5836        );
5837        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
5838            return false;
5839        };
5840        let values = stmt.query_row(rusqlite::params![row_id], |row| {
5841            let mut values = Vec::with_capacity(table.scope_variables.len());
5842            for index in 0..table.scope_variables.len() {
5843                values.push(row.get::<_, Option<String>>(index)?);
5844            }
5845            Ok(values)
5846        });
5847        let Ok(values) = values else {
5848            return false;
5849        };
5850        let mut recorded = false;
5851        for (scope, value) in table.scope_variables.iter().zip(values) {
5852            if let Some(value) = value {
5853                batch.scope(table_name, format!("{}:{value}", scope.prefix));
5854                recorded = true;
5855            }
5856        }
5857        recorded
5858    }
5859
5860    fn record_commit_changes(
5861        &self,
5862        batch: &mut ChangeAccumulator,
5863        tables: &[String],
5864        changes: &[ssp2::model::Change],
5865    ) {
5866        #[cfg(feature = "bench-internals")]
5867        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
5868        for change in changes {
5869            let Some(table_name) = tables.get(change.table_index as usize) else {
5870                continue;
5871            };
5872            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
5873            if let Some(table) = self.schema.table(table_name) {
5874                for (variable, value) in &change.scopes {
5875                    if let Some(scope) = table
5876                        .scope_variables
5877                        .iter()
5878                        .find(|scope| &scope.variable == variable)
5879                    {
5880                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
5881                        precise = true;
5882                    }
5883                }
5884            }
5885            if !precise {
5886                batch.table(table_name);
5887            }
5888        }
5889    }
5890
5891    fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
5892        if effective.is_empty() {
5893            return false;
5894        }
5895        let Some(table) = self.schema.table(table_name) else {
5896            return false;
5897        };
5898        let mut clauses = Vec::new();
5899        let mut params = Vec::new();
5900        for (variable, values) in effective {
5901            let Some(column) = table.scope_column(variable) else {
5902                return false;
5903            };
5904            if values.is_empty() {
5905                return false;
5906            }
5907            let placeholders = values
5908                .iter()
5909                .map(|value| {
5910                    params.push(SqlValue::Text(value.clone()));
5911                    "?"
5912                })
5913                .collect::<Vec<_>>()
5914                .join(", ");
5915            clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
5916        }
5917        let sql = format!(
5918            "SELECT 1 FROM {} WHERE {} LIMIT 1",
5919            base_table(table_name),
5920            clauses.join(" AND ")
5921        );
5922        self.conn
5923            .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
5924            .is_ok()
5925    }
5926
5927    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
5928    pub fn upgrading(&self) -> bool {
5929        self.upgrading
5930    }
5931
5932    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
5933    /// keeping this client's local database (identity, outbox, tables). The
5934    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
5935    /// version changed. Mirrors the TS client's boot-time detection —
5936    /// the Rust core has no persistent restart, so recreation IS the boot.
5937    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
5938        let new_schema = parse_schema_json(schema_json)?;
5939        let marker: Option<i32> = self
5940            .get_meta(LOCAL_SCHEMA_VERSION_KEY)
5941            .and_then(|v| v.parse().ok());
5942        self.schema = new_schema;
5943        self.create_bookkeeping_tables()?;
5944        if marker != Some(self.schema.version) {
5945            self.run_schema_reset()?;
5946        }
5947        self.prune_unknown_subscriptions()?;
5948        // The conformance recreate is the in-memory equivalent of reopening a
5949        // durable client. Apply the same startup catch-up contract even when
5950        // the schema itself did not change.
5951        self.enqueue_startup_sync_if_needed()?;
5952        Ok(())
5953    }
5954
5955    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
5956        let startup_work = !self.stopped
5957            && (!self.outbox.is_empty()
5958                || !self.load_pending_evictions()?.is_empty()
5959                || self.subs.iter().any(|sub| sub.state == SubState::Active));
5960        if startup_work {
5961            self.sync_needed = true;
5962            self.sync_intent_queue.push_back(SyncIntent::Interactive);
5963        }
5964        Ok(())
5965    }
5966
5967    /// A native client persists schema-floor stops across process restarts.
5968    /// Once the running app already satisfies that floor, the persisted stop
5969    /// is only stale evidence from an older server round (for example, the
5970    /// server was restarted after catching up to an app that was ahead).
5971    /// Clear it and let the normal startup pull re-negotiate. If the server is
5972    /// still incompatible it will return the floor again in that first round.
5973    fn clear_satisfied_persisted_schema_floor(&mut self) {
5974        let satisfied = self
5975            .schema_floor
5976            .as_ref()
5977            .and_then(|floor| floor.required_schema_version)
5978            .is_some_and(|required| self.schema.version >= required);
5979        if !satisfied {
5980            return;
5981        }
5982        self.schema_floor = None;
5983        self.stopped = false;
5984        self.delete_meta(SCHEMA_FLOOR_KEY);
5985    }
5986
5987    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
5988    /// and leaseState. Drops/recreates every synced table from the new
5989    /// schema, resets subscription sync-state (keeping registrations), clears
5990    /// the schema-floor stop state, rewrites the marker, drops outbox commits
5991    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
5992    fn run_schema_reset(&mut self) -> Result<(), String> {
5993        self.begin_observation("syncular_schema_reset")?;
5994        let mut batch = ChangeAccumulator::default();
5995        let result = self.run_schema_reset_observed(&mut batch, true);
5996        if let Err(error) = result {
5997            self.rollback_observation("syncular_schema_reset");
5998            return Err(error);
5999        }
6000        if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
6001            self.rollback_observation("syncular_schema_reset");
6002            return Err(error);
6003        }
6004        Ok(())
6005    }
6006
6007    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
6008        let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
6009        self.begin_observation("syncular_log_epoch_reset")?;
6010        let mut batch = ChangeAccumulator::default();
6011        let result = self.run_schema_reset_observed(&mut batch, false);
6012        if let Err(error) = result {
6013            self.rollback_observation("syncular_log_epoch_reset");
6014            return Err(error);
6015        }
6016        self.set_meta(LOG_EPOCH_KEY, log_epoch);
6017        self.sync_needed = true;
6018        batch.status = true;
6019        if let Err(error) = self.finish_observation("syncular_log_epoch_reset", batch) {
6020            self.rollback_observation("syncular_log_epoch_reset");
6021            return Err(error);
6022        }
6023        self.sync_intent_queue.push_back(SyncIntent::Interactive);
6024        Ok(resets)
6025    }
6026
6027    fn run_schema_reset_observed(
6028        &mut self,
6029        batch: &mut ChangeAccumulator,
6030        drop_incompatible: bool,
6031    ) -> Result<(), String> {
6032        self.upgrading = true;
6033        batch.status = true;
6034        for table in &self.schema.tables {
6035            batch.table(&table.name);
6036        }
6037        for (base_key, unit, table) in self.load_registered_window_units() {
6038            batch.window(&base_key, &table, &unit);
6039        }
6040        // The per-table insert SQL is derived from the OLD column lists.
6041        self.insert_sql.borrow_mut().clear();
6042        self.overlay_dirty.set(true);
6043        // Drop every synced table (base + visible) that currently exists —
6044        // discovered from sqlite_master so a bump that adds/removes tables is
6045        // handled. Bookkeeping tables (`_syncular_outbox/_subscriptions/_meta`
6046        // and the blob cache) are preserved; base tables are `_syncular_base_*`
6047        // so they are matched explicitly, not by the bookkeeping filter.
6048        // Drop virtual tables first so SQLite removes every FTS shadow table
6049        // atomically instead of the generic discovery tearing it apart.
6050        let virtual_tables: Vec<String> = {
6051            let mut stmt = self
6052                .conn
6053                .prepare(
6054                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
6055                )
6056                .map_err(|e| e.to_string())?;
6057            let rows = stmt
6058                .query_map([], |row| row.get::<_, String>(0))
6059                .map_err(|e| e.to_string())?;
6060            rows.filter_map(Result::ok)
6061                .filter(|name| is_synced_table_name(name))
6062                .collect()
6063        };
6064        for name in virtual_tables {
6065            self.conn
6066                .execute_batch(&format!(
6067                    "DROP TABLE IF EXISTS {}",
6068                    quote_ident(&format!("_syncular_fts_{name}"))
6069                ))
6070                .map_err(|e| e.to_string())?;
6071            self.conn
6072                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
6073                .map_err(|e| e.to_string())?;
6074        }
6075        let existing: Vec<String> = {
6076            let mut stmt = self
6077                .conn
6078                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
6079                .map_err(|e| e.to_string())?;
6080            let rows = stmt
6081                .query_map([], |row| row.get::<_, String>(0))
6082                .map_err(|e| e.to_string())?;
6083            rows.filter_map(Result::ok)
6084                .filter(|name| is_synced_table_name(name))
6085                .collect()
6086        };
6087        for name in &existing {
6088            if let Some(table) = name.strip_prefix("_syncular_base_") {
6089                batch.table(table);
6090            } else if !name.starts_with("_syncular_") {
6091                batch.table(name);
6092            }
6093        }
6094        for name in existing {
6095            self.conn
6096                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
6097                .map_err(|e| e.to_string())?;
6098        }
6099        // Recreate the synced tables from the NEW schema.
6100        self.create_synced_tables()?;
6101        // Reset every subscription's sync-state, keeping the registration.
6102        for sub in &mut self.subs {
6103            sub.cursor = -1;
6104            sub.bootstrap_state = None;
6105            sub.effective = None;
6106            sub.state = SubState::Active;
6107            sub.reason_code = None;
6108            sub.synced_once = false;
6109        }
6110        let subs = self.subs.clone();
6111        for sub in &subs {
6112            self.persist_sub(sub)?;
6113        }
6114        // The stop state is over: this client now ships a servable schema.
6115        self.stopped = false;
6116        self.schema_floor = None;
6117        self.delete_meta(SCHEMA_FLOOR_KEY);
6118        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
6119        self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
6120        // §7.4.4: drop outbox commits that cannot re-encode under the new
6121        // schema (a referenced column/table the bump removed), surfacing each
6122        // as a `sync.outbox_incompatible` rejection.
6123        if drop_incompatible && self.drop_incompatible_outbox()? {
6124            batch.rejections = true;
6125            batch.status = true;
6126            batch.outcomes = true;
6127        }
6128        // Re-apply the surviving outbox optimistically over the empty tables.
6129        self.rebuild_overlay();
6130        Ok(())
6131    }
6132
6133    /// §7.4.4: a persisted upsert whose values reference a column the current
6134    /// schema lacks (or a removed table) cannot be encoded. Drop the commit
6135    /// and raise a client-local `sync.outbox_incompatible` rejection.
6136    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
6137        let schema = &self.schema;
6138        let incompatible = self
6139            .outbox
6140            .iter()
6141            .filter(|commit| {
6142                commit.ops.iter().any(|op| {
6143                    if !op.upsert {
6144                        return false;
6145                    }
6146                    match schema.table(&op.table) {
6147                        None => true,
6148                        Some(table) => op.values.as_ref().is_some_and(|values| {
6149                            values
6150                                .keys()
6151                                .any(|key| !table.columns.iter().any(|c| &c.name == key))
6152                        }),
6153                    }
6154                })
6155            })
6156            .cloned()
6157            .collect::<Vec<_>>();
6158        if incompatible.is_empty() {
6159            return Ok(false);
6160        }
6161        let mut rejections = Vec::new();
6162        for commit in &incompatible {
6163            let results = commit
6164                .ops
6165                .iter()
6166                .enumerate()
6167                .map(|(op_index, operation)| {
6168                    let rejection = RejectionRecord {
6169                        client_commit_id: commit.client_commit_id.clone(),
6170                        op_index: op_index as i32,
6171                        code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
6172                        message: "the persisted commit cannot encode under the current schema"
6173                            .to_owned(),
6174                        retryable: false,
6175                        details: None,
6176                        operation: Some(CommitOperation::from(operation)),
6177                    };
6178                    rejections.push(rejection.clone());
6179                    CommitOperationOutcome::Error { rejection }
6180                })
6181                .collect::<Vec<_>>();
6182            self.persist_commit_outcome(
6183                &commit.client_commit_id,
6184                CommitOutcomeStatus::Rejected,
6185                &results,
6186                Some(&commit.ops),
6187            )?;
6188            self.delete_outbox_persisted(&commit.client_commit_id)?;
6189        }
6190        self.prune_commit_outcomes()?;
6191        let incompatible_ids = incompatible
6192            .iter()
6193            .map(|commit| commit.client_commit_id.as_str())
6194            .collect::<BTreeSet<_>>();
6195        self.outbox
6196            .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
6197        self.rejections.extend(rejections);
6198        Ok(true)
6199    }
6200
6201    /// §7.4.3: (re)create the base + visible table pair for every synced
6202    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
6203    fn create_synced_tables(&self) -> Result<(), String> {
6204        for table in &self.schema.tables {
6205            // The base half + the visible half form the synced-table pair. An
6206            // index name is global in SQLite, so the base half's indexes are
6207            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
6208            for (full, index_prefix) in [
6209                (base_table(&table.name), "_syncular_base_"),
6210                (visible_table(&table.name), ""),
6211            ] {
6212                let mut cols: Vec<String> =
6213                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
6214                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
6215                let sql = format!(
6216                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
6217                    cols.join(", "),
6218                    quote_ident(&table.primary_key)
6219                );
6220                self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
6221                // Local secondary indexes (CREATE INDEX subset). Created on
6222                // both halves so mirror reads hit an index on either. Runs on
6223                // both the initial create and the §7.4.3 reset recreate path.
6224                for index in &table.indexes {
6225                    let unique = if index.unique { "UNIQUE " } else { "" };
6226                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
6227                    let cols_sql = index
6228                        .columns
6229                        .iter()
6230                        .map(|c| quote_ident(c))
6231                        .collect::<Vec<_>>()
6232                        .join(", ");
6233                    let index_sql = format!(
6234                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
6235                    );
6236                    self.conn
6237                        .execute(&index_sql, [])
6238                        .map_err(|e| e.to_string())?;
6239                }
6240            }
6241        }
6242        // FTS exists only on the visible half. It is a local search
6243        // projection, never a synced/base table or a wire-schema member.
6244        for table in &self.schema.tables {
6245            for index in &table.fts_indexes {
6246                self.create_fts_projection(table, index)?;
6247            }
6248        }
6249        Ok(())
6250    }
6251
6252    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
6253        let count: i64 = self
6254            .conn
6255            .query_row(
6256                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
6257                rusqlite::params![index.name],
6258                |row| row.get(0),
6259            )
6260            .map_err(|e| e.to_string())?;
6261        Ok(count > 0)
6262    }
6263
6264    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
6265        for suffix in ["ai", "ad", "au"] {
6266            self.conn
6267                .execute(
6268                    &format!(
6269                        "DROP TRIGGER IF EXISTS {}",
6270                        quote_ident(&format!("{}_{suffix}", index.name))
6271                    ),
6272                    [],
6273                )
6274                .map_err(|e| e.to_string())?;
6275        }
6276        Ok(())
6277    }
6278
6279    fn create_fts_triggers(
6280        &self,
6281        table: &TableSchema,
6282        index: &FtsIndexSchema,
6283    ) -> Result<(), String> {
6284        let fts = quote_ident(&index.name);
6285        let source = visible_table(&table.name);
6286        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
6287        let pk = quote_ident(&table.primary_key);
6288        let projection_columns = std::iter::once(source_id.clone())
6289            .chain(index.columns.iter().map(|column| quote_ident(column)))
6290            .collect::<Vec<_>>()
6291            .join(", ");
6292        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
6293            .chain(
6294                index
6295                    .columns
6296                    .iter()
6297                    .map(|column| format!("new.{}", quote_ident(column))),
6298            )
6299            .collect::<Vec<_>>()
6300            .join(", ");
6301        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
6302        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
6303        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
6304        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
6305        let bi = quote_ident(&format!("{}_bi", index.name));
6306        let ai = quote_ident(&format!("{}_ai", index.name));
6307        let ad = quote_ident(&format!("{}_ad", index.name));
6308        let au = quote_ident(&format!("{}_au", index.name));
6309        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
6310        let sql = format!(
6311            "DROP TRIGGER IF EXISTS {bi};
6312             DROP TRIGGER IF EXISTS {ai};
6313             DROP TRIGGER IF EXISTS {ad};
6314             DROP TRIGGER IF EXISTS {au};
6315             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
6316             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
6317             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
6318             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
6319        );
6320        self.conn.execute_batch(&sql).map_err(|e| e.to_string())
6321    }
6322
6323    fn rebuild_fts_projection(
6324        &self,
6325        table: &TableSchema,
6326        index: &FtsIndexSchema,
6327    ) -> Result<(), String> {
6328        let fts = quote_ident(&index.name);
6329        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
6330        let pk = quote_ident(&table.primary_key);
6331        let indexed_columns = index
6332            .columns
6333            .iter()
6334            .map(|column| quote_ident(column))
6335            .collect::<Vec<_>>();
6336        let projection_columns = std::iter::once(source_id.clone())
6337            .chain(indexed_columns.iter().cloned())
6338            .collect::<Vec<_>>()
6339            .join(", ");
6340        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
6341        let sql = format!(
6342            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
6343            columns = indexed_columns.join(", "),
6344            source = visible_table(&table.name),
6345        );
6346        self.conn.execute_batch(&sql).map_err(|e| e.to_string())
6347    }
6348
6349    fn create_fts_projection(
6350        &self,
6351        table: &TableSchema,
6352        index: &FtsIndexSchema,
6353    ) -> Result<(), String> {
6354        self.begin_observation("syncular_fts_schema")?;
6355        let result = (|| {
6356            let existed = self.fts_projection_exists(index)?;
6357            let tokenizer = index.tokenize.replace('\'', "''");
6358            let columns = index
6359                .columns
6360                .iter()
6361                .map(|column| quote_ident(column))
6362                .collect::<Vec<_>>()
6363                .join(", ");
6364            let sql = format!(
6365            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
6366            fts = quote_ident(&index.name),
6367            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
6368        );
6369            self.conn.execute(&sql, []).map_err(|error| {
6370                format!(
6371                    "cannot create local FTS5 projection {:?}: {error}",
6372                    index.name
6373                )
6374            })?;
6375            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
6376            let mapped: bool = self
6377                .conn
6378                .query_row(
6379                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
6380                    [format!("_syncular_fts_{}", index.name)],
6381                    |row| row.get(0),
6382                )
6383                .map_err(|e| e.to_string())?;
6384            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|e| e.to_string())?;
6385            if !mapped && existed {
6386                self.conn
6387                    .execute_batch(&format!(
6388                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
6389                        quote_ident(FTS_SOURCE_ID_COLUMN),
6390                        quote_ident(&index.name)
6391                    ))
6392                    .map_err(|e| e.to_string())?;
6393            }
6394            self.create_fts_triggers(table, index)?;
6395            if !existed {
6396                self.rebuild_fts_projection(table, index)?;
6397            }
6398            Ok(())
6399        })();
6400        if let Err(error) = result {
6401            self.rollback_observation("syncular_fts_schema");
6402            return Err(error);
6403        }
6404        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
6405            self.rollback_observation("syncular_fts_schema");
6406            return Err(error.to_string());
6407        }
6408        Ok(())
6409    }
6410
6411    // -- persistence write-through --------------------------------------------
6412
6413    fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
6414        let state = serde_json::json!({
6415            "requested": scope_map_to_json(&sub.requested),
6416            "params": sub.params,
6417            "cursor": sub.cursor,
6418            "bootstrapState": sub.bootstrap_state,
6419            "status": sub.state.name(),
6420            "reasonCode": sub.reason_code,
6421            "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
6422            "syncedOnce": sub.synced_once,
6423        });
6424        self.conn.execute(
6425            "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
6426            rusqlite::params![sub.id, sub.table, state.to_string()],
6427        ).map(|_| ()).map_err(|error| error.to_string())
6428    }
6429
6430    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
6431        let ops: Vec<Value> = commit
6432            .ops
6433            .iter()
6434            .map(|op| {
6435                serde_json::json!({
6436                    "op": if op.upsert { "upsert" } else { "delete" },
6437                    "table": op.table,
6438                    "rowId": op.row_id,
6439                    "baseVersion": op.base_version,
6440                    "values": op.values.clone().map(Value::Object),
6441                })
6442            })
6443            .collect();
6444        self.conn
6445            .execute(
6446                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
6447                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
6448            )
6449            .map_err(|error| error.to_string())?;
6450        for operation in &commit.ops {
6451            let Some(values) = &operation.values else {
6452                continue;
6453            };
6454            let Some(table) = self.schema.table(&operation.table) else {
6455                continue;
6456            };
6457            for column in table
6458                .columns
6459                .iter()
6460                .filter(|column| column.ty == ColumnType::BlobRef)
6461            {
6462                let Some(Value::String(raw)) = values.get(&column.name) else {
6463                    continue;
6464                };
6465                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
6466                    continue;
6467                };
6468                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
6469                    continue;
6470                };
6471                self.conn
6472                    .execute(
6473                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
6474                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
6475                        rusqlite::params![commit.client_commit_id, blob_id],
6476                    )
6477                    .map_err(|error| error.to_string())?;
6478            }
6479        }
6480        Ok(())
6481    }
6482
6483    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
6484        self.conn
6485            .execute(
6486                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
6487                rusqlite::params![client_commit_id],
6488            )
6489            .map_err(|error| error.to_string())?;
6490        self.conn
6491            .execute(
6492                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
6493                rusqlite::params![client_commit_id],
6494            )
6495            .map(|_| ())
6496            .map_err(|error| error.to_string())
6497    }
6498
6499    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
6500        match status {
6501            CommitOutcomeStatus::Applied => "applied",
6502            CommitOutcomeStatus::Cached => "cached",
6503            CommitOutcomeStatus::Conflict => "conflict",
6504            CommitOutcomeStatus::Rejected => "rejected",
6505        }
6506    }
6507
6508    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
6509        match resolution {
6510            CommitOutcomeResolution::Active => "active",
6511            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
6512            CommitOutcomeResolution::Superseded => "superseded",
6513            CommitOutcomeResolution::Dismissed => "dismissed",
6514        }
6515    }
6516
6517    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
6518        match value {
6519            "applied" => Ok(CommitOutcomeStatus::Applied),
6520            "cached" => Ok(CommitOutcomeStatus::Cached),
6521            "conflict" => Ok(CommitOutcomeStatus::Conflict),
6522            "rejected" => Ok(CommitOutcomeStatus::Rejected),
6523            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
6524        }
6525    }
6526
6527    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
6528        match value {
6529            "active" => Ok(CommitOutcomeResolution::Active),
6530            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
6531            "superseded" => Ok(CommitOutcomeResolution::Superseded),
6532            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
6533            _ => Err(format!(
6534                "invalid persisted commit outcome resolution {value:?}"
6535            )),
6536        }
6537    }
6538
6539    fn persist_commit_outcome(
6540        &self,
6541        client_commit_id: &str,
6542        status: CommitOutcomeStatus,
6543        results: &[CommitOperationOutcome],
6544        operations: Option<&[OutboxOp]>,
6545    ) -> Result<(), String> {
6546        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
6547        let operations_json = operations
6548            .map(|items| {
6549                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
6550            })
6551            .transpose()
6552            .map_err(|error| error.to_string())?;
6553        self.conn
6554            .execute(
6555                "INSERT INTO _syncular_commit_outcomes (
6556                   client_commit_id, status, recorded_at_ms, results_json,
6557                   operations_json, resolution
6558                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
6559                rusqlite::params![
6560                    client_commit_id,
6561                    Self::outcome_status_name(status),
6562                    self.clock_now_ms(),
6563                    results_json,
6564                    operations_json
6565                ],
6566            )
6567            .map(|_| ())
6568            .map_err(|error| error.to_string())
6569    }
6570
6571    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
6572        let StoredCommitOutcomeRow {
6573            sequence,
6574            client_commit_id,
6575            status,
6576            recorded_at_ms,
6577            results_json,
6578            operations_json,
6579            resolution,
6580            resolved_at_ms,
6581            replacement_client_commit_id,
6582        } = row;
6583        Ok(CommitOutcome {
6584            sequence,
6585            client_commit_id,
6586            status: Self::parse_outcome_status(&status)?,
6587            recorded_at_ms,
6588            results: serde_json::from_str(&results_json)
6589                .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
6590            operations: operations_json
6591                .map(|value| {
6592                    serde_json::from_str(&value).map_err(|error| {
6593                        format!("invalid persisted commit outcome operations: {error}")
6594                    })
6595                })
6596                .transpose()?,
6597            resolution: Self::parse_outcome_resolution(&resolution)?,
6598            resolved_at_ms,
6599            replacement_client_commit_id,
6600        })
6601    }
6602
6603    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
6604        let row = self
6605            .conn
6606            .query_row(
6607                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
6608                        resolution, resolved_at_ms, replacement_client_commit_id
6609                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
6610                rusqlite::params![client_commit_id],
6611                |row| {
6612                    Ok(StoredCommitOutcomeRow {
6613                        sequence: row.get(0)?,
6614                        client_commit_id: row.get(1)?,
6615                        status: row.get(2)?,
6616                        recorded_at_ms: row.get(3)?,
6617                        results_json: row.get(4)?,
6618                        operations_json: row.get(5)?,
6619                        resolution: row.get(6)?,
6620                        resolved_at_ms: row.get(7)?,
6621                        replacement_client_commit_id: row.get(8)?,
6622                    })
6623                },
6624            )
6625            .optional()
6626            .map_err(|error| error.to_string())?;
6627        row.map(Self::outcome_from_row).transpose()
6628    }
6629
6630    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
6631        if query.limit == Some(0) {
6632            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
6633        }
6634        let mut sql = String::from(
6635            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
6636                    resolution, resolved_at_ms, replacement_client_commit_id
6637               FROM _syncular_commit_outcomes",
6638        );
6639        if query.active_only {
6640            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
6641        }
6642        sql.push_str(" ORDER BY seq DESC");
6643        if let Some(limit) = query.limit {
6644            sql.push_str(&format!(" LIMIT {limit}"));
6645        }
6646        let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
6647        let rows = stmt
6648            .query_map([], |row| {
6649                Ok(StoredCommitOutcomeRow {
6650                    sequence: row.get(0)?,
6651                    client_commit_id: row.get(1)?,
6652                    status: row.get(2)?,
6653                    recorded_at_ms: row.get(3)?,
6654                    results_json: row.get(4)?,
6655                    operations_json: row.get(5)?,
6656                    resolution: row.get(6)?,
6657                    resolved_at_ms: row.get(7)?,
6658                    replacement_client_commit_id: row.get(8)?,
6659                })
6660            })
6661            .map_err(|error| error.to_string())?;
6662        let mut outcomes = Vec::new();
6663        for row in rows {
6664            outcomes.push(Self::outcome_from_row(
6665                row.map_err(|error| error.to_string())?,
6666            )?);
6667        }
6668        Ok(outcomes)
6669    }
6670
6671    fn prune_commit_outcomes(&self) -> Result<(), String> {
6672        #[cfg(test)]
6673        self.outcome_prune_count
6674            .set(self.outcome_prune_count.get() + 1);
6675        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
6676        let count = self
6677            .conn
6678            .query_row(
6679                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
6680                [],
6681                |row| row.get::<_, i64>(0),
6682            )
6683            .map_err(|error| error.to_string())? as usize;
6684        let excess = count.saturating_sub(max_entries);
6685        if excess == 0 {
6686            return Ok(());
6687        }
6688        let mut stmt = self
6689            .conn
6690            .prepare(
6691                "SELECT seq FROM _syncular_commit_outcomes
6692                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
6693                  ORDER BY seq ASC LIMIT ?1",
6694            )
6695            .map_err(|error| error.to_string())?;
6696        let rows = stmt
6697            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
6698            .map_err(|error| error.to_string())?;
6699        let sequences = rows
6700            .collect::<Result<Vec<_>, _>>()
6701            .map_err(|error| error.to_string())?;
6702        drop(stmt);
6703        for sequence in sequences {
6704            self.conn
6705                .execute(
6706                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
6707                    rusqlite::params![sequence],
6708                )
6709                .map_err(|error| error.to_string())?;
6710        }
6711        Ok(())
6712    }
6713
6714    // -- driver surface ---------------------------------------------------------
6715
6716    pub fn subscribe(
6717        &mut self,
6718        id: String,
6719        table: String,
6720        scopes: Vec<(String, Vec<String>)>,
6721        params: Option<String>,
6722    ) -> Result<(), String> {
6723        if self.schema.table(&table).is_none() {
6724            return Err(format!("unknown table {table:?}"));
6725        }
6726        if let Some(existing) = self.subs.iter().find(|subscription| subscription.id == id) {
6727            let same_intent = existing.table == table
6728                && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
6729                && existing.params == params;
6730            if same_intent {
6731                return Ok(());
6732            }
6733            return Err(
6734                "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
6735                    .to_owned(),
6736            );
6737        }
6738        let sub = Subscription {
6739            id: id.clone(),
6740            table,
6741            requested: scopes,
6742            params,
6743            cursor: -1,
6744            bootstrap_state: None,
6745            state: SubState::Active,
6746            reason_code: None,
6747            effective: None,
6748            synced_once: false,
6749        };
6750        self.persist_sub(&sub)?;
6751        self.subs.push(sub);
6752        Ok(())
6753    }
6754
6755    pub fn unsubscribe(&mut self, id: &str) {
6756        self.subs.retain(|s| s.id != id);
6757        let _ = self.conn.execute(
6758            "DELETE FROM _syncular_subscriptions WHERE id = ?1",
6759            rusqlite::params![id],
6760        );
6761    }
6762
6763    // -- windowed subscriptions (§4.8) ------------------------------------------
6764
6765    /// §4.8: set the live window units for a base — a value-sharded family
6766    /// of subscriptions, one per unit. Added units get fresh subscriptions
6767    /// (image-lane bootstrap on the next sync); removed units are
6768    /// unsubscribed with their first eviction chunk in one transaction (E1–E4).
6769    /// Idempotent; re-entry cancels any deferred eviction.
6770    pub fn set_window(
6771        &mut self,
6772        base: &WindowBase,
6773        units: &[String],
6774    ) -> Result<CommandEffects, String> {
6775        let table = self
6776            .schema
6777            .table(&base.table)
6778            .ok_or_else(|| format!("unknown table {:?}", base.table))?;
6779        if table.scope_column(&base.variable).is_none() {
6780            return Err(format!(
6781                "setWindow: table {:?} has no scope variable {:?} (§4.8)",
6782                base.table, base.variable
6783            ));
6784        }
6785        let base_key = window_base_key(base);
6786        let wanted: std::collections::HashSet<&String> = units.iter().collect();
6787        let live = self.load_window_units(&base_key);
6788        self.begin_observation("syncular_window")?;
6789        let mut batch = ChangeAccumulator::default();
6790        let mut changed = false;
6791
6792        let prior_subs = self.subs.clone();
6793        let widened = (|| {
6794            // Widen: units wanted but not live → fresh subscription + registry row.
6795            for unit in units {
6796                if live.iter().any(|(u, _)| u == unit) {
6797                    continue;
6798                }
6799                let sub_id = derive_sub_id(base, unit);
6800                self.delete_pending_evict(&sub_id)?;
6801                self.insert_window_unit(&base_key, unit, &sub_id)?;
6802                self.subscribe(
6803                    sub_id,
6804                    base.table.clone(),
6805                    unit_scopes(base, unit),
6806                    base.params.clone(),
6807                )?;
6808                batch.window(&base_key, &base.table, unit);
6809                changed = true;
6810            }
6811
6812            self.finish_observation("syncular_window", batch)
6813        })();
6814        if let Err(error) = widened {
6815            self.rollback_observation("syncular_window");
6816            self.subs = prior_subs;
6817            return Err(error);
6818        }
6819        for (unit, sub_id) in live {
6820            if wanted.contains(&unit) {
6821                continue;
6822            }
6823            self.rebuild_overlay_if_dirty();
6824            let prior_subs = self.subs.clone();
6825            self.begin_observation("syncular_window")?;
6826            let effective = self
6827                .subs
6828                .iter()
6829                .find(|sub| sub.id == sub_id)
6830                .and_then(|sub| sub.effective.clone())
6831                .unwrap_or_else(|| unit_scopes(base, &unit));
6832            let mut batch = ChangeAccumulator::default();
6833            self.record_scope_map(&mut batch, &base.table, &effective);
6834            batch.window(&base_key, &base.table, &unit);
6835            let result = self.evict_unit(&base_key, base, &unit, &sub_id);
6836            let remaining = match result {
6837                Ok(remaining) => remaining,
6838                Err(error) => {
6839                    self.rollback_observation("syncular_window");
6840                    self.subs = prior_subs;
6841                    return Err(error);
6842                }
6843            };
6844            if let Err(error) = self.finish_observation("syncular_window", batch) {
6845                self.rollback_observation("syncular_window");
6846                self.subs = prior_subs;
6847                return Err(error);
6848            }
6849            if remaining {
6850                self.drain_pending_evictions()?;
6851            }
6852            changed = true;
6853        }
6854        Ok(if changed {
6855            CommandEffects::interactive()
6856        } else {
6857            CommandEffects::none()
6858        })
6859    }
6860
6861    /// §4.8 completeness oracle (I3): the windowed-in units for a base plus
6862    /// the subset still bootstrap-pending. Registration alone is not
6863    /// completeness — a unit is pending until its subscription completes a
6864    /// bootstrap round (cursor advances past -1 with no resume token held).
6865    pub fn window_state(&self, base: &WindowBase) -> WindowState {
6866        let mut units = Vec::new();
6867        let mut pending = Vec::new();
6868        for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
6869            let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
6870                Some(sub) => {
6871                    sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
6872                }
6873                None => true,
6874            };
6875            if is_pending {
6876                pending.push(unit.clone());
6877            }
6878            units.push(unit);
6879        }
6880        WindowState { units, pending }
6881    }
6882
6883    fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
6884        let mut stmt = match self
6885            .conn
6886            .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
6887        {
6888            Ok(stmt) => stmt,
6889            Err(_) => return Vec::new(),
6890        };
6891        let rows = stmt.query_map(rusqlite::params![base_key], |row| {
6892            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
6893        });
6894        match rows {
6895            Ok(rows) => rows.filter_map(Result::ok).collect(),
6896            Err(_) => Vec::new(),
6897        }
6898    }
6899
6900    fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
6901        let mut stmt = match self.conn.prepare(
6902            "SELECT windows.base, windows.unit, subscriptions.tbl
6903               FROM _syncular_windows AS windows
6904               JOIN _syncular_subscriptions AS subscriptions
6905                 ON subscriptions.id = windows.sub_id
6906               ORDER BY windows.base, windows.unit",
6907        ) {
6908            Ok(stmt) => stmt,
6909            Err(_) => return Vec::new(),
6910        };
6911        let rows = stmt.query_map([], |row| {
6912            Ok((
6913                row.get::<_, String>(0)?,
6914                row.get::<_, String>(1)?,
6915                row.get::<_, String>(2)?,
6916            ))
6917        });
6918        match rows {
6919            Ok(rows) => rows.filter_map(Result::ok).collect(),
6920            Err(_) => Vec::new(),
6921        }
6922    }
6923
6924    fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
6925        self.conn
6926            .query_row(
6927                "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
6928                rusqlite::params![sub_id],
6929                |row| Ok((row.get(0)?, row.get(1)?)),
6930            )
6931            .ok()
6932    }
6933
6934    fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
6935        self.conn
6936            .execute(
6937                "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
6938                rusqlite::params![base_key, unit, sub_id],
6939            )
6940            .map(|_| ())
6941            .map_err(|error| error.to_string())
6942    }
6943
6944    fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
6945        self.conn
6946            .execute(
6947                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
6948                rusqlite::params![base_key, unit],
6949            )
6950            .map(|_| ())
6951            .map_err(|error| error.to_string())
6952    }
6953
6954    /// §4.8 E1–E4: evict one departing unit, fused with unsubscription.
6955    /// Deletes the unit's rows except those pinned by a pending outbox
6956    /// commit (E1); records a deferred eviction if any pin remains; discards
6957    /// the subscription's cursor/resume/effective-echo (E3) and version
6958    /// state with the rows (E2). Fail-closed: no local mapping ⇒ evict
6959    /// nothing.
6960    fn evict_unit(
6961        &mut self,
6962        base_key: &str,
6963        base: &WindowBase,
6964        unit: &str,
6965        sub_id: &str,
6966    ) -> Result<bool, String> {
6967        let effective = self
6968            .subs
6969            .iter()
6970            .find(|s| s.id == sub_id)
6971            .and_then(|s| s.effective.clone())
6972            .unwrap_or_else(|| unit_scopes(base, unit));
6973        let pinned = self.pinned_row_ids(&base.table);
6974        let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
6975        self.delete_window_unit(base_key, unit)?;
6976        self.conn
6977            .execute(
6978                "DELETE FROM _syncular_subscriptions WHERE id = ?1",
6979                [sub_id],
6980            )
6981            .map_err(|error| error.to_string())?;
6982        self.subs.retain(|sub| sub.id != sub_id);
6983        if deferred {
6984            self.save_pending_evict(sub_id, &base.table, &effective)?;
6985        } else {
6986            self.delete_pending_evict(sub_id)?;
6987        }
6988        Ok(remaining)
6989    }
6990
6991    /// §4.8 E1: delete base rows matching effective scopes EXCEPT pinned
6992    /// primary keys. Returns (more unpinned work, any scoped rows remain).
6993    /// A missing scope-column mapping fails without deleting rows.
6994    fn evict_scope_rows(
6995        &self,
6996        table_name: &str,
6997        effective: &[(String, Vec<String>)],
6998        pinned: &std::collections::HashSet<String>,
6999    ) -> Result<(bool, bool), String> {
7000        if effective.is_empty() {
7001            return Ok((false, false));
7002        }
7003        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
7004        let mut clauses = Vec::new();
7005        let mut params: Vec<SqlValue> = Vec::new();
7006        for (variable, values) in effective {
7007            let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
7008            if values.is_empty() {
7009                return Ok((false, false));
7010            }
7011            let holes = values
7012                .iter()
7013                .map(|v| {
7014                    params.push(SqlValue::Text(v.clone()));
7015                    "?"
7016                })
7017                .collect::<Vec<_>>()
7018                .join(", ");
7019            clauses.push(format!("{} IN ({holes})", quote_ident(column)));
7020        }
7021        let scope = clauses.join(" AND ");
7022        let scope_params = params.len();
7023        let pk = quote_ident(&table.primary_key);
7024        if !pinned.is_empty() {
7025            let holes = pinned
7026                .iter()
7027                .map(|id| {
7028                    params.push(SqlValue::Text(id.clone()));
7029                    "?"
7030                })
7031                .collect::<Vec<_>>()
7032                .join(", ");
7033            clauses.push(format!("{pk} NOT IN ({holes})"));
7034        }
7035        let predicate = clauses.join(" AND ");
7036        let base = base_table(table_name);
7037        let visible = visible_table(table_name);
7038        let mut ids: Vec<SqlValue> = Vec::new();
7039        for source in [&base, &visible] {
7040            let mut stmt = self
7041                .conn
7042                .prepare_cached(&format!(
7043                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
7044                ))
7045                .map_err(|e| e.to_string())?;
7046            ids = stmt
7047                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
7048                .map_err(|e| e.to_string())?
7049                .collect::<Result<_, _>>()
7050                .map_err(|e| e.to_string())?;
7051            if !ids.is_empty() {
7052                break;
7053            }
7054        }
7055        if !ids.is_empty() {
7056            let holes = vec!["?"; ids.len()].join(", ");
7057            for source in [&base, &visible] {
7058                self.conn
7059                    .execute(
7060                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
7061                        rusqlite::params_from_iter(&ids),
7062                    )
7063                    .map_err(|e| e.to_string())?;
7064            }
7065        }
7066        let mut remaining = false;
7067        let mut deferred = false;
7068        for source in [&base, &visible] {
7069            remaining |= self
7070                .conn
7071                .query_row(
7072                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
7073                    rusqlite::params_from_iter(&params),
7074                    |row| row.get::<_, bool>(0),
7075                )
7076                .map_err(|e| e.to_string())?;
7077            deferred |= self
7078                .conn
7079                .query_row(
7080                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
7081                    rusqlite::params_from_iter(&params[..scope_params]),
7082                    |row| row.get::<_, bool>(0),
7083                )
7084                .map_err(|e| e.to_string())?;
7085        }
7086        Ok((remaining, deferred))
7087    }
7088
7089    /// Resume committed eviction chunks, retaining outbox pins and their durable marker.
7090    fn drain_pending_evictions(&mut self) -> Result<(), String> {
7091        let pending = self.load_pending_evictions()?;
7092        if pending.is_empty() {
7093            return Ok(());
7094        }
7095        self.rebuild_overlay_if_dirty();
7096        for (sub_id, table_name, effective) in pending {
7097            if self.schema.table(&table_name).is_none() {
7098                self.delete_pending_evict(&sub_id)?;
7099                continue;
7100            }
7101            loop {
7102                std::thread::yield_now();
7103                self.begin_observation("syncular_evict")?;
7104                let pinned = self.pinned_row_ids(&table_name);
7105                let result = self.evict_scope_rows(&table_name, &effective, &pinned);
7106                let (remaining, deferred) = match result {
7107                    Ok(result) => result,
7108                    Err(error) => {
7109                        self.rollback_observation("syncular_evict");
7110                        return Err(error);
7111                    }
7112                };
7113                if !deferred {
7114                    if let Err(error) = self.delete_pending_evict(&sub_id) {
7115                        self.rollback_observation("syncular_evict");
7116                        return Err(error);
7117                    }
7118                }
7119                let mut batch = ChangeAccumulator::default();
7120                self.record_scope_map(&mut batch, &table_name, &effective);
7121                if let Err(error) = self.finish_observation("syncular_evict", batch) {
7122                    self.rollback_observation("syncular_evict");
7123                    return Err(error);
7124                }
7125                if !remaining {
7126                    break;
7127                }
7128            }
7129        }
7130        Ok(())
7131    }
7132
7133    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
7134    /// commit — rows that MUST NOT be evicted until the commit drains.
7135    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
7136        let mut pinned = std::collections::HashSet::new();
7137        for commit in &self.outbox {
7138            for op in &commit.ops {
7139                if op.table == table {
7140                    pinned.insert(op.row_id.clone());
7141                }
7142            }
7143        }
7144        pinned
7145    }
7146
7147    fn save_pending_evict(
7148        &self,
7149        sub_id: &str,
7150        table: &str,
7151        effective: &[(String, Vec<String>)],
7152    ) -> Result<(), String> {
7153        self.conn.execute(
7154            "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
7155            rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
7156        ).map(|_| ()).map_err(|error| error.to_string())
7157    }
7158
7159    fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
7160        self.conn
7161            .execute(
7162                "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
7163                rusqlite::params![sub_id],
7164            )
7165            .map(|_| ())
7166            .map_err(|error| error.to_string())
7167    }
7168
7169    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
7170        let mut stmt = self
7171            .conn
7172            .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
7173            .map_err(|error| error.to_string())?;
7174        let rows = stmt
7175            .query_map([], |row| {
7176                Ok((
7177                    row.get::<_, String>(0)?,
7178                    row.get::<_, String>(1)?,
7179                    row.get::<_, String>(2)?,
7180                ))
7181            })
7182            .map_err(|error| error.to_string())?;
7183        rows.map(|row| {
7184            let (sub_id, table, json) = row.map_err(|error| error.to_string())?;
7185            let value: Value = serde_json::from_str(&json)
7186                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
7187            let effective = json_to_scope_map(&value)
7188                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
7189            Ok((sub_id, table, effective))
7190        })
7191        .collect()
7192    }
7193
7194    /// Record one atomic local commit (§7.1) and apply it optimistically.
7195    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
7196        if mutations.is_empty() {
7197            return Err("a commit must contain at least one operation (§6.1)".to_owned());
7198        }
7199        let mut ops = Vec::with_capacity(mutations.len());
7200        for mutation in mutations {
7201            match mutation {
7202                Mutation::Upsert {
7203                    table,
7204                    values,
7205                    base_version,
7206                } => {
7207                    let schema_table = self
7208                        .schema
7209                        .table(&table)
7210                        .ok_or_else(|| format!("unknown table {table:?}"))?;
7211                    // §5: value keys are accepted in snake_case AND the
7212                    // generated row types' camelCase; normalize to SQL truth
7213                    // before the pk lookup / codec see them.
7214                    let values = normalize_values_casing(schema_table, values)?;
7215                    let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
7216                    // §6.7 `mutate` marks every column present: fill missing
7217                    // nullable columns with NULL before the payload is built.
7218                    let values = full_row_values(schema_table, values)?;
7219                    // §5.11: validate the payload encodes with the current
7220                    // codec. A key-selection or unknown-key failure is a
7221                    // durable rejection at the push seam (§10.3), never an
7222                    // author-time error. Full-row `mutate` presents every
7223                    // column, so no stored fallback is needed.
7224                    self.validate_author_encode(schema_table, &row_id, &values)?;
7225                    ops.push(OutboxOp {
7226                        upsert: true,
7227                        table,
7228                        row_id,
7229                        base_version,
7230                        values: Some(values),
7231                    });
7232                }
7233                Mutation::Delete {
7234                    table,
7235                    row_id,
7236                    base_version,
7237                } => {
7238                    if self.schema.table(&table).is_none() {
7239                        return Err(format!("unknown table {table:?}"));
7240                    }
7241                    ops.push(OutboxOp {
7242                        upsert: false,
7243                        table,
7244                        row_id,
7245                        base_version,
7246                        values: None,
7247                    });
7248                }
7249            }
7250        }
7251        self.record_outbox_commit(ops)
7252    }
7253
7254    fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
7255        let commit = OutboxCommit {
7256            client_commit_id: uuid::Uuid::new_v4().to_string(),
7257            ops,
7258        };
7259        self.begin_observation("syncular_mutation")?;
7260        let mut batch = ChangeAccumulator::default();
7261        for op in &commit.ops {
7262            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
7263            if let Some(values) = &op.values {
7264                if let Some(table) = self.schema.table(&op.table) {
7265                    for scope in &table.scope_variables {
7266                        if let Some(Value::String(value)) = values.get(&scope.column) {
7267                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
7268                            precise = true;
7269                        }
7270                    }
7271                }
7272            }
7273            if !precise {
7274                batch.table(&op.table);
7275            }
7276        }
7277        if let Err(error) = self.persist_outbox_insert(&commit) {
7278            self.rollback_observation("syncular_mutation");
7279            return Err(error);
7280        }
7281        let was_dirty = self.overlay_dirty.get();
7282        // A clean overlay already contains the FIFO fold of every previous
7283        // commit. Appending a commit only needs its own operations applied.
7284        if !was_dirty {
7285            self.apply_outbox_ops(&commit.ops);
7286        }
7287        let id = commit.client_commit_id.clone();
7288        self.outbox.push(commit);
7289        if was_dirty {
7290            self.rebuild_overlay();
7291        }
7292        batch.status = true;
7293        if let Err(error) = self.finish_observation("syncular_mutation", batch) {
7294            self.rollback_observation("syncular_mutation");
7295            self.outbox.pop();
7296            self.overlay_dirty.set(was_dirty);
7297            return Err(error);
7298        }
7299        Ok(id)
7300    }
7301
7302    /// Record one §6.1 sparse upsert: the primary key plus the supplied
7303    /// non-scope columns are present; every other column is absent and stays
7304    /// untouched on the server and in the local overlay. The row need not be
7305    /// locally present — an absent local row leaves the overlay absent (§7.1)
7306    /// and the server answers per §5.2.
7307    pub fn patch(
7308        &mut self,
7309        table: &str,
7310        row_id: &str,
7311        partial: Map<String, Value>,
7312        base_version: Option<i64>,
7313    ) -> Result<String, String> {
7314        let schema_table = self
7315            .schema
7316            .table(table)
7317            .ok_or_else(|| format!("unknown table {table:?}"))?;
7318        let partial = normalize_values_casing(schema_table, partial)?;
7319        if let Some(pk) = partial.get(&schema_table.primary_key) {
7320            if pk.as_str() != Some(row_id) {
7321                return Err(format!(
7322                    "sync.invalid_request: table {table:?}: patch cannot change the primary key"
7323                ));
7324            }
7325        }
7326        // User's scope columns are server-emitted only (§3.4); a client patch
7327        // never carries one.
7328        for scope in &schema_table.scope_variables {
7329            if partial.contains_key(&scope.column) {
7330                return Err(format!(
7331                    "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
7332                    scope.column
7333                ));
7334            }
7335        }
7336        let mut values = partial;
7337        values.insert(
7338            schema_table.primary_key.clone(),
7339            Value::from(row_id.to_owned()),
7340        );
7341        // §5.11: structural validation only; an absent key-id selector is
7342        // resolved at the push seam from the stored local row, and a
7343        // key-selection or unknown-key failure becomes a durable
7344        // `client.encrypt_failed` rejection there (§10.3), never an
7345        // author-time error.
7346        self.validate_author_encode(schema_table, row_id, &values)?;
7347        self.record_outbox_commit(vec![OutboxOp {
7348            upsert: true,
7349            table: table.to_owned(),
7350            row_id: row_id.to_owned(),
7351            base_version,
7352            values: Some(values),
7353        }])
7354    }
7355
7356    /// §5.11 key-id fallback: the stored local row's selector value as a
7357    /// positional row (only the selector slot filled). Returns None when no
7358    /// selector is configured, the row is locally absent, or the stored
7359    /// value is not a usable key id.
7360    fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
7361        let selector = self.encryption.key_id_columns.get(&table.name)?;
7362        let index = table
7363            .columns
7364            .iter()
7365            .position(|column| &column.name == selector)?;
7366        let sql = format!(
7367            "SELECT {} FROM {} WHERE {}",
7368            quote_ident(selector),
7369            visible_table(&table.name),
7370            row_id_predicate(table)
7371        );
7372        let key: Option<String> = self
7373            .conn
7374            .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
7375            .ok()?;
7376        let key = key.filter(|key| !key.is_empty())?;
7377        let mut fallback: Row = vec![None; table.columns.len()];
7378        fallback[index] = Some(ColumnValue::String(key));
7379        Some(fallback)
7380    }
7381
7382    /// §5.11/§10.3 author-time encode check: a structural failure (a
7383    /// non-nullable NULL, a value the codec rejects) is an author-time
7384    /// error; a key-selection or unknown-key failure is deferred to the push
7385    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
7386    fn validate_author_encode(
7387        &self,
7388        table: &TableSchema,
7389        row_id: &str,
7390        values: &Map<String, Value>,
7391    ) -> Result<(), String> {
7392        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
7393            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
7394            result => result.map(|_| ()),
7395        }
7396    }
7397
7398    /// §5.11/§10.3: drop every pending commit whose push encode fails with
7399    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
7400    /// raise one durable rejection per commit. `sync()` never aborts for an
7401    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
7402    /// undoing the dropped commit's purely-optimistic rows.
7403    fn drop_unencodable_outbox(&mut self) -> Result<bool, String> {
7404        if !self
7405            .schema
7406            .tables
7407            .iter()
7408            .any(|table| table.has_encrypted_columns())
7409        {
7410            return Ok(false);
7411        }
7412        let failures = self
7413            .outbox
7414            .iter()
7415            .filter_map(|commit| {
7416                commit
7417                    .ops
7418                    .iter()
7419                    .find_map(|op| {
7420                        let values = op.values.as_ref().filter(|_| op.upsert)?;
7421                        let table = self.schema.table(&op.table)?;
7422                        let fallback = self.stored_key_fallback(table, &op.row_id);
7423                        match encode_sparse_row_json(
7424                            table,
7425                            &op.row_id,
7426                            values,
7427                            &self.encryption,
7428                            fallback.as_ref(),
7429                        ) {
7430                            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Some(error),
7431                            _ => None,
7432                        }
7433                    })
7434                    .map(|error| (commit.clone(), error))
7435            })
7436            .collect::<Vec<_>>();
7437        if failures.is_empty() {
7438            return Ok(false);
7439        }
7440        for (commit, message) in failures {
7441            let rejection = RejectionRecord {
7442                client_commit_id: commit.client_commit_id.clone(),
7443                op_index: 0,
7444                code: ENCRYPT_FAILED_CODE.to_owned(),
7445                message,
7446                retryable: false,
7447                details: None,
7448                operation: commit.ops.first().map(CommitOperation::from),
7449            };
7450            self.persist_commit_outcome(
7451                &commit.client_commit_id,
7452                CommitOutcomeStatus::Rejected,
7453                &[CommitOperationOutcome::Error {
7454                    rejection: rejection.clone(),
7455                }],
7456                Some(&commit.ops),
7457            )?;
7458            self.delete_outbox_persisted(&commit.client_commit_id)?;
7459            self.outbox
7460                .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
7461            self.rejections.push(rejection);
7462        }
7463        self.prune_commit_outcomes()?;
7464        self.overlay_dirty.set(true);
7465        self.rebuild_overlay();
7466        Ok(true)
7467    }
7468
7469    pub fn pending_commit_ids(&self) -> Vec<String> {
7470        self.outbox
7471            .iter()
7472            .map(|c| c.client_commit_id.clone())
7473            .collect()
7474    }
7475
7476    /// §6.1: the sparse push payload bytes of every pending upsert operation,
7477    /// FIFO — exactly what the next push round would put on the wire. Used by
7478    /// conformance B.21(h) to pin cross-core byte identity.
7479    pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
7480        let mut payloads = Vec::new();
7481        for commit in &self.outbox {
7482            for op in &commit.ops {
7483                let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
7484                    continue;
7485                };
7486                let Some(table) = self.schema.table(&op.table) else {
7487                    continue;
7488                };
7489                let fallback = self.stored_key_fallback(table, &op.row_id);
7490                if let Ok(payload) = encode_sparse_row_json(
7491                    table,
7492                    &op.row_id,
7493                    values,
7494                    &self.encryption,
7495                    fallback.as_ref(),
7496                ) {
7497                    payloads.push(payload);
7498                }
7499            }
7500        }
7501        payloads
7502    }
7503
7504    pub fn conflicts(&self) -> &[ConflictRecord] {
7505        &self.conflicts
7506    }
7507
7508    pub fn rejections(&self) -> &[RejectionRecord] {
7509        &self.rejections
7510    }
7511
7512    pub fn resolve_commit_outcome(
7513        &mut self,
7514        input: ResolveCommitOutcomeInput,
7515    ) -> Result<CommitOutcome, String> {
7516        let current = self
7517            .commit_outcome(&input.client_commit_id)?
7518            .ok_or_else(|| {
7519                format!(
7520                    "sync.outcome_not_found: no durable outcome exists for {:?}",
7521                    input.client_commit_id
7522                )
7523            })?;
7524        if current.resolution != CommitOutcomeResolution::Active {
7525            return Ok(current);
7526        }
7527        if input.resolution == CommitOutcomeResolution::Active {
7528            return Err("sync.invalid_request: resolution must leave active state".to_owned());
7529        }
7530        match input.resolution {
7531            CommitOutcomeResolution::Superseded => {
7532                let replacement = input
7533                    .replacement_client_commit_id
7534                    .as_deref()
7535                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
7536                    .ok_or_else(|| {
7537                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
7538                            .to_owned()
7539                    })?;
7540                let _ = replacement;
7541            }
7542            _ if input.replacement_client_commit_id.is_some() => {
7543                return Err(
7544                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
7545                        .to_owned(),
7546                );
7547            }
7548            _ => {}
7549        }
7550        let allowed = match current.status {
7551            CommitOutcomeStatus::Conflict => matches!(
7552                input.resolution,
7553                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
7554            ),
7555            CommitOutcomeStatus::Rejected => {
7556                input.resolution == CommitOutcomeResolution::Superseded
7557            }
7558            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
7559                input.resolution == CommitOutcomeResolution::Dismissed
7560            }
7561        };
7562        if !allowed {
7563            return Err(format!(
7564                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
7565                input.resolution, current.status
7566            ));
7567        }
7568
7569        self.begin_observation("syncular_outcome_resolution")?;
7570        let result = (|| {
7571            self.conn
7572                .execute(
7573                    "UPDATE _syncular_commit_outcomes
7574                        SET resolution = ?1, resolved_at_ms = ?2,
7575                            replacement_client_commit_id = ?3
7576                      WHERE client_commit_id = ?4 AND resolution = 'active'",
7577                    rusqlite::params![
7578                        Self::outcome_resolution_name(input.resolution),
7579                        self.clock_now_ms(),
7580                        input.replacement_client_commit_id,
7581                        input.client_commit_id
7582                    ],
7583                )
7584                .map_err(|error| error.to_string())?;
7585            let resolved = self
7586                .commit_outcome(&current.client_commit_id)?
7587                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
7588            self.prune_commit_outcomes()?;
7589            Ok(resolved)
7590        })();
7591        let resolved = match result {
7592            Ok(outcome) => outcome,
7593            Err(error) => {
7594                self.rollback_observation("syncular_outcome_resolution");
7595                return Err(error);
7596            }
7597        };
7598        self.conflicts
7599            .retain(|record| record.client_commit_id != current.client_commit_id);
7600        self.rejections
7601            .retain(|record| record.client_commit_id != current.client_commit_id);
7602        let batch = ChangeAccumulator {
7603            conflicts: current.status == CommitOutcomeStatus::Conflict,
7604            rejections: current.status == CommitOutcomeStatus::Rejected,
7605            outcomes: true,
7606            ..ChangeAccumulator::default()
7607        };
7608        if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
7609            self.rollback_observation("syncular_outcome_resolution");
7610            return Err(error);
7611        }
7612        Ok(resolved)
7613    }
7614
7615    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
7616        self.schema_floor.as_ref()
7617    }
7618
7619    /// §7.3.5: the client's opaque auth-lease state, if any.
7620    pub fn lease_state(&self) -> Option<&LeaseState> {
7621        self.lease_state.as_ref()
7622    }
7623
7624    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
7625    /// the two lease codes set it; other errors leave leaseState untouched.
7626    fn record_lease_error(&mut self, code: &str) {
7627        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
7628            return;
7629        }
7630        let mut next = self.lease_state.clone().unwrap_or_default();
7631        next.error_code = Some(code.to_owned());
7632        self.set_lease_state(Some(next));
7633    }
7634
7635    fn set_lease_state(&mut self, next: Option<LeaseState>) {
7636        if self.lease_state == next {
7637            return;
7638        }
7639        if self.begin_observation("syncular_lease").is_err() {
7640            return;
7641        }
7642        self.lease_state = next;
7643        if let Some(lease) = &self.lease_state {
7644            if let Ok(json) = serde_json::to_string(lease) {
7645                self.set_meta(LEASE_STATE_KEY, &json);
7646            }
7647        } else {
7648            self.delete_meta(LEASE_STATE_KEY);
7649        }
7650        let batch = ChangeAccumulator {
7651            status: true,
7652            ..ChangeAccumulator::default()
7653        };
7654        if self.finish_observation("syncular_lease", batch).is_err() {
7655            self.rollback_observation("syncular_lease");
7656        }
7657    }
7658
7659    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
7660        if self.schema_floor == next {
7661            return;
7662        }
7663        if self.begin_observation("syncular_schema_floor").is_err() {
7664            return;
7665        }
7666        self.schema_floor = next;
7667        self.stopped = self.schema_floor.is_some();
7668        if let Some(floor) = &self.schema_floor {
7669            if let Ok(json) = serde_json::to_string(floor) {
7670                self.set_meta(SCHEMA_FLOOR_KEY, &json);
7671            }
7672        } else {
7673            self.delete_meta(SCHEMA_FLOOR_KEY);
7674        }
7675        let batch = ChangeAccumulator {
7676            status: true,
7677            ..ChangeAccumulator::default()
7678        };
7679        if self
7680            .finish_observation("syncular_schema_floor", batch)
7681            .is_err()
7682        {
7683            self.rollback_observation("syncular_schema_floor");
7684        }
7685    }
7686
7687    fn set_upgrading(&mut self, value: bool) {
7688        if self.upgrading == value {
7689            return;
7690        }
7691        if self.begin_observation("syncular_upgrading").is_err() {
7692            return;
7693        }
7694        self.upgrading = value;
7695        let batch = ChangeAccumulator {
7696            status: true,
7697            ..ChangeAccumulator::default()
7698        };
7699        if self
7700            .finish_observation("syncular_upgrading", batch)
7701            .is_err()
7702        {
7703            self.rollback_observation("syncular_upgrading");
7704        }
7705    }
7706
7707    pub fn sync_needed(&self) -> bool {
7708        self.sync_needed
7709    }
7710
7711    pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
7712        let sub = self.subs.iter().find(|s| s.id == id)?;
7713        Some(SubscriptionStateView {
7714            id: sub.id.clone(),
7715            table: sub.table.clone(),
7716            status: sub.state.name().to_owned(),
7717            cursor: sub.cursor,
7718            has_resume_token: sub.bootstrap_state.is_some(),
7719            effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
7720            reason_code: sub.reason_code.clone(),
7721        })
7722    }
7723
7724    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
7725        let schema_table = self
7726            .schema
7727            .table(table)
7728            .ok_or_else(|| format!("unknown table {table:?}"))?;
7729        let sql = format!(
7730            "SELECT * FROM {} ORDER BY {} ASC",
7731            visible_table(table),
7732            quote_ident(&schema_table.primary_key)
7733        );
7734        let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
7735        let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
7736        let mut out = Vec::new();
7737        while let Some(row) = rows.next().map_err(|e| e.to_string())? {
7738            let mut values = Map::new();
7739            for (i, column) in schema_table.columns.iter().enumerate() {
7740                let value = row.get_ref(i).map_err(|e| e.to_string())?;
7741                values.insert(column.name.clone(), sql_ref_to_json(column, value));
7742            }
7743            let version: i64 = row
7744                .get(schema_table.columns.len())
7745                .map_err(|e| e.to_string())?;
7746            let row_id = match values.get(&schema_table.primary_key) {
7747                Some(Value::String(s)) => s.clone(),
7748                Some(Value::Number(n)) => n.to_string(),
7749                Some(Value::Bool(b)) => b.to_string(),
7750                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
7751            };
7752            out.push(RowState {
7753                row_id,
7754                version,
7755                values,
7756            });
7757        }
7758        Ok(out)
7759    }
7760
7761    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
7762    //
7763    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
7764    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
7765    // the op with `yrs`, re-encodes the whole doc state, and records a
7766    // baseVersion-less sparse upsert carrying only the primary key and the
7767    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
7768    // the overlay re-materializes the edit immediately (optimistic apply,
7769    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
7770    // Byte-compatible with `@syncular/crdt-yjs`.
7771
7772    /// The current stored value of a `crdt` column for one row — the visible
7773    /// (optimistic) bytes, or `None` when the row is absent or the column is
7774    /// NULL (the empty document, §5.10.1). Errors if the column is not a
7775    /// `crdt` column (guards the app against a typo'd column name).
7776    #[cfg(feature = "crdt-yjs")]
7777    fn crdt_column_bytes(
7778        &self,
7779        table: &str,
7780        row_id: &str,
7781        column: &str,
7782    ) -> Result<Option<Vec<u8>>, String> {
7783        let schema_table = self
7784            .schema
7785            .table(table)
7786            .ok_or_else(|| format!("unknown table {table:?}"))?;
7787        let col = schema_table
7788            .columns
7789            .iter()
7790            .find(|c| c.name == column)
7791            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
7792        if col.ty != ColumnType::Crdt {
7793            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
7794        }
7795        let sql = format!(
7796            "SELECT {} FROM {} WHERE {}",
7797            quote_ident(column),
7798            visible_table(table),
7799            row_id_predicate(schema_table)
7800        );
7801        let bytes: Option<Vec<u8>> = self
7802            .conn
7803            .prepare_cached(&sql)
7804            .map_err(|error| error.to_string())?
7805            .query_row(rusqlite::params![row_id], |row| {
7806                row.get::<_, Option<Vec<u8>>>(0)
7807            })
7808            .map_err(|e| match e {
7809                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
7810                other => other.to_string(),
7811            })?;
7812        Ok(bytes)
7813    }
7814
7815    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
7816    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
7817    /// An absent row / NULL column is the empty document (empty string).
7818    #[cfg(feature = "crdt-yjs")]
7819    pub fn crdt_text(
7820        &self,
7821        table: &str,
7822        row_id: &str,
7823        column: &str,
7824        name: &str,
7825    ) -> Result<String, String> {
7826        let bytes = self
7827            .crdt_column_bytes(table, row_id, column)?
7828            .unwrap_or_default();
7829        crate::crdt::text(&bytes, name)
7830    }
7831
7832    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
7833    /// record the resulting crdt-only sparse operation. Returns the enqueued
7834    /// `clientCommitId`.
7835    #[cfg(feature = "crdt-yjs")]
7836    pub fn crdt_insert_text(
7837        &mut self,
7838        table: &str,
7839        row_id: &str,
7840        column: &str,
7841        name: &str,
7842        index: u32,
7843        value: &str,
7844    ) -> Result<String, String> {
7845        let current = self
7846            .crdt_column_bytes(table, row_id, column)?
7847            .unwrap_or_default();
7848        let update = crate::crdt::insert_text(&current, name, index, value)?;
7849        self.crdt_push_update(table, row_id, column, &update)
7850    }
7851
7852    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
7853    /// record the resulting crdt-only sparse operation. Returns the enqueued
7854    /// `clientCommitId`.
7855    #[cfg(feature = "crdt-yjs")]
7856    pub fn crdt_delete_text(
7857        &mut self,
7858        table: &str,
7859        row_id: &str,
7860        column: &str,
7861        name: &str,
7862        index: u32,
7863        len: u32,
7864    ) -> Result<String, String> {
7865        let current = self
7866            .crdt_column_bytes(table, row_id, column)?
7867            .unwrap_or_default();
7868        let update = crate::crdt::delete_text(&current, name, index, len)?;
7869        self.crdt_push_update(table, row_id, column, &update)
7870    }
7871
7872    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
7873    /// `crdt` column's current state and record the resulting crdt-only sparse
7874    /// operation. The app authored the update with its own `yrs` model.
7875    /// Returns the enqueued `clientCommitId`.
7876    #[cfg(feature = "crdt-yjs")]
7877    pub fn crdt_apply_update(
7878        &mut self,
7879        table: &str,
7880        row_id: &str,
7881        column: &str,
7882        update: &[u8],
7883    ) -> Result<String, String> {
7884        let current = self
7885            .crdt_column_bytes(table, row_id, column)?
7886            .unwrap_or_default();
7887        let next = crate::crdt::apply_update(&current, update)?;
7888        self.crdt_push_update(table, row_id, column, &next)
7889    }
7890
7891    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7892    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7893    /// enqueue it. The server merges the crdt column and leaves every other
7894    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7895    /// clobber a concurrent edit to another column. A locally absent row
7896    /// records the same partial operation; the server answers per §6.2.
7897    #[cfg(feature = "crdt-yjs")]
7898    fn crdt_push_update(
7899        &mut self,
7900        table: &str,
7901        row_id: &str,
7902        column: &str,
7903        crdt_bytes: &[u8],
7904    ) -> Result<String, String> {
7905        let mut bytes_obj = Map::new();
7906        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7907        let mut partial = Map::new();
7908        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7909        self.patch(table, row_id, partial, None)
7910    }
7911
7912    /// Run an arbitrary read-only SQL query against the local database and
7913    /// return each row as a `column-name → JSON value` map. This is the seam
7914    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7915    /// SQL over the visible tables/views, not a fixed table read like
7916    /// [`read_rows`]).
7917    ///
7918    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7919    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7920    /// same envelope the command surface uses everywhere else. Output BLOB
7921    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7922    ///
7923    /// The result column typing is dynamic (SQLite's stored affinity), because
7924    /// arbitrary SQL can alias, join, and compute — there is no schema column
7925    /// to consult per output cell, unlike [`read_rows`].
7926    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7927        query_connection(&self.conn, sql, params)
7928    }
7929
7930    /// Repository benchmark and conformance access, present only with `bench-internals`.
7931    #[cfg(feature = "bench-internals")]
7932    #[doc(hidden)]
7933    pub fn benchmark_connection(&mut self) -> &mut Connection {
7934        &mut self.conn
7935    }
7936
7937    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7938    #[cfg(feature = "bench-internals")]
7939    #[doc(hidden)]
7940    pub fn benchmark_phases(
7941        &self,
7942        enabled: Option<bool>,
7943        reset: bool,
7944    ) -> Result<Option<Value>, String> {
7945        self.benchmark_phases.configure(enabled, reset)?;
7946        Ok(self.benchmark_phases.snapshot())
7947    }
7948
7949    /// Rows, coverage, and local revision from one SQLite read snapshot.
7950    pub fn query_snapshot(
7951        &mut self,
7952        sql: &str,
7953        params: &[QueryValue],
7954        coverage: &[WindowCoverage],
7955    ) -> Result<QuerySnapshot, String> {
7956        snapshot_connection(&self.conn, sql, params, coverage)
7957    }
7958
7959    // -- request building ---------------------------------------------------------
7960
7961    fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
7962        #[cfg(feature = "bench-internals")]
7963        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7964        let log_epoch = self.get_meta(LOG_EPOCH_KEY);
7965        let mut frames = vec![Frame::ReqHeader {
7966            client_id: self.client_id.clone(),
7967            schema_version: self.schema.version,
7968            log_epoch: log_epoch.clone(),
7969        }];
7970        let mut pushed_ids = Vec::new();
7971        let mut ops_in_request = 0usize;
7972        let mut deferred_commits = 0usize;
7973        #[cfg(feature = "bench-internals")]
7974        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7975        for (index, commit) in self
7976            .outbox
7977            .iter()
7978            .take(if log_epoch.is_some() {
7979                self.outbox.len()
7980            } else {
7981                0
7982            })
7983            .enumerate()
7984        {
7985            // §6.1 splitBatch: stop at the operation cap — commits apply in
7986            // order, so everything from the first non-fitting commit on is
7987            // deferred to the next round. A single over-cap commit still goes
7988            // alone (commits are atomic and cannot be split).
7989            if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
7990                deferred_commits = self.outbox.len() - index;
7991                break;
7992            }
7993            ops_in_request += commit.ops.len();
7994            let operations = commit
7995                .ops
7996                .iter()
7997                .map(|op| {
7998                    let payload = op.values.as_ref().and_then(|values| {
7999                        let table = self.schema.table(&op.table)?;
8000                        // §0: outbox entries encode at send time with the
8001                        // current codec (validated at mutate()). §6.1: the
8002                        // values map's keys are the sparse presence set.
8003                        // §5.11: encrypted columns are encrypted here, with
8004                        // an absent key-id selector read from the stored row.
8005                        // `sync_inner` runs `drop_unencodable_outbox` first,
8006                        // so an encrypt failure cannot reach this encode; the
8007                        // author seam validated every other failure.
8008                        let fallback = self.stored_key_fallback(table, &op.row_id);
8009                        encode_sparse_row_json(
8010                            table,
8011                            &op.row_id,
8012                            values,
8013                            &self.encryption,
8014                            fallback.as_ref(),
8015                        )
8016                        .ok()
8017                    });
8018                    ssp2::model::Operation {
8019                        table: op.table.clone(),
8020                        row_id: op.row_id.clone(),
8021                        op: if op.upsert { Op::Upsert } else { Op::Delete },
8022                        base_version: op.base_version,
8023                        payload,
8024                    }
8025                })
8026                .collect();
8027            frames.push(Frame::PushCommit {
8028                client_commit_id: commit.client_commit_id.clone(),
8029                operations,
8030            });
8031            pushed_ids.push(commit.client_commit_id.clone());
8032        }
8033        #[cfg(feature = "bench-internals")]
8034        drop(outbox_phase);
8035        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
8036        // bare URL — capability negotiation, decided per transport.
8037        let accept = self.limits.accept.unwrap_or(if url_capable {
8038            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
8039        } else {
8040            DEFAULT_ACCEPT
8041        });
8042        frames.push(Frame::PullHeader {
8043            limit_commits: self.limits.limit_commits.unwrap_or(0),
8044            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
8045            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
8046            accept,
8047        });
8048        let mut fresh = Vec::new();
8049        for sub in &self.subs {
8050            if sub.state != SubState::Active {
8051                continue;
8052            }
8053            let mut scopes = sub.requested.clone();
8054            sort_scope_map(&mut scopes);
8055            frames.push(Frame::Subscription {
8056                id: sub.id.clone(),
8057                table: sub.table.clone(),
8058                scopes,
8059                params: sub.params.clone().map(RawJson),
8060                cursor: sub.cursor,
8061                bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
8062            });
8063            fresh.push((
8064                sub.id.clone(),
8065                sub.cursor < 0 && sub.bootstrap_state.is_none(),
8066            ));
8067        }
8068        let message = Message {
8069            wire_version: WIRE_VERSION,
8070            msg_kind: MsgKind::Request,
8071            frames,
8072        };
8073        (
8074            message,
8075            RequestMeta {
8076                pushed_ids,
8077                fresh,
8078                accept,
8079                deferred_commits,
8080            },
8081        )
8082    }
8083
8084    // -- sync -------------------------------------------------------------------
8085
8086    /// Cloneable observation handle; no client lock is needed to read it.
8087    pub fn progress(&self) -> ProgressObserver {
8088        self.progress.clone()
8089    }
8090
8091    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
8092        self.progress.start();
8093        let started_at_ms = self.clock_now_ms();
8094        let outcome = self.sync_inner(transport);
8095        self.progress.update(|p| match &outcome {
8096            SyncOutcome::Ok(report) if report.failed.is_empty() => {
8097                p.state = ProgressState::Complete
8098            }
8099            SyncOutcome::Ok(_) => {
8100                p.state = ProgressState::Failed;
8101                p.error_code = Some("sync.scope_revoked".into());
8102            }
8103            SyncOutcome::Failed { error_code, .. } => {
8104                p.state = ProgressState::Failed;
8105                p.error_code = Some(Self::diagnostic_code(error_code));
8106            }
8107        });
8108        let completed_at_ms = self.clock_now_ms();
8109        self.last_round = Some(match &outcome {
8110            SyncOutcome::Ok(report) => DiagnosticLastRound {
8111                status: "succeeded".to_owned(),
8112                started_at_ms,
8113                completed_at_ms,
8114                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8115                counters: Some(DiagnosticRoundCounters {
8116                    pushed: report.pushed,
8117                    applied: report.applied.len(),
8118                    rejected: report.rejected.len(),
8119                    retryable: report.retryable.len(),
8120                    conflicts: report.conflicts,
8121                    commits_applied: report.commits_applied,
8122                    segment_rows_applied: report.segment_rows_applied,
8123                    bootstrapping: report.bootstrapping.len(),
8124                    resets: report.resets.len(),
8125                    revoked: report.revoked.len(),
8126                    failed: report.failed.len(),
8127                    deferred_commits: report.deferred_commits,
8128                }),
8129                error_code: None,
8130            },
8131            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
8132                status: "failed".to_owned(),
8133                started_at_ms,
8134                completed_at_ms,
8135                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8136                counters: None,
8137                error_code: Some(Self::diagnostic_code(error_code)),
8138            },
8139        });
8140        outcome
8141    }
8142
8143    fn sync_inner(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
8144        if self.stopped {
8145            // §1.6: the client stopped at the schema floor; syncing is inert
8146            // until an upgrade. The outbox is preserved for replay.
8147            return SyncOutcome::Ok(SyncReport {
8148                schema_floor: self.schema_floor.clone(),
8149                ..SyncReport::default()
8150            });
8151        }
8152        // §8.4: the coalesced sync-needed signal clears when a pull round
8153        // BEGINS, so a wake-up landing mid-round survives it.
8154        self.set_sync_needed(false, false);
8155        // §5.9.7 B4: upload pending blobs before pushing the referencing
8156        // rows, so the server-side existence check (§6.6) passes.
8157        if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
8158            if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
8159                if Self::retryable_transport_code(&code) {
8160                    self.schedule_background_retry();
8161                }
8162                return SyncOutcome::Failed {
8163                    error_code: code,
8164                    message,
8165                };
8166            }
8167        }
8168        if let Err(message) = self.drain_pending_evictions() {
8169            return SyncOutcome::Failed {
8170                error_code: "storage.failed".into(),
8171                message,
8172            };
8173        }
8174        // §5.11/§10.3: an unresolvable key id or an unknown key is a durable
8175        // per-commit rejection, never a reason to abort the round. Drop those
8176        // commits before the request is built.
8177        if let Err(message) = self.drop_unencodable_outbox() {
8178            return SyncOutcome::Failed {
8179                error_code: "storage.failed".into(),
8180                message,
8181            };
8182        }
8183        let (message, meta) = self.build_request(transport.supports_url_fetch());
8184        let request_bytes = {
8185            #[cfg(feature = "bench-internals")]
8186            let _phase = self.benchmark_phases.start(Phase::RequestEncode);
8187            encode_message(&message)
8188        };
8189        // §8.7: rounds ride the socket whenever it is connected (one
8190        // loop, no fallback pair); the transport seam stays bytes-in /
8191        // bytes-out either way. Registration-at-round-end is server-side.
8192        let round = if self.realtime_connected {
8193            transport.realtime_sync(&request_bytes)
8194        } else {
8195            transport.sync(&request_bytes)
8196        };
8197        let response_bytes = match round {
8198            Ok(bytes) => bytes,
8199            Err(TransportError { code, message }) => {
8200                // §7.3.5: a request-level lease code stops-and-surfaces —
8201                // record it in leaseState (no local-data purge, §7.3.4).
8202                self.record_lease_error(&code);
8203                if Self::retryable_transport_code(&code) {
8204                    self.schedule_background_retry();
8205                }
8206                return SyncOutcome::Failed {
8207                    error_code: code,
8208                    message,
8209                };
8210            }
8211        };
8212        #[cfg(feature = "bench-internals")]
8213        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
8214        let response = match decode_message(&response_bytes) {
8215            Ok(message) => message,
8216            Err(error) => {
8217                // §1.2 rule 1 / §1.4 rule 5: truncated or malformed
8218                // responses abort without persisting anything.
8219                return SyncOutcome::Failed {
8220                    error_code: error.code.as_str().to_owned(),
8221                    message: error.detail,
8222                };
8223            }
8224        };
8225        #[cfg(feature = "bench-internals")]
8226        drop(decode_phase);
8227        if response.msg_kind != MsgKind::Response {
8228            return SyncOutcome::Failed {
8229                error_code: "sync.invalid_request".to_owned(),
8230                message: "expected a response message".to_owned(),
8231            };
8232        }
8233        let mut outcome = self.process_response(transport, response, &meta);
8234        if let SyncOutcome::Ok(report) = &mut outcome {
8235            report.deferred_commits = meta.deferred_commits;
8236        }
8237        match &outcome {
8238            SyncOutcome::Ok(_) => self.reset_background_retry(),
8239            SyncOutcome::Failed { error_code, .. }
8240                if Self::retryable_transport_code(error_code) =>
8241            {
8242                self.schedule_background_retry();
8243            }
8244            SyncOutcome::Failed { .. } => {}
8245        }
8246        if meta.deferred_commits > 0 {
8247            // §6.1 splitBatch: commits past the operation cap wait for the
8248            // next round — keep the host's sync signal raised until then.
8249            self.set_sync_needed(true, true);
8250        }
8251        outcome
8252    }
8253
8254    pub fn sync_until_idle(
8255        &mut self,
8256        transport: &mut dyn Transport,
8257        max_rounds: Option<u32>,
8258    ) -> SyncOutcome {
8259        let rounds = max_rounds.unwrap_or(20).max(1);
8260        let mut aggregate = SyncReport::default();
8261        let mut spent = 0;
8262        while spent < rounds {
8263            spent += 1;
8264            let before = self
8265                .subs
8266                .iter()
8267                .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
8268                .collect::<Vec<_>>();
8269            match self.sync(transport) {
8270                SyncOutcome::Failed {
8271                    error_code,
8272                    message,
8273                } => {
8274                    return SyncOutcome::Failed {
8275                        error_code,
8276                        message,
8277                    };
8278                }
8279                SyncOutcome::Ok(report) => {
8280                    if max_rounds.is_none()
8281                        && report.segment_rows_applied > 0
8282                        && !report.bootstrapping.is_empty()
8283                        && before
8284                            != self
8285                                .subs
8286                                .iter()
8287                                .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
8288                                .collect::<Vec<_>>()
8289                    {
8290                        spent = 0;
8291                    }
8292                    aggregate.pushed += report.pushed;
8293                    aggregate.applied.extend(report.applied.iter().cloned());
8294                    aggregate.rejected.extend(report.rejected.iter().cloned());
8295                    aggregate.retryable.extend(report.retryable.iter().cloned());
8296                    aggregate.conflicts += report.conflicts;
8297                    aggregate.commits_applied += report.commits_applied;
8298                    aggregate.segment_rows_applied += report.segment_rows_applied;
8299                    aggregate.bootstrapping = report.bootstrapping.clone();
8300                    aggregate.resets.extend(report.resets.iter().cloned());
8301                    aggregate.revoked.extend(report.revoked.iter().cloned());
8302                    aggregate.failed.extend(report.failed.iter().cloned());
8303                    aggregate.deferred_commits = report.deferred_commits;
8304                    if report.schema_floor.is_some() {
8305                        aggregate.schema_floor = report.schema_floor.clone();
8306                    }
8307                    // §4.5: pull again whenever the response contained
8308                    // commits or segments; resets re-bootstrap; a pending
8309                    // resume token continues paging (§4.7); a raised
8310                    // sync-needed signal covers §6.1 splitBatch remainders
8311                    // (deferred outbox commits push on the next round).
8312                    let more = !report.bootstrapping.is_empty()
8313                        || report.commits_applied > 0
8314                        || report.segment_rows_applied > 0
8315                        || !report.resets.is_empty()
8316                        || self.sync_needed;
8317                    if !more {
8318                        return SyncOutcome::Ok(aggregate);
8319                    }
8320                }
8321            }
8322        }
8323        SyncOutcome::Failed {
8324            error_code: "sync.invalid_request".into(),
8325            message: "sync did not reach idle within the round budget".into(),
8326        }
8327    }
8328
8329    fn process_response(
8330        &mut self,
8331        transport: &mut dyn Transport,
8332        response: Message,
8333        meta: &RequestMeta,
8334    ) -> SyncOutcome {
8335        #[cfg(feature = "bench-internals")]
8336        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8337        let mut report = SyncReport {
8338            pushed: meta.pushed_ids.len() as u32,
8339            ..SyncReport::default()
8340        };
8341        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8342            HashMap::new();
8343        let pushed_ids = meta
8344            .pushed_ids
8345            .iter()
8346            .map(String::as_str)
8347            .collect::<HashSet<_>>();
8348        let mut last_final_push_result_id: Option<String> = None;
8349        for frame in &response.frames {
8350            match frame {
8351                Frame::PushResultDetails {
8352                    client_commit_id,
8353                    entries,
8354                } => {
8355                    let details = rejection_details_by_commit
8356                        .entry(client_commit_id.clone())
8357                        .or_default();
8358                    for entry in entries {
8359                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8360                            Ok(value) => value,
8361                            Err(message) => {
8362                                return SyncOutcome::Failed {
8363                                    error_code: "sync.invalid_request".to_owned(),
8364                                    message,
8365                                };
8366                            }
8367                        };
8368                        details.insert(entry.op_index, parsed);
8369                    }
8370                }
8371                Frame::PushResult {
8372                    client_commit_id,
8373                    status,
8374                    results,
8375                    ..
8376                } if pushed_ids.contains(client_commit_id.as_str())
8377                    && Self::push_result_is_final(*status, results) =>
8378                {
8379                    last_final_push_result_id = Some(client_commit_id.clone());
8380                }
8381                _ => {}
8382            }
8383        }
8384        let mut frames = response.frames.into_iter().peekable();
8385        if response.wire_version < 2 {
8386            return SyncOutcome::Failed {
8387                error_code: "client.invalid_host_response".to_owned(),
8388                message: "server response does not carry wire version 2 log-epoch state".to_owned(),
8389            };
8390        }
8391        match frames.next() {
8392            Some(Frame::RespHeader {
8393                required_schema_version,
8394                latest_schema_version,
8395                log_epoch,
8396                reset_required,
8397            }) => {
8398                let Some(log_epoch) = log_epoch else {
8399                    return SyncOutcome::Failed {
8400                        error_code: "client.invalid_host_response".to_owned(),
8401                        message: "response header omits logEpoch".to_owned(),
8402                    };
8403                };
8404                let Some(reset_required) = reset_required else {
8405                    return SyncOutcome::Failed {
8406                        error_code: "client.invalid_host_response".to_owned(),
8407                        message: "response header omits resetRequired".to_owned(),
8408                    };
8409                };
8410                if let Some(required) = required_schema_version {
8411                    // §1.6 schema-floor response: nothing else is processed;
8412                    // stop syncing and surface the upgrade requirement.
8413                    let floor = SchemaFloor {
8414                        required_schema_version: Some(required),
8415                        latest_schema_version,
8416                    };
8417                    self.set_schema_floor(Some(floor.clone()));
8418                    report.schema_floor = Some(floor);
8419                    return SyncOutcome::Ok(report);
8420                }
8421                if reset_required {
8422                    if frames.next().is_some() {
8423                        return SyncOutcome::Failed {
8424                            error_code: "client.invalid_host_response".to_owned(),
8425                            message: "log-epoch reset response contains body frames".to_owned(),
8426                        };
8427                    }
8428                    match self.run_log_epoch_reset(&log_epoch) {
8429                        Ok(resets) => {
8430                            report.resets = resets;
8431                            return SyncOutcome::Ok(report);
8432                        }
8433                        Err(message) => {
8434                            return SyncOutcome::Failed {
8435                                error_code: "sync.local_corrupt".to_owned(),
8436                                message,
8437                            };
8438                        }
8439                    }
8440                }
8441                if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
8442                    return SyncOutcome::Failed {
8443                        error_code: "client.invalid_host_response".to_owned(),
8444                        message: "server changed logEpoch without requiring a reset".to_owned(),
8445                    };
8446                }
8447            }
8448            _ => {
8449                return SyncOutcome::Failed {
8450                    error_code: "sync.invalid_request".to_owned(),
8451                    message: "response does not start with RESP_HEADER".to_owned(),
8452                };
8453            }
8454        }
8455
8456        let mut failure: Option<(String, String)> = None;
8457        while let Some(frame) = frames.next() {
8458            match frame {
8459                frame @ Frame::PushResult { .. } => {
8460                    let successful = matches!(
8461                        &frame,
8462                        Frame::PushResult {
8463                            status: PushStatus::Applied | PushStatus::Cached,
8464                            ..
8465                        }
8466                    );
8467                    let mut results = vec![frame];
8468                    if successful {
8469                        while matches!(
8470                            frames.peek(),
8471                            Some(Frame::PushResult {
8472                                status: PushStatus::Applied | PushStatus::Cached,
8473                                ..
8474                            })
8475                        ) {
8476                            if let Some(next) = frames.next() {
8477                                results.push(next);
8478                            }
8479                        }
8480                    }
8481                    if self
8482                        .handle_push_results(
8483                            &results,
8484                            &pushed_ids,
8485                            &rejection_details_by_commit,
8486                            &mut report,
8487                            last_final_push_result_id.as_deref(),
8488                        )
8489                        .is_err()
8490                    {
8491                        failure = Some((
8492                            "client.outcome_persistence_failed".to_owned(),
8493                            "local commit outcome could not be persisted".to_owned(),
8494                        ));
8495                        break;
8496                    }
8497                }
8498                Frame::PushResultDetails { .. } => {}
8499                Frame::SubStart {
8500                    id,
8501                    status,
8502                    reason_code,
8503                    effective_scopes,
8504                    bootstrap: _,
8505                } => {
8506                    let mut body = Vec::new();
8507                    let mut sub_end: Option<(i64, Option<String>)> = None;
8508                    for inner in frames.by_ref() {
8509                        match inner {
8510                            Frame::SubEnd {
8511                                next_cursor,
8512                                bootstrap_state,
8513                            } => {
8514                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8515                                break;
8516                            }
8517                            Frame::Unknown { .. } => {}
8518                            other => body.push(other),
8519                        }
8520                    }
8521                    if let Err(SectionError::Abort(code, message)) = self.process_section(
8522                        transport,
8523                        &id,
8524                        status,
8525                        &reason_code,
8526                        effective_scopes,
8527                        body,
8528                        sub_end,
8529                        meta,
8530                        &mut report,
8531                    ) {
8532                        failure = Some((code, message));
8533                        break;
8534                    }
8535                }
8536                Frame::Lease {
8537                    lease_id,
8538                    expires_at_ms,
8539                } => {
8540                    // §7.3.5: persist the opaque lease; a fresh lease clears
8541                    // any prior lease error (the outage/revocation is over).
8542                    self.set_lease_state(Some(LeaseState {
8543                        lease_id: Some(lease_id),
8544                        expires_at_ms: Some(expires_at_ms),
8545                        error_code: None,
8546                    }));
8547                }
8548                Frame::Error { code, message, .. } => {
8549                    // §1.6: the whole request failed; already-completed
8550                    // subscriptions keep their applied data and cursors.
8551                    failure = Some((code, message));
8552                    break;
8553                }
8554                Frame::Unknown { .. } => {}
8555                _ => {
8556                    failure = Some((
8557                        "sync.invalid_request".to_owned(),
8558                        "unexpected frame in response".to_owned(),
8559                    ));
8560                    break;
8561                }
8562            }
8563        }
8564
8565        // §7.1: reconcile the visible overlay once at the response boundary
8566        // after all outbox acknowledgements and server data applied so a
8567        // batch of PUSH_RESULT frames cannot trigger repeated full-table
8568        // rebuilds. This also covers a round that aborted mid-way.
8569        if self.overlay_dirty.get() {
8570            self.rebuild_overlay();
8571        }
8572        if let Some((error_code, message)) = failure {
8573            return SyncOutcome::Failed {
8574                error_code,
8575                message,
8576            };
8577        }
8578        // §4.8 E1: the push half may have drained commits that pinned rows of
8579        // a shrunk window unit — retry any deferred evictions now.
8580        if let Err(message) = self.drain_pending_evictions() {
8581            return SyncOutcome::Failed {
8582                error_code: "storage.failed".into(),
8583                message,
8584            };
8585        }
8586        self.ack_after_pull(transport);
8587        // §7.4.5: the reset is over once the first post-reset pull round
8588        // leaves no subscription mid-bootstrap — the tables are rebuilt.
8589        if self.upgrading && report.bootstrapping.is_empty() {
8590            self.set_upgrading(false);
8591        }
8592        SyncOutcome::Ok(report)
8593    }
8594
8595    // -- push results (§6.3, §7.2) ------------------------------------------------
8596
8597    fn handle_push_results(
8598        &mut self,
8599        frames: &[Frame],
8600        pushed_ids: &HashSet<&str>,
8601        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
8602        report: &mut SyncReport,
8603        last_final_push_result_id: Option<&str>,
8604    ) -> Result<(), String> {
8605        self.begin_observation("syncular_push_results")?;
8606        let conflict_count = self.conflicts.len();
8607        let rejection_count = self.rejections.len();
8608        let applied_count = report.applied.len();
8609        let rejected_count = report.rejected.len();
8610        let retryable_count = report.retryable.len();
8611        let was_dirty = self.overlay_dirty.get();
8612        let mut removed = Vec::new();
8613        let mut batch = ChangeAccumulator::default();
8614        let persisted = (|| {
8615            for frame in frames {
8616                let Frame::PushResult {
8617                    client_commit_id,
8618                    status,
8619                    results,
8620                    ..
8621                } = frame
8622                else {
8623                    return Err("unexpected acknowledgement frame".to_owned());
8624                };
8625                if !pushed_ids.contains(client_commit_id.as_str()) {
8626                    continue;
8627                }
8628                let Some(index) = self
8629                    .outbox
8630                    .iter()
8631                    .position(|commit| commit.client_commit_id == *client_commit_id)
8632                else {
8633                    continue;
8634                };
8635                let status = *status;
8636                let rejection_details = rejection_details_by_commit.get(client_commit_id);
8637                let operations = self.outbox[index].ops.clone();
8638                match status {
8639                    PushStatus::Applied | PushStatus::Cached => {
8640                        // §7.2: a lost ack replays as `cached` — proceed as if the
8641                        // ack had arrived.
8642                        let journal_results = results
8643                            .iter()
8644                            .map(|result| {
8645                                let op_index = match result {
8646                                    OpResult::Applied { op_index }
8647                                    | OpResult::Conflict { op_index, .. }
8648                                    | OpResult::Error { op_index, .. } => *op_index,
8649                                };
8650                                CommitOperationOutcome::Applied { op_index }
8651                            })
8652                            .collect::<Vec<_>>();
8653                        let outcome_status = if status == PushStatus::Applied {
8654                            CommitOutcomeStatus::Applied
8655                        } else {
8656                            CommitOutcomeStatus::Cached
8657                        };
8658                        let persisted = self
8659                            .persist_commit_outcome(
8660                                client_commit_id,
8661                                outcome_status,
8662                                &journal_results,
8663                                None,
8664                            )
8665                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
8666                        persisted?;
8667                        self.overlay_dirty.set(true);
8668                        batch.status = true;
8669                        batch.outcomes = true;
8670                    }
8671                    PushStatus::Rejected => {
8672                        if results.iter().any(|result| {
8673                            matches!(
8674                                result,
8675                                OpResult::Error {
8676                                    code,
8677                                    retryable: true,
8678                                    ..
8679                                } if code == "sync.idempotency_cache_miss"
8680                            )
8681                        }) {
8682                            // §6.3/§7.2: a serving failure, not an outcome — keep the
8683                            // exact commit queued for an identical retry.
8684                            report.retryable.push(client_commit_id.to_owned());
8685                            continue;
8686                        }
8687
8688                        let mut journal_results = Vec::with_capacity(results.len());
8689                        let mut conflicts = Vec::new();
8690                        let mut rejections = Vec::new();
8691                        for result in results {
8692                            match result {
8693                                OpResult::Applied { op_index } => {
8694                                    journal_results.push(CommitOperationOutcome::Applied {
8695                                        op_index: *op_index,
8696                                    });
8697                                }
8698                                OpResult::Conflict {
8699                                    op_index,
8700                                    code,
8701                                    message,
8702                                    server_version,
8703                                    server_row,
8704                                    conflict_columns: conflict_columns_bitmap,
8705                                } => {
8706                                    let operation = operations
8707                                        .get(*op_index as usize)
8708                                        .map(CommitOperation::from);
8709                                    let (table, row_id) = operation
8710                                        .as_ref()
8711                                        .map(|op| (op.table.clone(), op.row_id.clone()))
8712                                        .unwrap_or_default();
8713                                    let server_row_json = self
8714                                        .schema
8715                                        .table(&table)
8716                                        .and_then(|t| {
8717                                            decode_row_bytes(t, server_row, &self.encryption)
8718                                                .ok()
8719                                                .map(|row| (t, row))
8720                                        })
8721                                        .map(|(t, row)| {
8722                                            let mut map = Map::new();
8723                                            for (i, column) in t.columns.iter().enumerate() {
8724                                                map.insert(
8725                                                    column.name.clone(),
8726                                                    column_value_to_json(
8727                                                        row.get(i).unwrap_or(&None),
8728                                                    ),
8729                                                );
8730                                            }
8731                                            map
8732                                        })
8733                                        .unwrap_or_default();
8734                                    // §6.3: decode the presence-layout bitmap
8735                                    // into the marked column names.
8736                                    let conflict_columns = self
8737                                        .schema
8738                                        .table(&table)
8739                                        .map(|t| {
8740                                            t.columns
8741                                                .iter()
8742                                                .enumerate()
8743                                                .filter(|(index, _)| {
8744                                                    conflict_columns_bitmap
8745                                                        .get(index / 8)
8746                                                        .is_some_and(|byte| {
8747                                                            byte & (1 << (index % 8)) != 0
8748                                                        })
8749                                                })
8750                                                .map(|(_, column)| column.name.clone())
8751                                                .collect::<Vec<_>>()
8752                                        })
8753                                        .unwrap_or_default();
8754                                    let conflict = ConflictRecord {
8755                                        client_commit_id: client_commit_id.to_owned(),
8756                                        op_index: *op_index,
8757                                        table,
8758                                        row_id,
8759                                        code: code.clone(),
8760                                        message: message.clone(),
8761                                        server_version: *server_version,
8762                                        server_row: server_row_json,
8763                                        conflict_columns,
8764                                        operation,
8765                                    };
8766                                    journal_results.push(CommitOperationOutcome::Conflict {
8767                                        conflict: conflict.clone(),
8768                                    });
8769                                    conflicts.push(conflict);
8770                                }
8771                                OpResult::Error {
8772                                    op_index,
8773                                    code,
8774                                    message,
8775                                    retryable,
8776                                } => {
8777                                    let rejection = RejectionRecord {
8778                                        client_commit_id: client_commit_id.to_owned(),
8779                                        op_index: *op_index,
8780                                        code: code.clone(),
8781                                        message: message.clone(),
8782                                        retryable: *retryable,
8783                                        details: rejection_details
8784                                            .and_then(|details| details.get(op_index))
8785                                            .cloned(),
8786                                        operation: operations
8787                                            .get(*op_index as usize)
8788                                            .map(CommitOperation::from),
8789                                    };
8790                                    journal_results.push(CommitOperationOutcome::Error {
8791                                        rejection: rejection.clone(),
8792                                    });
8793                                    rejections.push(rejection);
8794                                }
8795                            }
8796                        }
8797                        let outcome_status = if conflicts.is_empty() {
8798                            CommitOutcomeStatus::Rejected
8799                        } else {
8800                            CommitOutcomeStatus::Conflict
8801                        };
8802                        let persisted = self
8803                            .persist_commit_outcome(
8804                                client_commit_id,
8805                                outcome_status,
8806                                &journal_results,
8807                                Some(&operations),
8808                            )
8809                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
8810                        persisted?;
8811                        batch.conflicts = !conflicts.is_empty();
8812                        batch.rejections = !rejections.is_empty();
8813                        batch.status = true;
8814                        batch.outcomes = true;
8815                        self.conflicts.extend(conflicts);
8816                        self.rejections.extend(rejections);
8817                        self.overlay_dirty.set(true);
8818                    }
8819                }
8820                removed.push((index, self.outbox.remove(index)));
8821                if batch.status {
8822                    for operation in &operations {
8823                        if !self.record_row_scopes(
8824                            &mut batch,
8825                            &operation.table,
8826                            &operation.row_id,
8827                            false,
8828                        ) {
8829                            batch.table(&operation.table);
8830                        }
8831                    }
8832                }
8833                match status {
8834                    PushStatus::Applied | PushStatus::Cached => {
8835                        report.applied.push(client_commit_id.to_owned())
8836                    }
8837                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
8838                }
8839            }
8840            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
8841                self.prune_commit_outcomes()?;
8842            }
8843            self.finish_observation("syncular_push_results", batch)
8844        })();
8845        if let Err(error) = persisted {
8846            self.rollback_observation("syncular_push_results");
8847            for (index, commit) in removed.into_iter().rev() {
8848                self.outbox.insert(index, commit);
8849            }
8850            self.conflicts.truncate(conflict_count);
8851            self.rejections.truncate(rejection_count);
8852            self.overlay_dirty.set(was_dirty);
8853            report.applied.truncate(applied_count);
8854            report.rejected.truncate(rejected_count);
8855            report.retryable.truncate(retryable_count);
8856            return Err(error);
8857        }
8858        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
8859        Ok(())
8860    }
8861
8862    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
8863        status != PushStatus::Rejected
8864            || !results.iter().any(|result| {
8865                matches!(
8866                    result,
8867                    OpResult::Error {
8868                        code,
8869                        retryable: true,
8870                        ..
8871                    } if code == "sync.idempotency_cache_miss"
8872                )
8873            })
8874    }
8875
8876    // -- subscription sections ------------------------------------------------------
8877
8878    #[allow(clippy::too_many_arguments)]
8879    fn process_section(
8880        &mut self,
8881        transport: &mut dyn Transport,
8882        id: &str,
8883        status: SubStatus,
8884        reason_code: &str,
8885        effective_scopes: Vec<(String, Vec<String>)>,
8886        body: Vec<Frame>,
8887        sub_end: Option<(i64, Option<String>)>,
8888        meta: &RequestMeta,
8889        report: &mut SyncReport,
8890    ) -> Result<(), SectionError> {
8891        let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
8892            return Ok(()); // unknown echo: ignore
8893        };
8894        if status != SubStatus::Active && sub_end.is_none() {
8895            return Err(SectionError::Abort(
8896                "sync.invalid_request".into(),
8897                "subscription section without SUB_END".into(),
8898            ));
8899        }
8900        match status {
8901            SubStatus::Revoked => {
8902                self.begin_observation("syncular_revocation")
8903                    .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
8904                let previous = self.subs[sub_index].clone();
8905                let was_dirty = self.overlay_dirty.get();
8906                let previous_outbox = self.outbox.clone();
8907                let rejection_count = self.rejections.len();
8908                let revoked_count = report.revoked.len();
8909                let failed_count = report.failed.len();
8910                let result =
8911                    (|| {
8912                        let mut batch = ChangeAccumulator::default();
8913                        let registered = self.window_unit_by_sub_id(id);
8914                        // §3.3: stop pulling, purge exactly the last effective grant.
8915                        let (table, effective) = {
8916                            let sub = &self.subs[sub_index];
8917                            (sub.table.clone(), sub.effective.clone().unwrap_or_default())
8918                        };
8919                        let purged = self.purge_scope_rows(&table, &effective);
8920                        match purged {
8921                            Ok(()) => {
8922                                self.record_scope_map(&mut batch, &table, &effective);
8923                                let sub = &mut self.subs[sub_index];
8924                                sub.state = SubState::Revoked;
8925                                sub.reason_code = Some(if reason_code.is_empty() {
8926                                    "sync.scope_revoked".to_owned()
8927                                } else {
8928                                    reason_code.to_owned()
8929                                });
8930                                report.revoked.push(id.to_owned());
8931                                let doomed_effective = effective;
8932                                let sub_table = table;
8933                                self.persist_sub(&self.subs[sub_index].clone()).map_err(
8934                                    |message| SectionError::Abort("storage.failed".into(), message),
8935                                )?;
8936                                let dropped = self
8937                                    .drop_doomed_outbox(&sub_table, &doomed_effective)
8938                                    .map_err(|message| {
8939                                        SectionError::Abort("storage.failed".to_owned(), message)
8940                                    })?;
8941                                if dropped {
8942                                    batch.status = true;
8943                                    batch.rejections = true;
8944                                    batch.outcomes = true;
8945                                }
8946                                // §5.9.7 B2: revocation deletes now-unauthorized blob
8947                                // bodies (evicted ≠ revoked).
8948                                self.delete_unreferenced_cached_blobs();
8949                            }
8950                            Err(()) => {
8951                                // §3.3 fail closed: no local mapping — never clear by
8952                                // approximation; fatal configuration error.
8953                                let sub = &mut self.subs[sub_index];
8954                                sub.state = SubState::Failed;
8955                                sub.reason_code = Some("sync.scope_revoked".to_owned());
8956                                report.failed.push(id.to_owned());
8957                                self.persist_sub(&self.subs[sub_index].clone()).map_err(
8958                                    |message| SectionError::Abort("storage.failed".into(), message),
8959                                )?;
8960                            }
8961                        }
8962                        if let Some((base_key, unit)) = registered {
8963                            batch.window(&base_key, &self.subs[sub_index].table, &unit);
8964                        }
8965                        self.rebuild_overlay_if_dirty();
8966                        self.finish_observation("syncular_revocation", batch)
8967                            .map_err(|message| {
8968                                SectionError::Abort("storage.failed".to_owned(), message)
8969                            })?;
8970                        Ok(())
8971                    })();
8972                if result.is_err() {
8973                    self.rollback_observation("syncular_revocation");
8974                    self.subs[sub_index] = previous;
8975                    self.overlay_dirty.set(was_dirty);
8976                    self.outbox = previous_outbox;
8977                    self.rejections.truncate(rejection_count);
8978                    report.revoked.truncate(revoked_count);
8979                    report.failed.truncate(failed_count);
8980                }
8981                result
8982            }
8983            SubStatus::Reset => {
8984                self.begin_observation("syncular_reset")
8985                    .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
8986                let previous = self.subs[sub_index].clone();
8987                let was_dirty = self.overlay_dirty.get();
8988                let reset_count = report.resets.len();
8989                let result = (|| {
8990                    let mut batch = ChangeAccumulator::default();
8991                    let registered = self.window_unit_by_sub_id(id);
8992                    // §4.6: discard cursor + bootstrap state, keep local rows —
8993                    // reset is a staleness signal, not a purge signal.
8994                    let sub = &mut self.subs[sub_index];
8995                    sub.cursor = -1;
8996                    sub.bootstrap_state = None;
8997                    report.resets.push(id.to_owned());
8998                    self.persist_sub(&self.subs[sub_index].clone())
8999                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9000                    if let Some((base_key, unit)) = registered {
9001                        batch.window(&base_key, &self.subs[sub_index].table, &unit);
9002                    }
9003                    self.finish_observation("syncular_reset", batch)
9004                        .map_err(|message| {
9005                            SectionError::Abort("storage.failed".to_owned(), message)
9006                        })?;
9007                    Ok(())
9008                })();
9009                if result.is_err() {
9010                    self.rollback_observation("syncular_reset");
9011                    self.subs[sub_index] = previous;
9012                    self.overlay_dirty.set(was_dirty);
9013                    report.resets.truncate(reset_count);
9014                }
9015                result
9016            }
9017            SubStatus::Active => {
9018                let fresh = meta
9019                    .fresh
9020                    .iter()
9021                    .find(|(fid, _)| fid == id)
9022                    .map(|(_, fresh)| *fresh)
9023                    .unwrap_or(false);
9024                let previous = self.subs[sub_index].clone();
9025                self.subs[sub_index].effective = Some(effective_scopes);
9026                let outcome = self
9027                    .apply_section_body(transport, sub_index, body, fresh, meta, report)
9028                    .and_then(|()| {
9029                        let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
9030                            SectionError::Abort(
9031                                "sync.invalid_request".into(),
9032                                "subscription section without SUB_END".into(),
9033                            )
9034                        })?;
9035                        self.apply_sub_end(sub_index, cursor, bootstrap_state)
9036                            .map_err(|message| {
9037                                SectionError::Abort("storage.failed".into(), message)
9038                            })
9039                    });
9040                match outcome {
9041                    Ok(()) => {
9042                        if self.subs[sub_index].bootstrap_state.is_some() {
9043                            report.bootstrapping.push(id.to_owned());
9044                        }
9045                        Ok(())
9046                    }
9047                    Err(SectionError::FailClosed) => {
9048                        self.subs[sub_index] = previous.clone();
9049                        self.begin_observation("syncular_section_failure")
9050                            .map_err(|message| {
9051                                SectionError::Abort("storage.failed".into(), message)
9052                            })?;
9053                        let persisted = (|| {
9054                            let mut batch = ChangeAccumulator::default();
9055                            if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
9056                                batch.window(&base_key, &self.subs[sub_index].table, &unit);
9057                            }
9058                            self.subs[sub_index].state = SubState::Failed;
9059                            self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
9060                            self.persist_sub(&self.subs[sub_index])?;
9061                            self.finish_observation("syncular_section_failure", batch)
9062                        })();
9063                        if let Err(message) = persisted {
9064                            self.rollback_observation("syncular_section_failure");
9065                            self.subs[sub_index] = previous;
9066                            return Err(SectionError::Abort("storage.failed".into(), message));
9067                        }
9068                        report.failed.push(id.to_owned());
9069                        Ok(())
9070                    }
9071                    Err(error) => {
9072                        // Frame/block transactions already completed. Preserve their
9073                        // rows and revisions, but leave this subscription cursor unchanged.
9074                        self.subs[sub_index] = previous;
9075                        Err(error)
9076                    }
9077                }
9078            }
9079        }
9080    }
9081
9082    fn apply_sub_end(
9083        &mut self,
9084        sub_index: usize,
9085        cursor: i64,
9086        bootstrap_state: Option<String>,
9087    ) -> Result<(), String> {
9088        #[cfg(feature = "bench-internals")]
9089        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9090        let previous = self.subs[sub_index].clone();
9091        self.begin_observation("syncular_sub_end")?;
9092        let persisted = (|| {
9093            let mut batch = ChangeAccumulator::default();
9094            let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
9095                && cursor >= 0
9096                && bootstrap_state.is_none();
9097            if completed {
9098                if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
9099                    batch.window(&base_key, &previous.table, &unit);
9100                }
9101            }
9102            let sub = &mut self.subs[sub_index];
9103            sub.cursor = cursor;
9104            sub.bootstrap_state = bootstrap_state;
9105            sub.synced_once = true;
9106            self.persist_sub(&self.subs[sub_index])?;
9107            self.finish_observation("syncular_sub_end", batch)
9108        })();
9109        if persisted.is_err() {
9110            self.rollback_observation("syncular_sub_end");
9111            self.subs[sub_index] = previous;
9112        }
9113        persisted
9114    }
9115
9116    fn apply_section_body(
9117        &mut self,
9118        transport: &mut dyn Transport,
9119        sub_index: usize,
9120        body: Vec<Frame>,
9121        fresh: bool,
9122        meta: &RequestMeta,
9123        report: &mut SyncReport,
9124    ) -> Result<(), SectionError> {
9125        let mut saw_segment = false;
9126        for frame in body {
9127            match frame {
9128                Frame::Commit {
9129                    tables, changes, ..
9130                } => {
9131                    self.apply_commit_frame(&tables, &changes)
9132                        .map_err(|(c, m)| SectionError::Abort(c, m))?;
9133                    report.commits_applied += 1;
9134                }
9135                Frame::SegmentInline { payload } => {
9136                    let segment = decode_rows_segment(&payload)
9137                        .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
9138                    self.progress.update(|p| {
9139                        p.phase = ProgressPhase::Import;
9140                        p.subscription_id = Some(self.subs[sub_index].id.clone());
9141                        p.table = Some(self.subs[sub_index].table.clone());
9142                        p.segment_id = None;
9143                        p.bytes_received = payload.len() as u64;
9144                        p.bytes_total = Some(payload.len() as u64);
9145                        p.rows_processed = 0;
9146                        p.rows_total =
9147                            Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9148                    });
9149                    let first = !saw_segment;
9150                    saw_segment = true;
9151                    let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
9152                    report.segment_rows_applied += applied;
9153                }
9154                Frame::SegmentRef {
9155                    segment_id,
9156                    byte_length,
9157                    media_type,
9158                    table,
9159                    row_count,
9160                    as_of_commit_seq,
9161                    scope_digest,
9162                    row_cursor,
9163                    next_row_cursor,
9164                    url,
9165                    url_expires_at_ms,
9166                    ..
9167                } => {
9168                    // §4.2: reject a descriptor whose mediaType was not
9169                    // advertised — never skip or guess.
9170                    let advertised = match media_type {
9171                        MediaType::Rows => {
9172                            meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9173                                || meta.accept & ACCEPT_INLINE_ROWS != 0
9174                        }
9175                        MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9176                    };
9177                    if !advertised {
9178                        return Err(SectionError::Abort(
9179                            "sync.invalid_request".to_owned(),
9180                            format!(
9181                                "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9182                                media_type.name()
9183                            ),
9184                        ));
9185                    }
9186                    self.progress.update(|p| {
9187                        p.phase = ProgressPhase::Download;
9188                        p.subscription_id = Some(self.subs[sub_index].id.clone());
9189                        p.table = Some(table.clone());
9190                        p.segment_id = Some(segment_id.clone());
9191                        p.bytes_received = 0;
9192                        p.bytes_total = u64::try_from(byte_length).ok();
9193                        p.rows_processed = 0;
9194                        p.rows_total = u64::try_from(row_count).ok();
9195                    });
9196                    let observer = self.progress.clone();
9197                    let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9198                    let bytes = if let Some(url) = url {
9199                        // §5.4: a url-carrying descriptor MUST be fetched
9200                        // from that URL; failure invalidates the whole
9201                        // descriptor (no fall-through to §5.5 — re-pull
9202                        // recovers, §1.4 rule 5).
9203                        if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9204                            return Err(SectionError::Abort(
9205                                "sync.invalid_request".to_owned(),
9206                                "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9207                                    .to_owned(),
9208                            ));
9209                        }
9210                        // §5.4: MUST NOT start a fetch at/past expiry.
9211                        if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
9212                            return Err(SectionError::Abort(
9213                                "sync.segment_expired".to_owned(),
9214                                format!(
9215                                    "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9216                                ),
9217                            ));
9218                        }
9219                        transport
9220                            .fetch_url(&url, &mut on_progress)
9221                            .map_err(|e| SectionError::Abort(e.code, e.message))?
9222                    } else {
9223                        let requested_scopes_json =
9224                            canonical_scope_json(&self.subs[sub_index].requested);
9225                        transport
9226                            .download_segment(
9227                                &SegmentRequest {
9228                                    segment_id: segment_id.clone(),
9229                                    table,
9230                                    requested_scopes_json,
9231                                },
9232                                &mut on_progress,
9233                            )
9234                            .map_err(|e| SectionError::Abort(e.code, e.message))?
9235                    };
9236                    // §5.1: verify the content address before applying.
9237                    on_progress(bytes.len() as u64);
9238                    let digest = Sha256::digest(&bytes);
9239                    let expected = segment_id
9240                        .strip_prefix("sha256:")
9241                        .unwrap_or(segment_id.as_str());
9242                    if bytes_to_hex(&digest) != expected {
9243                        return Err(SectionError::Abort(
9244                            "sync.invalid_request".to_owned(),
9245                            "segment bytes do not match the content address (§5.1)".to_owned(),
9246                        ));
9247                    }
9248                    self.progress.update(|p| p.phase = ProgressPhase::Import);
9249                    if media_type == MediaType::Sqlite {
9250                        // §5.3: images are whole-table — a paged sqlite
9251                        // descriptor is invalid.
9252                        if row_cursor.is_some() || next_row_cursor.is_some() {
9253                            return Err(SectionError::Abort(
9254                                "sync.invalid_request".to_owned(),
9255                                "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9256                                    .to_owned(),
9257                            ));
9258                        }
9259                        let first = !saw_segment;
9260                        saw_segment = true;
9261                        let applied = self.apply_sqlite_segment(
9262                            sub_index,
9263                            &bytes,
9264                            fresh && first,
9265                            row_count,
9266                            as_of_commit_seq,
9267                            &scope_digest,
9268                        )?;
9269                        report.segment_rows_applied += applied;
9270                    } else {
9271                        let segment = decode_rows_segment(&bytes).map_err(|e| {
9272                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9273                        })?;
9274                        let first = row_cursor.is_none();
9275                        saw_segment = true;
9276                        let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
9277                        report.segment_rows_applied += applied;
9278                    }
9279                }
9280                Frame::Unknown { .. } => {}
9281                Frame::Error { code, message, .. } => {
9282                    return Err(SectionError::Abort(code, message))
9283                }
9284                _ => {
9285                    return Err(SectionError::Abort(
9286                        "sync.invalid_request".to_owned(),
9287                        "unexpected frame inside a subscription section".to_owned(),
9288                    ));
9289                }
9290            }
9291        }
9292        Ok(())
9293    }
9294
9295    // Both pull and realtime frames use the same durable observation boundary.
9296    fn apply_commit_frame(
9297        &mut self,
9298        tables: &[String],
9299        changes: &[ssp2::model::Change],
9300    ) -> Result<(), (String, String)> {
9301        #[cfg(feature = "bench-internals")]
9302        let _phase = self.benchmark_phases.start(Phase::CommitApply);
9303        let was_dirty = self.overlay_dirty.get();
9304        self.begin_observation("syncular_commit")
9305            .map_err(|message| ("storage.failed".into(), message))?;
9306        let applied = (|| {
9307            let mut batch = ChangeAccumulator::default();
9308            self.record_commit_changes(&mut batch, tables, changes);
9309            self.apply_commit_changes(tables, changes)?;
9310            self.rebuild_overlay_if_dirty();
9311            self.finish_observation("syncular_commit", batch)
9312                .map_err(|message| ("storage.failed".into(), message))
9313        })();
9314        if applied.is_err() {
9315            self.rollback_observation("syncular_commit");
9316            self.overlay_dirty.set(was_dirty);
9317        }
9318        applied
9319    }
9320
9321    fn apply_commit_changes(
9322        &mut self,
9323        tables: &[String],
9324        changes: &[ssp2::model::Change],
9325    ) -> Result<(), (String, String)> {
9326        // A clean overlay can reconcile changed primary keys independently
9327        // when their tables have no secondary unique constraints. Such a
9328        // constraint can change whether another row's pending write succeeds,
9329        // so those frames still require the complete FIFO replay below.
9330        let mirror_visible = !self.overlay_dirty.get()
9331            && (self.outbox.is_empty()
9332                || changes.iter().all(|change| {
9333                    tables
9334                        .get(change.table_index as usize)
9335                        .and_then(|name| self.schema.table(name))
9336                        .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
9337                }));
9338        for change in changes {
9339            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
9340                (
9341                    "sync.invalid_request".to_owned(),
9342                    "change tableIndex out of range".to_owned(),
9343                )
9344            })?;
9345            let table = self.schema.table(table_name).ok_or_else(|| {
9346                (
9347                    "sync.schema_mismatch".to_owned(),
9348                    format!("change targets unknown table {table_name:?}"),
9349                )
9350            })?;
9351            match change.op {
9352                Op::Upsert => {
9353                    let payload = change.row.as_ref().ok_or_else(|| {
9354                        (
9355                            "sync.invalid_request".to_owned(),
9356                            "upsert change without row payload".to_owned(),
9357                        )
9358                    })?;
9359                    // §5.11: decrypt encrypted columns on apply.
9360                    let row = {
9361                        #[cfg(feature = "bench-internals")]
9362                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
9363                        decode_row_bytes(table, payload, &self.encryption)
9364                            .map_err(|m| ("sync.invalid_request".to_owned(), m))?
9365                    };
9366                    let version = change.row_version.unwrap_or(0);
9367                    let table_name = table.name.clone();
9368                    self.write_base_row(&table_name, &row, version)
9369                        .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
9370                    if mirror_visible {
9371                        self.write_row(&visible_table(&table_name), &table_name, &row, version)
9372                            .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
9373                    }
9374                }
9375                Op::Delete => {
9376                    self.delete_base_row(table_name, &change.row_id)
9377                        .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
9378                    if mirror_visible {
9379                        let sql = format!(
9380                            "DELETE FROM {} WHERE {}",
9381                            visible_table(table_name),
9382                            row_id_predicate(table)
9383                        );
9384                        self.conn
9385                            .prepare_cached(&sql)
9386                            .and_then(|mut statement| {
9387                                statement.execute(rusqlite::params![change.row_id])
9388                            })
9389                            .map_err(|error| {
9390                                ("sync.invalid_request".to_owned(), error.to_string())
9391                            })?;
9392                    }
9393                }
9394            }
9395        }
9396        if mirror_visible {
9397            if !self.outbox.is_empty() {
9398                let changed_rows: HashSet<(&str, &str)> = changes
9399                    .iter()
9400                    .map(|change| {
9401                        (
9402                            tables[change.table_index as usize].as_str(),
9403                            change.row_id.as_str(),
9404                        )
9405                    })
9406                    .collect();
9407                self.apply_outbox_ops(
9408                    self.outbox
9409                        .iter()
9410                        .flat_map(|commit| &commit.ops)
9411                        .filter(|op| {
9412                            changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
9413                        }),
9414                );
9415            }
9416            self.overlay_dirty.set(false);
9417        }
9418        Ok(())
9419    }
9420
9421    /// §5.6 segment application: validate against the generated schema,
9422    /// clear the grant on a fresh bootstrap's first page (fail closed
9423    /// without a mapping), then replace-or-upsert each row with its
9424    /// segment-carried server version (§5.2).
9425    fn apply_segment(
9426        &mut self,
9427        sub_index: usize,
9428        segment: &RowsSegment,
9429        first_fresh_page: bool,
9430    ) -> Result<u32, SectionError> {
9431        let (sub_table, effective) = {
9432            let sub = &self.subs[sub_index];
9433            (sub.table.clone(), sub.effective.clone().unwrap_or_default())
9434        };
9435        let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
9436            SectionError::Abort(
9437                "sync.schema_mismatch".to_owned(),
9438                format!("subscription table {sub_table:?} is not in the client schema"),
9439            )
9440        })?;
9441        // §5.2: the column table validates against the generated schema —
9442        // order, names, types, nullability; mismatch is fatal. §5.11: the
9443        // server sends the WIRE types (bytes for an encrypted column), so
9444        // validate against wire_columns.
9445        let matches = segment.table == table.name
9446            && segment.schema_version == self.schema.version
9447            && segment.columns.len() == table.wire_columns.len()
9448            && segment
9449                .columns
9450                .iter()
9451                .zip(table.wire_columns.iter())
9452                .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
9453        if !matches {
9454            return Err(SectionError::Abort(
9455                "sync.schema_mismatch".to_owned(),
9456                "segment column table does not match the generated schema (§5.2)".to_owned(),
9457            ));
9458        }
9459        let mut applied = 0u32;
9460        let empty = Vec::new();
9461        let blocks = if segment.blocks.is_empty() {
9462            std::slice::from_ref(&empty)
9463        } else {
9464            &segment.blocks
9465        };
9466        for (index, block) in blocks.iter().enumerate() {
9467            let clear = first_fresh_page && index == 0;
9468            if !clear && block.is_empty() {
9469                continue;
9470            }
9471            let was_dirty = self.overlay_dirty.get();
9472            let reconcile_pending = table.indexes.iter().any(|index| index.unique)
9473                && self
9474                    .outbox
9475                    .iter()
9476                    .flat_map(|commit| &commit.ops)
9477                    .any(|op| op.table == table.name);
9478            let mirror_visible = !reconcile_pending;
9479            self.begin_observation("syncular_segment_block")
9480                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9481            let outcome = (|| {
9482                self.rebuild_overlay_if_dirty();
9483                let mut changed_keys = Vec::new();
9484                let mut batch = ChangeAccumulator::default();
9485                if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
9486                    batch.table(&table.name);
9487                }
9488                if clear {
9489                    self.purge_scope_rows(&table.name, &effective)
9490                        .map_err(|()| SectionError::FailClosed)?;
9491                }
9492                for (row_index, row) in block.iter().enumerate() {
9493                    let decrypted;
9494                    let values = if table.has_encrypted_columns() {
9495                        let mut values = row.values.clone();
9496                        crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
9497                            .map_err(|message| {
9498                            SectionError::Abort("client.decrypt_failed".into(), message)
9499                        })?;
9500                        decrypted = values;
9501                        &decrypted
9502                    } else {
9503                        &row.values
9504                    };
9505                    self.write_base_row(&table.name, values, row.server_version)
9506                        .map_err(|message| {
9507                            SectionError::Abort("sync.invalid_request".into(), message)
9508                        })?;
9509                    if reconcile_pending {
9510                        changed_keys.push(
9511                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
9512                                |message| SectionError::Abort("storage.failed".into(), message),
9513                            )?,
9514                        );
9515                    }
9516                    if mirror_visible {
9517                        self.write_row(
9518                            &visible_table(&table.name),
9519                            &table.name,
9520                            values,
9521                            row.server_version,
9522                        )
9523                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9524                    }
9525                    let processed = u64::from(applied) + row_index as u64 + 1;
9526                    if processed.is_multiple_of(1024) {
9527                        self.progress.update(|p| p.rows_processed = processed);
9528                    }
9529                }
9530                if reconcile_pending {
9531                    self.reconcile_imported_rows(&table, &mut changed_keys)
9532                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9533                }
9534                self.apply_outbox_ops(
9535                    self.outbox
9536                        .iter()
9537                        .flat_map(|commit| &commit.ops)
9538                        .filter(|op| op.table == table.name),
9539                );
9540                self.overlay_dirty.set(false);
9541                self.finish_observation("syncular_segment_block", batch)
9542                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
9543            })();
9544            if let Err(error) = outcome {
9545                self.rollback_observation("syncular_segment_block");
9546                self.overlay_dirty.set(was_dirty);
9547                return Err(error);
9548            }
9549            applied += block.len() as u32;
9550            self.progress
9551                .update(|p| p.rows_processed = u64::from(applied));
9552            std::thread::yield_now();
9553        }
9554        self.progress
9555            .update(|p| p.rows_processed = u64::from(applied));
9556        Ok(applied)
9557    }
9558
9559    /// §5.3 sqlite-image application: validate the in-file metadata
9560    /// against the descriptor, validate column names/order against the
9561    /// generated schema, run the §5.6 first-page clear when fresh, then
9562    /// replace-or-upsert every image row with its `_syncular_version`.
9563    /// Mechanics: the image lands in a temp file read through a second
9564    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
9565    /// Each image chunk commits its writes and revision before readers continue.
9566    fn apply_sqlite_segment(
9567        &mut self,
9568        sub_index: usize,
9569        bytes: &[u8],
9570        first_fresh_page: bool,
9571        row_count: i64,
9572        as_of_commit_seq: i64,
9573        scope_digest: &str,
9574    ) -> Result<u32, SectionError> {
9575        let invalid = |detail: &str| {
9576            SectionError::Abort(
9577                "sync.invalid_request".to_owned(),
9578                format!("sqlite segment rejected: {detail} (§5.3)"),
9579            )
9580        };
9581        let (sub_table, effective) = {
9582            let sub = &self.subs[sub_index];
9583            (sub.table.clone(), sub.effective.clone().unwrap_or_default())
9584        };
9585        let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
9586            SectionError::Abort(
9587                "sync.schema_mismatch".to_owned(),
9588                format!("subscription table {sub_table:?} is not in the client schema"),
9589            )
9590        })?;
9591
9592        let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
9593        std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
9594        let img = match rusqlite::Connection::open_with_flags(
9595            &path,
9596            rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
9597        ) {
9598            Ok(conn) => conn,
9599            Err(_) => {
9600                let _ = std::fs::remove_file(&path);
9601                return Err(invalid("bytes do not open as a SQLite database"));
9602            }
9603        };
9604        let outcome = self.apply_sqlite_image(
9605            &img,
9606            &table,
9607            first_fresh_page,
9608            &effective,
9609            row_count,
9610            as_of_commit_seq,
9611            scope_digest,
9612        );
9613        drop(img);
9614        let _ = std::fs::remove_file(&path);
9615        outcome
9616    }
9617
9618    #[allow(clippy::too_many_arguments)]
9619    fn apply_sqlite_image(
9620        &mut self,
9621        img: &rusqlite::Connection,
9622        table: &crate::schema::TableSchema,
9623        first_fresh_page: bool,
9624        effective: &[(String, Vec<String>)],
9625        row_count: i64,
9626        as_of_commit_seq: i64,
9627        scope_digest: &str,
9628    ) -> Result<u32, SectionError> {
9629        let invalid = |detail: String| {
9630            SectionError::Abort(
9631                "sync.invalid_request".to_owned(),
9632                format!("sqlite segment rejected: {detail} (§5.3)"),
9633            )
9634        };
9635
9636        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
9637        //    one row).
9638        type MetaRow = (i64, String, i64, i64, String, i64, i64);
9639        let meta: MetaRow = img
9640            .query_row(
9641                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
9642                        \"scopeDigest\", \"rowCount\",
9643                        (SELECT count(*) FROM _syncular_segment)
9644                 FROM _syncular_segment",
9645                [],
9646                |row| {
9647                    Ok((
9648                        row.get(0)?,
9649                        row.get(1)?,
9650                        row.get(2)?,
9651                        row.get(3)?,
9652                        row.get(4)?,
9653                        row.get(5)?,
9654                        row.get(6)?,
9655                    ))
9656                },
9657            )
9658            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
9659        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
9660        if meta_count != 1 {
9661            return Err(invalid(format!(
9662                "_syncular_segment must contain exactly one row, found {meta_count}"
9663            )));
9664        }
9665        if format != 1 {
9666            return Err(invalid(format!("format {format}")));
9667        }
9668        if meta_table != table.name {
9669            return Err(invalid(format!("image table {meta_table:?}")));
9670        }
9671        if schema_version != i64::from(self.schema.version) {
9672            return Err(invalid(format!("schemaVersion {schema_version}")));
9673        }
9674        if pin != as_of_commit_seq {
9675            return Err(invalid(format!("asOfCommitSeq {pin}")));
9676        }
9677        if digest != scope_digest {
9678            return Err(invalid("scopeDigest mismatch".to_owned()));
9679        }
9680        if meta_rows != row_count {
9681            return Err(invalid(format!("rowCount {meta_rows}")));
9682        }
9683
9684        // 2. Column names and order vs the generated schema (§5.3 rule 3).
9685        let mut names: Vec<String> = Vec::new();
9686        let mut primary_keys: Vec<String> = Vec::new();
9687        {
9688            let mut stmt = img
9689                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
9690                .map_err(|_| invalid("image data table missing".to_owned()))?;
9691            let mut rows = stmt
9692                .query([])
9693                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
9694            while let Some(row) = rows
9695                .next()
9696                .map_err(|_| invalid("image data table unreadable".to_owned()))?
9697            {
9698                if row.get::<_, i64>(5).map_err(|e| invalid(e.to_string()))? > 0 {
9699                    primary_keys.push(row.get(1).map_err(|e| invalid(e.to_string()))?);
9700                }
9701                names.push(
9702                    row.get::<_, String>(1)
9703                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
9704                );
9705            }
9706        }
9707        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
9708        expected.push("_syncular_version");
9709        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
9710            return Err(SectionError::Abort(
9711                "sync.schema_mismatch".to_owned(),
9712                "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
9713            ));
9714        }
9715
9716        if primary_keys != vec![table.primary_key.clone()] {
9717            return Err(invalid(
9718                "image primary key does not match generated schema".into(),
9719            ));
9720        }
9721        let actual_count: i64 = img
9722            .query_row(
9723                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
9724                [],
9725                |row| row.get(0),
9726            )
9727            .map_err(|e| invalid(e.to_string()))?;
9728        if actual_count != row_count {
9729            return Err(invalid("image row count does not match descriptor".into()));
9730        }
9731        let insert = self.insert_row_sql(&base_table(&table.name), table);
9732        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
9733        let column_list = names
9734            .iter()
9735            .map(|n| quote_ident(n))
9736            .collect::<Vec<_>>()
9737            .join(", ");
9738        let mut stmt = img
9739            .prepare(&format!(
9740                "SELECT {column_list} FROM {} ORDER BY {}",
9741                quote_ident(&table.name),
9742                quote_ident(&table.primary_key)
9743            ))
9744            .map_err(|e| invalid(e.to_string()))?;
9745        let mut rows = stmt.query([]).map_err(|e| invalid(e.to_string()))?;
9746        let mut applied = 0u32;
9747        loop {
9748            let clear = applied == 0 && first_fresh_page;
9749            let was_dirty = self.overlay_dirty.get();
9750            let reconcile_pending = table.indexes.iter().any(|index| index.unique)
9751                && self
9752                    .outbox
9753                    .iter()
9754                    .flat_map(|commit| &commit.ops)
9755                    .any(|op| op.table == table.name);
9756            let mirror_visible = !reconcile_pending;
9757            self.begin_observation("syncular_image_chunk")
9758                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9759            let result = (|| {
9760                self.rebuild_overlay_if_dirty();
9761                let mut changed_keys = Vec::new();
9762                let mut batch = ChangeAccumulator::default();
9763                if clear {
9764                    if self.scoped_rows_exist(&table.name, effective) {
9765                        batch.table(&table.name);
9766                    }
9767                    self.purge_scope_rows(&table.name, effective)
9768                        .map_err(|()| SectionError::FailClosed)?;
9769                }
9770                let mut processed = 0u32;
9771                {
9772                    let mut ins = self
9773                        .conn
9774                        .prepare_cached(&insert)
9775                        .map_err(|e| invalid(e.to_string()))?;
9776                    let mut mirror = if mirror_visible {
9777                        Some(
9778                            self.conn
9779                                .prepare_cached(&visible_insert)
9780                                .map_err(|e| invalid(e.to_string()))?,
9781                        )
9782                    } else {
9783                        None
9784                    };
9785                    for _ in 0..1024 {
9786                        let Some(row) = rows.next().map_err(|e| invalid(e.to_string()))? else {
9787                            break;
9788                        };
9789                        for (i, column) in table.columns.iter().enumerate() {
9790                            let cell = row.get_ref(i).map_err(|e| invalid(e.to_string()))?;
9791                            let param = image_cell_param(column, cell).map_err(&invalid)?;
9792                            ins.raw_bind_parameter(i + 1, &param)
9793                                .map_err(|e| invalid(e.to_string()))?;
9794                            if let Some(mirror) = &mut mirror {
9795                                mirror
9796                                    .raw_bind_parameter(i + 1, &param)
9797                                    .map_err(|e| invalid(e.to_string()))?;
9798                            }
9799                        }
9800                        let version_index = table.columns.len();
9801                        let version: i64 =
9802                            row.get(version_index).map_err(|e| invalid(e.to_string()))?;
9803                        if version < 1 {
9804                            return Err(invalid("image row version must be positive".into()));
9805                        }
9806                        ins.raw_bind_parameter(version_index + 1, version)
9807                            .map_err(|e| invalid(e.to_string()))?;
9808                        ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
9809                        if let Some(mirror) = &mut mirror {
9810                            mirror
9811                                .raw_bind_parameter(version_index + 1, version)
9812                                .map_err(|e| invalid(e.to_string()))?;
9813                            mirror.raw_execute().map_err(|e| invalid(e.to_string()))?;
9814                        }
9815                        if reconcile_pending {
9816                            changed_keys.push(
9817                                row.get::<_, SqlValue>(table.pk_index)
9818                                    .map_err(|error| invalid(error.to_string()))?,
9819                            );
9820                        }
9821                        processed += 1;
9822                    }
9823                }
9824                if processed > 0 {
9825                    batch.table(&table.name);
9826                    self.overlay_dirty.set(true);
9827                }
9828                if reconcile_pending {
9829                    self.reconcile_imported_rows(table, &mut changed_keys)
9830                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9831                }
9832                self.apply_outbox_ops(
9833                    self.outbox
9834                        .iter()
9835                        .flat_map(|commit| &commit.ops)
9836                        .filter(|op| op.table == table.name),
9837                );
9838                self.overlay_dirty.set(false);
9839                self.finish_observation("syncular_image_chunk", batch)
9840                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9841                Ok(processed)
9842            })();
9843            let processed = match result {
9844                Ok(processed) => processed,
9845                Err(error) => {
9846                    self.rollback_observation("syncular_image_chunk");
9847                    self.overlay_dirty.set(was_dirty);
9848                    return Err(error);
9849                }
9850            };
9851            applied += processed;
9852            self.progress
9853                .update(|p| p.rows_processed = u64::from(applied));
9854            std::thread::yield_now();
9855            if i64::from(applied) == actual_count {
9856                break;
9857            }
9858        }
9859        Ok(applied)
9860    }
9861
9862    /// Restore only imported and pending row identities before FIFO replay. All pending
9863    /// identities participate: a server row can free a unique value for a different edit.
9864    /// The caller owns the chunk transaction and publishes its revision after replay.
9865    fn reconcile_imported_rows(
9866        &self,
9867        table: &TableSchema,
9868        keys: &mut Vec<SqlValue>,
9869    ) -> Result<(), String> {
9870        let visible = visible_table(&table.name);
9871        let base = base_table(&table.name);
9872        let pk = quote_ident(&table.primary_key);
9873        let predicate = row_id_predicate(table);
9874        let mut lookup = self.conn.prepare_cached(&format!(
9875            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
9876        )).map_err(|error| error.to_string())?;
9877        let mut deleted = HashSet::new();
9878        for op in self
9879            .outbox
9880            .iter()
9881            .flat_map(|commit| &commit.ops)
9882            .filter(|op| op.table == table.name)
9883        {
9884            if op.upsert {
9885                if let Some(values) = &op.values {
9886                    let key = json_to_column_value(
9887                        &table.columns[table.pk_index],
9888                        values.get(&table.primary_key),
9889                    )?;
9890                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
9891                }
9892            } else if deleted.insert(&op.row_id) {
9893                keys.extend(
9894                    lookup
9895                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
9896                        .map_err(|error| error.to_string())?
9897                        .collect::<Result<Vec<_>, _>>()
9898                        .map_err(|error| error.to_string())?,
9899                );
9900            }
9901        }
9902        // Delete the complete affected set before restoring any base value, so
9903        // old optimistic unique values cannot collide with the new server rows.
9904        for chunk in keys.chunks(400) {
9905            let holes = vec!["?"; chunk.len()].join(",");
9906            self.conn
9907                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
9908                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
9909                .map_err(|error| error.to_string())?;
9910        }
9911        let updates = table
9912            .columns
9913            .iter()
9914            .map(|column| column.name.as_str())
9915            .chain(std::iter::once("_syncular_version"))
9916            .filter(|name| *name != table.primary_key)
9917            .map(|name| {
9918                let name = quote_ident(name);
9919                format!("{name}=excluded.{name}")
9920            })
9921            .collect::<Vec<_>>()
9922            .join(",");
9923        for chunk in keys.chunks(400) {
9924            let holes = vec!["?"; chunk.len()].join(",");
9925            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
9926                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
9927                .map_err(|error| error.to_string())?;
9928        }
9929        Ok(())
9930    }
9931
9932    // -- application-authorized local purge ---------------------------------------
9933
9934    fn compile_local_data_purge(
9935        &self,
9936        input: &LocalDataPurgeInput,
9937    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
9938        let invalid = |message: String| format!("sync.invalid_request: {message}");
9939        if input.purge_id.is_empty()
9940            || input.purge_id.len() > 128
9941            || !is_local_operation_code_like(&input.purge_id)
9942        {
9943            return Err(invalid(
9944                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
9945            ));
9946        }
9947        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
9948            return Err(invalid(format!(
9949                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
9950            )));
9951        }
9952
9953        let mut deduplicated: BTreeMap<
9954            String,
9955            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
9956        > = BTreeMap::new();
9957        for target in &input.targets {
9958            let table = self.schema.table(&target.table).ok_or_else(|| {
9959                invalid(format!(
9960                    "local purge names unknown table {:?}",
9961                    target.table
9962                ))
9963            })?;
9964            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
9965                return Err(invalid(format!(
9966                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
9967                    target.table
9968                )));
9969            }
9970            let mut selectors = Vec::with_capacity(target.selectors.len());
9971            let mut canonical_selectors = BTreeMap::new();
9972            for (column_name, raw_values) in &target.selectors {
9973                let Some((column_index, column)) = table
9974                    .columns
9975                    .iter()
9976                    .enumerate()
9977                    .find(|(_, column)| column.name == *column_name)
9978                else {
9979                    return Err(invalid(format!(
9980                        "local purge target {:?} names unknown column {:?}",
9981                        target.table, column_name
9982                    )));
9983                };
9984                let encrypted = table
9985                    .encrypted_columns
9986                    .iter()
9987                    .any(|candidate| candidate.index == column_index);
9988                if column.ty != ColumnType::String || encrypted {
9989                    return Err(invalid(format!(
9990                        "local purge selector {:?}.{:?} must be a plaintext string column",
9991                        target.table, column_name
9992                    )));
9993                }
9994                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
9995                    return Err(invalid(format!(
9996                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
9997                        target.table, column_name
9998                    )));
9999                }
10000                let mut values = raw_values.clone();
10001                values.sort();
10002                values.dedup();
10003                if values.iter().any(|value| {
10004                    value.is_empty()
10005                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
10006                        || !is_local_operation_code_like(value)
10007                }) {
10008                    return Err(invalid(format!(
10009                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
10010                    )));
10011                }
10012                selectors.push((column_name.clone(), values.clone()));
10013                canonical_selectors.insert(column_name.clone(), values);
10014            }
10015            let canonical = LocalDataPurgeTarget {
10016                table: target.table.clone(),
10017                selectors: canonical_selectors,
10018            };
10019            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10020            deduplicated.insert(
10021                key,
10022                (
10023                    CompiledLocalDataPurgeTarget {
10024                        table: target.table.clone(),
10025                        selectors,
10026                    },
10027                    canonical,
10028                ),
10029            );
10030        }
10031        let targets = deduplicated
10032            .values()
10033            .map(|(compiled, _)| compiled.clone())
10034            .collect::<Vec<_>>();
10035        let canonical = deduplicated
10036            .values()
10037            .map(|(_, target)| target.clone())
10038            .collect::<Vec<_>>();
10039        let canonical_plan =
10040            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10041        Ok((targets, canonical_plan))
10042    }
10043
10044    fn local_purge_base_row_ids(
10045        &self,
10046        targets: &[CompiledLocalDataPurgeTarget],
10047    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
10048        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
10049        for target in targets {
10050            let table = self
10051                .schema
10052                .table(&target.table)
10053                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
10054            let mut clauses = Vec::with_capacity(target.selectors.len());
10055            let mut params = Vec::new();
10056            for (column, values) in &target.selectors {
10057                clauses.push(format!(
10058                    "{} IN ({})",
10059                    quote_ident(column),
10060                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
10061                ));
10062                params.extend(values.iter().cloned().map(SqlValue::Text));
10063            }
10064            let sql = format!(
10065                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
10066                quote_ident(&table.primary_key),
10067                base_table(&target.table),
10068                clauses.join(" AND ")
10069            );
10070            let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
10071            let rows = statement
10072                .query_map(rusqlite::params_from_iter(params), |row| {
10073                    row.get::<_, String>(0)
10074                })
10075                .map_err(|error| error.to_string())?;
10076            let ids = by_table.entry(target.table.clone()).or_default();
10077            for row in rows {
10078                ids.insert(row.map_err(|error| error.to_string())?);
10079            }
10080        }
10081        Ok(by_table)
10082    }
10083
10084    fn local_purge_values_match(
10085        target: &CompiledLocalDataPurgeTarget,
10086        values: &Map<String, Value>,
10087    ) -> bool {
10088        target.selectors.iter().all(|(column, allowed)| {
10089            values
10090                .get(column)
10091                .and_then(Value::as_str)
10092                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
10093        })
10094    }
10095
10096    /// Apply one host-authorized local security purge. The host MUST gate the
10097    /// corresponding subscriptions first; this primitive owns local SQLite
10098    /// cleanup and never grants/revokes server authority by itself.
10099    pub fn purge_local_data(
10100        &mut self,
10101        input: &LocalDataPurgeInput,
10102    ) -> Result<LocalDataPurgeResult, String> {
10103        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
10104        let meta_key = format!("localPurge:{}", input.purge_id);
10105        let applied_plan = self
10106            .conn
10107            .query_row(
10108                "SELECT value FROM _syncular_meta WHERE key = ?1",
10109                rusqlite::params![meta_key],
10110                |row| row.get::<_, String>(0),
10111            )
10112            .optional()
10113            .map_err(|error| error.to_string())?;
10114        if let Some(applied_plan) = applied_plan {
10115            if applied_plan != canonical_plan {
10116                return Err(format!(
10117                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
10118                    input.purge_id
10119                ));
10120            }
10121            return Ok(LocalDataPurgeResult {
10122                already_applied: true,
10123                purged_rows: 0,
10124                dropped_commits: 0,
10125            });
10126        }
10127
10128        let prior_outbox = self.outbox.clone();
10129        let prior_rejection_count = self.rejections.len();
10130        let prior_overlay_dirty = self.overlay_dirty.get();
10131        self.begin_observation("syncular_local_purge")?;
10132        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
10133            let row_ids = self.local_purge_base_row_ids(&targets)?;
10134            let doomed = self
10135                .outbox
10136                .iter()
10137                .filter(|commit| {
10138                    commit.ops.iter().any(|operation| {
10139                        let matching_targets = targets
10140                            .iter()
10141                            .filter(|target| target.table == operation.table)
10142                            .collect::<Vec<_>>();
10143                        if matching_targets.is_empty() {
10144                            return false;
10145                        }
10146                        if row_ids
10147                            .get(&operation.table)
10148                            .is_some_and(|ids| ids.contains(&operation.row_id))
10149                        {
10150                            return true;
10151                        }
10152                        operation.values.as_ref().is_some_and(|values| {
10153                            matching_targets
10154                                .iter()
10155                                .any(|target| Self::local_purge_values_match(target, values))
10156                        })
10157                    })
10158                })
10159                .cloned()
10160                .collect::<Vec<_>>();
10161            let doomed_ids = doomed
10162                .iter()
10163                .map(|commit| commit.client_commit_id.clone())
10164                .collect::<BTreeSet<_>>();
10165            let mut batch = ChangeAccumulator::default();
10166            let mut rejections = Vec::new();
10167            for commit in &doomed {
10168                for operation in &commit.ops {
10169                    batch.table(&operation.table);
10170                }
10171                let results = commit
10172                    .ops
10173                    .iter()
10174                    .enumerate()
10175                    .map(|(op_index, operation)| {
10176                        let rejection = RejectionRecord {
10177                            client_commit_id: commit.client_commit_id.clone(),
10178                            op_index: op_index as i32,
10179                            code: "client.local_data_purged".to_owned(),
10180                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
10181                            retryable: false,
10182                            details: None,
10183                            operation: Some(CommitOperation::from(operation)),
10184                        };
10185                        rejections.push(rejection.clone());
10186                        CommitOperationOutcome::Error { rejection }
10187                    })
10188                    .collect::<Vec<_>>();
10189                self.persist_commit_outcome(
10190                    &commit.client_commit_id,
10191                    CommitOutcomeStatus::Rejected,
10192                    &results,
10193                    Some(&commit.ops),
10194                )?;
10195                self.delete_outbox_persisted(&commit.client_commit_id)?;
10196            }
10197            if !doomed.is_empty() {
10198                self.outbox
10199                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
10200                self.rejections.extend(rejections);
10201                self.prune_commit_outcomes()?;
10202                self.overlay_dirty.set(true);
10203                batch.status = true;
10204                batch.rejections = true;
10205                batch.outcomes = true;
10206            }
10207
10208            let mut purged_rows = 0usize;
10209            for (table, ids) in &row_ids {
10210                if ids.is_empty() {
10211                    continue;
10212                }
10213                batch.table(table);
10214                for row_id in ids {
10215                    self.delete_base_row(table, row_id)?;
10216                    purged_rows += 1;
10217                }
10218            }
10219            self.rebuild_overlay_if_dirty();
10220            self.delete_unreferenced_cached_blobs();
10221            self.conn
10222                .execute(
10223                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
10224                    rusqlite::params![meta_key, canonical_plan],
10225                )
10226                .map_err(|error| error.to_string())?;
10227            Ok((
10228                batch,
10229                LocalDataPurgeResult {
10230                    already_applied: false,
10231                    purged_rows,
10232                    dropped_commits: doomed.len(),
10233                },
10234            ))
10235        })();
10236        let (batch, result) = match applied {
10237            Ok(value) => value,
10238            Err(error) => {
10239                self.rollback_observation("syncular_local_purge");
10240                self.outbox = prior_outbox;
10241                self.rejections.truncate(prior_rejection_count);
10242                self.overlay_dirty.set(prior_overlay_dirty);
10243                return Err(error);
10244            }
10245        };
10246        if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
10247            self.rollback_observation("syncular_local_purge");
10248            self.outbox = prior_outbox;
10249            self.rejections.truncate(prior_rejection_count);
10250            self.overlay_dirty.set(prior_overlay_dirty);
10251            return Err(error);
10252        }
10253        Ok(result)
10254    }
10255
10256    /// Application-authorized recovery of the replicated projection. Unlike a
10257    /// local security purge, this retains the entire outbox, device identity,
10258    /// lease, outcomes, subscription registrations, and protected bookkeeping.
10259    /// The projection reset, subscription rewind, optimistic replay, and
10260    /// idempotency marker share one savepoint for interruption safety.
10261    pub fn rebootstrap_local_data(
10262        &mut self,
10263        input: &LocalDataRebootstrapInput,
10264    ) -> Result<LocalDataRebootstrapResult, String> {
10265        if input.rebootstrap_id.is_empty()
10266            || input.rebootstrap_id.len() > 128
10267            || !is_local_operation_code_like(&input.rebootstrap_id)
10268        {
10269            return Err(
10270                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
10271                    .to_owned(),
10272            );
10273        }
10274        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
10275        if let Some(persisted) = self.get_meta_strict(&meta_key)? {
10276            let (retained_commits, reset_subscriptions) =
10277                decode_local_rebootstrap_receipt(&persisted)?;
10278            return Ok(LocalDataRebootstrapResult {
10279                already_applied: true,
10280                retained_commits,
10281                reset_subscriptions,
10282            });
10283        }
10284        if self.stopped || self.schema_floor.is_some() {
10285            return Err(
10286                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
10287                    .to_owned(),
10288            );
10289        }
10290
10291        let retained_commits = self.outbox.len();
10292        let reset_subscriptions = self.subs.len();
10293        let prior_subs = self.subs.clone();
10294        let prior_upgrading = self.upgrading;
10295        let prior_stopped = self.stopped;
10296        let prior_schema_floor = self.schema_floor.clone();
10297        let prior_overlay_dirty = self.overlay_dirty.get();
10298        let prior_sync_needed = self.sync_needed;
10299        let prior_sync_intents = self.sync_intent_queue.clone();
10300        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
10301
10302        self.begin_observation("syncular_local_rebootstrap")?;
10303        let mut batch = ChangeAccumulator::default();
10304        let applied = (|| -> Result<(), String> {
10305            self.run_schema_reset_observed(&mut batch, false)?;
10306            self.conn
10307                .execute(
10308                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
10309                    rusqlite::params![meta_key, receipt],
10310                )
10311                .map_err(|error| error.to_string())?;
10312            self.sync_needed = true;
10313            self.sync_intent_queue.push_back(SyncIntent::Interactive);
10314            batch.status = true;
10315            Ok(())
10316        })();
10317        if let Err(error) = applied {
10318            self.rollback_observation("syncular_local_rebootstrap");
10319            self.subs = prior_subs;
10320            self.upgrading = prior_upgrading;
10321            self.stopped = prior_stopped;
10322            self.schema_floor = prior_schema_floor;
10323            self.overlay_dirty.set(prior_overlay_dirty);
10324            self.sync_needed = prior_sync_needed;
10325            self.sync_intent_queue = prior_sync_intents;
10326            return Err(error);
10327        }
10328        if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
10329            self.rollback_observation("syncular_local_rebootstrap");
10330            self.subs = prior_subs;
10331            self.upgrading = prior_upgrading;
10332            self.stopped = prior_stopped;
10333            self.schema_floor = prior_schema_floor;
10334            self.overlay_dirty.set(prior_overlay_dirty);
10335            self.sync_needed = prior_sync_needed;
10336            self.sync_intent_queue = prior_sync_intents;
10337            return Err(error);
10338        }
10339
10340        Ok(LocalDataRebootstrapResult {
10341            already_applied: false,
10342            retained_commits,
10343            reset_subscriptions,
10344        })
10345    }
10346
10347    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
10348
10349    /// Delete base rows matching the effective scopes; `Err(())` = no local
10350    /// scope-column mapping for a key (the fail-closed case).
10351    fn purge_scope_rows(
10352        &mut self,
10353        table_name: &str,
10354        effective: &[(String, Vec<String>)],
10355    ) -> Result<(), ()> {
10356        if effective.is_empty() {
10357            return Ok(());
10358        }
10359        let table = self.schema.table(table_name).ok_or(())?.clone();
10360        let mut clauses = Vec::new();
10361        let mut params: Vec<SqlValue> = Vec::new();
10362        for (variable, values) in effective {
10363            let column = table.scope_column(variable).ok_or(())?;
10364            let placeholders: Vec<String> = values
10365                .iter()
10366                .map(|v| {
10367                    params.push(SqlValue::Text(v.clone()));
10368                    "?".to_owned()
10369                })
10370                .collect();
10371            clauses.push(format!(
10372                "{} IN ({})",
10373                quote_ident(column),
10374                placeholders.join(", ")
10375            ));
10376        }
10377        let sql = format!(
10378            "DELETE FROM {} WHERE {}",
10379            base_table(table_name),
10380            clauses.join(" AND ")
10381        );
10382        self.overlay_dirty.set(true);
10383        self.conn
10384            .execute(&sql, rusqlite::params_from_iter(&params))
10385            .map_err(|_| ())?;
10386        self.conn
10387            .execute(
10388                &format!(
10389                    "DELETE FROM {} WHERE {}",
10390                    visible_table(table_name),
10391                    clauses.join(" AND ")
10392                ),
10393                rusqlite::params_from_iter(&params),
10394            )
10395            .map_err(|_| ())?;
10396        Ok(())
10397    }
10398
10399    /// §3.3: drop pending commits whose upserts provably land in the
10400    /// revoked effective scopes — whole-commit, never per-operation.
10401    fn drop_doomed_outbox(
10402        &mut self,
10403        table_name: &str,
10404        effective: &[(String, Vec<String>)],
10405    ) -> Result<bool, String> {
10406        if effective.is_empty() {
10407            return Ok(false);
10408        }
10409        let Some(table) = self.schema.table(table_name).cloned() else {
10410            return Ok(false);
10411        };
10412        let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
10413        for (variable, values) in effective {
10414            match table.scope_column(variable) {
10415                Some(column) => mappings.push((column, values)),
10416                None => return Ok(false), // not provable without a mapping
10417            }
10418        }
10419        let doomed: Vec<OutboxCommit> = self
10420            .outbox
10421            .iter()
10422            .filter(|commit| {
10423                commit.ops.iter().any(|op| {
10424                    op.upsert
10425                        && op.table == table_name
10426                        && op.values.as_ref().is_some_and(|values| {
10427                            mappings
10428                                .iter()
10429                                .all(|(column, allowed)| match values.get(*column) {
10430                                    Some(Value::String(s)) => allowed.contains(s),
10431                                    Some(Value::Number(n)) => allowed.contains(&n.to_string()),
10432                                    _ => false,
10433                                })
10434                        })
10435                })
10436            })
10437            .cloned()
10438            .collect();
10439        if doomed.is_empty() {
10440            return Ok(false);
10441        }
10442        let mut rejections = Vec::new();
10443        for commit in &doomed {
10444            let results = commit
10445                .ops
10446                .iter()
10447                .enumerate()
10448                .map(|(op_index, operation)| {
10449                    let rejection = RejectionRecord {
10450                        client_commit_id: commit.client_commit_id.clone(),
10451                        op_index: op_index as i32,
10452                        code: "sync.scope_revoked".to_owned(),
10453                        message: "the commit was dropped because its effective scope was revoked"
10454                            .to_owned(),
10455                        retryable: false,
10456                        details: None,
10457                        operation: Some(CommitOperation::from(operation)),
10458                    };
10459                    rejections.push(rejection.clone());
10460                    CommitOperationOutcome::Error { rejection }
10461                })
10462                .collect::<Vec<_>>();
10463            self.persist_commit_outcome(
10464                &commit.client_commit_id,
10465                CommitOutcomeStatus::Rejected,
10466                &results,
10467                Some(&commit.ops),
10468            )?;
10469            self.delete_outbox_persisted(&commit.client_commit_id)?;
10470        }
10471        self.prune_commit_outcomes()?;
10472        let doomed_ids = doomed
10473            .iter()
10474            .map(|commit| commit.client_commit_id.as_str())
10475            .collect::<BTreeSet<_>>();
10476        self.outbox
10477            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
10478        self.rejections.extend(rejections);
10479        self.overlay_dirty.set(true);
10480        Ok(true)
10481    }
10482
10483    // -- blobs (§5.9) ----------------------------------------------------------------
10484
10485    /// §5.9.7: hash bytes into the content address, cache them, queue the
10486    /// upload (flushed before the next push, B4). Returns the canonical
10487    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
10488    /// column value.
10489    pub fn upload_blob(
10490        &mut self,
10491        bytes: &[u8],
10492        media_type: Option<String>,
10493        name: Option<String>,
10494    ) -> Result<Value, String> {
10495        let blob_id = blob_id_for(bytes);
10496        let now = self.clock_now_ms();
10497        let tx = self.conn.transaction().map_err(|e| e.to_string())?;
10498        tx.execute(
10499            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
10500             ON CONFLICT(blob_id) DO NOTHING",
10501            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
10502        )
10503        .map_err(|e| e.to_string())?;
10504        tx.execute(
10505            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
10506            rusqlite::params![blob_id, media_type, now],
10507        )
10508        .map_err(|e| e.to_string())?;
10509        tx.commit().map_err(|e| e.to_string())?;
10510        self.enforce_blob_cache_cap();
10511        let mut obj = Map::new();
10512        obj.insert("blobId".to_owned(), Value::from(blob_id));
10513        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
10514        if let Some(mt) = media_type {
10515            obj.insert("mediaType".to_owned(), Value::from(mt));
10516        }
10517        if let Some(n) = name {
10518            obj.insert("name".to_owned(), Value::from(n));
10519        }
10520        Ok(Value::Object(obj))
10521    }
10522
10523    /// §5.9.7: resolve an owned blob body without driver encoding. A
10524    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
10525    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
10526    pub fn fetch_blob_bytes(
10527        &mut self,
10528        transport: &mut dyn Transport,
10529        blob_id_or_ref: &str,
10530    ) -> Result<FetchedBlob, (String, String)> {
10531        let simple = |m: String| ("client.failed".to_owned(), m);
10532        let blob_id = if blob_id_or_ref.starts_with("sha256:") {
10533            blob_id_or_ref.to_owned()
10534        } else {
10535            let value: Value = serde_json::from_str(blob_id_or_ref)
10536                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
10537            value
10538                .get("blobId")
10539                .and_then(Value::as_str)
10540                .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
10541                .to_owned()
10542        };
10543        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
10544            return Ok(cached);
10545        }
10546        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
10547        // blob.not_found) verbatim so the harness can assert on it. The
10548        // authorized endpoint serves bytes inline OR (always-issue, presign
10549        // configured) a signed url the client fetches directly — no host auth,
10550        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
10551        #[cfg(feature = "bench-internals")]
10552        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
10553        let bytes = match transport
10554            .blob_download(&blob_id)
10555            .map_err(|e| (e.code, e.message))?
10556        {
10557            BlobDownload::Bytes(bytes) => bytes,
10558            BlobDownload::Url {
10559                url,
10560                url_expires_at_ms,
10561            } => {
10562                // §5.9.5: MUST NOT start a fetch at/past expiry.
10563                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
10564                    return Err((
10565                        "sync.segment_expired".to_owned(),
10566                        format!(
10567                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
10568                        ),
10569                    ));
10570                }
10571                transport
10572                    .fetch_blob_url(&url)
10573                    .map_err(|e| (e.code, e.message))?
10574            }
10575        };
10576        #[cfg(feature = "bench-internals")]
10577        drop(download_phase);
10578        #[cfg(feature = "bench-internals")]
10579        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
10580        // §5.9.5 inherits §5.1: verify the content address, reject mismatch.
10581        if blob_id_for(&bytes) != blob_id {
10582            return Err(simple(format!(
10583                "blob content address mismatch for {blob_id}"
10584            )));
10585        }
10586        #[cfg(feature = "bench-internals")]
10587        drop(validate_phase);
10588        #[cfg(feature = "bench-internals")]
10589        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
10590        let now = self.clock_now_ms();
10591        self.conn
10592            .execute(
10593                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
10594                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
10595            )
10596            .map_err(|e| simple(e.to_string()))?;
10597        self.conn
10598            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10599            .map_err(|e| simple(e.to_string()))?;
10600        self.enforce_blob_cache_cap();
10601        #[cfg(feature = "bench-internals")]
10602        drop(insert_phase);
10603        self.get_cached_blob_bytes(&blob_id)
10604            .map_err(simple)?
10605            .ok_or_else(|| simple("blob cache write failed".to_owned()))
10606    }
10607
10608    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
10609        #[cfg(feature = "bench-internals")]
10610        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
10611        let mut stmt = self
10612            .conn
10613            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
10614            .map_err(|e| e.to_string())?;
10615        let mut rows = stmt
10616            .query(rusqlite::params![blob_id])
10617            .map_err(|e| e.to_string())?;
10618        if let Some(row) = rows.next().map_err(|e| e.to_string())? {
10619            let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
10620            let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
10621            let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
10622            return Ok(Some(FetchedBlob {
10623                blob_id: blob_id.to_owned(),
10624                byte_length,
10625                bytes,
10626                media_type,
10627            }));
10628        }
10629        Ok(None)
10630    }
10631
10632    /// §5.9.7 B4: upload every queued blob before push.
10633    fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
10634        let pending: Vec<(String, Option<String>)> = {
10635            let mut stmt = self
10636                .conn
10637                .prepare(
10638                    "SELECT blob_id, media_type FROM (
10639                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
10640                      UNION ALL
10641                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
10642                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
10643                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
10644                    ) ORDER BY created_at_ms, blob_id",
10645                )
10646                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10647            let rows = stmt
10648                .query_map([], |row| {
10649                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
10650                })
10651                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10652            rows.collect::<Result<_, _>>()
10653                .map_err(|error| match error {
10654                    rusqlite::Error::InvalidColumnType(..)
10655                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
10656                        "sync.local_corrupt",
10657                        "Pending blob upload metadata is invalid",
10658                    ),
10659                    _ => TransportError::new("client.failed", error.to_string()),
10660                })?
10661        };
10662        for (blob_id, media_type) in pending {
10663            let stored: Option<(SqlValue, SqlValue)> = self
10664                .conn
10665                .query_row(
10666                    "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
10667                    rusqlite::params![blob_id],
10668                    |row| Ok((row.get(0)?, row.get(1)?)),
10669                )
10670                .optional()
10671                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10672            let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
10673                return Err(TransportError::new(
10674                    "sync.local_corrupt",
10675                    "Pending blob upload body is missing or corrupt",
10676                ));
10677            };
10678            if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
10679                return Err(TransportError::new(
10680                    "sync.local_corrupt",
10681                    "Pending blob upload body is missing or corrupt",
10682                ));
10683            }
10684            self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
10685            self.conn
10686                .execute(
10687                    "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
10688                    rusqlite::params![blob_id],
10689                )
10690                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10691        }
10692        Ok(())
10693    }
10694
10695    /// §5.9.3: upload one blob, preferring the presigned direct-to-storage
10696    /// grant when the transport supports it, else streaming through the direct
10697    /// host-authenticated endpoint (capability, not fallback). A `Url` grant
10698    /// PUTs direct with no host auth; on a grant PUT failure the client streams
10699    /// through the direct endpoint — a *different, host-authenticated
10700    /// capability*, not a fall-through of the grant's authority.
10701    fn upload_one(
10702        &self,
10703        transport: &mut dyn Transport,
10704        blob_id: &str,
10705        bytes: &[u8],
10706        media_type: Option<&str>,
10707    ) -> Result<(), TransportError> {
10708        match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
10709            BlobUploadGrant::Present => return Ok(()), // idempotent §5.9.3
10710            BlobUploadGrant::Url {
10711                url,
10712                url_expires_at_ms,
10713            } => {
10714                let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
10715                if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
10716                    return Ok(());
10717                }
10718                // Failed/expired grant PUT — stream through the direct endpoint.
10719            }
10720            BlobUploadGrant::None => {
10721                // No presign store — stream through the direct endpoint.
10722            }
10723        }
10724        transport.blob_upload(blob_id, bytes, media_type)
10725    }
10726
10727    fn visible_blob_ids_sql(&self) -> String {
10728        let mut selects = Vec::new();
10729        for table in &self.schema.tables {
10730            for column in table
10731                .columns
10732                .iter()
10733                .filter(|column| column.ty == ColumnType::BlobRef)
10734            {
10735                let identifier = quote_ident(&column.name);
10736                selects.push(format!(
10737                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
10738                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
10739                     AND json_type({identifier}, '$.blobId') = 'text'",
10740                    quote_ident(&table.name),
10741                ));
10742            }
10743        }
10744        if selects.is_empty() {
10745            "SELECT NULL AS blob_id WHERE 0".to_owned()
10746        } else {
10747            selects.join(" UNION ")
10748        }
10749    }
10750
10751    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
10752    /// staged uploads pin bodies directly; no derived refcount is stored.
10753    fn enforce_blob_cache_cap(&self) {
10754        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
10755            return;
10756        };
10757        let mut total: i64 = self
10758            .conn
10759            .query_row(
10760                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
10761                [],
10762                |row| row.get(0),
10763            )
10764            .unwrap_or(0);
10765        if total <= max_bytes {
10766            return;
10767        }
10768        let query = format!(
10769            "SELECT blob_id, byte_length FROM _syncular_blobs
10770             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
10771               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
10772               AND blob_id NOT IN ({})
10773             ORDER BY created_at_ms ASC, blob_id ASC",
10774            self.visible_blob_ids_sql(),
10775        );
10776        let candidates: Vec<(String, i64)> = {
10777            let Ok(mut stmt) = self.conn.prepare(&query) else {
10778                return;
10779            };
10780            let Ok(rows) = stmt.query_map([], |row| {
10781                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
10782            }) else {
10783                return;
10784            };
10785            rows.filter_map(Result::ok).collect()
10786        };
10787        for (blob_id, byte_length) in candidates {
10788            if total <= max_bytes {
10789                break;
10790            }
10791            let _ = self.conn.execute(
10792                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
10793                rusqlite::params![blob_id],
10794            );
10795            total -= byte_length;
10796        }
10797    }
10798
10799    fn delete_unreferenced_cached_blobs(&mut self) {
10800        #[cfg(feature = "bench-internals")]
10801        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
10802        if !self.schema_has_blobs() {
10803            return;
10804        }
10805        self.rebuild_overlay_if_dirty();
10806        let _ = self.conn.execute(
10807            &format!(
10808                "DELETE FROM _syncular_blobs
10809                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
10810                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
10811                   AND blob_id NOT IN ({})",
10812                self.visible_blob_ids_sql(),
10813            ),
10814            [],
10815        );
10816    }
10817
10818    // -- local row storage ----------------------------------------------------------
10819
10820    /// The cached per-table primary-key upsert SQL for `full_table` (see
10821    /// the `insert_sql` field: built once, reused per row).
10822    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
10823        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
10824            return sql.clone();
10825        }
10826        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
10827        columns.push(quote_ident("_syncular_version"));
10828        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
10829        let primary_key = quote_ident(&table.primary_key);
10830        let updates = columns
10831            .iter()
10832            .filter(|column| **column != primary_key)
10833            .map(|column| format!("{column}=excluded.{column}"))
10834            .collect::<Vec<_>>()
10835            .join(", ");
10836        let sql = format!(
10837            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
10838            columns.join(", "),
10839            placeholders.join(", ")
10840        );
10841        self.insert_sql
10842            .borrow_mut()
10843            .insert(full_table.to_owned(), sql.clone());
10844        sql
10845    }
10846
10847    fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
10848        self.overlay_dirty.set(true);
10849        self.write_row(&base_table(table_name), table_name, row, version)
10850    }
10851
10852    fn write_row(
10853        &self,
10854        full_table: &str,
10855        table_name: &str,
10856        row: &Row,
10857        version: i64,
10858    ) -> Result<(), String> {
10859        #[cfg(feature = "bench-internals")]
10860        let _phase = self.benchmark_phases.start(Phase::RowWrite);
10861        let table = self
10862            .schema
10863            .table(table_name)
10864            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
10865        let sql = self.insert_row_sql(full_table, table);
10866        let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
10867        let params = row
10868            .iter()
10869            .map(RowParam::Cell)
10870            .chain(std::iter::once(RowParam::Version(version)));
10871        stmt.execute(rusqlite::params_from_iter(params))
10872            .map_err(|e| e.to_string())?;
10873        Ok(())
10874    }
10875
10876    fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
10877        let table = self
10878            .schema
10879            .table(table_name)
10880            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
10881        self.overlay_dirty.set(true);
10882        let sql = format!(
10883            "DELETE FROM {} WHERE {}",
10884            base_table(table_name),
10885            row_id_predicate(table)
10886        );
10887        let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
10888        stmt.execute(rusqlite::params![row_id])
10889            .map_err(|e| e.to_string())?;
10890        Ok(())
10891    }
10892
10893    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
10894    /// the outbox changed since the last rebuild (the no-op sync round).
10895    fn rebuild_overlay_if_dirty(&mut self) {
10896        if self.overlay_dirty.get() {
10897            self.rebuild_overlay();
10898        }
10899    }
10900
10901    /// §7.1: local reads see outbox state applied optimistically — rebuild
10902    /// every visible table as (base server state) + (pending outbox replay
10903    /// on top). Optimistic rows carry version `-1`.
10904    fn rebuild_overlay(&mut self) {
10905        #[cfg(feature = "bench-internals")]
10906        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
10907        #[cfg(test)]
10908        self.overlay_rebuild_count
10909            .set(self.overlay_rebuild_count.get() + 1);
10910        self.exec("SAVEPOINT syncular_overlay");
10911        for table in self.schema.tables.clone() {
10912            for index in &table.fts_indexes {
10913                let _ = self.drop_fts_triggers(index);
10914            }
10915            let visible = visible_table(&table.name);
10916            let base = base_table(&table.name);
10917            self.exec(&format!("DELETE FROM {visible}"));
10918            self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
10919        }
10920        for commit in &self.outbox {
10921            self.apply_outbox_ops(&commit.ops);
10922        }
10923        for table in self.schema.tables.clone() {
10924            for index in &table.fts_indexes {
10925                let _ = self.rebuild_fts_projection(&table, index);
10926                let _ = self.create_fts_triggers(&table, index);
10927            }
10928        }
10929        self.exec("RELEASE syncular_overlay");
10930        self.overlay_dirty.set(false);
10931    }
10932
10933    fn apply_outbox_ops<'a>(&self, ops: impl IntoIterator<Item = &'a OutboxOp>) {
10934        #[cfg(feature = "bench-internals")]
10935        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
10936        for op in ops {
10937            #[cfg(feature = "bench-internals")]
10938            if let Some(phase) = &mut phase {
10939                phase.unit();
10940            }
10941            let Some(table) = self.schema.table(&op.table) else {
10942                continue;
10943            };
10944            if op.upsert {
10945                let Some(values) = op.values.as_ref() else {
10946                    continue;
10947                };
10948                // §7.1: the overlay applies the operation's PRESENT columns
10949                // over the current local row. A partial operation over an
10950                // absent local row leaves it absent — the server answers it
10951                // with sync.row_deleted or sync.row_missing (§6.2).
10952                let full = table
10953                    .columns
10954                    .iter()
10955                    .all(|column| values.contains_key(&column.name));
10956                let local = self.visible_row(table, &op.row_id);
10957                if local.is_none() && !full {
10958                    continue;
10959                }
10960                let mut row: Row = Vec::with_capacity(table.columns.len());
10961                let mut ok = true;
10962                for (index, column) in table.columns.iter().enumerate() {
10963                    let incoming = match values.get(&column.name) {
10964                        Some(raw) => json_to_column_value(column, Some(raw)),
10965                        None => Ok(local
10966                            .as_ref()
10967                            .and_then(|(values, _)| values.get(index).cloned())
10968                            .flatten()),
10969                    };
10970                    match incoming {
10971                        Ok(value) => row.push(value),
10972                        Err(_) => {
10973                            ok = false;
10974                            break;
10975                        }
10976                    }
10977                }
10978                if ok {
10979                    let version = local.as_ref().map_or(-1, |(_, version)| *version);
10980                    let _ = self.write_row(&visible_table(&table.name), &table.name, &row, version);
10981                }
10982            } else {
10983                let sql = format!(
10984                    "DELETE FROM {} WHERE {}",
10985                    visible_table(&table.name),
10986                    row_id_predicate(table)
10987                );
10988                let _ = self
10989                    .conn
10990                    .prepare_cached(&sql)
10991                    .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]));
10992            }
10993        }
10994    }
10995
10996    /// The visible (optimistic) row's values plus version, or `None` when the
10997    /// row is absent from the overlay.
10998    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Option<(Row, i64)> {
10999        let sql = format!(
11000            "SELECT * FROM {} WHERE {}",
11001            visible_table(&table.name),
11002            row_id_predicate(table)
11003        );
11004        let mut stmt = self.conn.prepare_cached(&sql).ok()?;
11005        let mut rows = stmt.query(rusqlite::params![row_id]).ok()?;
11006        let row = rows.next().ok()??;
11007        let mut values = Vec::with_capacity(table.columns.len());
11008        for (index, column) in table.columns.iter().enumerate() {
11009            let raw = row.get_ref(index).ok()?;
11010            let json = sql_ref_to_json(column, raw);
11011            values.push(json_to_column_value(column, Some(&json)).ok()?);
11012        }
11013        let version: i64 = row.get(table.columns.len()).ok()?;
11014        Some((values, version))
11015    }
11016
11017    fn exec(&self, sql: &str) {
11018        let _ = self.conn.execute_batch(sql);
11019    }
11020
11021    // -- realtime (§8) ---------------------------------------------------------------
11022
11023    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
11024        if self.realtime_connected {
11025            return Ok(());
11026        }
11027        transport
11028            .realtime_connect_for_client(&self.client_id)
11029            .map_err(|e| format!("{}: {}", e.code, e.message))?;
11030        self.realtime_connected = true;
11031        Ok(())
11032    }
11033
11034    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
11035        if !self.realtime_connected {
11036            return;
11037        }
11038        let _ = transport.realtime_close();
11039        self.realtime_connected = false;
11040        self.presence.clear(); // §8.6.1: presence is per-connection
11041    }
11042
11043    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
11044    /// document for `scope_key`. Requires a live socket; the document is
11045    /// ephemeral (lost on disconnect). Authorization is the connection's
11046    /// registration (§8.6.3) — an unheld key is rejected loudly by the
11047    /// server with `presence.forbidden`.
11048    pub fn set_presence(
11049        &mut self,
11050        transport: &mut dyn Transport,
11051        scope_key: &str,
11052        doc: Option<&Value>,
11053    ) -> Result<(), String> {
11054        if !self.realtime_connected {
11055            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
11056        }
11057        let text = encode_presence_publish(scope_key, doc);
11058        transport
11059            .realtime_send(&text)
11060            .map_err(|e| format!("{}: {}", e.code, e.message))
11061    }
11062
11063    /// §8.6: the peers currently present on a scope key (ephemeral).
11064    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
11065        self.presence
11066            .get(scope_key)
11067            .map(|peers| peers.values().cloned().collect())
11068            .unwrap_or_default()
11069    }
11070
11071    /// §8.6 apply an inbound presence fanout to the local map.
11072    fn apply_presence(
11073        &mut self,
11074        scope_key: String,
11075        kind: Option<PresenceKind>,
11076        actor_id: Option<String>,
11077        client_id: Option<String>,
11078        doc: Option<Value>,
11079        error: Option<String>,
11080    ) {
11081        // The publisher-directed error variant is out-of-band; nothing to
11082        // record in the peer map.
11083        if error.is_some() {
11084            return;
11085        }
11086        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
11087            return;
11088        };
11089        let peer_key = format!("{actor_id} {client_id}");
11090        match kind {
11091            PresenceKind::Leave => {
11092                if let Some(peers) = self.presence.get_mut(&scope_key) {
11093                    peers.remove(&peer_key);
11094                    if peers.is_empty() {
11095                        self.presence.remove(&scope_key);
11096                    }
11097                }
11098            }
11099            _ => {
11100                let doc = match doc {
11101                    Some(Value::Object(_)) => doc.unwrap(),
11102                    _ => return,
11103                };
11104                self.presence.entry(scope_key).or_default().insert(
11105                    peer_key,
11106                    PresencePeer {
11107                        actor_id,
11108                        client_id,
11109                        doc,
11110                    },
11111                );
11112            }
11113        }
11114    }
11115
11116    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
11117    pub fn on_realtime_text(&mut self, text: &str) {
11118        match parse_control(text) {
11119            Ok(ControlMessage::Hello { requires_sync, .. }) => {
11120                if requires_sync {
11121                    // §8.1: pull before trusting the socket for continuity.
11122                    self.set_sync_needed(true, true);
11123                }
11124            }
11125            Ok(ControlMessage::Presence {
11126                scope_key,
11127                kind,
11128                actor_id,
11129                client_id,
11130                doc,
11131                error,
11132                ..
11133            }) => {
11134                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
11135            }
11136            Ok(ControlMessage::Wake { .. }) => {
11137                // §8.3: any wake-up means "run a pull soon", never data.
11138                self.set_sync_needed(true, true);
11139            }
11140            _ => {}
11141        }
11142    }
11143
11144    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
11145    /// a pull response per section; an unapplied delta is a wake-up.
11146    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
11147        if self.stopped {
11148            return;
11149        }
11150        #[cfg(feature = "bench-internals")]
11151        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
11152        let message = match decode_message(bytes) {
11153            Ok(m) if m.msg_kind == MsgKind::Response => m,
11154            _ => {
11155                self.set_sync_needed(true, true);
11156                return;
11157            }
11158        };
11159        #[cfg(feature = "bench-internals")]
11160        drop(decode_phase);
11161        let mut frames = message.frames.into_iter();
11162        let mut applied_cursor: Option<i64> = None;
11163        let mut any_covered = false;
11164        let mut dropped = false;
11165        while let Some(frame) = frames.next() {
11166            let Frame::SubStart {
11167                id,
11168                status,
11169                effective_scopes,
11170                ..
11171            } = frame
11172            else {
11173                continue;
11174            };
11175            let mut body = Vec::new();
11176            let mut next_cursor: Option<i64> = None;
11177            for inner in frames.by_ref() {
11178                match inner {
11179                    Frame::SubEnd {
11180                        next_cursor: nc, ..
11181                    } => {
11182                        next_cursor = Some(nc);
11183                        break;
11184                    }
11185                    Frame::Unknown { .. } => {}
11186                    other => body.push(other),
11187                }
11188            }
11189            let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
11190                dropped = true;
11191                continue;
11192            };
11193            let sub = &self.subs[sub_index];
11194            // §8.2: only locally active, not mid-bootstrap subscriptions
11195            // apply; skipped sections are repaired by the next pull.
11196            if status != SubStatus::Active
11197                || sub.state != SubState::Active
11198                || sub.bootstrap_state.is_some()
11199                || !sub.synced_once
11200            {
11201                dropped = true;
11202                continue;
11203            }
11204            if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
11205                // Idempotent redelivery of an already-covered window.
11206                any_covered = true;
11207                continue;
11208            }
11209            let previous_effective = self.subs[sub_index].effective.clone();
11210            let previous_cursor = self.subs[sub_index].cursor;
11211            self.subs[sub_index].effective = Some(effective_scopes);
11212            let mut failed = false;
11213            for inner in body {
11214                match inner {
11215                    Frame::Commit {
11216                        tables, changes, ..
11217                    } => {
11218                        if self.apply_commit_frame(&tables, &changes).is_err() {
11219                            failed = true;
11220                            break;
11221                        }
11222                    }
11223                    _ => {
11224                        failed = true;
11225                        break;
11226                    }
11227                }
11228            }
11229            let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
11230                self.apply_sub_end(sub_index, cursor, None)
11231                    .ok()
11232                    .map(|()| cursor)
11233            });
11234            let Some(next_cursor) = committed else {
11235                self.subs[sub_index].effective = previous_effective;
11236                self.subs[sub_index].cursor = previous_cursor;
11237                dropped = true;
11238                continue;
11239            };
11240            applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
11241        }
11242        if let Some(cursor) = applied_cursor {
11243            // §8.2 ack point: the highest applied SUB_END.nextCursor.
11244            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
11245            let _ = transport.realtime_send(&ack);
11246        } else if !any_covered || dropped {
11247            // §8.2: a delta not applied at all is treated as a wake-up.
11248            self.set_sync_needed(true, true);
11249        }
11250    }
11251
11252    /// §8.2 ack point after an HTTP pull on a live connection: the minimum
11253    /// cursor across active, non-bootstrapping subscriptions that have
11254    /// synced at least once. No such subscription, no ack.
11255    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
11256        if !self.realtime_connected {
11257            return;
11258        }
11259        let floor = self
11260            .subs
11261            .iter()
11262            .filter(|s| {
11263                s.state == SubState::Active
11264                    && s.bootstrap_state.is_none()
11265                    && s.synced_once
11266                    && s.cursor >= 0
11267            })
11268            .map(|s| s.cursor)
11269            .min();
11270        if let Some(cursor) = floor {
11271            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
11272            let _ = transport.realtime_send(&ack);
11273        }
11274    }
11275}