1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12#[cfg(test)]
13use std::cell::Cell;
14use std::cell::RefCell;
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
16
17use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
18use rusqlite::{Connection, OpenFlags, OptionalExtension};
19use serde::{Deserialize, Serialize};
20use serde_json::{json, Map, Value};
21use sha2::{Digest, Sha256};
22use ssp2::decode::WIRE_VERSION;
23use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
24use ssp2::primitives::RawJson;
25use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
26use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
27
28use crate::api::{
29 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
30 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
31 ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects, CommitDelivery,
32 CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
33 CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
34 DiagnosticLastChange, DiagnosticLastRound, DiagnosticQueryFailure, DiagnosticRoundCounters,
35 DiagnosticSubscription, FetchedBlob, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
36 LocalDataPurgeTarget, LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation,
37 PresencePeer, PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot,
38 QueryValue, RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
39 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SubscriptionCatchup,
40 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
41 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
42 CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
43 MAX_DIAGNOSTIC_QUERY_FAILURES,
44};
45use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
46#[cfg(feature = "bench-internals")]
47use crate::bench::{Phase, Recorder};
48use crate::previous_version::{
49 capture_previous_version_from_replica, first_incompatibility,
50 reconcile_previous_version_at_boot, set_local_schema_descriptor,
51 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
52 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
53};
54use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
55use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
56use crate::values::{
57 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
58 encode_sparse_row_json, full_row_values, is_canonical_bytes_envelope, json_to_column_value,
59 json_to_scope_map, normalize_values_casing, render_row_id_json, scope_map_to_json,
60 sort_scope_map,
61};
62
63const DEFAULT_ACCEPT: u8 = 0b0111;
67const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
68const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
69const ACCEPT_SQLITE: u8 = 1 << 2;
70const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
71const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
72
73pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
75const LOCAL_REVISION_KEY: &str = "localRevision";
76const CLIENT_ID_KEY: &str = "clientId";
77const LEASE_STATE_KEY: &str = "leaseState";
78const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
79const LOG_EPOCH_KEY: &str = "logEpoch";
80const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
85const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
86const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
87const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
90const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
93const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
94const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
96 "the persisted commit carries values the current codec refuses";
97const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
99
100#[derive(Debug, Serialize, Deserialize)]
101#[serde(rename_all = "camelCase", deny_unknown_fields)]
102struct PersistedLocalDataRebootstrapReceipt {
103 version: u8,
104 retained_commits: u64,
105 reset_subscriptions: u64,
106}
107
108fn invalid_local_rebootstrap_receipt() -> String {
109 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
110}
111
112fn encode_local_rebootstrap_receipt(
113 retained_commits: usize,
114 reset_subscriptions: usize,
115) -> Result<String, String> {
116 let retained_commits =
117 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
118 let reset_subscriptions =
119 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
120 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
121 return Err(invalid_local_rebootstrap_receipt());
122 }
123 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
124 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
125 retained_commits,
126 reset_subscriptions,
127 })
128 .map_err(|_| invalid_local_rebootstrap_receipt())
129}
130
131fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
132 if value == "v1" {
134 return Ok((0, 0));
135 }
136 let receipt: PersistedLocalDataRebootstrapReceipt =
137 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
138 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
139 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
140 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
141 {
142 return Err(invalid_local_rebootstrap_receipt());
143 }
144 Ok((
145 usize::try_from(receipt.retained_commits)
146 .map_err(|_| invalid_local_rebootstrap_receipt())?,
147 usize::try_from(receipt.reset_subscriptions)
148 .map_err(|_| invalid_local_rebootstrap_receipt())?,
149 ))
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153enum SubState {
154 Active,
155 Revoked,
156 Failed,
157}
158
159#[cfg(test)]
160mod observation_tests {
161 use super::*;
162 use crate::native_transport::HostTransport;
163 use crate::values::encode_row_json;
164 use serde_json::json;
165
166 #[test]
167 fn authority_policy_refuses_encrypted_and_undeclared_columns_before_open() {
168 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
169 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "secret", "type": "bytes", "encrypted": true, "declaredType": "string", "nullable": true}
170 ]}]});
171 for name in [
172 "secret",
173 "missing",
174 "*",
175 "id FROM authority; DELETE FROM authority",
176 "_syncular_version",
177 ] {
178 let limits = ClientLimits {
179 authority_reads: vec![crate::api::AuthorityReadDeclaration {
180 table: "authority".into(),
181 columns: vec!["id".into(), "actor_id".into(), name.into()],
182 scopes: [("actor_id".into(), vec!["a".into()])].into(),
183 }],
184 ..Default::default()
185 };
186 let error = SyncClient::new("authority".into(), &schema, limits)
187 .err()
188 .expect("forbidden policy");
189 assert!(
190 error.starts_with("client.authority_read_forbidden:"),
191 "{error}"
192 );
193 }
194 }
195
196 #[test]
197 fn authority_snapshot_observes_one_file_transaction_during_concurrent_writes() {
198 let path = std::env::temp_dir().join(format!(
199 "syncular-authority-snapshot-{}.db",
200 uuid::Uuid::new_v4()
201 ));
202 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
203 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "revision", "type": "integer", "nullable": false}
204 ]}]});
205 let limits = ClientLimits {
206 authority_reads: vec![crate::api::AuthorityReadDeclaration {
207 table: "authority".into(),
208 columns: vec!["id".into(), "actor_id".into(), "revision".into()],
209 scopes: [("actor_id".into(), vec!["a".into()])].into(),
210 }],
211 ..Default::default()
212 };
213 let mut client =
214 SyncClient::open_path("authority".into(), &schema, limits, path.to_str().unwrap())
215 .unwrap();
216 client.begin_security_preflight();
217 let update = |conn: &Connection, revision: i64| {
218 conn.execute_batch("BEGIN IMMEDIATE").unwrap();
219 conn.execute(
220 "INSERT OR REPLACE INTO _syncular_base_authority VALUES('one','a',?1,1)",
221 [revision],
222 )
223 .unwrap();
224 conn.execute(
225 "UPDATE _syncular_meta SET value=?1 WHERE key='localRevision'",
226 [revision.to_string()],
227 )
228 .unwrap();
229 conn.execute("INSERT OR REPLACE INTO _syncular_subscriptions VALUES('s','authority',?1)", [json!({"requested": {"actor_id": ["a"]}, "effectiveScopes": {"actor_id": ["a"]}, "status": "active", "cursor": revision}).to_string()]).unwrap();
230 conn.execute_batch("COMMIT").unwrap();
231 };
232 update(&client.conn, 1);
233 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
234 let writer_barrier = barrier.clone();
235 let writer_path = path.clone();
236 let writer = std::thread::spawn(move || {
237 let conn = Connection::open(writer_path).unwrap();
238 conn.busy_timeout(std::time::Duration::from_secs(5))
239 .unwrap();
240 writer_barrier.wait();
241 for revision in 2..=100 {
242 update(&conn, revision);
243 }
244 });
245 barrier.wait();
246 for _ in 0..100 {
247 let snapshot = client.authority_snapshot().unwrap();
248 let revision = snapshot["revision"]
249 .as_str()
250 .unwrap()
251 .parse::<i64>()
252 .unwrap();
253 assert_eq!(
254 snapshot["tables"][0]["rows"][0]["values"]["revision"],
255 revision
256 );
257 assert_eq!(snapshot["tables"][0]["persisted"][0]["cursor"], revision);
258 assert_eq!(snapshot["complete"], true);
259 }
260 writer.join().unwrap();
261 assert_eq!(client.authority_snapshot().unwrap()["revision"], "100");
262 drop(client);
263 let _ = std::fs::remove_file(path);
264 }
265
266 fn client() -> SyncClient {
267 SyncClient::new(
268 "retry-test".to_owned(),
269 &json!({
270 "version": 1,
271 "tables": [{
272 "name": "tasks",
273 "primaryKey": "id",
274 "columns": [
275 { "name": "id", "type": "string", "nullable": false },
276 { "name": "project_id", "type": "string", "nullable": false }
277 ],
278 "scopes": [{ "pattern": "project:{project_id}" }]
279 }]
280 }),
281 ClientLimits::default(),
282 )
283 .expect("test client")
284 }
285
286 #[cfg(feature = "e2ee")]
287 #[test]
288 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
289 let schema = json!({
290 "version": 1,
291 "tables": [{
292 "name": "secrets",
293 "primaryKey": "id",
294 "columns": [
295 { "name": "id", "type": "string", "nullable": false },
296 { "name": "encryption_key_id", "type": "string", "nullable": true },
297 { "name": "note", "type": "bytes", "nullable": true,
298 "encrypted": true, "declaredType": "string" }
299 ],
300 "scopes": []
301 }]
302 });
303 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
304 .expect("test client");
305 let mut config = crate::values::EncryptionConfig::default();
306 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
307 config
308 .key_id_columns
309 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
310 client.set_encryption(config);
311 client
314 .write_base_row(
315 "secrets",
316 &vec![Some(ColumnValue::String("ghost".into())), None, None],
317 1,
318 )
319 .unwrap();
320 client.rebuild_overlay().expect("rebuild overlay");
321 let commit_id = client
322 .patch(
323 "secrets",
324 "ghost",
325 Map::from_iter([("note".to_owned(), json!("x"))]),
326 None,
327 )
328 .expect("patch records without an author-time encrypt failure");
329 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
330 assert!(client.drop_unencodable_outbox(true, None).unwrap());
332 assert!(client.pending_commit_ids().is_empty());
333 let rejections = client.rejections();
334 assert_eq!(rejections.len(), 1);
335 assert_eq!(rejections[0].code, "client.encrypt_failed");
336 assert_eq!(rejections[0].client_commit_id, commit_id);
337 assert_eq!(
338 client
339 .commit_outcome(&commit_id)
340 .unwrap()
341 .map(|outcome| outcome.status),
342 Some(CommitOutcomeStatus::Rejected)
343 );
344 }
345
346 #[test]
347 fn patch_scope_read_preserves_sqlite_failure() {
348 let mut client = client();
349 client
350 .conn
351 .execute_batch(
352 "DROP TABLE tasks;
353 CREATE VIEW tasks AS SELECT 't1' AS id, abs(-9223372036854775808) AS project_id;",
354 )
355 .unwrap();
356 let error = client
357 .patch(
358 "tasks",
359 "t1",
360 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
361 None,
362 )
363 .expect_err("the scope read must fail");
364 assert_ne!(error.code, "sync.invalid_request");
365 assert_eq!(error.code, "client.failed");
366 assert!(error.details.as_ref().unwrap()["legacyCause"]
367 .as_str()
368 .unwrap()
369 .contains("integer overflow"));
370 assert!(client.pending_commit_ids().is_empty());
371 assert!(client.conn.is_autocommit());
372 }
373
374 #[test]
375 fn patch_with_a_scope_column_matches_the_stored_local_row() {
376 let mut client = client();
377 client.create_synced_tables().unwrap();
378 client
379 .conn
380 .execute(
381 "INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
382 [],
383 )
384 .unwrap();
385
386 client
389 .patch(
390 "tasks",
391 "t1",
392 Map::from_iter([
393 ("project_id".to_owned(), json!("p1")),
394 ("id".to_owned(), json!("t1")),
395 ]),
396 None,
397 )
398 .expect("an equal scope column is a no-op");
399 let values = client.outbox[0].ops[0]
400 .values
401 .clone()
402 .expect("the patch records values");
403 assert_eq!(values.get("id"), Some(&json!("t1")));
404 assert!(!values.contains_key("project_id"));
405
406 let differing = client.patch(
408 "tasks",
409 "t1",
410 Map::from_iter([("project_id".to_owned(), json!("p2"))]),
411 None,
412 );
413 let differing = differing.expect_err("a differing scope column is rejected");
414 assert_eq!(differing.code, "sync.invalid_request");
415 assert_eq!(differing.message, "the authoring request is invalid");
416 assert!(
417 differing
418 .details
419 .as_ref()
420 .and_then(|details| details["legacyCause"].as_str())
421 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
422 "{differing:?}"
423 );
424
425 let absent = client.patch(
428 "tasks",
429 "ghost",
430 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
431 None,
432 );
433 let absent = absent.expect_err("an absent local row is rejected");
434 assert_eq!(absent.code, "sync.row_missing");
435 assert!(absent.message.contains("requires a local row"));
436 assert_eq!(client.pending_commit_ids().len(), 1);
437 }
438
439 #[test]
440 fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
441 let schema = json!({
442 "version": 1,
443 "tables": [
444 {
445 "name": "tenants",
446 "primaryKey": "tenant_id",
447 "columns": [
448 { "name": "tenant_id", "type": "string", "nullable": false },
449 { "name": "body", "type": "string", "nullable": false }
450 ],
451 "scopes": [{ "pattern": "tenant:{tenant_id}" }]
452 },
453 {
454 "name": "buckets",
455 "primaryKey": "id",
456 "columns": [
457 { "name": "id", "type": "string", "nullable": false },
458 { "name": "bucket", "type": "float", "nullable": false },
459 { "name": "body", "type": "string", "nullable": false }
460 ],
461 "scopes": [{ "pattern": "bucket:{bucket}" }]
462 }
463 ]
464 });
465 let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
466 .expect("test client");
467 client.create_synced_tables().unwrap();
468
469 client
471 .conn
472 .execute(
473 "INSERT INTO tenants (tenant_id, body, _syncular_version) VALUES ('t1', 'seed', 1)",
474 [],
475 )
476 .unwrap();
477 client
478 .patch(
479 "tenants",
480 "t1",
481 Map::from_iter([
482 ("tenant_id".to_owned(), json!("t1")),
483 ("body".to_owned(), json!("created")),
484 ]),
485 None,
486 )
487 .expect("the primary-key scope column is proven equal by construction");
488
489 client
491 .conn
492 .execute(
493 "INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
494 [],
495 )
496 .unwrap();
497 client
498 .patch(
499 "buckets",
500 "b1",
501 Map::from_iter([
502 ("bucket".to_owned(), json!(2)),
503 ("body".to_owned(), json!("coerced")),
504 ]),
505 None,
506 )
507 .expect("the stored REAL value coerces from a supplied integer");
508 let values = client.outbox[1].ops[0]
509 .values
510 .clone()
511 .expect("the patch records values");
512 assert!(!values.contains_key("bucket"));
513
514 let differing = client.patch(
516 "buckets",
517 "b1",
518 Map::from_iter([("bucket".to_owned(), json!(3))]),
519 None,
520 );
521 let differing = differing.expect_err("a differing scope value is rejected");
522 assert_eq!(differing.code, "sync.invalid_request");
523 assert_eq!(differing.message, "the authoring request is invalid");
524 assert!(
525 differing
526 .details
527 .as_ref()
528 .and_then(|details| details["legacyCause"].as_str())
529 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
530 "{differing:?}"
531 );
532 }
533
534 #[cfg(feature = "e2ee")]
535 #[test]
536 fn stored_key_fallback_resolves_an_integer_primary_key() {
537 let schema = json!({
538 "version": 1,
539 "tables": [{
540 "name": "patients",
541 "primaryKey": "id",
542 "columns": [
543 { "name": "id", "type": "integer", "nullable": false },
544 { "name": "encryption_key_id", "type": "string", "nullable": true },
545 { "name": "note", "type": "bytes", "nullable": true,
546 "encrypted": true, "declaredType": "string" }
547 ],
548 "scopes": []
549 }]
550 });
551 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
552 .expect("test client");
553 client.create_synced_tables().unwrap();
554 client
555 .conn
556 .execute(
557 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
558 VALUES (5, 'k1', NULL, 1)",
559 [],
560 )
561 .unwrap();
562 let mut config = crate::values::EncryptionConfig::default();
563 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
564 config
565 .key_id_columns
566 .insert("patients".to_owned(), "encryption_key_id".to_owned());
567 client.set_encryption(config);
568 let table = client.schema.table("patients").unwrap();
569 let fallback = client
570 .stored_key_fallback(table, "5")
571 .expect("the stored integer key resolves the fallback");
572 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
573 }
574
575 #[test]
576 fn patch_types_an_integer_primary_key_from_its_canonical_row_id() {
577 let schema = json!({
578 "version": 1,
579 "tables": [{
580 "name": "counters",
581 "primaryKey": "id",
582 "columns": [
583 { "name": "id", "type": "integer", "nullable": false },
584 { "name": "title", "type": "string", "nullable": false }
585 ],
586 "scopes": []
587 }]
588 });
589 let mut client = SyncClient::new("int-patch".into(), &schema, ClientLimits::default())
590 .expect("test client");
591 client
592 .write_base_row(
593 "counters",
594 &vec![
595 Some(ColumnValue::Integer(42)),
596 Some(ColumnValue::String("before".into())),
597 ],
598 1,
599 )
600 .unwrap();
601 client.rebuild_overlay().expect("rebuild overlay");
602 client
603 .patch(
604 "counters",
605 "42",
606 Map::from_iter([
607 ("id".to_owned(), json!(42)),
608 ("title".to_owned(), json!("after")),
609 ]),
610 None,
611 )
612 .expect("an integer key that renders as the rowId is not a change");
613 let rows = client.query("SELECT id, title FROM counters", &[]).unwrap();
614 assert_eq!(Value::from(rows), json!([{"id": 42, "title": "after"}]));
615 for row_id in ["042", "+42", "42.0"] {
616 let error = client
617 .patch(
618 "counters",
619 row_id,
620 Map::from_iter([("title".to_owned(), json!("x"))]),
621 None,
622 )
623 .expect_err("a non-canonical integer rowId names no row");
624 assert_eq!(error.code, "sync.row_missing", "{row_id}");
625 }
626 }
627
628 #[test]
629 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
630 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
631 {"name":"id","type":"string","nullable":false},
632 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
633 for state in ["absent", "cached", "pinned"] {
634 let path = std::env::temp_dir().join(format!(
635 "syncular-blob-stage-{}.sqlite",
636 uuid::Uuid::new_v4()
637 ));
638 let mut client = SyncClient::open_path(
639 "stage-test".into(),
640 &schema,
641 Default::default(),
642 path.to_str().unwrap(),
643 )
644 .unwrap();
645 let bytes = b"atomic stage";
646 if state != "absent" {
647 client
648 .upload_blob(bytes, Some("text/plain".into()), None)
649 .unwrap();
650 if state == "cached" {
651 client
652 .conn
653 .execute("DELETE FROM _syncular_blob_uploads", [])
654 .unwrap();
655 }
656 }
657 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
658 client
659 .conn
660 .execute_batch(
661 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
662 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
663 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
664 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
665 )
666 .unwrap();
667 let error = client
668 .upload_blob(bytes, Some("application/octet-stream".into()), None)
669 .unwrap_err();
670 assert!(error.contains("injected stage failure"), "{state}: {error}");
671 assert!(client.conn.is_autocommit());
672 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
673 drop(client);
674 let mut reopened = SyncClient::open_path(
675 "stage-test".into(),
676 &schema,
677 Default::default(),
678 path.to_str().unwrap(),
679 )
680 .unwrap();
681 let durable = reopened
682 .query("SELECT * FROM _syncular_blobs", &[])
683 .unwrap();
684 reopened
685 .conn
686 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
687 .unwrap();
688 let retried = reopened.upload_blob(bytes, None, None).unwrap();
689 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
690 assert_eq!(
691 reopened
692 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
693 .unwrap(),
694 vec![Map::from_iter([(
695 "blob_id".into(),
696 Value::from(blob_id_for(bytes)),
697 )])]
698 );
699 drop(reopened);
700 std::fs::remove_file(&path).unwrap();
701 assert_eq!(durable, bodies, "{state}: reopened body state");
702 assert_eq!(immediate, bodies, "{state}: immediate body state");
703 }
704 }
705
706 #[test]
707 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
708 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
709 {"name":"id","type":"string","nullable":false},
710 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
711 for fault in [
712 "missing",
713 "body-type",
714 "length",
715 "hash",
716 "pin-metadata",
717 "read",
718 "pin-delete",
719 ] {
720 let path = std::env::temp_dir().join(format!(
721 "syncular-blob-upload-{}.sqlite",
722 uuid::Uuid::new_v4()
723 ));
724 let mut client = SyncClient::open_path(
725 "upload-test".into(),
726 &schema,
727 Default::default(),
728 path.to_str().unwrap(),
729 )
730 .unwrap();
731 let bytes = b"durable upload";
732 let ref_value = client
733 .upload_blob(bytes, Some("text/plain".into()), None)
734 .unwrap();
735 let commit = client
736 .mutate(vec![Mutation::Upsert {
737 table: "attachments".into(),
738 values: Map::from_iter([
739 ("id".into(), Value::from("pending")),
740 (
741 "file".into(),
742 Value::from(format!(
743 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
744 blob_id_for(bytes),
745 bytes.len()
746 )),
747 ),
748 ]),
749 base_version: None,
750 }])
751 .unwrap();
752 client
753 .conn
754 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
755 .unwrap();
756 client.conn.execute_batch(match fault {
757 "missing" => "DELETE FROM _syncular_blobs",
758 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
759 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
760 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
761 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
762 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
763 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
764 }).unwrap();
765 let state = if fault == "read" {
766 Vec::new()
767 } else {
768 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
769 };
770 let mut transport = CountingRealtimeTransport::default();
771 let error = client.flush_blob_uploads(&mut transport).unwrap_err();
772 assert_eq!(
773 error.code,
774 if matches!(fault, "read" | "pin-delete") {
775 "client.failed"
776 } else {
777 "sync.local_corrupt"
778 },
779 "{fault}"
780 );
781 assert_eq!(
782 transport.blob_uploads.len(),
783 usize::from(fault == "pin-delete"),
784 "{fault}: corrupt bytes must never be uploaded"
785 );
786 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
787 if fault != "read" {
788 assert_eq!(
789 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
790 state
791 );
792 } else {
793 client
794 .conn
795 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
796 .unwrap();
797 }
798 drop(client);
799 let mut reopened = SyncClient::open_path(
800 "upload-test".into(),
801 &schema,
802 Default::default(),
803 path.to_str().unwrap(),
804 )
805 .unwrap();
806 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
807 if fault != "read" {
808 assert_eq!(
809 reopened
810 .query("SELECT * FROM _syncular_blobs", &[])
811 .unwrap(),
812 state
813 );
814 }
815 if fault == "pin-delete" {
816 reopened
817 .conn
818 .execute_batch("DROP TRIGGER fail_pin_delete")
819 .unwrap();
820 }
821 if fault == "pin-metadata" {
822 reopened
823 .conn
824 .execute(
825 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
826 [],
827 )
828 .unwrap();
829 }
830 reopened
831 .conn
832 .execute_batch(
833 "DELETE FROM _syncular_blobs;
834 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
835 )
836 .unwrap();
837 transport.blob_uploads.clear();
838 reopened.flush_blob_uploads(&mut transport).unwrap();
839 assert_eq!(
840 transport.blob_uploads,
841 [(blob_id_for(bytes), bytes.to_vec())]
842 );
843 assert_eq!(
844 reopened
845 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
846 .unwrap(),
847 Vec::<Map<String, Value>>::new()
848 );
849 assert_eq!(reopened.pending_commit_ids(), [commit]);
850 assert_eq!(
851 reopened
852 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
853 .unwrap()
854 .unwrap()
855 .bytes,
856 bytes
857 );
858 drop(reopened);
859 std::fs::remove_file(path).unwrap();
860 }
861 }
862
863 fn coalesced_fixture(sections: usize, commits: usize, rows: usize) -> (SyncClient, Message) {
867 let mut client = client();
868 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
869 let mut response = Message {
870 wire_version: WIRE_VERSION,
871 msg_kind: MsgKind::Response,
872 frames: vec![Frame::RespHeader {
873 required_schema_version: None,
874 latest_schema_version: None,
875 log_epoch: Some("epoch-1".into()),
876 reset_required: Some(false),
877 }],
878 };
879 let mut seq = 0;
880 for section in 0..sections {
881 let project = format!("p{section}");
882 let scopes = vec![("project_id".to_owned(), vec![project.clone()])];
883 client
884 .subscribe(format!("s{section}"), "tasks".into(), scopes.clone(), None)
885 .unwrap();
886 let sub = client.subs.last_mut().unwrap();
887 sub.cursor = 0;
888 sub.synced_once = true;
889 sub.effective = Some(scopes.clone());
890 let sub = sub.clone();
891 client.persist_sub(&sub).unwrap();
892 response.frames.push(Frame::SubStart {
893 id: format!("s{section}"),
894 status: SubStatus::Active,
895 reason_code: "".into(),
896 effective_scopes: scopes,
897 bootstrap: false,
898 });
899 for _ in 0..commits {
900 seq += 1;
901 let changes = (0..rows)
902 .map(|row| {
903 let id = format!("{project}-{seq}-{row}");
904 let values = Map::from_iter([
905 ("id".into(), Value::from(id.clone())),
906 ("project_id".into(), Value::from(project.clone())),
907 ]);
908 ssp2::model::Change {
909 table_index: 0,
910 row_id: id.clone(),
911 op: Op::Upsert,
912 row_version: Some(seq),
913 scopes: vec![("project_id".into(), project.clone())],
914 row: Some(
915 encode_row_json(
916 client.schema.table("tasks").unwrap(),
917 &id,
918 &values,
919 &client.encryption,
920 )
921 .unwrap(),
922 ),
923 }
924 })
925 .collect();
926 response.frames.push(Frame::Commit {
927 commit_seq: seq,
928 actor_id: "writer".into(),
929 created_at_ms: 1,
930 tables: vec!["tasks".into()],
931 changes,
932 });
933 }
934 response.frames.push(Frame::SubEnd {
935 next_cursor: seq,
936 bootstrap_state: None,
937 });
938 }
939 client.drain_change_batches();
940 (client, response)
941 }
942
943 fn count_commits(client: &SyncClient) -> std::sync::Arc<std::sync::atomic::AtomicUsize> {
944 let commits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
945 let counter = commits.clone();
946 client
947 .conn
948 .commit_hook(Some(move || {
949 counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
950 false
951 }))
952 .expect("commit hook");
953 commits
954 }
955
956 #[test]
957 fn a_response_of_many_small_sections_commits_once() {
958 for realtime in [false, true] {
959 let (mut client, response) = coalesced_fixture(70, 1, 1);
960 client.realtime_state = RealtimeState::Connected;
961 client.realtime_reason_code = None;
962 let mut transport = CountingRealtimeTransport::default();
963 let (_, meta) = client.build_request(false).unwrap();
964 let commits = count_commits(&client);
965 if realtime {
966 client.on_realtime_binary(&mut transport, &encode_message(&response));
967 } else {
968 assert!(matches!(
969 client.process_response(&mut transport, response, &meta),
970 SyncOutcome::Ok(_)
971 ));
972 }
973 assert_eq!(
974 commits.load(std::sync::atomic::Ordering::SeqCst),
975 1,
976 "{realtime}"
977 );
978 assert!(client.conn.is_autocommit());
979 assert!(client.subs.iter().all(|sub| sub.cursor > 0));
980 let revisions = client
982 .drain_change_batches()
983 .into_iter()
984 .filter(|batch| !batch.tables.is_empty())
985 .map(|batch| batch.revision)
986 .collect::<Vec<_>>();
987 assert_eq!(
988 revisions,
989 (1..=70).map(|r: u64| r.to_string()).collect::<Vec<_>>()
990 );
991 assert_eq!(client.query("SELECT id FROM tasks", &[]).unwrap().len(), 70);
992 assert_eq!(
994 transport
995 .messages
996 .iter()
997 .filter(|m| m.contains("\"ack\""))
998 .count(),
999 1,
1000 "{realtime}"
1001 );
1002 }
1003 }
1004
1005 #[test]
1006 fn a_response_past_the_row_bound_commits_per_bound() {
1007 let (mut client, response) = coalesced_fixture(1, 3, 600);
1009 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1010 let (_, meta) = client.build_request(false).unwrap();
1011 let commits = count_commits(&client);
1012 assert!(matches!(
1013 client.process_response(&mut transport, response, &meta),
1014 SyncOutcome::Ok(_)
1015 ));
1016 assert_eq!(commits.load(std::sync::atomic::Ordering::SeqCst), 3);
1017 assert_eq!(
1018 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1019 1800
1020 );
1021 assert_eq!(client.subs[0].cursor, 3);
1022 }
1023
1024 #[test]
1025 fn a_failed_coalesced_commit_restores_memory_and_publishes_nothing() {
1026 let (mut client, response) = coalesced_fixture(2, 2, 600);
1030 client
1031 .conn
1032 .execute_batch(
1033 "PRAGMA foreign_keys = ON; CREATE TABLE fail_parent(id INTEGER PRIMARY KEY); CREATE TABLE fail_child(id INTEGER REFERENCES fail_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_commit AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'p0-1-0' BEGIN INSERT INTO fail_child VALUES(1); END",
1034 )
1035 .unwrap();
1036 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1037 let (_, meta) = client.build_request(false).unwrap();
1038 let subs = client.subs.clone();
1039 let revision = client.local_revision();
1040 let outcome = client.process_response(&mut transport, response.clone(), &meta);
1041 assert!(
1042 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == "storage.failed"),
1043 "{outcome:?}"
1044 );
1045 assert!(client.conn.is_autocommit());
1046 assert!(client.coalesced.is_none() && client.coalesced_failure.is_none());
1047 assert_eq!(client.local_revision(), revision);
1048 assert!(client
1049 .drain_change_batches()
1050 .iter()
1051 .all(|batch| batch.tables.is_empty()));
1052 assert!(client
1053 .query("SELECT id FROM tasks", &[])
1054 .unwrap()
1055 .is_empty());
1056 assert_eq!(
1057 client.subs.iter().map(|s| s.cursor).collect::<Vec<_>>(),
1058 subs.iter().map(|s| s.cursor).collect::<Vec<_>>()
1059 );
1060 client
1062 .conn
1063 .execute_batch("DROP TRIGGER fail_commit")
1064 .unwrap();
1065 let (_, meta) = client.build_request(false).unwrap();
1066 assert!(matches!(
1067 client.process_response(&mut transport, response, &meta),
1068 SyncOutcome::Ok(_)
1069 ));
1070 assert_eq!(
1071 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1072 2400
1073 );
1074 }
1075
1076 #[test]
1077 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
1078 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1079 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
1080 "scopes":[{"pattern":"project:{project_id}"}]}]});
1081 for realtime in [false, true] {
1082 for fault in [
1083 "none",
1084 "row",
1085 "revision",
1086 "commit",
1087 "cursor",
1088 "error",
1089 "missing-end",
1090 ] {
1091 if realtime && fault == "missing-end" {
1092 continue;
1093 }
1094 let path = std::env::temp_dir()
1095 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
1096 let mut client = SyncClient::open_path(
1097 "frame-test".into(),
1098 &schema,
1099 Default::default(),
1100 path.to_str().unwrap(),
1101 )
1102 .unwrap();
1103 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1104 client
1105 .subscribe(
1106 "tasks".into(),
1107 "tasks".into(),
1108 vec![("project_id".into(), vec!["p1".into()])],
1109 None,
1110 )
1111 .unwrap();
1112 client.subs[0].cursor = 0;
1113 client.subs[0].synced_once = true;
1114 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1115 client.persist_sub(&client.subs[0]).unwrap();
1116 client.realtime_state = RealtimeState::Connected;
1117 client.realtime_reason_code = None;
1118 client.drain_change_batches();
1119 let mut transport = CountingRealtimeTransport::default();
1120 let (_, meta) = client.build_request(false).unwrap();
1121 let mut response = Message {
1122 wire_version: WIRE_VERSION,
1123 msg_kind: MsgKind::Response,
1124 frames: vec![
1125 Frame::RespHeader {
1126 required_schema_version: None,
1127 latest_schema_version: None,
1128 log_epoch: Some("epoch-1".into()),
1129 reset_required: Some(false),
1130 },
1131 Frame::SubStart {
1132 id: "tasks".into(),
1133 status: SubStatus::Active,
1134 reason_code: "".into(),
1135 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1136 bootstrap: false,
1137 },
1138 ],
1139 };
1140 for (index, id) in ["first", "second"].iter().enumerate() {
1141 let values = Map::from_iter([
1142 ("id".into(), Value::from(*id)),
1143 ("project_id".into(), Value::from("p1")),
1144 ]);
1145 response.frames.push(Frame::Commit {
1146 commit_seq: index as i64 + 1,
1147 actor_id: "writer".into(),
1148 created_at_ms: 1,
1149 tables: vec!["tasks".into()],
1150 changes: vec![ssp2::model::Change {
1151 table_index: 0,
1152 row_id: (*id).into(),
1153 op: Op::Upsert,
1154 row_version: Some(index as i64 + 1),
1155 scopes: vec![("project_id".into(), "p1".into())],
1156 row: Some(
1157 encode_row_json(
1158 client.schema.table("tasks").unwrap(),
1159 id,
1160 &values,
1161 &client.encryption,
1162 )
1163 .unwrap(),
1164 ),
1165 }],
1166 });
1167 }
1168 response.frames.push(Frame::SubEnd {
1169 next_cursor: 2,
1170 bootstrap_state: None,
1171 });
1172 let valid = response.clone();
1173 match fault {
1174 "row" => {
1175 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
1176 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
1177 },
1178 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
1179 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
1180 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
1181 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
1182 "missing-end" => {response.frames.truncate(3);},
1183 _ => {}
1184 }
1185 if realtime {
1186 client.on_realtime_binary(&mut transport, &encode_message(&response));
1187 } else {
1188 assert_eq!(
1189 matches!(
1190 client.process_response(&mut transport, response, &meta),
1191 SyncOutcome::Ok(_)
1192 ),
1193 fault == "none",
1194 "{fault}"
1195 );
1196 }
1197 assert!(
1198 client.conn.is_autocommit(),
1199 "no open transaction after {realtime}/{fault}"
1200 );
1201 let count = match fault {
1205 "none" | "cursor" => 2,
1206 "commit" => 0,
1207 _ => 1,
1208 };
1209 let expected_revision = count + u64::from(realtime && fault != "none");
1210 assert_eq!(
1211 client.local_revision(),
1212 expected_revision,
1213 "{realtime}/{fault}"
1214 );
1215 let batches = client.drain_change_batches();
1216 assert_eq!(
1217 batches.len(),
1218 expected_revision as usize,
1219 "{realtime}/{fault}"
1220 );
1221 let row_batches: Vec<_> = batches
1222 .iter()
1223 .filter(|batch| !batch.tables.is_empty())
1224 .collect();
1225 assert_eq!(row_batches.len(), count as usize);
1226 assert_eq!(
1227 row_batches
1228 .iter()
1229 .map(|b| b.revision.clone())
1230 .collect::<Vec<_>>(),
1231 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
1232 );
1233 for table in ["tasks", "_syncular_base_tasks"] {
1234 assert_eq!(
1235 client
1236 .conn
1237 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
1238 .get::<_, u64>(0))
1239 .unwrap(),
1240 count,
1241 "{realtime}/{fault}/{table}"
1242 );
1243 }
1244 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1245 assert_eq!(
1246 transport
1247 .messages
1248 .iter()
1249 .filter(|m| m.contains("\"ack\""))
1250 .count(),
1251 usize::from(fault == "none")
1252 );
1253 if matches!(fault, "revision" | "commit" | "cursor") {
1254 client
1255 .conn
1256 .execute_batch("DROP TRIGGER fail_frame")
1257 .unwrap();
1258 }
1259 drop(client);
1260 let mut reopened = SyncClient::open_path(
1261 "frame-test".into(),
1262 &schema,
1263 Default::default(),
1264 path.to_str().unwrap(),
1265 )
1266 .unwrap();
1267 assert_eq!(reopened.local_revision(), expected_revision);
1268 assert_eq!(
1269 reopened
1270 .query("SELECT id FROM tasks ORDER BY id", &[])
1271 .unwrap()
1272 .len(),
1273 count as usize
1274 );
1275 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1276 let (_, meta) = reopened.build_request(false).unwrap();
1277 assert!(matches!(
1278 reopened.process_response(&mut transport, valid, &meta),
1279 SyncOutcome::Ok(_)
1280 ));
1281 assert_eq!(
1282 reopened
1283 .query("SELECT id FROM tasks ORDER BY id", &[])
1284 .unwrap()
1285 .len(),
1286 2
1287 );
1288 assert_eq!(reopened.subs[0].cursor, 2);
1289 drop(reopened);
1290 std::fs::remove_file(path).unwrap();
1291 }
1292 }
1293 }
1294
1295 #[test]
1296 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
1297 for status in [SubStatus::Reset, SubStatus::Revoked] {
1298 for fault in ["subscription", "revision"] {
1299 let mut client = client();
1300 client
1301 .subscribe(
1302 "tasks".into(),
1303 "tasks".into(),
1304 vec![("project_id".into(), vec!["p1".into()])],
1305 None,
1306 )
1307 .unwrap();
1308 client.subs[0].cursor = 41;
1309 client.subs[0].synced_once = true;
1310 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1311 client.persist_sub(&client.subs[0]).unwrap();
1312 client
1313 .mutate(vec![Mutation::Upsert {
1314 table: "tasks".into(),
1315 values: Map::from_iter([
1316 ("id".into(), Value::from("pending")),
1317 ("project_id".into(), Value::from("p1")),
1318 ]),
1319 base_version: None,
1320 }])
1321 .unwrap();
1322 let ids = client.pending_commit_ids();
1323 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
1324 let revision = client.local_revision();
1325 client.drain_change_batches();
1326 let (_, meta) = client.build_request(false).unwrap();
1327 client.conn.execute_batch(if fault == "subscription" {
1328 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
1329 } else {
1330 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
1331 }).unwrap();
1332 let mut transport = CountingRealtimeTransport::default();
1333 let mut report = SyncReport::default();
1334 if status == SubStatus::Reset {
1336 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
1337 }
1338 let result = client.process_section(
1339 &mut transport,
1340 "tasks",
1341 status,
1342 "sync.scope_revoked",
1343 vec![],
1344 vec![],
1345 Some((42, None)),
1346 &meta,
1347 &mut report,
1348 );
1349 assert!(result.is_err(), "{status:?}/{fault}");
1350 assert!(client.conn.is_autocommit());
1351 assert_eq!(client.subs[0].cursor, 41);
1352 assert_eq!(client.subs[0].state, SubState::Active);
1353 assert_eq!(client.pending_commit_ids(), ids);
1354 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
1355 assert!(client.rejections.is_empty());
1356 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
1357 assert_eq!(client.local_revision(), revision);
1358 assert!(client.drain_change_batches().is_empty());
1359 assert!(report.revoked.is_empty() && report.resets.is_empty());
1360 }
1361 }
1362 }
1363
1364 #[test]
1365 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
1366 let mut client = client();
1367 client
1368 .subscribe(
1369 "tasks".into(),
1370 "tasks".into(),
1371 vec![("project_id".into(), vec!["p1".into()])],
1372 None,
1373 )
1374 .unwrap();
1375 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1376 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
1377 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
1378 let row = |id: &str| ssp2::segment::SegmentRow {
1379 server_version: 1,
1380 values: vec![
1381 Some(ColumnValue::String(id.into())),
1382 Some(ColumnValue::String("p1".into())),
1383 ],
1384 };
1385 let segment = RowsSegment {
1386 table: "tasks".into(),
1387 schema_version: 1,
1388 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
1389 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
1390 };
1391 assert!(client.apply_segment(0, &segment, true).is_err());
1392 assert!(client.conn.is_autocommit());
1393 assert_eq!(
1394 client
1395 .query("SELECT id FROM tasks ORDER BY id", &[])
1396 .unwrap(),
1397 vec![Map::from_iter([("id".into(), Value::from("first"))])]
1398 );
1399 assert_eq!(client.local_revision(), 1);
1400 assert_eq!(client.drain_change_batches().len(), 1);
1401 assert_eq!(client.subs[0].cursor, -1);
1402 client
1403 .conn
1404 .execute_batch("DROP TRIGGER fail_block")
1405 .unwrap();
1406 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
1407 assert_eq!(client.local_revision(), 3);
1408 assert_eq!(client.drain_change_batches().len(), 2);
1409 }
1410
1411 #[test]
1412 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
1413 assert_eq!(
1414 client().diagnostics_storage().blob_cache_bytes_approx,
1415 Some(0)
1416 );
1417 let client = SyncClient::new(
1418 "diagnostic-storage".to_owned(),
1419 &json!({
1420 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
1421 "columns": [{"name": "id", "type": "string", "nullable": false},
1422 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
1423 }),
1424 ClientLimits {
1425 blob_cache_max_bytes: Some(5),
1426 ..Default::default()
1427 },
1428 )
1429 .unwrap();
1430 let empty = client.diagnostics_storage();
1431 assert_eq!(empty.status, "healthy");
1432 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
1433 assert_eq!(empty.retained_outcome_entries, Some(0));
1434 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
1435 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
1436 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
1437 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
1438 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
1439 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
1440 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
1441 let written = client.diagnostics_storage();
1442 assert_eq!(written.status, "pressure");
1443 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
1444 assert_eq!(written.retained_outcome_entries, Some(2));
1445 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
1446 assert_eq!(written.blob_cache_bytes_approx, Some(8));
1447 assert_eq!(
1448 written.pressure_reason_code.as_deref(),
1449 Some("client.blob_cache_over_limit")
1450 );
1451 client
1452 .conn
1453 .execute_batch(
1454 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
1455 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
1456 )
1457 .unwrap();
1458 let staged = client.diagnostics_storage();
1459 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
1460 assert_eq!(staged.retained_outcome_entries, Some(0));
1461 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
1462 assert_eq!(staged.status, "healthy");
1463 client.conn.execute_batch("ROLLBACK").unwrap();
1464 assert_eq!(client.diagnostics_storage(), written);
1465 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
1466 let unreadable = client.diagnostics_storage();
1467 assert_eq!(unreadable.status, "unreadable");
1468 assert!(unreadable.database_bytes_approx.is_none());
1469 assert!(unreadable.pending_outbox_bytes_approx.is_none());
1470 assert!(unreadable.retained_outcome_entries.is_none());
1471 assert!(unreadable.retained_outcome_bytes_approx.is_none());
1472 assert!(unreadable.blob_cache_bytes_approx.is_none());
1473 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
1474 let restored = client.diagnostics_storage();
1475 assert_eq!(restored.status, written.status);
1476 assert_eq!(
1477 restored.retained_outcome_bytes_approx,
1478 written.retained_outcome_bytes_approx
1479 );
1480 client.conn.set_prepared_statement_cache_capacity(0);
1481 assert_eq!(client.diagnostics_storage(), restored);
1482 }
1483
1484 struct ServerErrorTransport {
1486 code: &'static str,
1487 retryable: bool,
1488 }
1489
1490 impl Transport for ServerErrorTransport {
1491 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1492 Ok(encode_message(&Message {
1493 wire_version: WIRE_VERSION,
1494 msg_kind: MsgKind::Response,
1495 frames: vec![
1496 Frame::RespHeader {
1497 required_schema_version: None,
1498 latest_schema_version: None,
1499 log_epoch: Some("epoch-1".to_owned()),
1500 reset_required: Some(false),
1501 },
1502 Frame::Error {
1503 code: self.code.to_owned(),
1504 message: "server error".to_owned(),
1505 category: "internal".to_owned(),
1506 retryable: self.retryable,
1507 recommended_action: "retryLater".to_owned(),
1508 details: None,
1509 },
1510 ],
1511 }))
1512 }
1513
1514 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1515 self.sync(request)
1516 }
1517
1518 fn download_segment(
1519 &mut self,
1520 _request: &SegmentRequest,
1521 _on_progress: &mut dyn FnMut(u64),
1522 ) -> Result<Vec<u8>, TransportError> {
1523 Err(TransportError::new("sync.transport_failed", "offline"))
1524 }
1525
1526 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1527 Ok(())
1528 }
1529
1530 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1531 Ok(())
1532 }
1533
1534 fn realtime_close(&mut self) -> Result<(), TransportError> {
1535 Ok(())
1536 }
1537 }
1538
1539 #[test]
1540 fn retryable_server_error_schedules_a_background_retry() {
1541 let mut client = client();
1542 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1543 let mut transport = ServerErrorTransport {
1544 code: "sync.internal_error",
1545 retryable: true,
1546 };
1547 assert!(matches!(
1548 client.sync(&mut transport),
1549 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
1550 ));
1551 assert!(matches!(
1552 client.drain_sync_intents().as_slice(),
1553 [SyncIntent::Background { delay_ms: 250 }]
1554 ));
1555 assert_eq!(
1556 client.progress().snapshot().unwrap().retry_delay_ms,
1557 Some(250)
1558 );
1559 let mut transport = ServerErrorTransport {
1560 code: "sync.invalid_request",
1561 retryable: false,
1562 };
1563 assert!(matches!(
1564 client.sync(&mut transport),
1565 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
1566 ));
1567 assert!(client.drain_sync_intents().is_empty());
1568 let progress = client.progress().snapshot().unwrap();
1569 assert_eq!(progress.state, ProgressState::Failed);
1570 assert_eq!(progress.retry_delay_ms, None);
1571 }
1572
1573 #[test]
1574 fn sync_until_idle_reports_budget_exhaustion_or_failure() {
1575 let exhausted = SyncOutcome::BudgetExhausted(SyncReport::default()).to_json();
1578 assert_eq!(exhausted["ok"], json!(true));
1579 assert_eq!(exhausted["budgetExhausted"], json!(true));
1580 assert!(exhausted["report"].is_object());
1581
1582 let mut client = client();
1583 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1584 let mut idle_transport = HostTransport::new_from_config(&json!({})).unwrap();
1586 let zero = client.sync_until_idle(&mut idle_transport, Some(0));
1587 assert!(
1588 matches!(zero, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
1589 "{zero:?}"
1590 );
1591 let mut erroring = ServerErrorTransport {
1593 code: "sync.internal_error",
1594 retryable: false,
1595 };
1596 let failed = client.sync_until_idle(&mut erroring, Some(3));
1597 assert!(
1598 matches!(failed, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1599 "{failed:?}"
1600 );
1601 }
1602
1603 struct ScriptedSyncTransport {
1606 responses: std::collections::VecDeque<Message>,
1607 calls: usize,
1608 }
1609
1610 impl Transport for ScriptedSyncTransport {
1611 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1612 self.calls += 1;
1613 let response = self.responses.pop_front().expect("scripted response");
1614 Ok(encode_message(&response))
1615 }
1616
1617 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1618 self.sync(request)
1619 }
1620
1621 fn download_segment(
1622 &mut self,
1623 _request: &SegmentRequest,
1624 _on_progress: &mut dyn FnMut(u64),
1625 ) -> Result<Vec<u8>, TransportError> {
1626 Err(TransportError::new("sync.transport_failed", "offline"))
1627 }
1628
1629 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1630 Ok(())
1631 }
1632
1633 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1634 Ok(())
1635 }
1636
1637 fn realtime_close(&mut self) -> Result<(), TransportError> {
1638 Ok(())
1639 }
1640 }
1641
1642 #[test]
1643 fn sync_until_idle_success_then_failure_keeps_earlier_work() {
1644 let path = std::env::temp_dir().join(format!(
1645 "syncular-budget-script-{}.sqlite",
1646 uuid::Uuid::new_v4()
1647 ));
1648 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1649 {"name":"id","type":"string","nullable":false},
1650 {"name":"project_id","type":"string","nullable":false}],
1651 "scopes":[{"pattern":"project:{project_id}"}]}]});
1652 let mut client = SyncClient::open_path(
1653 "budget-script".into(),
1654 &schema,
1655 ClientLimits::default(),
1656 path.to_str().unwrap(),
1657 )
1658 .expect("open client");
1659 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1660 client
1661 .subscribe(
1662 "tasks".into(),
1663 "tasks".into(),
1664 vec![("project_id".into(), vec!["p1".into()])],
1665 None,
1666 )
1667 .unwrap();
1668 client.subs[0].cursor = 0;
1669 client.subs[0].synced_once = true;
1670 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1671 client.persist_sub(&client.subs[0]).unwrap();
1672 client.drain_change_batches();
1673
1674 let row = encode_row_json(
1675 client.schema.table("tasks").unwrap(),
1676 "kept",
1677 &Map::from_iter([
1678 ("id".into(), json!("kept")),
1679 ("project_id".into(), json!("p1")),
1680 ]),
1681 &client.encryption,
1682 )
1683 .unwrap();
1684 let success = Message {
1685 wire_version: WIRE_VERSION,
1686 msg_kind: MsgKind::Response,
1687 frames: vec![
1688 Frame::RespHeader {
1689 required_schema_version: None,
1690 latest_schema_version: None,
1691 log_epoch: Some("epoch-1".into()),
1692 reset_required: Some(false),
1693 },
1694 Frame::SubStart {
1695 id: "tasks".into(),
1696 status: SubStatus::Active,
1697 reason_code: String::new(),
1698 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1699 bootstrap: false,
1700 },
1701 Frame::Commit {
1702 commit_seq: 1,
1703 created_at_ms: 1,
1704 actor_id: "writer".into(),
1705 tables: vec!["tasks".into()],
1706 changes: vec![ssp2::model::Change {
1707 table_index: 0,
1708 row_id: "kept".into(),
1709 op: Op::Upsert,
1710 row_version: Some(1),
1711 scopes: vec![("project_id".into(), "p1".into())],
1712 row: Some(row),
1713 }],
1714 },
1715 Frame::SubEnd {
1716 next_cursor: 1,
1717 bootstrap_state: None,
1718 },
1719 ],
1720 };
1721 let failure = Message {
1722 wire_version: WIRE_VERSION,
1723 msg_kind: MsgKind::Response,
1724 frames: vec![
1725 Frame::RespHeader {
1726 required_schema_version: None,
1727 latest_schema_version: None,
1728 log_epoch: Some("epoch-1".into()),
1729 reset_required: Some(false),
1730 },
1731 Frame::Error {
1732 code: "sync.internal_error".into(),
1733 message: "second round failed".into(),
1734 category: "internal".into(),
1735 retryable: false,
1736 recommended_action: "retryLater".into(),
1737 details: None,
1738 },
1739 ],
1740 };
1741 let mut transport = ScriptedSyncTransport {
1742 responses: std::collections::VecDeque::from([success, failure]),
1743 calls: 0,
1744 };
1745 let outcome = client.sync_until_idle(&mut transport, Some(2));
1746 assert!(
1747 matches!(outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1748 "{outcome:?}"
1749 );
1750 assert_eq!(
1751 transport.calls, 2,
1752 "one successful round then the failing round"
1753 );
1754 assert_eq!(
1755 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
1756 json!("kept"),
1757 "the earlier round's durable work survives the later failure"
1758 );
1759 drop(client);
1760 let _ = std::fs::remove_file(&path);
1761 }
1762
1763 #[test]
1764 fn sync_until_idle_stops_on_a_schema_floor() {
1765 let mut client = client();
1766 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1767 let floor = Message {
1768 wire_version: WIRE_VERSION,
1769 msg_kind: MsgKind::Response,
1770 frames: vec![Frame::RespHeader {
1771 required_schema_version: Some(2),
1772 latest_schema_version: Some(2),
1773 log_epoch: Some("epoch-1".into()),
1774 reset_required: Some(false),
1775 }],
1776 };
1777 let mut transport = ScriptedSyncTransport {
1778 responses: std::collections::VecDeque::from([floor.clone(), floor]),
1779 calls: 0,
1780 };
1781 let outcome = client.sync_until_idle(&mut transport, Some(3));
1782 match outcome {
1783 SyncOutcome::Ok(report) => assert_eq!(
1784 report
1785 .schema_floor
1786 .as_ref()
1787 .and_then(|floor| floor.required_schema_version),
1788 Some(2),
1789 "the stop retains the latest schema floor"
1790 ),
1791 other => panic!("expected a schema-floor stop, got {other:?}"),
1792 }
1793 assert_eq!(
1794 transport.calls, 1,
1795 "a schema floor stops the budget loop after one round"
1796 );
1797 }
1798
1799 #[test]
1800 fn sync_report_merge_schema_floor_is_latest_state() {
1801 let mut aggregate = SyncReport {
1802 schema_floor: Some(SchemaFloor {
1803 required_schema_version: Some(2),
1804 latest_schema_version: Some(2),
1805 }),
1806 ..SyncReport::default()
1807 };
1808 aggregate.merge(&SyncReport::default());
1809 assert!(
1810 aggregate.schema_floor.is_none(),
1811 "a later round without a floor clears it"
1812 );
1813 }
1814
1815 #[test]
1816 fn push_capacity_limits_are_hard_and_keep_intent() {
1817 let mut c = client();
1819 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
1820 c.limits.max_push_operations_per_request = Some(1);
1821 let id = c
1822 .mutate(vec![
1823 Mutation::Upsert {
1824 table: "tasks".into(),
1825 values: Map::from_iter([
1826 ("id".into(), json!("a")),
1827 ("project_id".into(), json!("p1")),
1828 ]),
1829 base_version: None,
1830 },
1831 Mutation::Upsert {
1832 table: "tasks".into(),
1833 values: Map::from_iter([
1834 ("id".into(), json!("b")),
1835 ("project_id".into(), json!("p1")),
1836 ]),
1837 base_version: None,
1838 },
1839 ])
1840 .unwrap();
1841 let error = c.build_request(false).unwrap_err();
1842 let SyncOutcome::Failed {
1843 error_code,
1844 details,
1845 ..
1846 } = *error
1847 else {
1848 panic!("expected a capacity failure");
1849 };
1850 assert_eq!(error_code, "client.push_request_too_large");
1851 let details = details.expect("capacity details");
1852 assert_eq!(details["kind"], json!("operations"));
1853 assert_eq!(details["limit"], json!(1));
1854 assert_eq!(details["size"], json!(2));
1855 assert_eq!(details["clientCommitId"], json!(id));
1856 assert_eq!(c.pending_commit_ids(), vec![id]);
1857
1858 let mut c = client();
1860 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
1861 c.limits.max_push_commits_per_request = Some(1);
1862 let first = c
1863 .mutate(vec![Mutation::Upsert {
1864 table: "tasks".into(),
1865 values: Map::from_iter([
1866 ("id".into(), json!("one")),
1867 ("project_id".into(), json!("p1")),
1868 ]),
1869 base_version: None,
1870 }])
1871 .unwrap();
1872 let second = c
1873 .mutate(vec![Mutation::Upsert {
1874 table: "tasks".into(),
1875 values: Map::from_iter([
1876 ("id".into(), json!("two")),
1877 ("project_id".into(), json!("p1")),
1878 ]),
1879 base_version: None,
1880 }])
1881 .unwrap();
1882 let (_, meta) = c.build_request(false).unwrap();
1883 assert_eq!(meta.pushed_ids, vec![first.clone()]);
1884 assert_eq!(meta.deferred_commits, 1);
1885 assert_eq!(c.pending_commit_ids(), vec![first, second]);
1886 }
1887
1888 #[test]
1889 fn push_request_byte_cap_defers_the_prefix_boundary_and_errors_the_head() {
1890 let mut c = client();
1893 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
1894 let first = c
1895 .mutate(vec![Mutation::Upsert {
1896 table: "tasks".into(),
1897 values: Map::from_iter([
1898 ("id".into(), json!("one")),
1899 ("project_id".into(), json!("p1")),
1900 ]),
1901 base_version: None,
1902 }])
1903 .unwrap();
1904 let second = c
1905 .mutate(vec![Mutation::Upsert {
1906 table: "tasks".into(),
1907 values: Map::from_iter([
1908 ("id".into(), json!("two")),
1909 ("project_id".into(), json!("p1")),
1910 ]),
1911 base_version: None,
1912 }])
1913 .unwrap();
1914 let (message, meta) = c.build_request(false).unwrap();
1915 assert_eq!(meta.pushed_ids.len(), 2);
1916 let full = encode_message(&message).len();
1917 c.limits.max_push_request_bytes = Some(full - 1);
1918 let (_, meta) = c.build_request(false).unwrap();
1919 assert_eq!(meta.pushed_ids, vec![first.clone()]);
1920 assert_eq!(meta.deferred_commits, 1);
1921 assert_eq!(c.pending_commit_ids(), vec![first, second]);
1922
1923 let mut c = client();
1926 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
1927 let huge = c
1928 .mutate(vec![Mutation::Upsert {
1929 table: "tasks".into(),
1930 values: Map::from_iter([
1931 ("id".into(), json!("x".repeat(20_000))),
1932 ("project_id".into(), json!("p1")),
1933 ]),
1934 base_version: None,
1935 }])
1936 .unwrap();
1937 let (message, meta) = c.build_request(false).unwrap();
1938 assert_eq!(meta.pushed_ids, vec![huge.clone()]);
1939 let with_huge = encode_message(&message).len();
1940 c.limits.max_push_request_bytes = Some(with_huge - 1);
1941 let error = c.build_request(false).unwrap_err();
1942 let SyncOutcome::Failed {
1943 error_code,
1944 details,
1945 ..
1946 } = *error
1947 else {
1948 panic!("expected a capacity failure");
1949 };
1950 assert_eq!(error_code, "client.push_request_too_large");
1951 let details = details.expect("capacity details");
1952 assert_eq!(details["kind"], json!("bytes"));
1953 assert_eq!(details["limit"], json!((with_huge - 1) as u64));
1954 assert_eq!(details["size"], json!(with_huge as u64));
1955 assert_eq!(details["clientCommitId"], json!(huge));
1956 assert_eq!(c.pending_commit_ids(), vec![huge]);
1957
1958 let mut c = client();
1960 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
1961 c.limits.max_push_request_bytes = Some(1);
1962 let error = c.build_request(false).unwrap_err();
1963 let SyncOutcome::Failed { details, .. } = *error else {
1964 panic!("expected a capacity failure");
1965 };
1966 let details = details.expect("capacity details");
1967 assert_eq!(details["kind"], json!("bytes"));
1968 assert!(details.get("clientCommitId").is_none(), "{details}");
1969 }
1970
1971 #[test]
1972 fn push_limits_validate_at_construction() {
1973 for limits in [
1974 ClientLimits {
1975 max_push_commits_per_request: Some(0),
1976 ..Default::default()
1977 },
1978 ClientLimits {
1979 max_push_operations_per_request: Some(0),
1980 ..Default::default()
1981 },
1982 ClientLimits {
1983 max_push_request_bytes: Some(0),
1984 ..Default::default()
1985 },
1986 ClientLimits {
1987 max_push_request_bytes: Some(u32::MAX as usize + 1),
1988 ..Default::default()
1989 },
1990 ] {
1991 let schema = json!({"version":1,"tables":[]});
1992 let error = SyncClient::with_connection(
1993 "bad-limits".into(),
1994 &schema,
1995 limits,
1996 Connection::open_in_memory().unwrap(),
1997 )
1998 .err();
1999 assert_eq!(
2000 error.as_deref(),
2001 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2002 );
2003 }
2004 }
2005
2006 #[test]
2007 fn open_path_rejects_invalid_push_limits_before_storage() {
2008 for identity in [false, true] {
2009 let path = std::env::temp_dir().join(format!(
2010 "syncular-bad-limits-{}.sqlite",
2011 uuid::Uuid::new_v4()
2012 ));
2013 let schema = json!({"version":1,"tables":[]});
2014 let limits = ClientLimits {
2015 max_push_request_bytes: Some(0),
2016 ..Default::default()
2017 };
2018 let error = if identity {
2019 SyncClient::open_path_with_identity(None, &schema, limits, path.to_str().unwrap())
2020 } else {
2021 SyncClient::open_path("bad-limits".into(), &schema, limits, path.to_str().unwrap())
2022 }
2023 .err();
2024 assert_eq!(
2025 error.as_deref(),
2026 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2027 );
2028 assert!(
2029 !path.exists(),
2030 "an invalid limit is rejected before the database file is created"
2031 );
2032 }
2033 }
2034
2035 #[test]
2036 fn failed_progress_retry_delay_doubles_to_the_cap() {
2037 let mut client = client();
2038 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2039 let mut transport = ServerErrorTransport {
2040 code: "sync.internal_error",
2041 retryable: true,
2042 };
2043 let delays: Vec<Option<u64>> = (0..9)
2044 .map(|_| {
2045 client.sync(&mut transport);
2046 client.progress().snapshot().unwrap().retry_delay_ms
2047 })
2048 .collect();
2049 assert_eq!(
2050 delays,
2051 [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
2052 );
2053 }
2054
2055 #[test]
2056 fn background_retry_deadlines_back_off_and_reset() {
2057 let mut client = client();
2058 client.schedule_background_retry();
2059 assert!(matches!(
2060 client.drain_sync_intents().as_slice(),
2061 [SyncIntent::Background { delay_ms: 250 }]
2062 ));
2063 client.schedule_background_retry();
2064 assert!(matches!(
2065 client.drain_sync_intents().as_slice(),
2066 [SyncIntent::Background { delay_ms: 500 }]
2067 ));
2068 client.reset_background_retry();
2069 client.schedule_background_retry();
2070 assert!(matches!(
2071 client.drain_sync_intents().as_slice(),
2072 [SyncIntent::Background { delay_ms: 250 }]
2073 ));
2074 }
2075
2076 #[test]
2077 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
2078 let mut client = client();
2079 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2080 client
2081 .subscribe(
2082 "epoch-tasks".to_owned(),
2083 "tasks".to_owned(),
2084 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2085 None,
2086 )
2087 .expect("subscribe");
2088 client.drain_sync_intents();
2089
2090 client
2091 .run_log_epoch_reset("epoch-2")
2092 .expect("reset partition log epoch");
2093
2094 assert!(matches!(
2095 client.drain_sync_intents().as_slice(),
2096 [SyncIntent::Interactive]
2097 ));
2098 }
2099
2100 #[derive(Default)]
2101 struct CountingRealtimeTransport {
2102 connects: usize,
2103 closes: usize,
2104 messages: Vec<String>,
2105 blob_uploads: Vec<(String, Vec<u8>)>,
2106 blob_download: Option<Vec<u8>>,
2107 sync_calls: usize,
2108 realtime_sync_calls: usize,
2109 refuse_connect: bool,
2110 }
2111
2112 impl Transport for CountingRealtimeTransport {
2113 fn blob_upload(
2114 &mut self,
2115 blob_id: &str,
2116 bytes: &[u8],
2117 _media_type: Option<&str>,
2118 ) -> Result<(), TransportError> {
2119 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
2120 Ok(())
2121 }
2122
2123 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
2124 self.blob_download
2125 .clone()
2126 .map(BlobDownload::Bytes)
2127 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
2128 }
2129
2130 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2131 self.sync_calls += 1;
2132 Err(TransportError::new("sync.transport_failed", "offline"))
2133 }
2134
2135 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2136 self.realtime_sync_calls += 1;
2137 Err(TransportError::new("sync.transport_failed", "offline"))
2138 }
2139
2140 fn download_segment(
2141 &mut self,
2142 _request: &SegmentRequest,
2143 _on_progress: &mut dyn FnMut(u64),
2144 ) -> Result<Vec<u8>, TransportError> {
2145 Err(TransportError::new("sync.transport_failed", "offline"))
2146 }
2147
2148 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2149 self.connects += 1;
2150 if self.refuse_connect {
2151 return Err(TransportError::new("transport.failed", "refused"));
2152 }
2153 Ok(())
2154 }
2155
2156 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2157 self.messages.push(text.to_owned());
2158 Ok(())
2159 }
2160
2161 fn realtime_close(&mut self) -> Result<(), TransportError> {
2162 self.closes += 1;
2163 Ok(())
2164 }
2165 }
2166
2167 #[test]
2168 fn blob_results_own_downloaded_bytes() {
2169 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
2170 {"name":"id","type":"string","nullable":false},
2171 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
2172 let bytes = b"owned typed result".to_vec();
2173 let blob_id = blob_id_for(&bytes);
2174 let mut transport = CountingRealtimeTransport {
2175 blob_download: Some(bytes.clone()),
2176 ..Default::default()
2177 };
2178 let result = {
2179 let mut client =
2180 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
2181 .unwrap();
2182 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
2183 assert_eq!(result.blob_id, blob_id);
2184 assert_eq!(result.byte_length, bytes.len() as i64);
2185 assert_eq!(result.bytes, bytes);
2186 assert_eq!(result.media_type, None);
2187 assert_eq!(
2188 client
2189 .conn
2190 .query_row(
2191 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
2192 rusqlite::params![blob_id],
2193 |row| row.get::<_, i64>(0),
2194 )
2195 .unwrap(),
2196 0
2197 );
2198 result
2199 };
2200 assert_eq!(result.bytes, bytes);
2201
2202 let mut client =
2203 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
2204 assert_eq!(
2205 client
2206 .fetch_blob_bytes(&mut transport, "not json")
2207 .unwrap_err(),
2208 (
2209 "client.failed".to_owned(),
2210 "blob ref is not JSON".to_owned()
2211 )
2212 );
2213 }
2214
2215 #[test]
2216 fn visible_blob_references_survive_cache_trimming() {
2217 let schema = json!({"version":1,"tables":[
2218 {"name":"attachments","primaryKey":"id","columns":[
2219 {"name":"id","type":"string","nullable":false},
2220 {"name":"body","type":"blob_ref","nullable":true},
2221 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
2222 {"name":"avatars","primaryKey":"id","columns":[
2223 {"name":"id","type":"string","nullable":false},
2224 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
2225 ]});
2226 let bytes = b"referenced above cap".to_vec();
2227 let blob_id = blob_id_for(&bytes);
2228 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
2229 let mut client = SyncClient::new(
2230 "visible-blob-refs".to_owned(),
2231 &schema,
2232 ClientLimits {
2233 blob_cache_max_bytes: Some(1),
2234 ..Default::default()
2235 },
2236 )
2237 .unwrap();
2238 client.conn.execute(
2239 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
2240 rusqlite::params![reference, reference, reference, reference],
2241 ).unwrap();
2242 client
2243 .conn
2244 .execute(
2245 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
2246 rusqlite::params![reference],
2247 )
2248 .unwrap();
2249 client.conn.execute(
2250 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
2251 [],
2252 ).unwrap();
2253 let mut transport = CountingRealtimeTransport {
2254 blob_download: Some(bytes.clone()),
2255 ..Default::default()
2256 };
2257
2258 assert_eq!(
2259 client
2260 .fetch_blob_bytes(&mut transport, &blob_id)
2261 .unwrap()
2262 .bytes,
2263 bytes
2264 );
2265 let rows = client
2266 .conn
2267 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
2268 .unwrap()
2269 .query_map([], |row| row.get::<_, String>(0))
2270 .unwrap()
2271 .collect::<Result<Vec<_>, _>>()
2272 .unwrap();
2273 assert_eq!(rows, vec![blob_id]);
2274 }
2275
2276 #[test]
2277 fn closed_transport_never_invokes_network_and_reopen_is_host_owned() {
2278 let mut transport = CountingRealtimeTransport::default();
2279 let path = std::env::temp_dir().join(format!(
2280 "syncular-transport-gate-{}.db",
2281 uuid::Uuid::new_v4()
2282 ));
2283 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
2284 {"name":"id","type":"string","nullable":false},
2285 {"name":"project_id","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
2286 let mut client = SyncClient::open_path(
2287 "gate".into(),
2288 &schema,
2289 ClientLimits::default(),
2290 path.to_str().unwrap(),
2291 )
2292 .unwrap();
2293 client.connect_realtime(&mut transport).unwrap();
2294 client.set_transport_enabled(&mut transport, false);
2295 assert_eq!(transport.closes, 1);
2296 client.begin_security_preflight();
2297 client.activate_security(Default::default()).unwrap();
2298 client
2299 .subscribe(
2300 "tasks".into(),
2301 "tasks".into(),
2302 vec![("project_id".into(), vec!["p1".into()])],
2303 None,
2304 )
2305 .unwrap();
2306 for id in ["one", "two"] {
2307 client
2308 .mutate(vec![Mutation::Upsert {
2309 table: "tasks".into(),
2310 values: Map::from_iter([
2311 ("id".into(), json!(id)),
2312 ("project_id".into(), json!("p1")),
2313 ]),
2314 base_version: None,
2315 }])
2316 .unwrap();
2317 }
2318 assert_eq!(
2319 client
2320 .query("SELECT id FROM tasks ORDER BY id", &[])
2321 .unwrap()
2322 .len(),
2323 2
2324 );
2325 assert!(
2326 matches!(client.sync(&mut transport), SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.offline")
2327 );
2328 assert!(client
2329 .connect_realtime(&mut transport)
2330 .unwrap_err()
2331 .starts_with("sync.offline:"));
2332 assert!(client
2333 .set_presence(&mut transport, "project:p1", None)
2334 .unwrap_err()
2335 .starts_with("sync.offline:"));
2336 assert_eq!(transport.sync_calls, 0);
2337 assert_eq!(transport.realtime_sync_calls, 0);
2338 assert_eq!(transport.connects, 1);
2339 assert!(transport.messages.is_empty());
2340 assert!(matches!(
2341 client.drain_sync_intents().as_slice(),
2342 [SyncIntent::None]
2343 ));
2344 let pending = client.pending_commit_ids();
2345 drop(client);
2346 let mut reopened = SyncClient::open_path(
2347 "gate".into(),
2348 &schema,
2349 ClientLimits::default(),
2350 path.to_str().unwrap(),
2351 )
2352 .unwrap();
2353 assert!(reopened.transport_enabled());
2354 reopened.set_transport_enabled(&mut transport, false);
2355 assert_eq!(reopened.pending_commit_ids(), pending);
2356 assert_eq!(
2357 reopened.query("SELECT id FROM tasks", &[]).unwrap().len(),
2358 2
2359 );
2360 reopened.set_transport_enabled(&mut transport, true);
2361 assert!(matches!(
2362 reopened.drain_sync_intents().as_slice(),
2363 [SyncIntent::Interactive]
2364 ));
2365 reopened.set_transport_enabled(&mut transport, true);
2366 assert!(reopened.drain_sync_intents().is_empty());
2367 drop(reopened);
2368 std::fs::remove_file(path).unwrap();
2369 }
2370
2371 #[test]
2372 fn realtime_connection_ownership_is_idempotent() {
2373 let mut client = client();
2374 let mut transport = CountingRealtimeTransport {
2375 connects: 0,
2376 closes: 0,
2377 ..Default::default()
2378 };
2379 client
2380 .connect_realtime(&mut transport)
2381 .expect("first connect");
2382 client
2383 .connect_realtime(&mut transport)
2384 .expect("idempotent connect");
2385 assert_eq!(transport.connects, 1);
2386 client.disconnect_realtime(&mut transport);
2387 client.disconnect_realtime(&mut transport);
2388 assert_eq!(transport.closes, 1);
2389 client
2390 .connect_realtime(&mut transport)
2391 .expect("deliberate reconnect");
2392 assert_eq!(transport.connects, 2);
2393 }
2394
2395 #[test]
2396 fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
2397 let mut client = client();
2398 client
2399 .set_realtime_policy(RealtimePolicy::Required)
2400 .expect("set policy");
2401 let mut transport = CountingRealtimeTransport::default();
2402 match client.sync(&mut transport) {
2403 SyncOutcome::RealtimeUnavailable {
2404 state,
2405 reason_code,
2406 retry_delay_ms,
2407 } => {
2408 assert_eq!(state, RealtimeState::Disconnected);
2409 assert_eq!(reason_code, None);
2410 assert_eq!(retry_delay_ms, 250);
2411 }
2412 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2413 }
2414 assert_eq!(transport.sync_calls, 0);
2415 assert!(matches!(
2416 client.drain_sync_intents().as_slice(),
2417 [SyncIntent::Background { delay_ms: 250 }]
2418 ));
2419 let snapshot = client
2420 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2421 .expect("diagnostics");
2422 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2423 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
2424 assert_eq!(snapshot.host.realtime_reason_code, None);
2425 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2426 assert_eq!(
2427 snapshot
2428 .last_round
2429 .as_ref()
2430 .and_then(|round| round.error_code.clone()),
2431 Some(REALTIME_UNAVAILABLE_CODE.to_owned())
2432 );
2433 }
2434
2435 #[test]
2436 fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
2437 let mut client = client();
2438 client
2439 .set_realtime_policy(RealtimePolicy::Required)
2440 .expect("set policy");
2441 let mut transport = CountingRealtimeTransport {
2442 refuse_connect: true,
2443 ..Default::default()
2444 };
2445 assert!(client.connect_realtime(&mut transport).is_err());
2446 assert_eq!(client.realtime_state(), RealtimeState::Refused);
2447 let snapshot = client
2448 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2449 .expect("diagnostics");
2450 assert_eq!(
2451 snapshot.host.realtime_reason_code.as_deref(),
2452 Some("transport.failed")
2453 );
2454 match client.sync(&mut transport) {
2455 SyncOutcome::RealtimeUnavailable {
2456 state, reason_code, ..
2457 } => {
2458 assert_eq!(state, RealtimeState::Refused);
2459 assert_eq!(reason_code.as_deref(), Some("transport.failed"));
2460 }
2461 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2462 }
2463 assert_eq!(transport.sync_calls, 0);
2464 assert_eq!(transport.realtime_sync_calls, 0);
2465 }
2466
2467 #[test]
2468 fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
2469 let mut client = client();
2470 client
2471 .set_realtime_policy(RealtimePolicy::Required)
2472 .expect("set policy");
2473 let mut transport = CountingRealtimeTransport::default();
2474 client.connect_realtime(&mut transport).expect("connect");
2475 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2476 assert!(
2477 client.set_realtime_policy(RealtimePolicy::Off).is_err(),
2478 "a policy change cannot silently re-route a connected socket"
2479 );
2480 assert!(matches!(
2481 client.sync(&mut transport),
2482 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
2483 ));
2484 assert_eq!(client.realtime_state(), RealtimeState::Lost);
2485 assert_eq!(transport.realtime_sync_calls, 1);
2486 assert_eq!(transport.sync_calls, 0);
2487 let snapshot = client
2488 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2489 .expect("diagnostics");
2490 assert_eq!(
2491 snapshot.host.realtime_reason_code.as_deref(),
2492 Some("sync.transport_failed")
2493 );
2494 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2495 match client.sync(&mut transport) {
2496 SyncOutcome::RealtimeUnavailable {
2497 state, reason_code, ..
2498 } => {
2499 assert_eq!(state, RealtimeState::Lost);
2500 assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
2501 }
2502 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2503 }
2504 assert_eq!(transport.sync_calls, 0);
2505 client.disconnect_realtime(&mut transport);
2506 assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
2507 assert_eq!(transport.closes, 1);
2508 client.connect_realtime(&mut transport).expect("reconnect");
2509 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2510 assert_eq!(transport.connects, 2);
2511 }
2512
2513 #[test]
2514 fn off_policy_uses_http_and_refuses_an_explicit_connect() {
2515 let mut client = client();
2516 client
2517 .set_realtime_policy(RealtimePolicy::Off)
2518 .expect("set policy");
2519 let mut transport = CountingRealtimeTransport::default();
2520 assert!(client.connect_realtime(&mut transport).is_err());
2521 assert_eq!(client.realtime_state(), RealtimeState::Disabled);
2522 assert!(matches!(
2523 client.sync(&mut transport),
2524 SyncOutcome::Failed { .. }
2525 ));
2526 assert_eq!(transport.sync_calls, 1);
2527 assert_eq!(transport.realtime_sync_calls, 0);
2528 let snapshot = client
2529 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2530 .expect("diagnostics");
2531 assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
2532 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
2533 }
2534
2535 #[test]
2536 fn optional_policy_keeps_the_http_round() {
2537 let mut client = client();
2538 let mut transport = CountingRealtimeTransport::default();
2539 assert!(matches!(
2540 client.sync(&mut transport),
2541 SyncOutcome::Failed { .. }
2542 ));
2543 assert_eq!(transport.sync_calls, 1);
2544 assert_eq!(transport.realtime_sync_calls, 0);
2545 let snapshot = client
2546 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2547 .expect("diagnostics");
2548 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2549 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
2550 }
2551
2552 #[test]
2553 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
2554 let mut client = client();
2555 client
2556 .subscribe(
2557 "repair-tasks".to_owned(),
2558 "tasks".to_owned(),
2559 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2560 None,
2561 )
2562 .expect("subscribe");
2563 {
2564 let sub = client
2565 .subs
2566 .iter_mut()
2567 .find(|sub| sub.id == "repair-tasks")
2568 .expect("subscription");
2569 sub.cursor = 42;
2570 sub.synced_once = true;
2571 let persisted = sub.clone();
2572 client.persist_sub(&persisted).unwrap();
2573 }
2574 for table in ["_syncular_base_tasks", "tasks"] {
2575 client
2576 .conn
2577 .execute(
2578 &format!(
2579 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
2580 ),
2581 rusqlite::params!["server-row", "p1"],
2582 )
2583 .expect("seed server row");
2584 }
2585 let pending = client
2586 .mutate(vec![Mutation::Upsert {
2587 table: "tasks".to_owned(),
2588 values: Map::from_iter([
2589 ("id".to_owned(), Value::from("offline-row")),
2590 ("project_id".to_owned(), Value::from("p1")),
2591 ]),
2592 base_version: None,
2593 }])
2594 .expect("queue offline work");
2595 client.drain_change_batches();
2596 client.drain_sync_intents();
2597
2598 assert_eq!(
2599 client
2600 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2601 rebootstrap_id: "support-case-001".to_owned(),
2602 })
2603 .expect("rebootstrap"),
2604 LocalDataRebootstrapResult {
2605 already_applied: false,
2606 retained_commits: 1,
2607 reset_subscriptions: 1,
2608 }
2609 );
2610 let visible_ids = client
2611 .conn
2612 .prepare("SELECT id FROM tasks ORDER BY id")
2613 .expect("prepare visible ids")
2614 .query_map([], |row| row.get::<_, String>(0))
2615 .expect("query visible ids")
2616 .collect::<Result<Vec<_>, _>>()
2617 .expect("collect visible ids");
2618 assert_eq!(visible_ids, vec!["offline-row"]);
2619 assert_eq!(client.pending_commit_ids(), vec![pending]);
2620 assert_eq!(
2621 client
2622 .subscription_state("repair-tasks")
2623 .expect("subscription")
2624 .cursor,
2625 -1
2626 );
2627 assert!(client.upgrading());
2628 assert!(client.sync_needed());
2629 assert!(matches!(
2630 client.drain_sync_intents().as_slice(),
2631 [SyncIntent::Interactive]
2632 ));
2633 assert_eq!(client.drain_change_batches().len(), 1);
2634
2635 assert_eq!(
2636 client
2637 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2638 rebootstrap_id: "support-case-001".to_owned(),
2639 })
2640 .expect("idempotent retry"),
2641 LocalDataRebootstrapResult {
2642 already_applied: true,
2643 retained_commits: 1,
2644 reset_subscriptions: 1,
2645 }
2646 );
2647 assert!(client.drain_change_batches().is_empty());
2648 }
2649
2650 #[test]
2651 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
2652 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
2653 assert_eq!(
2654 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
2655 (3, 4)
2656 );
2657 assert_eq!(
2658 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
2659 (0, 0)
2660 );
2661 for malformed in [
2662 "",
2663 "{}",
2664 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
2665 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
2666 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
2667 ] {
2668 assert_eq!(
2669 decode_local_rebootstrap_receipt(malformed)
2670 .expect_err("malformed receipt must fail"),
2671 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
2672 );
2673 }
2674 }
2675
2676 #[test]
2677 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
2678 let path = std::env::temp_dir().join(format!(
2679 "syncular-rebootstrap-receipt-{}.db",
2680 uuid::Uuid::new_v4()
2681 ));
2682 let schema = json!({
2683 "version": 1,
2684 "tables": [{
2685 "name": "tasks",
2686 "primaryKey": "id",
2687 "columns": [
2688 { "name": "id", "type": "string", "nullable": false },
2689 { "name": "project_id", "type": "string", "nullable": false }
2690 ],
2691 "scopes": [{ "pattern": "project:{project_id}" }]
2692 }]
2693 });
2694 let path_string = path.to_str().expect("UTF-8 temp path");
2695
2696 {
2697 let mut first = SyncClient::open_path(
2698 "repair-restart-client".to_owned(),
2699 &schema,
2700 ClientLimits::default(),
2701 path_string,
2702 )
2703 .expect("first open");
2704 first
2705 .subscribe(
2706 "repair-tasks".to_owned(),
2707 "tasks".to_owned(),
2708 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2709 None,
2710 )
2711 .expect("subscribe");
2712 first
2713 .mutate(vec![Mutation::Upsert {
2714 table: "tasks".to_owned(),
2715 values: Map::from_iter([
2716 ("id".to_owned(), Value::from("offline-row")),
2717 ("project_id".to_owned(), Value::from("p1")),
2718 ]),
2719 base_version: None,
2720 }])
2721 .expect("queue offline work");
2722 assert_eq!(
2723 first
2724 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2725 rebootstrap_id: "restart-receipt".to_owned(),
2726 })
2727 .expect("first rebootstrap"),
2728 LocalDataRebootstrapResult {
2729 already_applied: false,
2730 retained_commits: 1,
2731 reset_subscriptions: 1,
2732 }
2733 );
2734 }
2735
2736 let mut reopened = SyncClient::open_path(
2737 "repair-restart-client".to_owned(),
2738 &schema,
2739 ClientLimits::default(),
2740 path_string,
2741 )
2742 .expect("reopen");
2743 reopened.drain_change_batches();
2744 reopened.drain_sync_intents();
2745 assert_eq!(
2746 reopened
2747 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2748 rebootstrap_id: "restart-receipt".to_owned(),
2749 })
2750 .expect("receipt replay"),
2751 LocalDataRebootstrapResult {
2752 already_applied: true,
2753 retained_commits: 1,
2754 reset_subscriptions: 1,
2755 }
2756 );
2757 assert!(reopened.drain_change_batches().is_empty());
2758 assert!(reopened.drain_sync_intents().is_empty());
2759 drop(reopened);
2760 std::fs::remove_file(path).expect("remove temp database");
2761 }
2762
2763 #[test]
2764 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
2765 let mut malformed = client();
2766 malformed
2767 .subscribe(
2768 "repair-tasks".to_owned(),
2769 "tasks".to_owned(),
2770 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2771 None,
2772 )
2773 .expect("subscribe");
2774 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
2775 malformed.drain_change_batches();
2776 malformed.drain_sync_intents();
2777 let malformed_error = malformed
2778 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2779 rebootstrap_id: "malformed".to_owned(),
2780 })
2781 .expect_err("malformed receipt must fail closed");
2782 assert_eq!(
2783 malformed_error,
2784 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
2785 );
2786 assert!(!malformed.upgrading());
2787 assert_eq!(
2788 malformed
2789 .subscription_state("repair-tasks")
2790 .expect("unchanged subscription")
2791 .cursor,
2792 -1
2793 );
2794 assert!(malformed.drain_change_batches().is_empty());
2795 assert!(malformed.drain_sync_intents().is_empty());
2796
2797 let mut unreadable = client();
2798 unreadable
2799 .conn
2800 .execute(
2801 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
2802 rusqlite::params!["server-row", "p1"],
2803 )
2804 .expect("seed visible row");
2805 unreadable
2806 .conn
2807 .execute("DROP TABLE _syncular_meta", [])
2808 .expect("break marker storage");
2809 let unreadable_error = unreadable
2810 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2811 rebootstrap_id: "unreadable".to_owned(),
2812 })
2813 .expect_err("unreadable marker storage must fail closed");
2814 assert_eq!(
2815 unreadable_error,
2816 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
2817 );
2818 let visible_rows = unreadable
2819 .conn
2820 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
2821 .expect("visible projection remains");
2822 assert_eq!(visible_rows, 1);
2823 assert!(!unreadable.upgrading());
2824 assert!(unreadable.drain_change_batches().is_empty());
2825 assert!(unreadable.drain_sync_intents().is_empty());
2826 }
2827
2828 #[test]
2829 fn local_rebootstrap_cannot_bypass_schema_floor() {
2830 let mut client = client();
2831 client.set_schema_floor(Some(SchemaFloor {
2832 required_schema_version: Some(2),
2833 latest_schema_version: Some(2),
2834 }));
2835 let error = client
2836 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2837 rebootstrap_id: "blocked-floor".to_owned(),
2838 })
2839 .expect_err("schema floor must block repair");
2840 assert!(error.contains("cannot bypass an active schema-floor stop"));
2841 }
2842
2843 #[test]
2844 fn diego_applied_sparse_moves_stay_visible_until_pull_image_arrives() {
2845 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
2846 "columns":[{"name":"id","type":"string","nullable":false},
2847 {"name":"project_id","type":"string","nullable":false},
2848 {"name":"starts_at_ms","type":"integer","nullable":false},
2849 {"name":"version","type":"integer","nullable":false}],
2850 "scopes":[{"pattern":"project:{project_id}"}]}]});
2851 let mut client =
2852 SyncClient::new("diego-ack-gap".into(), &schema, ClientLimits::default()).unwrap();
2853 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2854 client
2855 .write_base_row(
2856 "tasks",
2857 &vec![
2858 Some(ColumnValue::String("t1".into())),
2859 Some(ColumnValue::String("p1".into())),
2860 Some(ColumnValue::Integer(100)),
2861 Some(ColumnValue::Integer(1)),
2862 ],
2863 1,
2864 )
2865 .unwrap();
2866 client.rebuild_overlay().expect("rebuild overlay");
2867 let first = client
2868 .patch(
2869 "tasks",
2870 "t1",
2871 Map::from_iter([
2872 ("starts_at_ms".into(), json!(200)),
2873 ("version".into(), json!(2)),
2874 ]),
2875 None,
2876 )
2877 .unwrap();
2878 let (_, first_request) = client.build_request(false).unwrap();
2879 let second = client
2881 .patch(
2882 "tasks",
2883 "t1",
2884 Map::from_iter([
2885 ("starts_at_ms".into(), json!(300)),
2886 ("version".into(), json!(3)),
2887 ]),
2888 None,
2889 )
2890 .unwrap();
2891 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2892 for (id, seq, meta) in [
2893 (first, 2, first_request),
2894 (second, 3, client.build_request(false).unwrap().1),
2895 ] {
2896 let response = Message {
2897 wire_version: WIRE_VERSION,
2898 msg_kind: MsgKind::Response,
2899 frames: vec![
2900 Frame::RespHeader {
2901 required_schema_version: None,
2902 latest_schema_version: None,
2903 log_epoch: Some("epoch-1".into()),
2904 reset_required: Some(false),
2905 },
2906 Frame::PushResult {
2907 client_commit_id: id,
2908 status: PushStatus::Applied,
2909 commit_seq: Some(seq),
2910 results: vec![OpResult::Applied { op_index: 0 }],
2911 },
2912 ],
2913 };
2914 assert!(matches!(
2915 client.process_response(&mut transport, response, &meta),
2916 SyncOutcome::Ok(_)
2917 ));
2918 let row = &client
2919 .query("SELECT starts_at_ms, version FROM tasks WHERE id='t1'", &[])
2920 .unwrap()[0];
2921 assert_eq!(
2922 row["starts_at_ms"],
2923 json!(300),
2924 "an applied ACK without a pull image must not expose the old base"
2925 );
2926 assert_eq!(
2927 row["version"],
2928 json!(3),
2929 "the queued domain version must remain monotonic"
2930 );
2931 }
2932 }
2933
2934 #[test]
2935 fn first_epoch_acquisition_preserves_tables_subscriptions_and_local_intent() {
2936 for stored in [None, Some("epoch-old")] {
2937 let mut client = client();
2938 client
2939 .subscribe(
2940 "tasks".into(),
2941 "tasks".into(),
2942 vec![("project_id".into(), vec!["p1".into()])],
2943 None,
2944 )
2945 .unwrap();
2946 client
2947 .conn
2948 .execute_batch(
2949 "CREATE TRIGGER epoch_table_identity AFTER INSERT ON tasks BEGIN SELECT 1; END",
2950 )
2951 .unwrap();
2952 let id = client
2953 .mutate(vec![Mutation::Upsert {
2954 table: "tasks".into(),
2955 values: Map::from_iter([
2956 ("id".into(), json!("offline")),
2957 ("project_id".into(), json!("p1")),
2958 ]),
2959 base_version: None,
2960 }])
2961 .unwrap();
2962 if let Some(epoch) = stored {
2963 client.set_meta(LOG_EPOCH_KEY, epoch);
2964 }
2965 let before = serde_json::to_value(client.subscription_state("tasks")).unwrap();
2966 let (_, meta) = client.build_request(false).unwrap();
2967 client.drain_change_batches();
2968 client.drain_sync_intents();
2969 let response = Message {
2970 wire_version: WIRE_VERSION,
2971 msg_kind: MsgKind::Response,
2972 frames: vec![Frame::RespHeader {
2973 required_schema_version: None,
2974 latest_schema_version: None,
2975 log_epoch: Some("epoch-new".into()),
2976 reset_required: Some(true),
2977 }],
2978 };
2979 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2980 let outcome = client.process_response(&mut transport, response, &meta);
2981 let SyncOutcome::Ok(report) = outcome else {
2982 panic!("epoch round failed: {outcome:?}");
2983 };
2984 assert_eq!(
2985 report.resets,
2986 if stored.is_none() {
2987 Vec::<String>::new()
2988 } else {
2989 vec!["tasks".into()]
2990 }
2991 );
2992 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
2993 assert_eq!(client.pending_commit_ids(), vec![id]);
2994 assert_eq!(
2995 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
2996 json!("offline")
2997 );
2998 let trigger: i64 = client
2999 .conn
3000 .query_row(
3001 "SELECT count(*) FROM sqlite_master WHERE name='epoch_table_identity'",
3002 [],
3003 |row| row.get(0),
3004 )
3005 .unwrap();
3006 if stored.is_none() {
3007 assert!(!client.upgrading);
3008 assert!(client
3009 .drain_change_batches()
3010 .iter()
3011 .all(|batch| batch.status.as_ref().is_none_or(|status| !status.upgrading)));
3012 assert_eq!(
3013 serde_json::to_value(client.subscription_state("tasks")).unwrap(),
3014 before
3015 );
3016 assert_eq!(trigger, 1, "acquisition must not drop the synced table");
3017 } else {
3018 assert!(client.upgrading);
3019 assert_eq!(trigger, 0, "an epoch change must drop the old table");
3020 }
3021 assert!(matches!(
3022 client.drain_sync_intents().as_slice(),
3023 [SyncIntent::Interactive]
3024 ));
3025 }
3026 }
3027
3028 #[test]
3029 fn epoch_acquisition_storage_failures_preserve_ready_state_and_intent() {
3030 for key in ["logEpoch", "localRevision"] {
3031 let mut client = client();
3032 let id = client
3033 .mutate(vec![Mutation::Upsert {
3034 table: "tasks".into(),
3035 values: Map::from_iter([
3036 ("id".into(), json!("offline")),
3037 ("project_id".into(), json!("p1")),
3038 ]),
3039 base_version: None,
3040 }])
3041 .unwrap();
3042 let revision = client.local_revision();
3043 let needed = client.sync_needed;
3044 client.drain_change_batches();
3045 client.drain_sync_intents();
3046 client.conn.execute_batch(&format!("CREATE TRIGGER fail_acquisition BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected acquisition failure'); END")).unwrap();
3047 assert!(client.run_log_epoch_reset("epoch-new").is_err());
3048 assert!(client.get_meta(LOG_EPOCH_KEY).is_none());
3049 assert!(!client.upgrading);
3050 assert_eq!(client.sync_needed, needed);
3051 assert_eq!(client.local_revision(), revision);
3052 assert_eq!(client.pending_commit_ids(), [id]);
3053 assert!(client.drain_change_batches().is_empty());
3054 assert!(client.drain_sync_intents().is_empty());
3055 }
3056 }
3057
3058 #[test]
3059 fn failed_log_epoch_resets_restore_memory_and_durable_state() {
3060 for key in [
3061 "localSchemaVersion",
3062 "localSchemaDescriptor",
3063 "logEpoch",
3064 "localRevision",
3065 ] {
3066 let mut client = client();
3067 client.set_meta(LOG_EPOCH_KEY, "epoch-old");
3068 let commit = client
3069 .mutate(vec![Mutation::Upsert {
3070 table: "tasks".into(),
3071 values: Map::from_iter([
3072 ("id".into(), json!("offline")),
3073 ("project_id".into(), json!("p1")),
3074 ]),
3075 base_version: None,
3076 }])
3077 .unwrap();
3078 client.active_round = Some(uuid::Uuid::new_v4());
3079 client.overlay_dirty.set(false);
3080 let round = client.active_round;
3081 let needed = client.sync_needed;
3082 let revision = client.local_revision();
3083 let rows = serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap();
3084 client.drain_change_batches();
3085 client.drain_sync_intents();
3086 client.conn.execute_batch(&format!("CREATE TRIGGER fail_reset BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected reset failure'); END")).unwrap();
3087 assert!(client.run_log_epoch_reset("epoch-new").is_err(), "{key}");
3088 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-old"));
3089 assert!(!client.upgrading);
3090 assert_eq!(client.active_round, round);
3091 assert!(!client.overlay_dirty.get());
3092 assert_eq!(client.sync_needed, needed);
3093 assert_eq!(client.local_revision(), revision);
3094 assert_eq!(
3095 client.pending_commit_ids().as_slice(),
3096 std::slice::from_ref(&commit)
3097 );
3098 assert_eq!(
3099 serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap(),
3100 rows
3101 );
3102 assert!(client.drain_change_batches().is_empty());
3103 assert!(client.drain_sync_intents().is_empty());
3104 assert!(client.conn.is_autocommit());
3105 client
3106 .conn
3107 .execute_batch("DROP TRIGGER fail_reset")
3108 .unwrap();
3109 client.run_log_epoch_reset("epoch-new").unwrap();
3110 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3111 assert_eq!(client.pending_commit_ids(), [commit]);
3112 }
3113 }
3114
3115 #[test]
3116 fn acknowledged_restore_and_replay_ignore_unrelated_subscription_count() {
3117 for subscriptions in [0, 2, 6, 12] {
3118 let mut client = SyncClient::new("ack-scope".into(), &json!({ "version": 1, "tables": [
3119 { "name": "tasks", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] },
3120 { "name": "docs", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] }
3121 ] }), ClientLimits::default()).unwrap();
3122 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3123 client.conn.execute_batch("CREATE TABLE ack_writes(kind TEXT); CREATE TRIGGER count_ack_restore AFTER DELETE ON tasks BEGIN INSERT INTO ack_writes VALUES('restore'); END; CREATE TRIGGER count_ack_replay AFTER INSERT ON tasks BEGIN INSERT INTO ack_writes VALUES('replay'); END;").unwrap();
3124 for index in 0..32 {
3125 client
3126 .mutate(vec![Mutation::Upsert {
3127 table: "tasks".into(),
3128 values: Map::from_iter([
3129 ("id".into(), json!(format!("task-{index}"))),
3130 ("project_id".into(), json!("p1")),
3131 ]),
3132 base_version: None,
3133 }])
3134 .unwrap();
3135 }
3136 let (_, meta) = client.build_request(false).unwrap();
3137 let mut frames = vec![Frame::RespHeader {
3138 required_schema_version: None,
3139 latest_schema_version: None,
3140 log_epoch: Some("epoch-1".into()),
3141 reset_required: Some(false),
3142 }];
3143 frames.extend(meta.pushed_ids.iter().enumerate().map(|(index, id)| {
3144 Frame::PushResult {
3145 client_commit_id: id.clone(),
3146 status: PushStatus::Applied,
3147 commit_seq: Some(index as i64 + 1),
3148 results: vec![OpResult::Applied { op_index: 0 }],
3149 }
3150 }));
3151 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3152 client.acknowledged_replay_count.set(0);
3153 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3154 assert!(matches!(
3155 client.process_response(
3156 &mut transport,
3157 Message {
3158 wire_version: WIRE_VERSION,
3159 msg_kind: MsgKind::Response,
3160 frames
3161 },
3162 &meta
3163 ),
3164 SyncOutcome::Ok(_)
3165 ));
3166 assert_eq!(client.acknowledged_replay_count.get(), 32);
3167 assert_eq!(
3168 client
3169 .query(
3170 "SELECT count(*) AS n FROM ack_writes WHERE kind='restore'",
3171 &[]
3172 )
3173 .unwrap()[0]["n"],
3174 json!(32)
3175 );
3176 assert_eq!(
3177 client
3178 .query(
3179 "SELECT count(*) AS n FROM ack_writes WHERE kind='replay'",
3180 &[]
3181 )
3182 .unwrap()[0]["n"],
3183 json!(32)
3184 );
3185 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3186 client.acknowledged_replay_count.set(0);
3187 for index in 0..subscriptions {
3188 client
3189 .subscribe(
3190 format!("docs-{index}"),
3191 "docs".into(),
3192 vec![("project_id".into(), vec![format!("scope-{index}")])],
3193 None,
3194 )
3195 .unwrap();
3196 let sub = client.subs.len() - 1;
3197 client.subs[sub].effective =
3198 Some(vec![("project_id".into(), vec![format!("scope-{index}")])]);
3199 let segment = RowsSegment {
3200 table: "docs".into(),
3201 schema_version: 1,
3202 columns: client.schema.table("docs").unwrap().wire_columns.clone(),
3203 blocks: if index % 2 == 0 {
3204 vec![vec![ssp2::segment::SegmentRow {
3205 values: vec![
3206 Some(ColumnValue::String(format!("doc-{index}"))),
3207 Some(ColumnValue::String(format!("scope-{index}"))),
3208 ],
3209 server_version: 1,
3210 }]]
3211 } else {
3212 vec![]
3213 },
3214 };
3215 assert!(client.apply_segment(sub, &segment, true).is_ok());
3216 }
3217 assert_eq!(
3218 client
3219 .query("SELECT count(*) AS n FROM ack_writes", &[])
3220 .unwrap()[0]["n"],
3221 json!(0),
3222 "unrelated bootstrap must not restore/replay tasks"
3223 );
3224 assert_eq!(
3225 client.acknowledged_replay_count.get(),
3226 0,
3227 "unrelated bootstrap must not decode task ACKs"
3228 );
3229 assert_eq!(
3230 client
3231 .query("SELECT count(*) AS n FROM tasks", &[])
3232 .unwrap()[0]["n"],
3233 json!(32)
3234 );
3235 assert_eq!(
3236 client
3237 .query("SELECT count(*) AS n FROM _syncular_acked_rows", &[])
3238 .unwrap()[0]["n"],
3239 json!(32)
3240 );
3241 }
3242 }
3243
3244 #[test]
3245 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
3246 let mut client = client();
3247 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3248 const COMMIT_COUNT: usize = 32;
3249
3250 for index in 0..COMMIT_COUNT {
3251 client
3252 .mutate(vec![Mutation::Upsert {
3253 table: "tasks".to_owned(),
3254 values: Map::from_iter([
3255 ("id".to_owned(), Value::from(format!("task-{index}"))),
3256 ("project_id".to_owned(), Value::from("project-1")),
3257 ]),
3258 base_version: None,
3259 }])
3260 .expect("queue commit");
3261 }
3262
3263 let (_, request_meta) = client.build_request(false).unwrap();
3264 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
3265 let mut frames = vec![Frame::RespHeader {
3266 required_schema_version: None,
3267 latest_schema_version: None,
3268 log_epoch: Some("epoch-1".to_owned()),
3269 reset_required: Some(false),
3270 }];
3271 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
3272 |(index, client_commit_id)| Frame::PushResult {
3273 client_commit_id: client_commit_id.clone(),
3274 status: PushStatus::Applied,
3275 commit_seq: Some(index as i64 + 1),
3276 results: vec![OpResult::Applied { op_index: 0 }],
3277 },
3278 ));
3279 let response = Message {
3280 wire_version: WIRE_VERSION,
3281 msg_kind: MsgKind::Response,
3282 frames,
3283 };
3284 let mut transport =
3285 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
3286
3287 client.overlay_rebuild_count.set(0);
3288 client.outcome_prune_count.set(0);
3289 let outcome = client.process_response(&mut transport, response, &request_meta);
3290 assert!(matches!(outcome, SyncOutcome::Ok(_)));
3291 assert!(client.pending_commit_ids().is_empty());
3292 assert_eq!(
3293 client.overlay_rebuild_count.get(),
3294 1,
3295 "one response must reconcile its acknowledged commits with one overlay rebuild"
3296 );
3297 assert_eq!(
3298 client.outcome_prune_count.get(),
3299 1,
3300 "one response must enforce outcome retention once"
3301 );
3302 }
3303
3304 #[test]
3305 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
3306 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
3307 "columns": [
3308 {"name":"id", "type":"string", "nullable":false},
3309 {"name":"project_id", "type":"string", "nullable":false},
3310 {"name":"title", "type":"string", "nullable":false}
3311 ], "scopes": [{"pattern":"project:{project_id}"}],
3312 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
3313 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
3314 }] });
3315 let mut incremental =
3316 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3317 let mut reference =
3318 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3319 let tables = vec!["tasks".to_owned()];
3320 for index in 0..32 {
3321 let id = format!("task-{}", index % 8);
3322 let values = Map::from_iter([
3323 ("id".to_owned(), Value::from(id.clone())),
3324 (
3325 "project_id".to_owned(),
3326 Value::from(format!("p{}", index % 2)),
3327 ),
3328 ("title".to_owned(), Value::from(format!("needle{index}"))),
3329 ]);
3330 let payload = encode_row_json(
3331 incremental.schema.table("tasks").unwrap(),
3332 &id,
3333 &values,
3334 &incremental.encryption,
3335 )
3336 .unwrap();
3337 let change = ssp2::model::Change {
3338 table_index: 0,
3339 row_id: id,
3340 op: if index % 5 == 0 {
3341 Op::Delete
3342 } else {
3343 Op::Upsert
3344 },
3345 row_version: Some(index + 1),
3346 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
3347 row: Some(payload),
3348 };
3349 reference.overlay_dirty.set(true);
3350 for instance in [&mut incremental, &mut reference] {
3351 instance.begin_observation("mirror_test").unwrap();
3352 let mut batch = ChangeAccumulator::default();
3353 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
3354 instance
3355 .apply_commit_changes(&tables, std::slice::from_ref(&change))
3356 .unwrap();
3357 instance
3358 .rebuild_overlay_if_dirty()
3359 .expect("rebuild overlay");
3360 instance.finish_observation("mirror_test", batch).unwrap();
3361 }
3362 for sql in [
3363 "SELECT * FROM tasks ORDER BY id",
3364 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
3365 ] {
3366 assert_eq!(
3367 incremental.query(sql, &[]).unwrap(),
3368 reference.query(sql, &[]).unwrap()
3369 );
3370 }
3371 assert_eq!(incremental.local_revision(), reference.local_revision());
3372 assert_eq!(
3373 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3374 serde_json::to_value(reference.drain_change_batches()).unwrap()
3375 );
3376 }
3377 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
3378 assert_eq!(reference.overlay_rebuild_count.get(), 32);
3379 }
3380
3381 #[test]
3382 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
3383 let cases = [
3384 (
3385 "string",
3386 vec![
3387 json!("1"),
3388 json!("01"),
3389 json!(""),
3390 json!("nul\u{0}id"),
3391 json!("é"),
3392 ],
3393 ),
3394 (
3395 "integer",
3396 vec![
3397 json!(i64::MIN),
3398 json!(-1),
3399 json!(0),
3400 json!(1),
3401 json!(i64::MAX),
3402 ],
3403 ),
3404 ("boolean", vec![json!(false), json!(true)]),
3405 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
3406 ];
3407 for (kind, ids) in cases {
3408 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3409 "columns":[{"name":"id","type":kind,"nullable":false},
3410 {"name":"project_id","type":"string","nullable":false}],
3411 "scopes":[{"pattern":"project:{project_id}"}]}]});
3412 let mut client =
3413 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
3414 for (index, id) in ids.iter().enumerate() {
3415 let table = client.schema.table("tasks").unwrap();
3416 client
3417 .write_base_row(
3418 "tasks",
3419 &vec![
3420 if kind == "integer" {
3421 Some(ColumnValue::Integer(id.as_i64().unwrap()))
3424 } else {
3425 json_to_column_value(&table.columns[0], Some(id)).unwrap()
3426 },
3427 Some(ColumnValue::String(format!("p{index}"))),
3428 ],
3429 1,
3430 )
3431 .unwrap();
3432 }
3433 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3434 for base in [true, false] {
3435 let full = if base {
3436 base_table("tasks")
3437 } else {
3438 visible_table("tasks")
3439 };
3440 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
3441 for row_id in [
3442 "1",
3443 "01",
3444 "1.0",
3445 "1.0000000000000002",
3446 "+1",
3447 "1e0",
3448 " 1",
3449 "-1",
3450 "0",
3451 "9223372036854775807",
3452 "-9223372036854775808",
3453 "true",
3454 "false",
3455 "",
3456 "nul\u{0}id",
3457 "é",
3458 "null",
3459 "{\"a\":1}",
3460 ] {
3461 let old_sql =
3462 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
3463 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
3464 let old = client
3465 .conn
3466 .prepare(&old_sql)
3467 .unwrap()
3468 .query_map([row_id], |row| row.get::<_, String>(0))
3469 .unwrap()
3470 .collect::<Result<Vec<_>, _>>()
3471 .unwrap();
3472 let new = client
3473 .conn
3474 .prepare_cached(&new_sql)
3475 .unwrap()
3476 .query_map([row_id], |row| row.get::<_, String>(0))
3477 .unwrap()
3478 .collect::<Result<Vec<_>, _>>()
3479 .unwrap();
3480 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
3481 let mut batch = ChangeAccumulator::default();
3482 assert_eq!(
3483 client.record_row_scopes(&mut batch, "tasks", row_id, base),
3484 !old.is_empty()
3485 );
3486 let expected = old.first().map(|scope| {
3487 BTreeMap::from([(
3488 "tasks".to_owned(),
3489 Some(BTreeSet::from([format!("project:{scope}")])),
3490 )])
3491 });
3492 assert_eq!(batch.tables, expected.unwrap_or_default());
3493 client
3494 .conn
3495 .execute_batch("SAVEPOINT lookup_delete")
3496 .unwrap();
3497 if base {
3498 client.delete_base_row("tasks", row_id).unwrap();
3499 } else {
3500 client
3501 .apply_outbox_ops(&[OutboxOp {
3502 upsert: false,
3503 table: "tasks".into(),
3504 row_id: row_id.into(),
3505 base_version: None,
3506 values: None,
3507 }])
3508 .expect("replay delete");
3509 }
3510 let remaining: i64 = client
3511 .conn
3512 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
3513 row.get(0)
3514 })
3515 .unwrap();
3516 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
3517 client
3518 .conn
3519 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
3520 .unwrap();
3521 client.overlay_dirty.set(false);
3522 }
3523 for (predicate, expected) in [
3524 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
3525 (predicate, "SEARCH"),
3526 ] {
3527 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
3528 let details = client
3529 .conn
3530 .prepare(&sql)
3531 .unwrap()
3532 .query_map(["1"], |row| row.get::<_, String>(3))
3533 .unwrap()
3534 .collect::<Result<Vec<_>, _>>()
3535 .unwrap();
3536 assert!(
3537 details.iter().any(|detail| detail.contains(expected)),
3538 "{kind}: {details:?}"
3539 );
3540 }
3541 }
3542 }
3543 }
3544
3545 #[test]
3546 fn primary_keys_reject_every_ineligible_column_type() {
3547 for ty in ["float", "bytes", "crdt", "blob_ref"] {
3551 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3552 "columns":[{"name":"id","type":ty,"nullable":false},
3553 {"name":"project_id","type":"string","nullable":false}],
3554 "scopes":[{"pattern":"project:{project_id}"}]}]});
3555 let error =
3556 match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
3557 Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
3558 Err(error) => error,
3559 };
3560 assert!(
3561 error.contains("has an ineligible column type (§2.4)"),
3562 "{ty}: {error}"
3563 );
3564 }
3565 for ty in ["string", "integer", "boolean", "json"] {
3566 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3567 "columns":[{"name":"id","type":ty,"nullable":false},
3568 {"name":"project_id","type":"string","nullable":false}],
3569 "scopes":[{"pattern":"project:{project_id}"}]}]});
3570 SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
3571 .unwrap_or_else(|error| panic!("{ty}: {error}"));
3572 }
3573 }
3574
3575 #[test]
3576 fn pending_row_reconciliation_matches_full_fifo_replay() {
3577 for unique in [false, true] {
3578 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
3579 "name": name, "primaryKey": "id", "columns": [
3580 {"name":"id", "type":"string", "nullable":false},
3581 {"name":"project_id", "type":"string", "nullable":false},
3582 {"name":"title", "type":"string", "nullable":false}],
3583 "scopes": [{"pattern":"project:{project_id}"}],
3584 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
3585 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
3586 })))});
3587 let mut incremental =
3588 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
3589 let mut reference =
3590 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
3591 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
3592 for instance in [&mut incremental, &mut reference] {
3593 for table in &tables {
3594 for index in 0..12 {
3595 instance
3596 .write_base_row(
3597 table,
3598 &vec![
3599 Some(ColumnValue::String(format!("row{index}"))),
3600 Some(ColumnValue::String("p0".into())),
3601 Some(ColumnValue::String(format!("base{index}"))),
3602 ],
3603 1,
3604 )
3605 .unwrap();
3606 }
3607 }
3608 instance
3609 .rebuild_overlay_if_dirty()
3610 .expect("rebuild overlay");
3611 }
3612 for index in 0..80 {
3613 let mutations = (0..3)
3614 .map(|offset| {
3615 let table = tables[(index + offset) % 2].clone();
3616 let id = format!("row{}", (index + offset) % 16);
3617 if (index + offset) % 7 == 0 {
3618 Mutation::Delete {
3619 table,
3620 row_id: id,
3621 base_version: None,
3622 }
3623 } else {
3624 Mutation::Upsert {
3625 table,
3626 values: Map::from_iter([
3627 ("id".into(), Value::from(id)),
3628 ("project_id".into(), Value::from(format!("p{}", index % 3))),
3629 (
3630 "title".into(),
3631 Value::from(format!("needle{}", (index + offset) % 16)),
3632 ),
3633 ]),
3634 base_version: None,
3635 }
3636 }
3637 })
3638 .collect::<Vec<_>>();
3639 for instance in [&mut incremental, &mut reference] {
3640 instance.mutate(mutations.clone()).unwrap();
3641 }
3642 let changes = (0..3)
3645 .map(|offset| {
3646 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
3647 let id = format!("row{}", (index + offset) % 16);
3648 let values = Map::from_iter([
3649 ("id".into(), Value::from(id.clone())),
3650 (
3651 "project_id".into(),
3652 Value::from(format!("p{}", (index + 1) % 3)),
3653 ),
3654 (
3655 "title".into(),
3656 Value::from(format!("remote{index}_{offset}")),
3657 ),
3658 ]);
3659 ssp2::model::Change {
3660 table_index: table_index as u16,
3661 row_id: id.clone(),
3662 op: if (index + offset) % 5 == 0 {
3663 Op::Delete
3664 } else {
3665 Op::Upsert
3666 },
3667 row_version: Some(index as i64 + 2),
3668 scopes: vec![],
3669 row: Some(
3670 encode_row_json(
3671 incremental.schema.table(&tables[table_index]).unwrap(),
3672 &id,
3673 &values,
3674 &incremental.encryption,
3675 )
3676 .unwrap(),
3677 ),
3678 }
3679 })
3680 .collect::<Vec<_>>();
3681 reference.overlay_dirty.set(true);
3682 incremental
3683 .apply_commit_frame(&tables, &changes, None)
3684 .unwrap();
3685 reference
3686 .apply_commit_frame(&tables, &changes, None)
3687 .unwrap();
3688 for table in &tables {
3689 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
3690 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
3691 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
3692 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
3693 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
3694 }
3695 }
3696 assert_eq!(incremental.local_revision(), reference.local_revision());
3697 assert_eq!(
3698 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3699 serde_json::to_value(reference.drain_change_batches()).unwrap()
3700 );
3701 }
3702 assert_eq!(
3703 incremental.overlay_rebuild_count.get(),
3704 if unique { 81 } else { 21 }
3705 );
3706 assert_eq!(reference.overlay_rebuild_count.get(), 81);
3707 }
3708 }
3709
3710 #[test]
3711 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
3712 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
3713 "columns":[{"name":"id", "type":"string", "nullable":false},
3714 {"name":"title", "type":"string", "nullable":false}],
3715 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
3716 let mut client =
3717 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
3718 for (id, title) in [("a", "original"), ("b", "occupied")] {
3719 client
3720 .write_base_row(
3721 "tasks",
3722 &vec![
3723 Some(ColumnValue::String(id.into())),
3724 Some(ColumnValue::String(title.into())),
3725 ],
3726 1,
3727 )
3728 .unwrap();
3729 }
3730 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3731 let pending = client
3732 .mutate(vec![Mutation::Upsert {
3733 table: "tasks".into(),
3734 values: Map::from_iter([
3735 ("id".into(), Value::from("a")),
3736 ("title".into(), Value::from("available")),
3737 ]),
3738 base_version: None,
3739 }])
3740 .unwrap();
3741 assert_eq!(
3742 client
3743 .conn
3744 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
3745 .get::<_, String>(
3746 0
3747 ))
3748 .unwrap(),
3749 "available"
3750 );
3751 for (title, expected) in [("available", "original"), ("occupied", "available")] {
3752 let change = ssp2::model::Change {
3753 table_index: 0,
3754 row_id: "b".into(),
3755 op: Op::Upsert,
3756 row_version: Some(2),
3757 scopes: vec![],
3758 row: Some(
3759 encode_row_json(
3760 client.schema.table("tasks").unwrap(),
3761 "b",
3762 &Map::from_iter([
3763 ("id".into(), Value::from("b")),
3764 ("title".into(), Value::from(title)),
3765 ]),
3766 &client.encryption,
3767 )
3768 .unwrap(),
3769 ),
3770 };
3771 client
3772 .apply_commit_frame(&["tasks".into()], &[change], None)
3773 .unwrap();
3774 assert_eq!(
3775 client
3776 .conn
3777 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
3778 .get::<_, String>(
3779 0
3780 ))
3781 .unwrap(),
3782 expected
3783 );
3784 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
3785 }
3786 assert_eq!(client.overlay_rebuild_count.get(), 3);
3787 }
3788
3789 fn measure_scoped_overlay(unchanged_rows: i64, rounds: i64) -> Vec<u128> {
3790 let schema = json!({ "version": 1, "tables": [
3791 { "name":"tasks", "primaryKey":"id", "columns":[
3792 {"name":"id","type":"string","nullable":false},
3793 {"name":"project_id","type":"string","nullable":false}],
3794 "scopes":[{"pattern":"project:{project_id}"}] },
3795 { "name":"catalogue", "primaryKey":"id", "columns":[
3796 {"name":"id","type":"string","nullable":false},
3797 {"name":"project_id","type":"string","nullable":false},
3798 {"name":"title","type":"string","nullable":false}],
3799 "scopes":[{"pattern":"project:{project_id}"}],
3800 "ftsIndexes":[{"name":"catalogue_fts","columns":["title"],"tokenize":"unicode61"}] }
3801 ] });
3802 let mut client =
3803 SyncClient::new("overlay-scope".into(), &schema, ClientLimits::default()).unwrap();
3804 if unchanged_rows > 0 {
3805 client.conn.execute_batch(&format!("WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i<{unchanged_rows}) INSERT INTO _syncular_base_catalogue SELECT 'c'||i,'p1','needle '||i,1 FROM n")).unwrap();
3806 }
3807 client.rebuild_overlay().expect("rebuild overlay");
3808 client.conn.execute_batch("CREATE TABLE audit(tbl TEXT); CREATE TRIGGER catalogue_changed AFTER DELETE ON catalogue BEGIN INSERT INTO audit VALUES ('catalogue'); END;").unwrap();
3809 let mut elapsed = Vec::new();
3810 for round in 1..=rounds {
3811 let pending = client
3812 .mutate(vec![Mutation::Upsert {
3813 table: "tasks".into(),
3814 values: Map::from_iter([
3815 ("id".into(), json!("t1")),
3816 ("project_id".into(), json!("p1")),
3817 ]),
3818 base_version: None,
3819 }])
3820 .unwrap();
3821 let started = std::time::Instant::now();
3822 client
3823 .write_base_row(
3824 "tasks",
3825 &vec![
3826 Some(ColumnValue::String("t1".into())),
3827 Some(ColumnValue::String("p1".into())),
3828 ],
3829 round,
3830 )
3831 .unwrap();
3832 let mut report = SyncReport::default();
3833 client
3834 .handle_push_results(
3835 &[Frame::PushResult {
3836 client_commit_id: pending.clone(),
3837 status: PushStatus::Applied,
3838 commit_seq: Some(round),
3839 results: vec![OpResult::Applied { op_index: 0 }],
3840 }],
3841 &HashSet::from([pending.as_str()]),
3842 &HashMap::new(),
3843 &mut report,
3844 Some(&pending),
3845 )
3846 .unwrap();
3847 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3848 elapsed.push(started.elapsed().as_nanos());
3849 }
3850 assert_eq!(
3851 client
3852 .conn
3853 .query_row("SELECT count(*) FROM audit", [], |r| r.get::<_, i64>(0))
3854 .unwrap(),
3855 0,
3856 "unchanged catalogue must never be copied"
3857 );
3858 assert_eq!(
3859 client
3860 .conn
3861 .query_row(
3862 "SELECT count(*) FROM catalogue_fts WHERE catalogue_fts MATCH 'needle'",
3863 [],
3864 |r| r.get::<_, i64>(0)
3865 )
3866 .unwrap(),
3867 unchanged_rows
3868 );
3869 assert_eq!(
3870 client.query("SELECT * FROM tasks", &[]).unwrap()[0]["_syncular_version"],
3871 json!(rounds)
3872 );
3873 elapsed
3874 }
3875
3876 #[test]
3877 fn changed_overlay_leaves_large_unchanged_table_and_fts_untouched() {
3878 let elapsed = measure_scoped_overlay(100_000, 1);
3879 eprintln!(
3880 "table-scoped overlay with 100k unchanged rows: {} ns",
3881 elapsed[0]
3882 );
3883 }
3884
3885 #[test]
3886 #[ignore = "isolated performance measurement; run with the shared heavy-check lock"]
3887 fn benchmark_scoped_overlay() {
3888 for rows in [0, 100_000, 200_000] {
3889 let mut elapsed = measure_scoped_overlay(rows, 30);
3890 elapsed.sort_unstable();
3891 eprintln!(
3892 "{}",
3893 json!({"unchangedRows": rows, "rounds": elapsed.len(),
3894 "medianNs": elapsed[elapsed.len()/2], "minNs": elapsed[0],
3895 "maxNs": elapsed[elapsed.len()-1], "unchangedDeletes": 0})
3896 );
3897 }
3898 }
3899
3900 #[test]
3901 fn released_context_discards_in_flight_ack_and_server_rows() {
3902 for barrier in [
3903 "preflight",
3904 "purge",
3905 "rebootstrap",
3906 "subscription",
3907 "failed-purge-replay",
3908 "failed-purge-publish",
3909 "failed-rebootstrap",
3910 ] {
3911 let mut client = client();
3912 client.set_meta(LOG_EPOCH_KEY, "epoch");
3913 client
3914 .subscribe(
3915 "tasks".into(),
3916 "tasks".into(),
3917 vec![("project_id".into(), vec!["p1".into()])],
3918 None,
3919 )
3920 .unwrap();
3921 let commit = client
3922 .mutate(vec![Mutation::Upsert {
3923 table: "tasks".into(),
3924 values: Map::from_iter([
3925 ("id".into(), json!("local")),
3926 ("project_id".into(), json!("p1")),
3927 ]),
3928 base_version: None,
3929 }])
3930 .unwrap();
3931 let prepared = client.prepare_sync_round(false).unwrap();
3932 client.set_transport_enabled(&mut CountingRealtimeTransport::default(), false);
3933 let payload = encode_row_json(
3934 &client.schema.tables[0],
3935 "server",
3936 &Map::from_iter([
3937 ("id".into(), json!("server")),
3938 ("project_id".into(), json!("p1")),
3939 ]),
3940 &client.encryption,
3941 )
3942 .unwrap();
3943 let completed = crate::CompletedSyncRound {
3944 prepared,
3945 exchange: crate::round::ExchangeResult::Reply {
3946 transport_failed: false,
3947 downloads: Default::default(),
3948 response: Ok(Message {
3949 wire_version: WIRE_VERSION,
3950 msg_kind: MsgKind::Response,
3951 frames: vec![
3952 Frame::RespHeader {
3953 required_schema_version: None,
3954 latest_schema_version: None,
3955 log_epoch: Some("epoch".into()),
3956 reset_required: Some(false),
3957 },
3958 Frame::PushResult {
3959 client_commit_id: commit.clone(),
3960 status: PushStatus::Applied,
3961 commit_seq: Some(1),
3962 results: vec![OpResult::Applied { op_index: 0 }],
3963 },
3964 Frame::SubStart {
3965 id: "tasks".into(),
3966 status: SubStatus::Active,
3967 reason_code: String::new(),
3968 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
3969 bootstrap: true,
3970 },
3971 Frame::Commit {
3972 commit_seq: 1,
3973 created_at_ms: 0,
3974 actor_id: "actor".into(),
3975 tables: vec!["tasks".into()],
3976 changes: vec![ssp2::model::Change {
3977 table_index: 0,
3978 row_id: "server".into(),
3979 op: Op::Upsert,
3980 row_version: Some(1),
3981 scopes: vec![("project_id".into(), "p1".into())],
3982 row: Some(payload),
3983 }],
3984 },
3985 Frame::SubEnd {
3986 next_cursor: 1,
3987 bootstrap_state: None,
3988 },
3989 ],
3990 }),
3991 },
3992 };
3993 match barrier {
3994 "preflight" => client.begin_security_preflight(),
3995 "purge" => {
3996 client
3997 .purge_local_data(&LocalDataPurgeInput {
3998 purge_id: "pending-round".into(),
3999 targets: vec![LocalDataPurgeTarget {
4000 table: "tasks".into(),
4001 selectors: BTreeMap::from([(
4002 "project_id".into(),
4003 vec!["p1".into()],
4004 )]),
4005 }],
4006 })
4007 .unwrap();
4008 }
4009 "rebootstrap" => {
4010 client
4011 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4012 rebootstrap_id: "pending-round".into(),
4013 })
4014 .unwrap();
4015 }
4016 "failed-purge-replay" | "failed-purge-publish" | "failed-rebootstrap" => {
4017 let trigger = if barrier == "failed-purge-replay" {
4018 "CREATE TRIGGER fail_local BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4019 } else {
4020 "CREATE TRIGGER fail_local BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4021 };
4022 client.conn.execute_batch(trigger).unwrap();
4023 let revision = client.local_revision();
4024 if barrier == "failed-rebootstrap" {
4025 assert!(client
4026 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4027 rebootstrap_id: "failed-round".into(),
4028 })
4029 .is_err());
4030 } else {
4031 assert!(client
4032 .purge_local_data(&LocalDataPurgeInput {
4033 purge_id: "failed-round".into(),
4034 targets: vec![LocalDataPurgeTarget {
4035 table: "tasks".into(),
4036 selectors: BTreeMap::from([(
4037 "project_id".into(),
4038 vec!["p1".into()]
4039 )]),
4040 }],
4041 })
4042 .is_err());
4043 }
4044 assert_eq!(client.local_revision(), revision);
4045 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
4046 client
4047 .conn
4048 .execute_batch("DROP TRIGGER fail_local")
4049 .unwrap();
4050 let applied = client.apply_sync_round(completed);
4051 assert!(
4052 matches!(
4053 applied,
4054 crate::AppliedSyncRound::Complete {
4055 outcome: SyncOutcome::Ok(_),
4056 ..
4057 }
4058 ),
4059 "{barrier}"
4060 );
4061 assert!(client.pending_commit_ids().is_empty());
4062 assert_eq!(
4063 client
4064 .query("SELECT id FROM tasks WHERE id = 'server'", &[])
4065 .unwrap()
4066 .len(),
4067 1
4068 );
4069 continue;
4070 }
4071 "subscription" => {
4072 client.unsubscribe("tasks");
4073 client
4074 .subscribe(
4075 "tasks".into(),
4076 "tasks".into(),
4077 vec![("project_id".into(), vec!["p2".into()])],
4078 None,
4079 )
4080 .unwrap();
4081 }
4082 _ => unreachable!(),
4083 }
4084 let pending_before_reply = client.pending_commit_ids();
4085 let revision = client.local_revision();
4086 let applied = client.apply_sync_round(completed);
4087 assert!(
4088 matches!(applied, crate::AppliedSyncRound::Complete { outcome: SyncOutcome::Failed { ref error_code, .. }, ref controls, .. } if error_code == "client.round_cancelled" && controls.is_empty())
4089 );
4090 assert_eq!(client.local_revision(), revision);
4091 assert!(client
4092 .conn
4093 .query_row(
4094 "SELECT NOT EXISTS(SELECT 1 FROM tasks WHERE id='server')",
4095 [],
4096 |r| r.get::<_, bool>(0)
4097 )
4098 .unwrap());
4099 assert_eq!(client.pending_commit_ids(), pending_before_reply);
4100 }
4101 }
4102
4103 #[test]
4104 fn sync_round_deadline_is_anchored_once_per_round_and_carried_across_continues() {
4105 struct DeadlineRecording {
4106 budget: std::time::Duration,
4107 seen: Vec<Option<std::time::Instant>>,
4108 }
4109 impl Transport for DeadlineRecording {
4110 fn round_deadline(&self) -> Option<std::time::Duration> {
4111 Some(self.budget)
4112 }
4113 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4114 self.seen.push(deadline);
4115 }
4116 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4117 Err(TransportError::new("transport.failed", "injected"))
4118 }
4119 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4120 self.sync(request)
4121 }
4122 fn download_segment(
4123 &mut self,
4124 _: &SegmentRequest,
4125 _: &mut dyn FnMut(u64),
4126 ) -> Result<Vec<u8>, TransportError> {
4127 Err(TransportError::new("transport.failed", "injected"))
4128 }
4129 fn blob_upload_grant(
4130 &mut self,
4131 _: &str,
4132 _: u64,
4133 _: Option<&str>,
4134 ) -> Result<crate::BlobUploadGrant, TransportError> {
4135 Ok(crate::BlobUploadGrant::Present)
4136 }
4137 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4138 Ok(())
4139 }
4140 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4141 Ok(())
4142 }
4143 fn realtime_close(&mut self) -> Result<(), TransportError> {
4144 Ok(())
4145 }
4146 }
4147
4148 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4149 {"name":"id","type":"string","nullable":false},
4150 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4151 let mut client =
4152 SyncClient::new("deadline-round".into(), &schema, Default::default()).unwrap();
4153 client.set_meta(LOG_EPOCH_KEY, "epoch");
4154 client
4155 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4156 .unwrap();
4157
4158 let mut transport = DeadlineRecording {
4159 budget: std::time::Duration::from_secs(30),
4160 seen: Vec::new(),
4161 };
4162 let prepared = client.prepare_sync_round(false).unwrap();
4163 assert_eq!(prepared.uploads.len(), 1, "one staged upload");
4164 let prepared = match client.apply_sync_round(prepared.exchange(&mut transport)) {
4166 crate::AppliedSyncRound::Continue(prepared) => prepared,
4167 crate::AppliedSyncRound::Complete { .. } => panic!("the upload must continue"),
4168 };
4169 let completed = prepared.exchange(&mut transport);
4171 let anchor = completed
4172 .prepared
4173 .round_deadline_at
4174 .expect("the upload round anchors a deadline");
4175 assert!(matches!(
4176 client.apply_sync_round(completed),
4177 crate::AppliedSyncRound::Complete { .. }
4178 ));
4179 assert_eq!(transport.seen.len(), 4, "one scope per exchange");
4180 assert_eq!(
4181 transport.seen[0],
4182 Some(anchor),
4183 "the upload anchors the round"
4184 );
4185 assert_eq!(
4186 transport.seen[1], None,
4187 "the upload exchange clears its scope"
4188 );
4189 assert_eq!(
4190 transport.seen[2],
4191 Some(anchor),
4192 "a continuation reuses the carried anchor instead of refreshing it"
4193 );
4194 assert_eq!(transport.seen[3], None, "the continuation clears its scope");
4195
4196 transport.budget = std::time::Duration::from_secs(60);
4199 let prepared = client.prepare_sync_round(false).unwrap();
4200 let completed = prepared.exchange(&mut transport);
4201 let fresh = completed
4202 .prepared
4203 .round_deadline_at
4204 .expect("the next round anchors a deadline");
4205 client.apply_sync_round(completed);
4206 assert!(
4207 fresh >= anchor + std::time::Duration::from_secs(30),
4208 "the next round anchors a fresh deadline from the new budget"
4209 );
4210 assert_eq!(transport.seen[4], Some(fresh));
4211 assert_eq!(
4212 transport.seen[5], None,
4213 "the new round clears its scope too"
4214 );
4215 }
4216
4217 #[derive(Default)]
4220 struct RoundDeadlineProbe {
4221 budget: Option<std::time::Duration>,
4222 budget_queries: std::cell::Cell<usize>,
4223 scoped: Vec<Option<std::time::Instant>>,
4224 }
4225
4226 impl Transport for RoundDeadlineProbe {
4227 fn round_deadline(&self) -> Option<std::time::Duration> {
4228 self.budget_queries.set(self.budget_queries.get() + 1);
4229 self.budget
4230 }
4231 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4232 self.scoped.push(deadline);
4233 }
4234 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4235 Err(TransportError::new("transport.failed", "injected"))
4236 }
4237 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4238 self.sync(request)
4239 }
4240 fn download_segment(
4241 &mut self,
4242 _: &SegmentRequest,
4243 _: &mut dyn FnMut(u64),
4244 ) -> Result<Vec<u8>, TransportError> {
4245 Err(TransportError::new("transport.failed", "injected"))
4246 }
4247 fn blob_upload_grant(
4248 &mut self,
4249 _: &str,
4250 _: u64,
4251 _: Option<&str>,
4252 ) -> Result<crate::BlobUploadGrant, TransportError> {
4253 Ok(crate::BlobUploadGrant::Present)
4254 }
4255 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4256 Ok(())
4257 }
4258 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4259 Ok(())
4260 }
4261 fn realtime_close(&mut self) -> Result<(), TransportError> {
4262 Ok(())
4263 }
4264 }
4265
4266 #[test]
4267 fn an_unusable_upload_body_does_not_anchor_the_round_deadline() {
4268 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4269 {"name":"id","type":"string","nullable":false},
4270 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4271 let mut client =
4272 SyncClient::new("deadline-unusable".into(), &schema, ClientLimits::default()).unwrap();
4273 client.set_meta(LOG_EPOCH_KEY, "epoch");
4274 client
4275 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4276 .unwrap();
4277 client
4278 .conn
4279 .execute_batch("DELETE FROM _syncular_blobs")
4280 .unwrap();
4281 let prepared = client.prepare_sync_round(false).unwrap();
4282 assert!(
4283 prepared.uploads.front().is_some_and(|u| u.bytes.is_err()),
4284 "the staged body must fail locally"
4285 );
4286 let mut transport = RoundDeadlineProbe {
4287 budget: Some(std::time::Duration::from_secs(30)),
4288 ..Default::default()
4289 };
4290 let completed = prepared.exchange(&mut transport);
4291 assert!(matches!(
4292 completed.exchange,
4293 crate::round::ExchangeResult::Upload { result: Err(_), .. }
4294 ));
4295 assert_eq!(
4296 transport.budget_queries.get(),
4297 0,
4298 "a round with no network work never asks for a budget"
4299 );
4300 assert_eq!(transport.scoped, vec![None]);
4301 }
4302
4303 #[test]
4304 fn a_transport_budget_the_clock_cannot_represent_fails_before_network() {
4305 struct HugeBudget;
4306 impl Transport for HugeBudget {
4307 fn round_deadline(&self) -> Option<std::time::Duration> {
4308 Some(std::time::Duration::MAX)
4309 }
4310 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4311 panic!("no network for an invalid budget")
4312 }
4313 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4314 self.sync(request)
4315 }
4316 fn download_segment(
4317 &mut self,
4318 _: &SegmentRequest,
4319 _: &mut dyn FnMut(u64),
4320 ) -> Result<Vec<u8>, TransportError> {
4321 panic!("no network for an invalid budget")
4322 }
4323 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4324 Ok(())
4325 }
4326 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4327 Ok(())
4328 }
4329 fn realtime_close(&mut self) -> Result<(), TransportError> {
4330 Ok(())
4331 }
4332 }
4333
4334 let mut client = client();
4335 client.set_meta(LOG_EPOCH_KEY, "epoch");
4336 let prepared = client.prepare_sync_round(false).unwrap();
4337 let outcome = match client.apply_sync_round(prepared.exchange(&mut HugeBudget)) {
4338 crate::AppliedSyncRound::Complete { outcome, .. } => outcome,
4339 crate::AppliedSyncRound::Continue(_) => {
4340 panic!("an unrepresentable budget must not continue")
4341 }
4342 };
4343 match outcome {
4344 SyncOutcome::Failed {
4345 error_code,
4346 message,
4347 ..
4348 } => {
4349 assert_eq!(error_code, "sync.invalid_request");
4350 assert_eq!(message, "transport round deadline is out of range");
4351 }
4352 _ => panic!("an unrepresentable budget must fail synchronously"),
4353 }
4354 }
4355
4356 #[test]
4357 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
4358 for failure in ["visible", "revision", "commit"] {
4359 let mut client = client();
4360 let id = client
4361 .mutate(vec![Mutation::Upsert {
4362 table: "tasks".into(),
4363 values: Map::from_iter([
4364 ("id".into(), Value::from("t1")),
4365 ("project_id".into(), Value::from("pending")),
4366 ]),
4367 base_version: None,
4368 }])
4369 .unwrap();
4370 client.drain_change_batches();
4371 let revision = client.local_revision();
4372 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
4373 let trigger = match failure {
4374 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
4375 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
4376 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
4377 };
4378 client.conn.execute_batch(&trigger).unwrap();
4379 let change = ssp2::model::Change {
4380 table_index: 0,
4381 row_id: "t1".into(),
4382 op: Op::Upsert,
4383 row_version: Some(2),
4384 scopes: vec![],
4385 row: Some(
4386 encode_row_json(
4387 client.schema.table("tasks").unwrap(),
4388 "t1",
4389 &Map::from_iter([
4390 ("id".into(), Value::from("t1")),
4391 ("project_id".into(), Value::from("remote")),
4392 ]),
4393 &client.encryption,
4394 )
4395 .unwrap(),
4396 ),
4397 };
4398 assert!(client
4399 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), None)
4400 .is_err());
4401 assert!(client.conn.is_autocommit());
4402 assert!(!client.overlay_dirty.get());
4403 assert_eq!(client.local_revision(), revision);
4404 assert!(client.drain_change_batches().is_empty());
4405 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
4406 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
4407 assert!(client
4408 .query("SELECT * FROM _syncular_base_tasks", &[])
4409 .unwrap()
4410 .is_empty());
4411 client
4412 .conn
4413 .execute_batch("DROP TRIGGER fail_pending")
4414 .unwrap();
4415 client
4416 .apply_commit_frame(&["tasks".into()], &[change], None)
4417 .unwrap();
4418 assert_eq!(client.local_revision(), revision + 1);
4419 assert_eq!(client.pending_commit_ids(), vec![id]);
4420 let mut expected = before[0].clone();
4423 expected.insert("_syncular_version".to_owned(), Value::from(2));
4424 assert_eq!(
4425 client.query("SELECT * FROM tasks", &[]).unwrap(),
4426 vec![expected]
4427 );
4428 assert_eq!(
4429 client
4430 .conn
4431 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
4432 .get::<_, String>(0))
4433 .unwrap(),
4434 "remote"
4435 );
4436 }
4437 }
4438
4439 #[test]
4440 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
4441 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
4442 "columns": [
4443 {"name":"id", "type":"string", "nullable":false},
4444 {"name":"project_id", "type":"string", "nullable":false},
4445 {"name":"title", "type":"string", "nullable":false}],
4446 "scopes": [{"pattern":"project:{project_id}"}],
4447 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
4448 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
4449 let mut incremental =
4450 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4451 let mut reference =
4452 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4453 for instance in [&mut incremental, &mut reference] {
4454 for index in 0..16 {
4455 instance
4456 .write_base_row(
4457 "tasks",
4458 &vec![
4459 Some(ColumnValue::String(format!("task{index}"))),
4460 Some(ColumnValue::String("p0".to_owned())),
4461 Some(ColumnValue::String(format!("base{index}"))),
4462 ],
4463 1,
4464 )
4465 .unwrap();
4466 }
4467 instance
4468 .rebuild_overlay_if_dirty()
4469 .expect("rebuild overlay");
4470 instance.overlay_rebuild_count.set(0);
4471 }
4472 for index in 0..100 {
4473 let mutations = (0..1 + index % 4)
4474 .map(|offset| {
4475 let id = format!("task{}", (index + offset) % 24);
4476 if (index + offset) % 7 == 0 {
4477 Mutation::Delete {
4478 table: "tasks".to_owned(),
4479 row_id: id,
4480 base_version: None,
4481 }
4482 } else {
4483 Mutation::Upsert {
4484 table: "tasks".to_owned(),
4485 values: Map::from_iter([
4486 ("id".to_owned(), Value::from(id)),
4487 (
4488 "project_id".to_owned(),
4489 Value::from(format!("p{}", index % 2)),
4490 ),
4491 (
4492 "title".to_owned(),
4493 Value::from(format!("needle{}", (index + offset) % 24)),
4494 ),
4495 ]),
4496 base_version: None,
4497 }
4498 }
4499 })
4500 .collect::<Vec<_>>();
4501 reference.overlay_dirty.set(true);
4502 incremental.mutate(mutations.clone()).unwrap();
4503 reference.mutate(mutations).unwrap();
4504 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
4505 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
4506 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
4507 }
4508 assert_eq!(incremental.local_revision(), reference.local_revision());
4509 assert_eq!(
4510 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4511 serde_json::to_value(reference.drain_change_batches()).unwrap()
4512 );
4513 }
4514 assert_eq!(incremental.outbox.len(), 100);
4515 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
4516 assert_eq!(reference.overlay_rebuild_count.get(), 100);
4517 }
4518
4519 #[test]
4520 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
4521 for fail_at in ["outbox", "revision"] {
4522 let mut client = client();
4523 let values = Map::from_iter([
4524 ("id".to_owned(), Value::from("t1")),
4525 ("project_id".to_owned(), Value::from("p1")),
4526 ]);
4527 let mutation = Mutation::Upsert {
4528 table: "tasks".to_owned(),
4529 values,
4530 base_version: None,
4531 };
4532 let trigger = if fail_at == "outbox" {
4533 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
4534 } else {
4535 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
4536 };
4537 client.conn.execute_batch(&trigger).unwrap();
4538 let revision = client.local_revision();
4539 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
4540 assert_eq!(client.local_revision(), revision);
4541 assert!(client.outbox.is_empty());
4542 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
4543 assert_eq!(
4544 client
4545 .conn
4546 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
4547 .get::<_, i64>(0))
4548 .unwrap(),
4549 0
4550 );
4551 assert!(client.drain_change_batches().is_empty());
4552 assert!(!client.overlay_dirty.get());
4553 client
4554 .conn
4555 .execute_batch("DROP TRIGGER fail_append")
4556 .unwrap();
4557 client.mutate(vec![mutation]).unwrap();
4558 assert_eq!(client.outbox.len(), 1);
4559 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
4560 }
4561 }
4562
4563 #[test]
4564 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
4565 let mut client = client();
4566 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
4567 let values = Map::from_iter([
4568 ("id".to_owned(), Value::from("t1")),
4569 ("project_id".to_owned(), Value::from("p1")),
4570 ]);
4571 let payload = encode_row_json(
4572 client.schema.table("tasks").unwrap(),
4573 "t1",
4574 &values,
4575 &client.encryption,
4576 )
4577 .unwrap();
4578 let first = ssp2::model::Change {
4579 table_index: 0,
4580 row_id: "t1".to_owned(),
4581 op: Op::Upsert,
4582 row_version: Some(1),
4583 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
4584 row: Some(payload),
4585 };
4586 let mut invalid = first.clone();
4587 invalid.table_index = 1;
4588 let revision = client.local_revision();
4589 client.begin_observation("failed_mirror").unwrap();
4590 assert!(client
4591 .apply_commit_changes(&tables, &[first, invalid])
4592 .is_err());
4593 client.rollback_observation("failed_mirror");
4594 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4595 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
4596 assert_eq!(
4597 client
4598 .conn
4599 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
4600 .get::<_, i64>(
4601 0
4602 ))
4603 .unwrap(),
4604 0
4605 );
4606 assert_eq!(client.local_revision(), revision);
4607 assert!(client.drain_change_batches().is_empty());
4608 }
4609
4610 #[test]
4611 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
4612 for failure in ["journal", "revision", "commit", "later-frame"] {
4613 let mut client = client();
4614 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4615 for id in ["first", "second"] {
4616 client
4617 .mutate(vec![Mutation::Upsert {
4618 table: "tasks".to_owned(),
4619 values: Map::from_iter([
4620 ("id".to_owned(), Value::from(id)),
4621 ("project_id".to_owned(), Value::from("p1")),
4622 ]),
4623 base_version: None,
4624 }])
4625 .unwrap();
4626 }
4627 let (_, meta) = client.build_request(false).unwrap();
4628 client
4630 .mutate(vec![Mutation::Upsert {
4631 table: "tasks".to_owned(),
4632 values: Map::from_iter([
4633 ("id".to_owned(), Value::from("later")),
4634 ("project_id".to_owned(), Value::from("p1")),
4635 ]),
4636 base_version: None,
4637 }])
4638 .unwrap();
4639 let ids = client.pending_commit_ids();
4640 let rows = client
4641 .query("SELECT * FROM tasks ORDER BY id", &[])
4642 .unwrap();
4643 let revision = client.local_revision();
4644 client.drain_change_batches();
4645 if failure != "later-frame" {
4646 client.conn.execute_batch(match failure {
4647 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
4648 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
4649 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
4650 }).unwrap();
4651 }
4652 let mut response = Message {
4653 wire_version: WIRE_VERSION,
4654 msg_kind: MsgKind::Response,
4655 frames: vec![Frame::RespHeader {
4656 required_schema_version: None,
4657 latest_schema_version: None,
4658 log_epoch: Some("epoch-1".to_owned()),
4659 reset_required: Some(false),
4660 }],
4661 };
4662 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
4663 response.frames.push(Frame::PushResult {
4664 client_commit_id: id.to_owned(),
4665 status: PushStatus::Applied,
4666 commit_seq: Some(1),
4667 results: vec![OpResult::Applied { op_index: 0 }],
4668 });
4669 }
4670 if failure == "later-frame" {
4671 response.frames.push(Frame::Error {
4672 code: "sync.invalid_request".to_owned(),
4673 message: "later error".to_owned(),
4674 category: "protocol".to_owned(),
4675 retryable: false,
4676 recommended_action: "retry".to_owned(),
4677 details: None,
4678 });
4679 }
4680 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
4681 let outcome = client.process_response(&mut transport, response.clone(), &meta);
4682 let expected = match failure {
4685 "later-frame" => "sync.invalid_request",
4686 "commit" => "storage.failed",
4687 _ => "client.outcome_persistence_failed",
4688 };
4689 assert!(
4690 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
4691 "{failure}: {outcome:?}"
4692 );
4693 if failure == "later-frame" {
4694 assert_eq!(client.pending_commit_ids(), ids[2..]);
4695 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
4696 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
4697 let batches = client.drain_change_batches();
4698 assert_eq!(batches.len(), 1);
4699 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
4700 assert_eq!(client.local_revision(), revision + 1);
4701 response.frames.pop();
4702 } else {
4703 assert_eq!(client.pending_commit_ids(), ids);
4704 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
4705 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
4706 assert_eq!(client.local_revision(), revision);
4707 assert!(client.drain_change_batches().is_empty());
4708 assert_eq!(
4709 client
4710 .query("SELECT * FROM tasks ORDER BY id", &[])
4711 .unwrap(),
4712 rows
4713 );
4714 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
4715 }
4716 for frame in &mut response.frames {
4717 if let Frame::PushResult { status, .. } = frame {
4718 *status = PushStatus::Cached;
4719 }
4720 }
4721 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
4722 else {
4723 panic!("retry failed");
4724 };
4725 assert_eq!(
4726 report.applied,
4727 if failure == "later-frame" {
4728 vec![]
4729 } else {
4730 ids[..2].to_vec()
4731 }
4732 );
4733 assert_eq!(client.pending_commit_ids(), ids[2..]);
4734 let batches = client.drain_change_batches();
4735 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
4736 if let Some(batch) = batches.first() {
4737 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
4738 }
4739 assert_eq!(client.local_revision(), revision + 1);
4740 }
4741 }
4742
4743 #[test]
4744 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
4745 for failure in ["revision", "journal", "commit"] {
4746 for status in ["applied", "cached", "rejected", "conflict"] {
4747 let mut client = client();
4748 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4749 for id in ["first", "later"] {
4750 client
4751 .mutate(vec![Mutation::Upsert {
4752 table: "tasks".to_owned(),
4753 values: Map::from_iter([
4754 ("id".to_owned(), Value::from(id)),
4755 ("project_id".to_owned(), Value::from("p1")),
4756 ]),
4757 base_version: None,
4758 }])
4759 .unwrap();
4760 }
4761 let (_, meta) = client.build_request(false).unwrap();
4762 let ids = client.pending_commit_ids();
4763 let rows = client
4764 .query("SELECT * FROM tasks ORDER BY id", &[])
4765 .unwrap();
4766 let revision = client.local_revision();
4767 client.drain_change_batches();
4768 let (push_status, results) = match status {
4769 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
4770 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
4771 "conflict" => (
4772 PushStatus::Rejected,
4773 vec![OpResult::Conflict {
4774 op_index: 0,
4775 code: "sync.version_conflict".to_owned(),
4776 message: "conflict".to_owned(),
4777 server_version: 1,
4778 server_row: encode_row_json(
4779 client.schema.table("tasks").unwrap(),
4780 "first",
4781 &Map::from_iter([
4782 ("id".to_owned(), Value::from("first")),
4783 ("project_id".to_owned(), Value::from("p1")),
4784 ]),
4785 &client.encryption,
4786 )
4787 .unwrap(),
4788 conflict_columns: Vec::new(),
4789 }],
4790 ),
4791 _ => (
4792 PushStatus::Rejected,
4793 vec![OpResult::Error {
4794 op_index: 0,
4795 code: "sync.validation_failed".to_owned(),
4796 message: "rejected".to_owned(),
4797 retryable: false,
4798 }],
4799 ),
4800 };
4801 let response = Message {
4802 wire_version: WIRE_VERSION,
4803 msg_kind: MsgKind::Response,
4804 frames: vec![
4805 Frame::RespHeader {
4806 required_schema_version: None,
4807 latest_schema_version: None,
4808 log_epoch: Some("epoch-1".to_owned()),
4809 reset_required: Some(false),
4810 },
4811 Frame::PushResult {
4812 client_commit_id: ids[0].clone(),
4813 status: push_status,
4814 commit_seq: Some(1),
4815 results,
4816 },
4817 ],
4818 };
4819 client.conn.execute_batch(match failure {
4820 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
4821 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
4822 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
4823 }).unwrap();
4824 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
4825 let outcome = client.process_response(&mut transport, response.clone(), &meta);
4826 assert_eq!(
4827 client.pending_commit_ids(),
4828 ids,
4829 "{failure}/{status}: pending identity"
4830 );
4831 assert_eq!(
4832 client.local_revision(),
4833 revision,
4834 "{failure}/{status}: revision"
4835 );
4836 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
4837 assert!(client.conflicts().is_empty());
4838 assert!(client.rejections().is_empty());
4839 assert!(client.drain_change_batches().is_empty());
4840 assert_eq!(
4841 client
4842 .query("SELECT * FROM tasks ORDER BY id", &[])
4843 .unwrap(),
4844 rows
4845 );
4846 let expected = if failure == "commit" {
4849 "storage.failed"
4850 } else {
4851 "client.outcome_persistence_failed"
4852 };
4853 assert!(
4854 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
4855 "{failure}/{status}: failed round"
4856 );
4857 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
4858 assert!(matches!(
4859 client.process_response(&mut transport, response, &meta),
4860 SyncOutcome::Ok(_)
4861 ));
4862 assert_eq!(client.pending_commit_ids(), ids[1..]);
4863 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
4864 assert_eq!(client.drain_change_batches().len(), 1);
4865 }
4866 }
4867 }
4868
4869 #[test]
4870 fn mixed_push_results_reconcile_each_boundary_and_prune_once_per_response() {
4871 let mut client = client();
4872 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4873 for index in 0..4 {
4874 client
4875 .mutate(vec![Mutation::Upsert {
4876 table: "tasks".to_owned(),
4877 values: Map::from_iter([
4878 ("id".to_owned(), Value::from(format!("task-{index}"))),
4879 ("project_id".to_owned(), Value::from("project-1")),
4880 ]),
4881 base_version: None,
4882 }])
4883 .expect("queue commit");
4884 }
4885
4886 let (_, request_meta) = client.build_request(false).unwrap();
4887 let ids = &request_meta.pushed_ids;
4888 assert_eq!(ids.len(), 4);
4889 let response = Message {
4890 wire_version: WIRE_VERSION,
4891 msg_kind: MsgKind::Response,
4892 frames: vec![
4893 Frame::RespHeader {
4894 required_schema_version: None,
4895 latest_schema_version: None,
4896 log_epoch: Some("epoch-1".to_owned()),
4897 reset_required: Some(false),
4898 },
4899 Frame::PushResult {
4900 client_commit_id: ids[0].clone(),
4901 status: PushStatus::Applied,
4902 commit_seq: Some(1),
4903 results: vec![OpResult::Applied { op_index: 0 }],
4904 },
4905 Frame::PushResult {
4906 client_commit_id: ids[1].clone(),
4907 status: PushStatus::Cached,
4908 commit_seq: Some(2),
4909 results: vec![OpResult::Applied { op_index: 0 }],
4910 },
4911 Frame::PushResult {
4912 client_commit_id: ids[2].clone(),
4913 status: PushStatus::Rejected,
4914 commit_seq: None,
4915 results: vec![OpResult::Error {
4916 op_index: 0,
4917 code: "sync.validation_failed".to_owned(),
4918 message: "rejected".to_owned(),
4919 retryable: false,
4920 }],
4921 },
4922 Frame::PushResult {
4923 client_commit_id: ids[3].clone(),
4924 status: PushStatus::Rejected,
4925 commit_seq: None,
4926 results: vec![OpResult::Error {
4927 op_index: 0,
4928 code: "sync.idempotency_cache_miss".to_owned(),
4929 message: "retry".to_owned(),
4930 retryable: true,
4931 }],
4932 },
4933 ],
4934 };
4935 let mut transport =
4936 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
4937
4938 client.overlay_rebuild_count.set(0);
4939 client.outcome_prune_count.set(0);
4940 let outcome = client.process_response(&mut transport, response, &request_meta);
4941 let SyncOutcome::Ok(report) = outcome else {
4942 panic!("mixed push-result response failed");
4943 };
4944
4945 assert_eq!(report.applied, ids[..2]);
4946 assert_eq!(report.rejected, ids[2..3]);
4947 assert_eq!(report.retryable, ids[3..4]);
4948 assert_eq!(client.pending_commit_ids(), ids[3..4]);
4949 assert_eq!(
4950 client
4951 .query("SELECT id FROM tasks ORDER BY id", &[])
4952 .expect("query visible overlay"),
4953 ["task-0", "task-1", "task-3"]
4954 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
4955 .to_vec()
4956 );
4957 assert_eq!(client.overlay_rebuild_count.get(), 2);
4959 assert_eq!(client.outcome_prune_count.get(), 1);
4960 }
4961
4962 fn replay_schema(fts: bool) -> Value {
4963 json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
4964 {"name":"id","type":"string","nullable":false},
4965 {"name":"project_id","type":"string","nullable":false},
4966 {"name":"title","type":"string","nullable":false},
4967 {"name":"done","type":"boolean","nullable":false}
4968 ],"scopes":[{"pattern":"project:{project_id}"}], "ftsIndexes": if fts { json!([{"name":"tasks_fts","columns":["title"],"tokenize":"unicode61"}]) } else { json!([]) }}]})
4969 }
4970
4971 #[test]
4972 fn aborted_schema_and_epoch_resets_restore_memory_and_durable_intent() {
4973 for schema_reset in [false, true] {
4974 let mut schema = replay_schema(false);
4975 let mut client =
4976 SyncClient::new("reset-failure".into(), &schema, ClientLimits::default()).unwrap();
4977 let id = client
4978 .mutate(vec![Mutation::Upsert {
4979 table: "tasks".into(),
4980 values: Map::from_iter([
4981 ("id".into(), json!("queued")),
4982 ("project_id".into(), json!("p1")),
4983 ("title".into(), json!("old")),
4984 ("done".into(), json!(false)),
4985 ]),
4986 base_version: None,
4987 }])
4988 .unwrap();
4989 client.set_meta(LOG_EPOCH_KEY, "old");
4990 let revision = client.local_revision();
4991 let needed = client.sync_needed;
4992 client.drain_change_batches();
4993 client.drain_sync_intents();
4994 client.conn.execute_batch("CREATE TRIGGER refuse_revision BEFORE INSERT ON _syncular_meta WHEN NEW.key='localRevision' BEGIN SELECT RAISE(ABORT,'reset completion failed'); END;").unwrap();
4995 if schema_reset {
4996 schema["version"] = json!(2);
4997 schema["tables"][0]["columns"]
4998 .as_array_mut()
4999 .unwrap()
5000 .retain(|column| column["name"] != "title");
5001 client.schema = parse_schema_json(&schema).unwrap();
5002 assert!(client.run_schema_reset().is_err());
5003 } else {
5004 assert!(client.run_log_epoch_reset("new").is_err());
5005 }
5006 assert!(!client.upgrading);
5007 assert_eq!(client.sync_needed, needed);
5008 assert_eq!(client.local_revision(), revision);
5009 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5010 assert!(client.rejections.is_empty());
5011 assert!(client.commit_outcome(&id).unwrap().is_none());
5012 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("old"));
5013 assert_eq!(
5014 client
5015 .query("SELECT title FROM tasks WHERE id='queued'", &[])
5016 .unwrap()[0]["title"],
5017 "old"
5018 );
5019 assert_eq!(
5020 client
5021 .conn
5022 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5023 .get::<_, i64>(0))
5024 .unwrap(),
5025 1
5026 );
5027 assert!(client.drain_change_batches().is_empty());
5028 assert!(client.drain_sync_intents().is_empty());
5029 assert!(client.conn.is_autocommit());
5030 }
5031 }
5032
5033 #[test]
5034 fn cached_blob_cleanup_propagates_delete_failures() {
5035 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5036 {"name":"id","type":"string","nullable":false},
5037 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5038 let mut client =
5039 SyncClient::new("blob-cleanup".into(), &schema, ClientLimits::default()).unwrap();
5040 client.conn.execute_batch("CREATE TRIGGER refuse_blob_cleanup BEFORE DELETE ON _syncular_blobs BEGIN SELECT RAISE(ABORT,'blob cleanup failed'); END;").unwrap();
5041 client.conn.execute("INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,created_at_ms) VALUES ('test',X'01',1,0)", []).unwrap();
5042 client
5043 .subscribe("attachments".into(), "attachments".into(), vec![], None)
5044 .unwrap();
5045 client.subs[0].effective = Some(vec![]);
5046 client.persist_sub(&client.subs[0]).unwrap();
5047 let revision = client.local_revision();
5048 let (_, meta) = client.build_request(false).unwrap();
5049 let mut transport = CountingRealtimeTransport::default();
5050 let mut report = SyncReport::default();
5051 assert!(client
5052 .process_section(
5053 &mut transport,
5054 "attachments",
5055 SubStatus::Revoked,
5056 "sync.scope_revoked",
5057 vec![],
5058 vec![],
5059 Some((0, None)),
5060 &meta,
5061 &mut report
5062 )
5063 .is_err());
5064 assert_eq!(client.subs[0].state, SubState::Active);
5065 assert_eq!(client.local_revision(), revision);
5066 assert!(report.revoked.is_empty());
5067 assert!(client.conn.is_autocommit());
5068 client
5069 .conn
5070 .execute_batch("DROP TRIGGER refuse_blob_cleanup")
5071 .unwrap();
5072 assert!(client
5073 .process_section(
5074 &mut transport,
5075 "attachments",
5076 SubStatus::Revoked,
5077 "sync.scope_revoked",
5078 vec![],
5079 vec![],
5080 Some((0, None)),
5081 &meta,
5082 &mut report
5083 )
5084 .is_ok());
5085 assert_eq!(client.subs[0].state, SubState::Revoked);
5086 assert_eq!(
5087 client
5088 .conn
5089 .query_row("SELECT count(*) FROM _syncular_blobs", [], |row| row
5090 .get::<_, i64>(0))
5091 .unwrap(),
5092 0
5093 );
5094 }
5095
5096 #[test]
5097 fn overlay_replay_trigger_refuses_reopen_and_preserves_durable_intent() {
5098 let path =
5099 std::env::temp_dir().join(format!("syncular-replay-{}.db", uuid::Uuid::new_v4()));
5100 let schema = replay_schema(false);
5101 let mut client = SyncClient::open_path(
5102 "replay".into(),
5103 &schema,
5104 ClientLimits::default(),
5105 path.to_str().unwrap(),
5106 )
5107 .unwrap();
5108 let ids = ["early", "bad"].map(|id| {
5109 client
5110 .mutate(vec![Mutation::Upsert {
5111 table: "tasks".into(),
5112 values: Map::from_iter([
5113 ("id".into(), json!(id)),
5114 ("project_id".into(), json!("p1")),
5115 ("title".into(), json!(id)),
5116 ("done".into(), json!(false)),
5117 ]),
5118 base_version: None,
5119 }])
5120 .unwrap()
5121 });
5122 client.conn.execute_batch("DELETE FROM tasks; CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.id='bad' BEGIN SELECT RAISE(ABORT,'replay failed'); END;").unwrap();
5123 let revision = client.local_revision();
5124 drop(client);
5125 let error = SyncClient::open_path(
5126 "replay".into(),
5127 &schema,
5128 ClientLimits::default(),
5129 path.to_str().unwrap(),
5130 )
5131 .err()
5132 .expect("reopen must fail");
5133 assert!(error.contains("replay failed"));
5134 let conn = Connection::open(&path).unwrap();
5135 assert_eq!(
5136 conn.query_row("SELECT count(*) FROM tasks", [], |row| row.get::<_, i64>(0))
5137 .unwrap(),
5138 0
5139 );
5140 assert_eq!(
5141 conn.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5142 .get::<_, i64>(0))
5143 .unwrap(),
5144 2
5145 );
5146 assert_eq!(
5147 meta_get(&conn, LOCAL_REVISION_KEY),
5148 Some(revision.to_string())
5149 );
5150 conn.execute_batch("DROP TRIGGER fail_replay").unwrap();
5151 drop(conn);
5152 let reopened = SyncClient::open_path(
5153 "replay".into(),
5154 &schema,
5155 ClientLimits::default(),
5156 path.to_str().unwrap(),
5157 )
5158 .unwrap();
5159 assert_eq!(reopened.pending_commit_ids(), ids);
5160 assert_eq!(
5161 reopened
5162 .query("SELECT id FROM tasks ORDER BY id", &[])
5163 .unwrap()
5164 .len(),
5165 2
5166 );
5167 drop(reopened);
5168 std::fs::remove_file(path).unwrap();
5169 }
5170
5171 #[test]
5172 fn overlay_replay_failure_rolls_back_push_and_pull_apply_boundaries() {
5173 let schema = replay_schema(false);
5174 let mut client =
5175 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5176 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5177 client
5178 .write_base_row(
5179 "tasks",
5180 &vec![
5181 Some(ColumnValue::String("queued".into())),
5182 Some(ColumnValue::String("p1".into())),
5183 Some(ColumnValue::String("server".into())),
5184 Some(ColumnValue::Boolean(false)),
5185 ],
5186 1,
5187 )
5188 .unwrap();
5189 client.rebuild_overlay().unwrap();
5190 let id = client
5191 .patch(
5192 "tasks",
5193 "queued",
5194 Map::from_iter([("title".into(), json!("local"))]),
5195 Some(1),
5196 )
5197 .unwrap();
5198 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
5199 let base = client
5200 .query("SELECT * FROM _syncular_base_tasks", &[])
5201 .unwrap();
5202 let revision = client.local_revision();
5203 client.drain_change_batches();
5204 client.conn.execute_batch("CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.title='local' BEGIN SELECT RAISE(ABORT,'replay failed'); END").unwrap();
5205 let remote = vec![
5206 Some(ColumnValue::String("queued".into())),
5207 Some(ColumnValue::String("p1".into())),
5208 Some(ColumnValue::String("remote".into())),
5209 Some(ColumnValue::Boolean(false)),
5210 ];
5211 let bytes = crate::values::encode_row_json(
5212 client.schema.table("tasks").unwrap(),
5213 "queued",
5214 &Map::from_iter(
5215 client
5216 .schema
5217 .table("tasks")
5218 .unwrap()
5219 .columns
5220 .iter()
5221 .zip(&remote)
5222 .map(|(column, value)| (column.name.clone(), column_value_to_json(value))),
5223 ),
5224 &client.encryption,
5225 )
5226 .unwrap();
5227 let change = ssp2::model::Change {
5228 table_index: 0,
5229 op: Op::Upsert,
5230 row_id: "queued".into(),
5231 row_version: Some(2),
5232 scopes: vec![("project_id".into(), "p1".into())],
5233 row: Some(bytes),
5234 };
5235 assert!(client
5236 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), Some(2))
5237 .is_err());
5238 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5239 assert_eq!(
5240 client
5241 .query("SELECT * FROM _syncular_base_tasks", &[])
5242 .unwrap(),
5243 base
5244 );
5245 assert_eq!(client.local_revision(), revision);
5246 assert!(client.drain_change_batches().is_empty());
5247 let response = Message {
5248 wire_version: 2,
5249 msg_kind: MsgKind::Response,
5250 frames: vec![
5251 Frame::RespHeader {
5252 required_schema_version: None,
5253 latest_schema_version: None,
5254 log_epoch: Some("epoch-1".into()),
5255 reset_required: Some(false),
5256 },
5257 Frame::PushResult {
5258 client_commit_id: id.clone(),
5259 status: PushStatus::Applied,
5260 commit_seq: Some(2),
5261 results: vec![OpResult::Applied { op_index: 0 }],
5262 },
5263 ],
5264 };
5265 let (_, meta) = client.build_request(false).unwrap();
5266 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5267 assert!(
5268 matches!(client.process_response(&mut transport,response.clone(),&meta),SyncOutcome::Failed { error_code,.. } if error_code=="client.outcome_persistence_failed")
5269 );
5270 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5271 assert!(client.commit_outcome(&id).unwrap().is_none());
5272 assert_eq!(
5273 client
5274 .conn
5275 .query_row("SELECT count(*) FROM _syncular_acked_rows", [], |row| row
5276 .get::<_, i64>(
5277 0
5278 ))
5279 .unwrap(),
5280 0
5281 );
5282 assert_eq!(
5283 client
5284 .conn
5285 .query_row("SELECT count(*) FROM _syncular_row_deliveries", [], |row| {
5286 row.get::<_, i64>(0)
5287 })
5288 .unwrap(),
5289 0
5290 );
5291 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5292 assert_eq!(client.local_revision(), revision);
5293 assert!(client.drain_change_batches().is_empty());
5294 client
5295 .conn
5296 .execute_batch("DROP TRIGGER fail_replay")
5297 .unwrap();
5298 assert!(matches!(
5299 client.process_response(&mut transport, response, &meta),
5300 SyncOutcome::Ok(_)
5301 ));
5302 client
5303 .apply_commit_frame(&["tasks".into()], &[change], Some(2))
5304 .unwrap();
5305 assert!(client.pending_commit_ids().is_empty());
5306 assert_eq!(
5307 client.query("SELECT title FROM tasks", &[]).unwrap()[0]["title"],
5308 json!("remote")
5309 );
5310 }
5311
5312 #[test]
5313 fn overlay_replay_distinguishes_absence_from_sql_and_value_decode_errors() {
5314 let client = SyncClient::new(
5315 "replay".into(),
5316 &replay_schema(false),
5317 ClientLimits::default(),
5318 )
5319 .unwrap();
5320 let table = client.schema.table("tasks").unwrap();
5321 let patch = OutboxOp {
5322 table: "tasks".into(),
5323 row_id: "absent".into(),
5324 upsert: true,
5325 base_version: Some(1),
5326 values: Some(Map::from_iter([
5327 ("id".into(), json!("absent")),
5328 ("title".into(), json!("patch")),
5329 ])),
5330 };
5331 assert!(client.visible_row(table, "absent").unwrap().is_none());
5332 client
5333 .apply_outbox_ops([&patch])
5334 .expect("sparse replay over truly absent row");
5335 client
5336 .conn
5337 .execute_batch("INSERT INTO tasks VALUES ('bad','p1','title','invalid',1)")
5338 .unwrap();
5339 assert!(client
5340 .visible_row(table, "bad")
5341 .unwrap_err()
5342 .starts_with("sync.local_corrupt:"));
5343 let mut invalid = patch.clone();
5344 invalid.values = Some(Map::from_iter([
5345 ("id".into(), json!("bad-value")),
5346 ("project_id".into(), json!("p1")),
5347 ("title".into(), json!("title")),
5348 ("done".into(), json!({"bad":true})),
5349 ]));
5350 assert!(client
5351 .apply_outbox_ops([&invalid])
5352 .unwrap_err()
5353 .starts_with("sync.local_corrupt:"));
5354 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5355 assert!(
5356 client.apply_outbox_ops([&patch]).is_err(),
5357 "SQL failure must not count as absence"
5358 );
5359 }
5360
5361 #[test]
5362 fn overlay_replay_keeps_secondary_unique_deferral_and_lazy_base_precedence() {
5363 let mut schema = replay_schema(false);
5364 schema["tables"][0]["indexes"] =
5365 json!([{"name":"tasks_project_title", "columns":["project_id","title"],"unique":true}]);
5366 let mut client =
5367 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5368 let id = client
5369 .mutate(vec![Mutation::Upsert {
5370 table: "tasks".into(),
5371 values: Map::from_iter([
5372 ("id".into(), json!("queued")),
5373 ("project_id".into(), json!("p1")),
5374 ("title".into(), json!("occupied")),
5375 ("done".into(), json!(false)),
5376 ]),
5377 base_version: None,
5378 }])
5379 .unwrap();
5380 client
5381 .write_base_row(
5382 "tasks",
5383 &vec![
5384 Some(ColumnValue::String("server".into())),
5385 Some(ColumnValue::String("p1".into())),
5386 Some(ColumnValue::String("occupied".into())),
5387 Some(ColumnValue::Boolean(false)),
5388 ],
5389 1,
5390 )
5391 .unwrap();
5392 client.rebuild_overlay_if_dirty().unwrap();
5393 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5394 assert!(client
5395 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5396 .unwrap()
5397 .is_none());
5398 assert!(client.rejections().is_empty());
5399 client.delete_base_row("tasks", "server").unwrap();
5400 client.rebuild_overlay_if_dirty().unwrap();
5401 assert!(client
5402 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5403 .unwrap()
5404 .is_some());
5405 client
5406 .write_base_row(
5407 "tasks",
5408 &vec![
5409 Some(ColumnValue::String("base".into())),
5410 Some(ColumnValue::String("p1".into())),
5411 Some(ColumnValue::String("base-title".into())),
5412 Some(ColumnValue::Boolean(false)),
5413 ],
5414 1,
5415 )
5416 .unwrap();
5417 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5418 client
5419 .retain_failed_commit(
5420 "retained",
5421 &[OutboxOp {
5422 table: "tasks".into(),
5423 row_id: "base".into(),
5424 upsert: true,
5425 base_version: Some(1),
5426 values: Some(Map::from_iter([
5427 ("id".into(), json!("base")),
5428 ("title".into(), json!("patch")),
5429 ])),
5430 }],
5431 )
5432 .expect("an existing base must not read the unavailable visible table");
5433 }
5434
5435 #[test]
5436 fn overlay_replay_fts_and_savepoint_release_failures_roll_back_all_visible_changes() {
5437 for failure in ["fts", "release"] {
5438 let mut client = SyncClient::new(
5439 "replay".into(),
5440 &replay_schema(true),
5441 ClientLimits::default(),
5442 )
5443 .unwrap();
5444 client
5445 .mutate(vec![Mutation::Upsert {
5446 table: "tasks".into(),
5447 values: Map::from_iter([
5448 ("id".into(), json!("queued")),
5449 ("project_id".into(), json!("p1")),
5450 ("title".into(), json!("needle")),
5451 ("done".into(), json!(false)),
5452 ]),
5453 base_version: None,
5454 }])
5455 .unwrap();
5456 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
5457 let fts_before = client
5458 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5459 .unwrap();
5460 let triggers_before: Vec<String> = client
5461 .conn
5462 .prepare("SELECT sql FROM sqlite_master WHERE type='trigger' ORDER BY name")
5463 .unwrap()
5464 .query_map([], |row| row.get(0))
5465 .unwrap()
5466 .collect::<Result<_, _>>()
5467 .unwrap();
5468 if failure == "fts" {
5469 client.conn.execute_batch("CREATE TRIGGER fail_fts BEFORE INSERT ON _syncular_fts_tasks_fts BEGIN SELECT RAISE(ABORT,'fts replay failed'); END;").unwrap();
5470 } else {
5471 client.conn.execute_batch("PRAGMA foreign_keys=ON; CREATE TABLE parent(id TEXT PRIMARY KEY); CREATE TABLE deferred(id TEXT REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_release AFTER INSERT ON tasks BEGIN INSERT INTO deferred VALUES ('missing'); END;").unwrap();
5472 }
5473 client.overlay_dirty.set(true);
5474 assert!(client.rebuild_overlay().is_err());
5475 assert!(client.overlay_dirty.get());
5476 assert!(client.conn.is_autocommit());
5477 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
5478 assert_eq!(
5479 client
5480 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5481 .unwrap(),
5482 fts_before
5483 );
5484 let original_triggers: Vec<String> = client.conn.prepare("SELECT sql FROM sqlite_master WHERE type='trigger' AND name NOT LIKE 'fail_%' ORDER BY name").unwrap().query_map([], |row| row.get(0)).unwrap().collect::<Result<_,_>>().unwrap();
5485 assert_eq!(original_triggers, triggers_before);
5486 assert_eq!(client.pending_commit_ids().len(), 1);
5487 client
5488 .conn
5489 .execute_batch(if failure == "fts" {
5490 "DROP TRIGGER fail_fts"
5491 } else {
5492 "DROP TRIGGER fail_release"
5493 })
5494 .unwrap();
5495 client
5496 .rebuild_overlay()
5497 .expect("retry after removing the fault");
5498 }
5499 }
5500
5501 #[test]
5502 fn secondary_unique_collision_preserves_existing_synced_row() {
5503 let client = SyncClient::new(
5504 "unique-upsert-test".to_owned(),
5505 &json!({
5506 "version": 1,
5507 "tables": [{
5508 "name": "tasks",
5509 "primaryKey": "id",
5510 "columns": [
5511 { "name": "id", "type": "string", "nullable": false },
5512 { "name": "project_id", "type": "string", "nullable": false },
5513 { "name": "title", "type": "string", "nullable": false }
5514 ],
5515 "scopes": [{ "pattern": "project:{project_id}" }],
5516 "indexes": [{
5517 "name": "tasks_by_project_title",
5518 "columns": ["project_id", "title"],
5519 "unique": true
5520 }]
5521 }]
5522 }),
5523 ClientLimits::default(),
5524 )
5525 .expect("test client");
5526 let table = client.schema.table("tasks").expect("tasks table");
5527 let sql = client.insert_row_sql(&base_table("tasks"), table);
5528
5529 client
5530 .conn
5531 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
5532 .expect("insert first row");
5533 client
5534 .conn
5535 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
5536 .expect("update same primary key");
5537 client
5538 .conn
5539 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
5540 .expect("insert second row");
5541 assert!(client
5542 .conn
5543 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
5544 .is_err());
5545
5546 let rows = client
5547 .conn
5548 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
5549 .expect("prepare rows")
5550 .query_map([], |row| {
5551 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
5552 })
5553 .expect("query rows")
5554 .collect::<Result<Vec<_>, _>>()
5555 .expect("collect rows");
5556 assert_eq!(
5557 rows,
5558 vec![
5559 ("t1".to_owned(), "updated".to_owned()),
5560 ("t2".to_owned(), "original".to_owned())
5561 ]
5562 );
5563 }
5564
5565 #[test]
5566 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
5567 let path = std::env::temp_dir().join(format!(
5568 "syncular-startup-intent-{}.db",
5569 uuid::Uuid::new_v4()
5570 ));
5571 let schema = json!({
5572 "version": 1,
5573 "tables": [{
5574 "name": "tasks",
5575 "primaryKey": "id",
5576 "columns": [
5577 { "name": "id", "type": "string", "nullable": false },
5578 { "name": "project_id", "type": "string", "nullable": false }
5579 ],
5580 "scopes": [{ "pattern": "project:{project_id}" }]
5581 }]
5582 });
5583
5584 {
5585 let mut first = SyncClient::open_path_with_identity(
5586 None,
5587 &schema,
5588 ClientLimits::default(),
5589 path.to_str().expect("UTF-8 temp path"),
5590 )
5591 .expect("first open");
5592 first
5593 .set_window(
5594 &WindowBase {
5595 table: "tasks".to_owned(),
5596 variable: "project_id".to_owned(),
5597 fixed_scopes: Vec::new(),
5598 params: None,
5599 },
5600 &["persisted".to_owned()],
5601 )
5602 .expect("persist window");
5603 }
5604
5605 let mut reopened = SyncClient::open_path_with_identity(
5606 None,
5607 &schema,
5608 ClientLimits::default(),
5609 path.to_str().expect("UTF-8 temp path"),
5610 )
5611 .expect("reopen");
5612 assert!(reopened.sync_needed());
5613 assert!(matches!(
5614 reopened.drain_sync_intents().as_slice(),
5615 [SyncIntent::Interactive]
5616 ));
5617 drop(reopened);
5618 std::fs::remove_file(path).expect("remove temp database");
5619 }
5620
5621 #[test]
5622 fn reopening_preserves_immutable_subscription_identity_and_progress() {
5623 let path = std::env::temp_dir().join(format!(
5624 "syncular-subscription-identity-{}.db",
5625 uuid::Uuid::new_v4()
5626 ));
5627 let schema = json!({
5628 "version": 1,
5629 "tables": [
5630 {
5631 "name": "tasks",
5632 "primaryKey": "id",
5633 "columns": [
5634 { "name": "id", "type": "string", "nullable": false },
5635 { "name": "project_id", "type": "string", "nullable": false }
5636 ],
5637 "scopes": [{ "pattern": "project:{project_id}" }]
5638 },
5639 {
5640 "name": "docs",
5641 "primaryKey": "id",
5642 "columns": [
5643 { "name": "id", "type": "string", "nullable": false },
5644 { "name": "org_id", "type": "string", "nullable": false },
5645 { "name": "project_id", "type": "string", "nullable": false }
5646 ],
5647 "scopes": [
5648 { "pattern": "org:{org_id}" },
5649 { "pattern": "project:{projectId}", "column": "project_id" }
5650 ]
5651 }
5652 ]
5653 });
5654
5655 {
5656 let mut first = SyncClient::open_path_with_identity(
5657 None,
5658 &schema,
5659 ClientLimits::default(),
5660 path.to_str().expect("UTF-8 temp path"),
5661 )
5662 .expect("first open");
5663 first
5664 .subscribe(
5665 "stable-subscription".to_owned(),
5666 "tasks".to_owned(),
5667 vec![(
5668 "project_id".to_owned(),
5669 vec!["p2".to_owned(), "p1".to_owned()],
5670 )],
5671 Some(r#"{"view":"v1"}"#.to_owned()),
5672 )
5673 .expect("persist subscription");
5674 let persisted = {
5675 let subscription = first
5676 .subs
5677 .iter_mut()
5678 .find(|subscription| subscription.id == "stable-subscription")
5679 .expect("subscription");
5680 subscription.cursor = 41;
5681 subscription.bootstrap_state = Some("resume-token".to_owned());
5682 subscription.effective = Some(vec![(
5683 "project_id".to_owned(),
5684 vec!["p1".to_owned(), "p2".to_owned()],
5685 )]);
5686 subscription.synced_once = true;
5687 subscription.clone()
5688 };
5689 first.persist_sub(&persisted).unwrap();
5690 }
5691
5692 let mut reopened = SyncClient::open_path_with_identity(
5693 None,
5694 &schema,
5695 ClientLimits::default(),
5696 path.to_str().expect("UTF-8 temp path"),
5697 )
5698 .expect("reopen");
5699 let progress = reopened
5700 .subscription_state("stable-subscription")
5701 .expect("persisted state");
5702 assert_eq!(progress.cursor, 41);
5703 assert!(progress.has_resume_token);
5704
5705 reopened
5706 .subscribe(
5707 "stable-subscription".to_owned(),
5708 "tasks".to_owned(),
5709 vec![(
5710 "project_id".to_owned(),
5711 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
5712 )],
5713 Some(r#"{"view":"v1"}"#.to_owned()),
5714 )
5715 .expect("canonical intent is idempotent");
5716 assert_eq!(
5717 reopened
5718 .subscription_state("stable-subscription")
5719 .expect("unchanged state")
5720 .cursor,
5721 progress.cursor
5722 );
5723
5724 for (table, scopes, params) in [
5725 (
5726 "tasks",
5727 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
5728 Some(r#"{"view":"v1"}"#.to_owned()),
5729 ),
5730 (
5731 "tasks",
5732 vec![(
5733 "project_id".to_owned(),
5734 vec!["p2".to_owned(), "p1".to_owned()],
5735 )],
5736 Some(r#"{"view":"v2"}"#.to_owned()),
5737 ),
5738 (
5739 "docs",
5740 vec![
5741 ("org_id".to_owned(), vec!["o1".to_owned()]),
5742 ("projectId".to_owned(), vec!["p1".to_owned()]),
5743 ],
5744 Some(r#"{"view":"v1"}"#.to_owned()),
5745 ),
5746 ] {
5747 let error = reopened
5748 .subscribe(
5749 "stable-subscription".to_owned(),
5750 table.to_owned(),
5751 scopes,
5752 params,
5753 )
5754 .expect_err("identity rebind must fail");
5755 assert!(error.starts_with("client.subscription_intent_mismatch:"));
5756 assert_eq!(
5757 reopened
5758 .subscription_state("stable-subscription")
5759 .expect("unchanged state")
5760 .cursor,
5761 progress.cursor
5762 );
5763 }
5764
5765 drop(reopened);
5766 std::fs::remove_file(path).expect("remove temp database");
5767 }
5768
5769 #[test]
5770 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
5771 let path = std::env::temp_dir().join(format!(
5772 "syncular-satisfied-schema-floor-{}.db",
5773 uuid::Uuid::new_v4()
5774 ));
5775 let schema = json!({
5776 "version": 23,
5777 "tables": [{
5778 "name": "tasks",
5779 "primaryKey": "id",
5780 "columns": [
5781 { "name": "id", "type": "string", "nullable": false },
5782 { "name": "project_id", "type": "string", "nullable": false }
5783 ],
5784 "scopes": [{ "pattern": "project:{project_id}" }]
5785 }]
5786 });
5787
5788 {
5789 let mut first = SyncClient::open_path_with_identity(
5790 None,
5791 &schema,
5792 ClientLimits::default(),
5793 path.to_str().expect("UTF-8 temp path"),
5794 )
5795 .expect("first open");
5796 first
5797 .subscribe(
5798 "tasks".to_owned(),
5799 "tasks".to_owned(),
5800 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
5801 None,
5802 )
5803 .expect("persist subscription");
5804 first.set_schema_floor(Some(SchemaFloor {
5805 required_schema_version: Some(22),
5806 latest_schema_version: Some(22),
5807 }));
5808 }
5809
5810 let mut reopened = SyncClient::open_path_with_identity(
5811 None,
5812 &schema,
5813 ClientLimits::default(),
5814 path.to_str().expect("UTF-8 temp path"),
5815 )
5816 .expect("reopen");
5817 assert!(reopened.schema_floor().is_none());
5818 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
5819 assert!(reopened.sync_needed());
5820 assert!(matches!(
5821 reopened.drain_sync_intents().as_slice(),
5822 [SyncIntent::Interactive]
5823 ));
5824 drop(reopened);
5825 std::fs::remove_file(path).expect("remove temp database");
5826 }
5827
5828 #[test]
5829 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
5830 let path = std::env::temp_dir().join(format!(
5831 "syncular-unsatisfied-schema-floor-{}.db",
5832 uuid::Uuid::new_v4()
5833 ));
5834 let schema = json!({
5835 "version": 1,
5836 "tables": [{
5837 "name": "tasks",
5838 "primaryKey": "id",
5839 "columns": [
5840 { "name": "id", "type": "string", "nullable": false },
5841 { "name": "project_id", "type": "string", "nullable": false }
5842 ],
5843 "scopes": [{ "pattern": "project:{project_id}" }]
5844 }]
5845 });
5846
5847 {
5848 let mut first = SyncClient::open_path_with_identity(
5849 None,
5850 &schema,
5851 ClientLimits::default(),
5852 path.to_str().expect("UTF-8 temp path"),
5853 )
5854 .expect("first open");
5855 first
5856 .subscribe(
5857 "tasks".to_owned(),
5858 "tasks".to_owned(),
5859 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
5860 None,
5861 )
5862 .expect("persist subscription");
5863 first.set_schema_floor(Some(SchemaFloor {
5864 required_schema_version: Some(2),
5865 latest_schema_version: Some(2),
5866 }));
5867 }
5868
5869 let reopened = SyncClient::open_path_with_identity(
5870 None,
5871 &schema,
5872 ClientLimits::default(),
5873 path.to_str().expect("UTF-8 temp path"),
5874 )
5875 .expect("reopen");
5876 assert_eq!(
5877 reopened.schema_floor(),
5878 Some(&SchemaFloor {
5879 required_schema_version: Some(2),
5880 latest_schema_version: Some(2),
5881 })
5882 );
5883 assert!(!reopened.sync_needed());
5884 drop(reopened);
5885 std::fs::remove_file(path).expect("remove temp database");
5886 }
5887
5888 #[test]
5889 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
5890 let path = std::env::temp_dir().join(format!(
5891 "syncular-indexed-column-bump-{}.db",
5892 uuid::Uuid::new_v4()
5893 ));
5894 let old_schema = json!({
5895 "version": 1,
5896 "tables": [
5897 {
5898 "name": "tasks",
5899 "primaryKey": "id",
5900 "columns": [
5901 { "name": "id", "type": "string", "nullable": false },
5902 { "name": "project_id", "type": "string", "nullable": false }
5903 ],
5904 "scopes": [{ "pattern": "project:{project_id}" }]
5905 },
5906 {
5907 "name": "legacy",
5908 "primaryKey": "id",
5909 "columns": [
5910 { "name": "id", "type": "string", "nullable": false },
5911 { "name": "project_id", "type": "string", "nullable": false }
5912 ],
5913 "scopes": [{ "pattern": "project:{project_id}" }]
5914 }
5915 ]
5916 });
5917 {
5918 let mut first = SyncClient::open_path_with_identity(
5919 None,
5920 &old_schema,
5921 ClientLimits::default(),
5922 path.to_str().expect("UTF-8 temp path"),
5923 )
5924 .expect("open old schema");
5925 first
5926 .subscribe(
5927 "legacy-sub".to_owned(),
5928 "legacy".to_owned(),
5929 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
5930 None,
5931 )
5932 .expect("persist legacy subscription");
5933 }
5934
5935 let new_schema = json!({
5936 "version": 2,
5937 "tables": [{
5938 "name": "tasks",
5939 "primaryKey": "id",
5940 "columns": [
5941 { "name": "id", "type": "string", "nullable": false },
5942 { "name": "project_id", "type": "string", "nullable": false },
5943 { "name": "facility_membership_id", "type": "string", "nullable": true }
5944 ],
5945 "scopes": [{ "pattern": "project:{project_id}" }],
5946 "indexes": [{
5947 "name": "tasks_by_membership",
5948 "columns": ["project_id", "facility_membership_id"],
5949 "unique": false
5950 }]
5951 }]
5952 });
5953 let upgraded = SyncClient::open_path_with_identity(
5954 None,
5955 &new_schema,
5956 ClientLimits::default(),
5957 path.to_str().expect("UTF-8 temp path"),
5958 )
5959 .expect("open upgraded schema");
5960 let columns = upgraded
5961 .conn
5962 .prepare("PRAGMA table_info(tasks)")
5963 .expect("prepare columns")
5964 .query_map([], |row| row.get::<_, String>(1))
5965 .expect("query columns")
5966 .collect::<Result<Vec<_>, _>>()
5967 .expect("collect columns");
5968 assert!(columns.contains(&"facility_membership_id".to_owned()));
5969 let index_count: i64 = upgraded
5970 .conn
5971 .query_row(
5972 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
5973 [],
5974 |row| row.get(0),
5975 )
5976 .expect("query index");
5977 assert_eq!(index_count, 1);
5978 assert!(upgraded.subscription_state("legacy-sub").is_none());
5979 drop(upgraded);
5980 std::fs::remove_file(path).expect("remove temp database");
5981 }
5982
5983 #[test]
5984 fn migrates_pre_envelope_outcome_journal_additively() {
5985 let path = std::env::temp_dir().join(format!(
5986 "syncular-outcome-migration-{}.db",
5987 uuid::Uuid::new_v4()
5988 ));
5989 let conn = Connection::open(&path).expect("open legacy database");
5990 conn.execute_batch(
5991 "CREATE TABLE _syncular_commit_outcomes (
5992 seq INTEGER PRIMARY KEY AUTOINCREMENT,
5993 client_commit_id TEXT NOT NULL UNIQUE,
5994 status TEXT NOT NULL,
5995 recorded_at_ms INTEGER NOT NULL,
5996 results_json TEXT NOT NULL,
5997 resolution TEXT NOT NULL DEFAULT 'active',
5998 resolved_at_ms INTEGER,
5999 replacement_client_commit_id TEXT);",
6000 )
6001 .expect("create legacy outcome journal");
6002 drop(conn);
6003 let schema = json!({
6004 "version": 1,
6005 "tables": [{
6006 "name": "tasks",
6007 "primaryKey": "id",
6008 "columns": [
6009 { "name": "id", "type": "string", "nullable": false },
6010 { "name": "project_id", "type": "string", "nullable": false }
6011 ],
6012 "scopes": [{ "pattern": "project:{project_id}" }]
6013 }]
6014 });
6015 let client = SyncClient::open_path(
6016 "migration-native".to_owned(),
6017 &schema,
6018 ClientLimits::default(),
6019 path.to_str().expect("UTF-8 temp path"),
6020 )
6021 .expect("migrate database");
6022 let has_operations = client
6023 .conn
6024 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
6025 .expect("prepare table info")
6026 .query_map([], |row| row.get::<_, String>(1))
6027 .expect("query table info")
6028 .filter_map(Result::ok)
6029 .any(|column| column == "operations_json");
6030 assert!(has_operations);
6031 drop(client);
6032 std::fs::remove_file(path).expect("remove temp database");
6033 }
6034
6035 #[test]
6036 fn retained_rejection_survives_file_reopen_and_security_purge() {
6037 let path =
6038 std::env::temp_dir().join(format!("syncular-retained-{}.db", uuid::Uuid::new_v4()));
6039 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id", "columns": [
6040 { "name": "id", "type": "string", "nullable": false },
6041 { "name": "project_id", "type": "string", "nullable": false },
6042 { "name": "title", "type": "string", "nullable": false }
6043 ], "scopes": [{ "pattern": "project:{project_id}" }] }] });
6044 {
6045 let mut first = SyncClient::open_path(
6046 "retained-native".to_owned(),
6047 &schema,
6048 ClientLimits::default(),
6049 path.to_str().unwrap(),
6050 )
6051 .unwrap();
6052 first
6053 .write_base_row(
6054 "tasks",
6055 &vec![
6056 Some(ColumnValue::String("t1".into())),
6057 Some(ColumnValue::String("p1".into())),
6058 Some(ColumnValue::String("server".into())),
6059 ],
6060 1,
6061 )
6062 .unwrap();
6063 first.rebuild_overlay().expect("rebuild overlay");
6064 let operation = OutboxOp {
6065 table: "tasks".into(),
6066 row_id: "t1".into(),
6067 upsert: true,
6068 base_version: Some(1),
6069 values: Some(Map::from_iter([
6070 ("id".into(), json!("t1")),
6071 ("title".into(), json!("mine")),
6072 ])),
6073 };
6074 first
6075 .retain_failed_commit("retained", std::slice::from_ref(&operation))
6076 .unwrap();
6077 first
6078 .persist_commit_outcome(
6079 "retained",
6080 CommitOutcomeStatus::Rejected,
6081 &[CommitOperationOutcome::Error {
6082 rejection: RejectionRecord {
6083 client_commit_id: "retained".into(),
6084 op_index: 0,
6085 code: "validation.denied".into(),
6086 message: "denied".into(),
6087 retryable: false,
6088 details: None,
6089 operation: None,
6090 },
6091 }],
6092 Some(&[operation]),
6093 )
6094 .unwrap();
6095 first.rebuild_overlay().expect("rebuild overlay");
6096 }
6097 {
6098 let mut reopened = SyncClient::open_path(
6099 "retained-native".to_owned(),
6100 &schema,
6101 ClientLimits::default(),
6102 path.to_str().unwrap(),
6103 )
6104 .unwrap();
6105 let row = reopened
6106 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6107 .expect("read visible row")
6108 .unwrap();
6109 assert_eq!(row.0[2], Some(ColumnValue::String("mine".into())));
6110 assert_eq!(
6111 reopened
6112 .commit_outcome("retained")
6113 .unwrap()
6114 .unwrap()
6115 .retained_rows
6116 .unwrap()[0]
6117 .server_row
6118 .as_ref()
6119 .unwrap()["title"],
6120 json!("server")
6121 );
6122 reopened
6123 .conn
6124 .execute("DELETE FROM _syncular_base_tasks WHERE id = 't1'", [])
6125 .unwrap();
6126 reopened.rebuild_overlay().expect("rebuild overlay");
6127 assert!(reopened
6128 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6129 .expect("read visible row")
6130 .is_none());
6131 let retained = reopened.commit_outcome("retained").unwrap().unwrap();
6132 assert!(retained.retained_rows.is_some());
6133 assert_eq!(
6134 retained.operations.unwrap()[0].values.as_ref().unwrap()["title"],
6135 json!("mine")
6136 );
6137 reopened
6138 .purge_local_data(&LocalDataPurgeInput {
6139 purge_id: "protected-purge".into(),
6140 targets: vec![LocalDataPurgeTarget {
6141 table: "tasks".into(),
6142 selectors: BTreeMap::from_iter([("project_id".into(), vec!["p1".into()])]),
6143 }],
6144 })
6145 .unwrap();
6146 assert!(reopened
6147 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6148 .expect("read visible row")
6149 .is_none());
6150 assert!(reopened
6151 .commit_outcome("retained")
6152 .unwrap()
6153 .unwrap()
6154 .retained_rows
6155 .is_none());
6156 }
6157 std::fs::remove_file(path).unwrap();
6158 }
6159
6160 #[test]
6161 fn durable_conflict_outcome_and_resolution_survive_reopen() {
6162 let path = std::env::temp_dir().join(format!(
6163 "syncular-durable-outcome-{}.db",
6164 uuid::Uuid::new_v4()
6165 ));
6166 let schema = json!({
6167 "version": 1,
6168 "tables": [{
6169 "name": "tasks",
6170 "primaryKey": "id",
6171 "columns": [
6172 { "name": "id", "type": "string", "nullable": false },
6173 { "name": "project_id", "type": "string", "nullable": false }
6174 ],
6175 "scopes": [{ "pattern": "project:{project_id}" }]
6176 }]
6177 });
6178 let conflict = ConflictRecord {
6179 client_commit_id: "losing-commit".to_owned(),
6180 op_index: 0,
6181 table: "tasks".to_owned(),
6182 row_id: "t1".to_owned(),
6183 code: "sync.version_conflict".to_owned(),
6184 message: "stale base version".to_owned(),
6185 server_version: 2,
6186 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6187 conflict_columns: vec!["project_id".to_owned()],
6188 operation: Some(CommitOperation {
6189 table: "tasks".to_owned(),
6190 row_id: "t1".to_owned(),
6191 op: "upsert".to_owned(),
6192 base_version: Some(1),
6193 values: Some(Map::from_iter([
6194 ("id".into(), json!("t1")),
6195 ("project_id".into(), json!("p1")),
6196 ])),
6197 }),
6198 };
6199 let failed_operations = vec![
6200 OutboxOp {
6201 upsert: true,
6202 table: "tasks".to_owned(),
6203 row_id: "t1".to_owned(),
6204 base_version: Some(1),
6205 values: Some(Map::from_iter([
6206 ("id".into(), json!("t1")),
6207 ("project_id".into(), json!("p1")),
6208 ])),
6209 },
6210 OutboxOp {
6211 upsert: true,
6212 table: "tasks".to_owned(),
6213 row_id: "status-event-1".to_owned(),
6214 base_version: Some(0),
6215 values: Some(Map::from_iter([
6216 ("id".into(), json!("status-event-1")),
6217 ("project_id".into(), json!("p1")),
6218 ])),
6219 },
6220 ];
6221
6222 {
6223 let mut first = SyncClient::open_path(
6224 "durable-native".to_owned(),
6225 &schema,
6226 ClientLimits::default(),
6227 path.to_str().expect("UTF-8 temp path"),
6228 )
6229 .expect("first open");
6230 first
6231 .begin_observation("test_outcome")
6232 .expect("begin outcome");
6233 first
6234 .persist_commit_outcome(
6235 "losing-commit",
6236 CommitOutcomeStatus::Conflict,
6237 &[CommitOperationOutcome::Conflict {
6238 conflict: conflict.clone(),
6239 }],
6240 Some(&failed_operations),
6241 )
6242 .expect("persist outcome");
6243 first.conflicts.push(conflict);
6244 first
6245 .finish_observation(
6246 "test_outcome",
6247 ChangeAccumulator {
6248 conflicts: true,
6249 outcomes: true,
6250 ..ChangeAccumulator::default()
6251 },
6252 )
6253 .expect("commit outcome");
6254 }
6255
6256 {
6257 let mut reopened = SyncClient::open_path(
6258 "durable-native".to_owned(),
6259 &schema,
6260 ClientLimits::default(),
6261 path.to_str().expect("UTF-8 temp path"),
6262 )
6263 .expect("reopen");
6264 assert_eq!(reopened.conflicts().len(), 1);
6265 assert_eq!(
6266 reopened.conflicts()[0].conflict_columns,
6267 vec!["project_id".to_owned()],
6268 "conflictColumns survives the journal round trip"
6269 );
6270 let outcome = reopened
6271 .commit_outcome("losing-commit")
6272 .expect("read outcome")
6273 .expect("outcome");
6274 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6275 let operations = outcome.operations.expect("aggregate envelope");
6276 assert_eq!(operations.len(), 2);
6277 assert_eq!(operations[1].row_id, "status-event-1");
6278 let resolved = reopened
6279 .resolve_commit_outcome(ResolveCommitOutcomeInput {
6280 client_commit_id: "losing-commit".to_owned(),
6281 resolution: CommitOutcomeResolution::ResolvedKeepServer,
6282 replacement_client_commit_id: None,
6283 })
6284 .expect("resolve");
6285 assert_eq!(
6286 resolved.resolution,
6287 CommitOutcomeResolution::ResolvedKeepServer
6288 );
6289 assert!(reopened.conflicts().is_empty());
6290 }
6291
6292 let reopened = SyncClient::open_path(
6293 "durable-native".to_owned(),
6294 &schema,
6295 ClientLimits::default(),
6296 path.to_str().expect("UTF-8 temp path"),
6297 )
6298 .expect("second reopen");
6299 assert!(reopened.conflicts().is_empty());
6300 assert_eq!(
6301 reopened
6302 .commit_outcome("losing-commit")
6303 .expect("read outcome")
6304 .expect("outcome")
6305 .resolution,
6306 CommitOutcomeResolution::ResolvedKeepServer
6307 );
6308 drop(reopened);
6309 std::fs::remove_file(path).expect("remove temp database");
6310 }
6311
6312 #[test]
6313 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
6314 let path =
6315 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6316 let schema = json!({
6317 "version": 1,
6318 "tables": [{
6319 "name": "tasks",
6320 "primaryKey": "id",
6321 "columns": [
6322 { "name": "id", "type": "string", "nullable": false },
6323 { "name": "project_id", "type": "string", "nullable": false }
6324 ],
6325 "scopes": [{ "pattern": "project:{project_id}" }]
6326 }]
6327 });
6328 let mut client = SyncClient::open_path_with_identity(
6329 Some("sidecar-client".to_owned()),
6330 &schema,
6331 ClientLimits::default(),
6332 path.to_str().expect("UTF-8 temp path"),
6333 )
6334 .expect("open owner");
6335 let base = WindowBase {
6336 table: "tasks".to_owned(),
6337 variable: "project_id".to_owned(),
6338 fixed_scopes: Vec::new(),
6339 params: None,
6340 };
6341 client
6342 .set_window(&base, &["one".to_owned()])
6343 .expect("set window");
6344 client
6345 .mutate(vec![Mutation::Upsert {
6346 table: "tasks".to_owned(),
6347 values: Map::from_iter([
6348 ("id".to_owned(), Value::from("t1")),
6349 ("project_id".to_owned(), Value::from("one")),
6350 ]),
6351 base_version: None,
6352 }])
6353 .expect("local mutate");
6354
6355 let coverage = [WindowCoverage {
6356 base,
6357 units: vec!["one".to_owned(), "missing".to_owned()],
6358 }];
6359 let owner = client
6360 .query_snapshot(
6361 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6362 &[],
6363 &coverage,
6364 None,
6365 )
6366 .expect("owner snapshot");
6367 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6368 let sidecar = reader
6369 .query_snapshot(
6370 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6371 &[],
6372 &coverage,
6373 )
6374 .expect("sidecar snapshot");
6375
6376 assert_eq!(sidecar.revision, owner.revision);
6377 assert_eq!(sidecar.rows, owner.rows);
6378 assert_eq!(
6379 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
6380 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
6381 );
6382 assert_eq!(sidecar.revision, "2");
6383 assert_eq!(sidecar.rows[0]["id"], "t1");
6384 assert_eq!(sidecar.rows[0]["server_version"], -1);
6385 assert!(!sidecar.coverage.complete);
6386 assert_eq!(sidecar.coverage.pending.len(), 1);
6387 assert_eq!(sidecar.coverage.missing.len(), 1);
6388
6389 drop(reader);
6390 drop(client);
6391 std::fs::remove_file(path).expect("remove temp database");
6392 }
6393
6394 #[test]
6395 fn file_snapshot_reader_reports_catchup_delivery_and_multi_statement() {
6396 let path =
6397 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6398 let schema = json!({
6399 "version": 1,
6400 "tables": [{
6401 "name": "tasks",
6402 "primaryKey": "id",
6403 "columns": [
6404 { "name": "id", "type": "string", "nullable": false },
6405 { "name": "project_id", "type": "string", "nullable": false }
6406 ],
6407 "scopes": [{ "pattern": "project:{project_id}" }]
6408 }]
6409 });
6410 let mut client = SyncClient::open_path_with_identity(
6411 Some("sidecar-catchup".to_owned()),
6412 &schema,
6413 ClientLimits::default(),
6414 path.to_str().expect("UTF-8 temp path"),
6415 )
6416 .expect("open owner");
6417 client
6418 .subscribe(
6419 "s1".into(),
6420 "tasks".into(),
6421 vec![("project_id".into(), vec!["one".into()])],
6422 None,
6423 )
6424 .expect("subscribe");
6425 let commit_id = client
6426 .mutate(vec![Mutation::Upsert {
6427 table: "tasks".to_owned(),
6428 values: Map::from_iter([
6429 ("id".to_owned(), Value::from("t1")),
6430 ("project_id".to_owned(), Value::from("one")),
6431 ]),
6432 base_version: None,
6433 }])
6434 .expect("mutate");
6435 let revision = client.local_revision().to_string();
6436
6437 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6438 assert_eq!(reader.local_revision().expect("revision"), revision);
6439
6440 let read = reader
6441 .snapshot_read(&SnapshotReadRequest {
6442 statements: vec![
6443 SnapshotStatement {
6444 sql: "SELECT 1 AS n",
6445 params: &[],
6446 },
6447 SnapshotStatement {
6448 sql: "SELECT id FROM tasks ORDER BY id",
6449 params: &[],
6450 },
6451 ],
6452 coverage: &[],
6453 subscriptions: vec!["s1".to_owned(), "absent".to_owned()],
6454 commit_ids: vec![commit_id.clone(), "never-known".to_owned()],
6455 })
6456 .expect("snapshot read");
6457 assert_eq!(read.revision, revision);
6458 assert_eq!(read.queries.len(), 2);
6459 assert_eq!(read.queries[0][0]["n"], 1);
6460 assert_eq!(read.queries[1][0]["id"], "t1");
6461 match &read.subscriptions[0] {
6462 SubscriptionCatchup::Known {
6463 status,
6464 cursor,
6465 bootstrap_complete,
6466 known_pending_pages,
6467 ..
6468 } => {
6469 assert_eq!(status, "active");
6470 assert_eq!(*cursor, -1);
6471 assert!(!bootstrap_complete);
6472 assert!(known_pending_pages);
6473 }
6474 other => panic!("expected a known subscription: {other:?}"),
6475 }
6476 assert!(matches!(
6477 &read.subscriptions[1],
6478 SubscriptionCatchup::Unknown { id } if id.as_str() == "absent"
6479 ));
6480 assert!(matches!(
6481 &read.deliveries[0],
6482 CommitDelivery::Pending { client_commit_id } if client_commit_id == &commit_id
6483 ));
6484 assert!(matches!(
6485 &read.deliveries[1],
6486 CommitDelivery::Unknown { client_commit_id } if client_commit_id.as_str() == "never-known"
6487 ));
6488
6489 assert!(reader
6491 .query_snapshot("SELECT * FROM does_not_exist", &[], &[])
6492 .is_err());
6493 assert_eq!(
6494 reader.local_revision().expect("revision after rollback"),
6495 revision
6496 );
6497 assert!(reader
6499 .query_snapshot("DELETE FROM tasks", &[], &[])
6500 .is_err());
6501
6502 client
6504 .conn
6505 .execute(
6506 "UPDATE _syncular_meta SET value = '+1' WHERE key = 'localRevision'",
6507 [],
6508 )
6509 .expect("corrupt revision");
6510 let failure = reader.local_revision().expect_err("corrupt revision fails");
6511 assert_eq!(failure.code, Some("sync.local_corrupt"));
6512
6513 drop(reader);
6514 drop(client);
6515 std::fs::remove_file(path).expect("remove temp database");
6516 }
6517
6518 #[test]
6519 fn snapshot_delivery_is_the_persisted_journal_view_without_retained_rows() {
6520 let path =
6521 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6522 let schema = json!({
6523 "version": 1,
6524 "tables": [{
6525 "name": "tasks",
6526 "primaryKey": "id",
6527 "columns": [
6528 { "name": "id", "type": "string", "nullable": false },
6529 { "name": "project_id", "type": "string", "nullable": false }
6530 ],
6531 "scopes": [{ "pattern": "project:{project_id}" }]
6532 }]
6533 });
6534 let mut client = SyncClient::open_path_with_identity(
6535 Some("sidecar-delivery".to_owned()),
6536 &schema,
6537 ClientLimits::default(),
6538 path.to_str().expect("UTF-8 temp path"),
6539 )
6540 .expect("open owner");
6541 client.begin_observation("test_delivery").expect("observe");
6542 let operations = [OutboxOp {
6543 upsert: true,
6544 table: "tasks".to_owned(),
6545 row_id: "t1".to_owned(),
6546 base_version: Some(1),
6547 values: Some(Map::from_iter([
6548 ("id".into(), json!("t1")),
6549 ("project_id".into(), json!("p1")),
6550 ])),
6551 }];
6552 client
6553 .persist_commit_outcome(
6554 "conflict-commit",
6555 CommitOutcomeStatus::Conflict,
6556 &[CommitOperationOutcome::Conflict {
6557 conflict: ConflictRecord {
6558 client_commit_id: "conflict-commit".to_owned(),
6559 op_index: 0,
6560 table: "tasks".to_owned(),
6561 row_id: "t1".to_owned(),
6562 code: "sync.version_conflict".to_owned(),
6563 message: "stale base version".to_owned(),
6564 server_version: 2,
6565 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6566 conflict_columns: vec!["project_id".to_owned()],
6567 operation: None,
6568 },
6569 }],
6570 Some(&operations),
6571 )
6572 .expect("persist conflict");
6573 client
6574 .persist_commit_outcome(
6575 "rejected-commit",
6576 CommitOutcomeStatus::Rejected,
6577 &[CommitOperationOutcome::Error {
6578 rejection: RejectionRecord {
6579 client_commit_id: "rejected-commit".to_owned(),
6580 op_index: 0,
6581 code: "sync.unknown_table".to_owned(),
6582 message: "unknown table".to_owned(),
6583 retryable: false,
6584 details: None,
6585 operation: None,
6586 },
6587 }],
6588 Some(&operations),
6589 )
6590 .expect("persist rejection");
6591 client
6592 .finish_observation(
6593 "test_delivery",
6594 ChangeAccumulator {
6595 conflicts: true,
6596 outcomes: true,
6597 ..ChangeAccumulator::default()
6598 },
6599 )
6600 .expect("commit outcomes");
6601
6602 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6603 match reader.commit_delivery("conflict-commit").expect("conflict") {
6604 CommitDelivery::Known { outcome, .. } => {
6605 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6606 assert!(matches!(
6607 outcome.results.as_slice(),
6608 [CommitOperationOutcome::Conflict { conflict }]
6609 if conflict.code == "sync.version_conflict"
6610 && conflict.server_version == 2
6611 ));
6612 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
6613 assert!(
6614 outcome.retained_rows.is_none(),
6615 "the sidecar reports the persisted view, not owner-derived retained rows"
6616 );
6617 }
6618 other => panic!("expected a known conflict: {other:?}"),
6619 }
6620 match reader
6621 .commit_delivery("rejected-commit")
6622 .expect("rejection")
6623 {
6624 CommitDelivery::Known { outcome, .. } => {
6625 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
6626 assert!(matches!(
6627 outcome.results.as_slice(),
6628 [CommitOperationOutcome::Error { rejection }]
6629 if rejection.code == "sync.unknown_table"
6630 ));
6631 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
6632 assert!(outcome.retained_rows.is_none());
6633 }
6634 other => panic!("expected a known rejection: {other:?}"),
6635 }
6636 drop(reader);
6637 drop(client);
6638 std::fs::remove_file(path).expect("remove temp database");
6639 }
6640
6641 #[test]
6642 fn snapshot_read_isolates_a_concurrent_writer_commit() {
6643 use std::sync::mpsc::channel;
6644 let path = std::env::temp_dir().join(format!(
6645 "syncular-read-concurrent-{}.db",
6646 uuid::Uuid::new_v4()
6647 ));
6648 let schema = json!({
6649 "version": 1,
6650 "tables": [{
6651 "name": "tasks",
6652 "primaryKey": "id",
6653 "columns": [
6654 { "name": "id", "type": "string", "nullable": false },
6655 { "name": "project_id", "type": "string", "nullable": false }
6656 ],
6657 "scopes": [{ "pattern": "project:{project_id}" }]
6658 }]
6659 });
6660 let mut client = SyncClient::open_path_with_identity(
6661 Some("sidecar-concurrent".to_owned()),
6662 &schema,
6663 ClientLimits::default(),
6664 path.to_str().expect("UTF-8 temp path"),
6665 )
6666 .expect("open owner");
6667 client
6668 .mutate(vec![Mutation::Upsert {
6669 table: "tasks".to_owned(),
6670 values: Map::from_iter([
6671 ("id".to_owned(), Value::from("t1")),
6672 ("project_id".to_owned(), Value::from("one")),
6673 ]),
6674 base_version: None,
6675 }])
6676 .expect("seed row");
6677 let revision_before = client.local_revision().to_string();
6678
6679 let (ready_tx, ready_rx) = channel::<()>();
6682 let (done_tx, done_rx) = channel::<()>();
6683 let writer_path = path.to_string_lossy().into_owned();
6684 let writer_schema = schema.clone();
6685 let writer = std::thread::spawn(move || {
6686 let mut writer = SyncClient::open_path_with_identity(
6687 Some("sidecar-concurrent".to_owned()),
6688 &writer_schema,
6689 ClientLimits::default(),
6690 &writer_path,
6691 )
6692 .expect("open writer");
6693 ready_rx.recv().expect("reader signalled");
6694 writer
6695 .mutate(vec![Mutation::Upsert {
6696 table: "tasks".to_owned(),
6697 values: Map::from_iter([
6698 ("id".to_owned(), Value::from("t2")),
6699 ("project_id".to_owned(), Value::from("one")),
6700 ]),
6701 base_version: None,
6702 }])
6703 .expect("writer commit");
6704 let _ = done_tx.send(());
6705 });
6706 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6707 SNAPSHOT_READ_HOOK.with(|slot| {
6708 *slot.borrow_mut() = Some(Box::new(move || {
6709 ready_tx.send(()).expect("signal writer");
6710 done_rx.recv().expect("writer finished");
6711 }));
6712 });
6713 let read = reader
6714 .snapshot_read(&SnapshotReadRequest {
6715 statements: vec![
6716 SnapshotStatement {
6717 sql: "SELECT count(*) AS n FROM tasks",
6718 params: &[],
6719 },
6720 SnapshotStatement {
6721 sql: "SELECT count(*) AS n FROM tasks",
6722 params: &[],
6723 },
6724 ],
6725 coverage: &[],
6726 subscriptions: Vec::new(),
6727 commit_ids: Vec::new(),
6728 })
6729 .expect("snapshot read");
6730 assert_eq!(
6731 read.queries[0][0]["n"], 1,
6732 "first statement sees the pre-commit row"
6733 );
6734 assert_eq!(
6735 read.queries[1][0]["n"], 1,
6736 "a later statement keeps the one read snapshot"
6737 );
6738 assert_eq!(
6739 read.revision, revision_before,
6740 "the read reports the snapshot revision, not the writer's"
6741 );
6742 writer.join().expect("join writer");
6743
6744 assert_ne!(
6745 reader.local_revision().expect("revision after commit"),
6746 revision_before,
6747 "the next read observes the committed revision"
6748 );
6749 let after = reader
6750 .query_snapshot("SELECT count(*) AS n FROM tasks", &[], &[])
6751 .expect("count after commit");
6752 assert_eq!(
6753 after.rows[0]["n"], 2,
6754 "the committed row is visible afterwards"
6755 );
6756
6757 drop(reader);
6758 drop(client);
6759 std::fs::remove_file(path).expect("remove temp database");
6760 }
6761
6762 #[test]
6763 fn snapshot_read_rejects_every_non_canonical_revision() {
6764 let path = std::env::temp_dir().join(format!(
6765 "syncular-read-revision-{}.db",
6766 uuid::Uuid::new_v4()
6767 ));
6768 let schema = json!({
6769 "version": 1,
6770 "tables": [{
6771 "name": "tasks",
6772 "primaryKey": "id",
6773 "columns": [
6774 { "name": "id", "type": "string", "nullable": false },
6775 { "name": "project_id", "type": "string", "nullable": false }
6776 ],
6777 "scopes": [{ "pattern": "project:{project_id}" }]
6778 }]
6779 });
6780 let client = SyncClient::open_path_with_identity(
6781 Some("sidecar-revision".to_owned()),
6782 &schema,
6783 ClientLimits::default(),
6784 path.to_str().expect("UTF-8 temp path"),
6785 )
6786 .expect("open owner");
6787 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6788 for bad in [
6789 "+1",
6790 "01",
6791 "-1",
6792 " 1",
6793 "1 ",
6794 "18446744073709551616",
6795 "1.0",
6796 "abc",
6797 "",
6798 ] {
6799 client
6800 .conn
6801 .execute(
6802 "UPDATE _syncular_meta SET value = ?1 WHERE key = 'localRevision'",
6803 rusqlite::params![bad],
6804 )
6805 .expect("corrupt revision");
6806 let failure = reader.local_revision().expect_err("corrupt marker fails");
6807 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad:?}");
6808 }
6809 for value in ["X'31'", "CAST(X'FF' AS TEXT)"] {
6810 client
6811 .conn
6812 .execute_batch(&format!(
6813 "UPDATE _syncular_meta SET value={value} WHERE key='localRevision'"
6814 ))
6815 .expect("corrupt stored revision type");
6816 assert_eq!(
6817 reader.local_revision().expect_err("invalid SQL field").code,
6818 Some("sync.local_corrupt")
6819 );
6820 }
6821 client
6823 .conn
6824 .execute("DELETE FROM _syncular_meta WHERE key = 'localRevision'", [])
6825 .expect("delete marker");
6826 assert_eq!(reader.local_revision().expect("legacy zero"), "0");
6827
6828 drop(reader);
6829 drop(client);
6830 std::fs::remove_file(path).expect("remove temp database");
6831 }
6832
6833 #[test]
6834 fn snapshot_read_rejects_corrupt_subscription_state() {
6835 let path = std::env::temp_dir().join(format!(
6836 "syncular-read-substate-{}.db",
6837 uuid::Uuid::new_v4()
6838 ));
6839 let schema = json!({
6840 "version": 1,
6841 "tables": [{
6842 "name": "tasks",
6843 "primaryKey": "id",
6844 "columns": [
6845 { "name": "id", "type": "string", "nullable": false },
6846 { "name": "project_id", "type": "string", "nullable": false }
6847 ],
6848 "scopes": [{ "pattern": "project:{project_id}" }]
6849 }]
6850 });
6851 let mut client = SyncClient::open_path_with_identity(
6852 Some("sidecar-substate".to_owned()),
6853 &schema,
6854 ClientLimits::default(),
6855 path.to_str().expect("UTF-8 temp path"),
6856 )
6857 .expect("open owner");
6858 client
6859 .subscribe(
6860 "s1".into(),
6861 "tasks".into(),
6862 vec![("project_id".into(), vec!["one".into()])],
6863 None,
6864 )
6865 .expect("subscribe");
6866 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6867 for bad in [
6868 "not json".to_owned(),
6869 "{}".to_owned(),
6870 json!({ "status": "bogus", "cursor": 0 }).to_string(),
6871 json!({ "status": "active", "cursor": "x" }).to_string(),
6872 json!({ "status": "active", "cursor": 0, "bootstrapState": 7 }).to_string(),
6873 json!({ "status": "active", "cursor": 0, "reasonCode": 7 }).to_string(),
6874 ] {
6875 client
6876 .conn
6877 .execute(
6878 "UPDATE _syncular_subscriptions SET state_json = ?1 WHERE id = 's1'",
6879 rusqlite::params![bad],
6880 )
6881 .expect("corrupt subscription state");
6882 let failure = reader
6883 .subscription_catchup("s1")
6884 .expect_err("corrupt state fails");
6885 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad}");
6886 }
6887
6888 for assignment in [
6889 "tbl=X'7461736b73'",
6890 "state_json=X'7b7d'",
6891 "state_json=CAST(X'FF' AS TEXT)",
6892 ] {
6893 client.conn.execute_batch(&format!(
6894 "UPDATE _syncular_subscriptions SET tbl='tasks',state_json='{{\"status\":\"active\",\"cursor\":0}}'; UPDATE _syncular_subscriptions SET {assignment} WHERE id='s1'"
6895 )).expect("corrupt stored subscription type");
6896 assert_eq!(
6897 reader
6898 .subscription_catchup("s1")
6899 .expect_err("invalid SQL field")
6900 .code,
6901 Some("sync.local_corrupt")
6902 );
6903 }
6904
6905 drop(reader);
6906 drop(client);
6907 std::fs::remove_file(path).expect("remove temp database");
6908 }
6909
6910 #[test]
6911 fn snapshot_read_rolls_back_a_later_invalid_statement_and_guards_every_write() {
6912 let path = std::env::temp_dir().join(format!(
6913 "syncular-read-rollback-{}.db",
6914 uuid::Uuid::new_v4()
6915 ));
6916 let schema = json!({
6917 "version": 1,
6918 "tables": [{
6919 "name": "tasks",
6920 "primaryKey": "id",
6921 "columns": [
6922 { "name": "id", "type": "string", "nullable": false },
6923 { "name": "project_id", "type": "string", "nullable": false }
6924 ],
6925 "scopes": [{ "pattern": "project:{project_id}" }]
6926 }]
6927 });
6928 let mut client = SyncClient::open_path_with_identity(
6929 Some("sidecar-rollback".to_owned()),
6930 &schema,
6931 ClientLimits::default(),
6932 path.to_str().expect("UTF-8 temp path"),
6933 )
6934 .expect("open owner");
6935 let commit_id = client
6936 .mutate(vec![Mutation::Upsert {
6937 table: "tasks".to_owned(),
6938 values: Map::from_iter([
6939 ("id".to_owned(), Value::from("t1")),
6940 ("project_id".to_owned(), Value::from("one")),
6941 ]),
6942 base_version: None,
6943 }])
6944 .expect("seed row");
6945 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6946 let revision = reader.local_revision().expect("revision");
6947
6948 let failure = reader
6951 .snapshot_read(&SnapshotReadRequest {
6952 statements: vec![
6953 SnapshotStatement {
6954 sql: "SELECT id FROM tasks",
6955 params: &[],
6956 },
6957 SnapshotStatement {
6958 sql: "SELECT * FROM does_not_exist",
6959 params: &[],
6960 },
6961 ],
6962 coverage: &[],
6963 subscriptions: Vec::new(),
6964 commit_ids: vec![commit_id.clone()],
6965 })
6966 .expect_err("second statement fails");
6967 assert_eq!(failure.code, None);
6968 assert_eq!(
6969 reader.local_revision().expect("revision after rollback"),
6970 revision,
6971 "a rolled-back read consumes no revision"
6972 );
6973 assert!(
6974 !matches!(
6975 reader
6976 .commit_delivery(&commit_id)
6977 .expect("delivery after rollback"),
6978 CommitDelivery::Unknown { .. }
6979 ),
6980 "the same reader serves the next read after releasing the connection"
6981 );
6982
6983 let guard = reader
6985 .snapshot_read(&SnapshotReadRequest {
6986 statements: vec![SnapshotStatement {
6987 sql: "DELETE FROM tasks",
6988 params: &[],
6989 }],
6990 coverage: &[],
6991 subscriptions: Vec::new(),
6992 commit_ids: Vec::new(),
6993 })
6994 .expect_err("write guard");
6995 assert!(guard.message.contains("read-only"), "{}", guard.message);
6996
6997 drop(reader);
6998 drop(client);
6999 std::fs::remove_file(path).expect("remove temp database");
7000 }
7001
7002 #[test]
7003 fn owned_query_failures_classify_order_bound_and_clear() {
7004 let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7005 rusqlite::ffi::Error::new(266),
7006 Some("private SQLite prose".to_owned()),
7007 ));
7008 assert_eq!(io.code, Some("client.storage_io"));
7009 assert_eq!(io.sqlite_code, Some(266));
7010 assert_eq!(io.message, "local SQLite storage I/O failed");
7011 let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7012 rusqlite::ffi::Error::new(26),
7013 None,
7014 ));
7015 assert_eq!(corrupt.code, Some("client.storage_corrupt"));
7016 assert!(!io.retryable());
7017 assert!(!corrupt.retryable());
7018
7019 for (raw, extended) in [(5, 5), (5 | (1 << 8), 261), (6, 6), (6 | (1 << 8), 262)] {
7022 let busy = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7023 rusqlite::ffi::Error::new(raw),
7024 Some("database is locked".to_owned()),
7025 ));
7026 assert_eq!(busy.code, Some("client.storage_busy"));
7027 assert_eq!(busy.sqlite_code, Some(extended));
7028 assert_eq!(busy.message, "local SQLite storage is busy");
7029 assert!(busy.retryable());
7030 }
7031 let unrelated = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7032 rusqlite::ffi::Error::new(1),
7033 Some("generic SQLite prose".to_owned()),
7034 ));
7035 assert_eq!(unrelated.code, None);
7036 assert!(!unrelated.retryable());
7037
7038 let mut client = client();
7039 let generic = QueryReadFailure::from("private query prose".to_owned());
7040 client.record_query_read(
7041 QueryOwner {
7042 id: "first",
7043 tables: &["tasks", "tasks"],
7044 },
7045 Some(&generic),
7046 );
7047 client.record_query_read(
7048 QueryOwner {
7049 id: "second",
7050 tables: &["tasks"],
7051 },
7052 Some(&generic),
7053 );
7054 client.record_query_read(
7055 QueryOwner {
7056 id: "first",
7057 tables: &["docs"],
7058 },
7059 Some(&io),
7060 );
7061 let failures = &client
7062 .diagnostics_snapshot(&Default::default())
7063 .expect("diagnostics")
7064 .query_failures;
7065 assert_eq!(
7066 failures
7067 .iter()
7068 .map(|entry| entry.id.as_str())
7069 .collect::<Vec<_>>(),
7070 vec!["second", "first"]
7071 );
7072 assert_eq!(failures[1].tables, vec!["docs"]);
7073 assert_eq!(failures[1].code, "client.storage_io");
7074
7075 for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
7076 client.record_query_read(
7077 QueryOwner {
7078 id: &format!("query-{index}"),
7079 tables: &["tasks"],
7080 },
7081 Some(&generic),
7082 );
7083 }
7084 let failures = &client
7085 .diagnostics_snapshot(&Default::default())
7086 .expect("bounded diagnostics")
7087 .query_failures;
7088 assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
7089 assert_eq!(failures.last().expect("last failure").id, "query-256");
7090
7091 client
7092 .query_snapshot(
7093 "SELECT id FROM tasks",
7094 &[],
7095 &[],
7096 Some(QueryOwner {
7097 id: "query-256",
7098 tables: &["tasks"],
7099 }),
7100 )
7101 .expect("successful owned read");
7102 assert!(client
7103 .diagnostics_snapshot(&Default::default())
7104 .expect("cleared diagnostics")
7105 .query_failures
7106 .iter()
7107 .all(|entry| entry.id != "query-256"));
7108 }
7109
7110 #[test]
7111 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
7112 let mut client = client();
7113 let image = Connection::open_in_memory().unwrap();
7114 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
7115 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7116 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
7117 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7118 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
7119 let table = client.schema.table("tasks").unwrap().clone();
7120 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7121 assert!(client
7122 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
7123 .is_err());
7124 assert!(client.conn.is_autocommit());
7125 assert_eq!(
7126 client
7127 .query("SELECT count(*) AS n FROM tasks", &[])
7128 .unwrap()[0]["n"],
7129 1024
7130 );
7131 assert_eq!(client.drain_change_batches().len(), 1);
7132 image
7133 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
7134 .unwrap();
7135 assert!(matches!(
7136 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
7137 Ok(2050)
7138 ));
7139 assert_eq!(
7140 client
7141 .query("SELECT count(*) AS n FROM tasks", &[])
7142 .unwrap()[0]["n"],
7143 2050
7144 );
7145 assert_eq!(client.drain_change_batches().len(), 3);
7146 }
7147
7148 #[test]
7149 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
7150 let mut client = client();
7151 let base = WindowBase {
7152 table: "tasks".into(),
7153 variable: "project_id".into(),
7154 fixed_scopes: Vec::new(),
7155 params: None,
7156 };
7157 client
7158 .set_window(&base, &["p1".into(), "p2".into()])
7159 .unwrap();
7160 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7161 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7162 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
7163 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7164 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7165 client.drain_change_batches();
7166 assert!(client.set_window(&base, &["p2".into()]).is_err());
7167 assert!(client.conn.is_autocommit());
7168 assert_eq!(
7169 client
7170 .query("SELECT count(*) AS n FROM tasks", &[])
7171 .unwrap()[0]["n"],
7172 1027
7173 );
7174 assert_eq!(client.window_state(&base).units, vec!["p2"]);
7175 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
7176 assert_eq!(client.drain_change_batches().len(), 1);
7177 client
7178 .conn
7179 .execute_batch("DROP TRIGGER interrupt_evict")
7180 .unwrap();
7181 client.drain_pending_evictions().unwrap();
7182 assert!(client.load_pending_evictions().unwrap().is_empty());
7183 assert_eq!(
7184 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
7185 "held"
7186 );
7187 assert_eq!(client.drain_change_batches().len(), 2);
7188 }
7189
7190 #[test]
7191 fn eviction_marker_failures_roll_back_window_changes() {
7192 let mut client = client();
7193 let base = WindowBase {
7194 table: "tasks".into(),
7195 variable: "project_id".into(),
7196 fixed_scopes: Vec::new(),
7197 params: None,
7198 };
7199 client.set_window(&base, &["p1".into()]).unwrap();
7200 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7201 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7202 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7203 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7204 let revision = client.local_revision();
7205 assert!(client.set_window(&base, &[]).is_err());
7206 assert!(client.conn.is_autocommit());
7207 assert_eq!(client.local_revision(), revision);
7208 assert_eq!(
7209 client
7210 .query("SELECT count(*) AS n FROM tasks", &[])
7211 .unwrap()[0]["n"],
7212 2050
7213 );
7214 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7215 client.conn.execute_batch("DROP TRIGGER fail_marker;
7216 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7217 assert!(client.set_window(&base, &[]).is_err());
7218 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
7219 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7220 assert!(client.drain_pending_evictions().is_err());
7221 assert!(client.conn.is_autocommit());
7222 assert_eq!(
7223 client
7224 .query("SELECT count(*) AS n FROM tasks", &[])
7225 .unwrap()[0]["n"],
7226 2
7227 );
7228 assert!(client.set_window(&base, &["p1".into()]).is_err());
7229 assert!(client.conn.is_autocommit());
7230 assert!(client.window_state(&base).units.is_empty());
7231 assert!(client.subs.is_empty());
7232 assert_eq!(
7233 client
7234 .query(
7235 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
7236 &[]
7237 )
7238 .unwrap()[0]["n"],
7239 1
7240 );
7241 client
7242 .conn
7243 .execute_batch("DROP TRIGGER fail_marker")
7244 .unwrap();
7245 client.set_window(&base, &["p1".into()]).unwrap();
7246 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7247 }
7248
7249 #[test]
7250 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
7251 let mut client = client();
7252 client
7253 .conn
7254 .execute(
7255 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
7256 [],
7257 )
7258 .unwrap();
7259 assert_eq!(
7260 client.drain_pending_evictions().unwrap_err(),
7261 "sync.local_corrupt: invalid pending eviction scopes"
7262 );
7263 assert!(client.conn.is_autocommit());
7264 assert!(client.enqueue_startup_sync_if_needed().is_err());
7265 }
7266
7267 #[test]
7268 fn image_chunks_reject_null_primary_keys() {
7269 let mut client = client();
7270 let image = Connection::open_in_memory().unwrap();
7271 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7272 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
7273 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
7274 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7275 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
7276 let table = client.schema.table("tasks").unwrap().clone();
7277 assert!(client
7278 .apply_sqlite_image(
7279 &image,
7280 &table,
7281 true,
7282 &[("project_id".into(), vec!["p1".into()])],
7283 1025,
7284 7,
7285 "digest"
7286 )
7287 .is_err());
7288 assert!(client.conn.is_autocommit());
7289 assert_eq!(
7290 client
7291 .query("SELECT count(*) AS n FROM tasks", &[])
7292 .unwrap()[0]["n"],
7293 0
7294 );
7295 }
7296
7297 #[test]
7298 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
7299 for images in [false, true] {
7300 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7301 "name":name,"primaryKey":"id","columns":[
7302 {"name":"id","type":"string","nullable":false},
7303 {"name":"project_id","type":"string","nullable":false},
7304 {"name":"title","type":"string","nullable":false}],
7305 "scopes":[{"pattern":"project:{project_id}"}],
7306 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
7307 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7308 })))});
7309 let mut actual =
7310 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
7311 let mut reference =
7312 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
7313 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7314 for client in [&mut actual, &mut reference] {
7315 client
7316 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
7317 .unwrap();
7318 client.subs[0].effective = Some(scopes.clone());
7319 for (table, id, title) in [
7320 ("tasks", "a", "original"),
7321 ("tasks", "b", "occupied"),
7322 ("tasks", "c", "deleted"),
7323 ("notes", "n", "untouched"),
7324 ] {
7325 client
7326 .write_base_row(
7327 table,
7328 &vec![
7329 Some(ColumnValue::String(id.into())),
7330 Some(ColumnValue::String("p1".into())),
7331 Some(ColumnValue::String(title.into())),
7332 ],
7333 1,
7334 )
7335 .unwrap();
7336 }
7337 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7338 client
7339 .mutate(vec![
7340 Mutation::Upsert {
7341 table: "tasks".into(),
7342 values: Map::from_iter([
7343 ("id".into(), Value::from("a")),
7344 ("project_id".into(), Value::from("p1")),
7345 ("title".into(), Value::from("available")),
7346 ]),
7347 base_version: None,
7348 },
7349 Mutation::Delete {
7350 table: "tasks".into(),
7351 row_id: "c".into(),
7352 base_version: None,
7353 },
7354 Mutation::Upsert {
7355 table: "tasks".into(),
7356 values: Map::from_iter([
7357 ("id".into(), Value::from("local")),
7358 ("project_id".into(), Value::from("p1")),
7359 ("title".into(), Value::from("local value")),
7360 ]),
7361 base_version: None,
7362 },
7363 Mutation::Upsert {
7364 table: "notes".into(),
7365 values: Map::from_iter([
7366 ("id".into(), Value::from("n")),
7367 ("project_id".into(), Value::from("p1")),
7368 ("title".into(), Value::from("pending note")),
7369 ]),
7370 base_version: None,
7371 },
7372 ])
7373 .unwrap();
7374 }
7375 let pending = actual.pending_commit_ids();
7376 actual.overlay_rebuild_count.set(0);
7377 for (id, title) in [
7378 ("b", "available"),
7379 ("b", "occupied"),
7380 ("a", "server original"),
7381 ("d", "local value"),
7382 ("d", "released"),
7383 ("c", "redelivered"),
7384 ] {
7385 let table = actual.schema.table("tasks").unwrap().clone();
7386 let values = vec![
7387 Some(ColumnValue::String(id.into())),
7388 Some(ColumnValue::String("p1".into())),
7389 Some(ColumnValue::String(title.into())),
7390 ];
7391 if images {
7392 let image = Connection::open_in_memory().unwrap();
7393 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7394 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7395 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7396 image
7397 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
7398 .unwrap();
7399 assert!(matches!(
7400 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
7401 Ok(1)
7402 ));
7403 } else {
7404 let segment = RowsSegment {
7405 table: "tasks".into(),
7406 schema_version: 1,
7407 columns: table.wire_columns.clone(),
7408 blocks: vec![vec![ssp2::segment::SegmentRow {
7409 server_version: 2,
7410 values: values.clone(),
7411 }]],
7412 };
7413 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
7414 }
7415 reference.write_base_row("tasks", &values, 2).unwrap();
7416 reference.rebuild_overlay().expect("rebuild overlay");
7417 for table in ["tasks", "notes"] {
7418 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
7419 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
7420 }
7421 }
7422 assert_eq!(actual.pending_commit_ids(), pending);
7423 }
7424 assert_eq!(
7425 actual.overlay_rebuild_count.get(),
7426 0,
7427 "each chunk must leave unrelated tables alone"
7428 );
7429 let before = actual
7430 .query("SELECT * FROM tasks ORDER BY id", &[])
7431 .unwrap();
7432 let fts_before = actual
7433 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7434 .unwrap();
7435 let revision = actual.local_revision();
7436 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
7437 let table = actual.schema.table("tasks").unwrap().clone();
7438 let segment = RowsSegment {
7439 table: "tasks".into(),
7440 schema_version: 1,
7441 columns: table.wire_columns.clone(),
7442 blocks: vec![vec![ssp2::segment::SegmentRow {
7443 server_version: 3,
7444 values: vec![
7445 Some(ColumnValue::String("b".into())),
7446 Some(ColumnValue::String("p1".into())),
7447 Some(ColumnValue::String("fresh".into())),
7448 ],
7449 }]],
7450 };
7451 assert!(actual.apply_segment(0, &segment, false).is_err());
7452 assert!(actual.conn.is_autocommit());
7453 assert_eq!(
7454 actual
7455 .query("SELECT * FROM tasks ORDER BY id", &[])
7456 .unwrap(),
7457 before
7458 );
7459 assert_eq!(
7460 actual
7461 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7462 .unwrap(),
7463 fts_before
7464 );
7465 assert_eq!(actual.local_revision(), revision);
7466 assert_eq!(actual.pending_commit_ids(), pending);
7467 }
7468 }
7469
7470 #[test]
7471 fn import_reconciliation_preserves_typed_pending_primary_keys() {
7472 for (kind, a, b) in [
7473 ("string", json!("a"), json!("b")),
7474 ("integer", json!(1), json!(2)),
7475 ("boolean", json!(true), json!(false)),
7476 ("json", json!("1"), json!("2")),
7477 ] {
7478 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
7479 for images in [false, true] {
7480 let mut client =
7481 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
7482 let table = client.schema.table("tasks").unwrap().clone();
7483 client
7484 .subscribe("tasks".into(), "tasks".into(), vec![], None)
7485 .unwrap();
7486 for (id, title) in [(&a, "original"), (&b, "occupied")] {
7487 client
7488 .write_base_row(
7489 "tasks",
7490 &vec![
7491 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
7492 Some(ColumnValue::String(title.into())),
7493 ],
7494 1,
7495 )
7496 .unwrap();
7497 }
7498 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7499 client
7500 .mutate(vec![Mutation::Upsert {
7501 table: "tasks".into(),
7502 values: Map::from_iter([
7503 ("id".into(), a.clone()),
7504 ("title".into(), json!("available")),
7505 ]),
7506 base_version: None,
7507 }])
7508 .unwrap();
7509 client.overlay_rebuild_count.set(0);
7510 for (title, expected) in [("available", "original"), ("occupied", "available")] {
7511 let values = vec![
7512 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
7513 Some(ColumnValue::String(title.into())),
7514 ];
7515 if images {
7516 let image = Connection::open_in_memory().unwrap();
7517 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7518 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7519 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7520 image
7521 .execute(
7522 "INSERT INTO tasks VALUES (?1,?2,2)",
7523 rusqlite::params![RowParam::Cell(&values[0]), title],
7524 )
7525 .unwrap();
7526 assert!(
7527 matches!(
7528 client.apply_sqlite_image(
7529 &image,
7530 &table,
7531 false,
7532 &[],
7533 1,
7534 7,
7535 "digest"
7536 ),
7537 Ok(1)
7538 ),
7539 "{kind}, images={images}"
7540 );
7541 } else {
7542 let segment = RowsSegment {
7543 table: "tasks".into(),
7544 schema_version: 1,
7545 columns: table.wire_columns.clone(),
7546 blocks: vec![vec![ssp2::segment::SegmentRow {
7547 server_version: 2,
7548 values,
7549 }]],
7550 };
7551 assert!(
7552 matches!(client.apply_segment(0, &segment, false), Ok(1)),
7553 "{kind}"
7554 );
7555 }
7556 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
7557 let actual: String = client
7558 .conn
7559 .query_row(
7560 "SELECT title FROM tasks WHERE id=?1",
7561 [RowParam::Cell(&key)],
7562 |row| row.get(0),
7563 )
7564 .unwrap();
7565 assert_eq!(actual, expected, "{kind}, images={images}");
7566 }
7567 assert_eq!(client.overlay_rebuild_count.get(), 0);
7568 }
7569 }
7570 }
7571
7572 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
7573 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7574 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
7575 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
7576 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7577 })))});
7578 let mut client =
7579 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
7580 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
7581 client.rebuild_overlay().expect("rebuild overlay");
7582 client
7583 .mutate(vec![Mutation::Upsert {
7584 table: "tasks".into(),
7585 values: Map::from_iter([
7586 ("id".into(), json!("offline")),
7587 ("project_id".into(), json!("p1")),
7588 ("title".into(), json!("pending edit")),
7589 ]),
7590 base_version: None,
7591 }])
7592 .unwrap();
7593 let image = Connection::open_in_memory().unwrap();
7594 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7595 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
7596 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
7597 image
7598 .execute(
7599 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
7600 [rows],
7601 )
7602 .unwrap();
7603 client.overlay_rebuild_count.set(0);
7604 let table = client.schema.table("tasks").unwrap().clone();
7605 (client, image, table)
7606 }
7607
7608 #[test]
7609 fn sqlite_full_image_import_retains_the_first_code_and_recovers() {
7610 let (mut client, image, table) = unique_import_fixture(1);
7611 image
7612 .execute("UPDATE tasks SET title = ?1", ["full ".repeat(32768)])
7613 .unwrap();
7614 client
7615 .conn
7616 .execute_batch("PRAGMA max_page_count = 1")
7617 .unwrap();
7618 let effective = [("project_id".into(), vec!["p1".into()])];
7619 assert!(client
7620 .apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest")
7621 .is_err());
7622 let failure = client.storage_failure.borrow().clone().unwrap();
7623 assert_eq!(failure.code, Some("client.storage_full"));
7624 assert_eq!(failure.sqlite_code, Some(13));
7625 assert!(!failure.message.contains("rollback"));
7626 assert!(client.conn.is_autocommit());
7627 client
7628 .conn
7629 .execute_batch("PRAGMA max_page_count = 1073741823")
7630 .unwrap();
7631 assert!(matches!(
7632 client.apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest"),
7633 Ok(1)
7634 ));
7635 assert!(client.conn.is_autocommit());
7636 }
7637
7638 #[test]
7639 fn authoring_storage_failures_are_structured_and_scoped() {
7640 let path = std::env::temp_dir().join(format!(
7641 "syncular-authoring-failure-{}.db",
7642 uuid::Uuid::new_v4()
7643 ));
7644 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
7645 {"name":"id","type":"string","nullable":false},
7646 {"name":"project_id","type":"string","nullable":false},
7647 {"name":"title","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
7648 let mut client = SyncClient::open_path(
7649 "authoring".into(),
7650 &schema,
7651 ClientLimits::default(),
7652 path.to_str().unwrap(),
7653 )
7654 .expect("open client");
7655 client.create_synced_tables().expect("create synced tables");
7656 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
7658
7659 let upsert = |id: &str| {
7660 vec![Mutation::Upsert {
7661 table: "tasks".into(),
7662 values: Map::from_iter([
7663 ("id".into(), json!(id)),
7664 ("project_id".into(), json!("p1")),
7665 ("title".into(), json!("first")),
7666 ]),
7667 base_version: None,
7668 }]
7669 };
7670
7671 let locker = Connection::open(&path).expect("second connection");
7672 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
7673 let busy = client.mutate(upsert("t1")).expect_err("busy write");
7674 assert_eq!(busy.code, "client.storage_busy");
7675 assert!(busy.retryable);
7676 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
7677 assert!(client.pending_commit_ids().is_empty());
7678 assert_eq!(client.local_revision(), 0);
7679
7680 let invalid = client.mutate(Vec::new()).expect_err("empty commit");
7682 assert_eq!(invalid.code, "sync.invalid_request");
7683 assert_eq!(invalid.message, "the authoring request is invalid");
7684 assert!(
7685 invalid.details.as_ref().unwrap()["legacyCause"]
7686 .as_str()
7687 .unwrap()
7688 .contains("at least one operation"),
7689 "{invalid:?}"
7690 );
7691
7692 locker.execute_batch("ROLLBACK").unwrap();
7693 let id = client.mutate(upsert("t1")).expect("retry after busy");
7694 assert_eq!(client.pending_commit_ids(), vec![id]);
7695 assert_eq!(client.local_revision(), 1);
7696
7697 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
7699 let busy_again = client.mutate(upsert("t2")).expect_err("busy write again");
7700 assert_eq!(busy_again.code, "client.storage_busy");
7701 assert!(busy_again.retryable);
7702 locker.execute_batch("ROLLBACK").unwrap();
7703
7704 let full = ClientError::from(QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7706 rusqlite::ffi::Error::new(13),
7707 Some("database or disk is full".to_owned()),
7708 )));
7709 assert_eq!(full.code, "client.storage_full");
7710 assert!(!full.retryable);
7711 assert_eq!(full.details.as_ref().unwrap()["sqliteCode"], 13);
7712
7713 drop(locker);
7714 drop(client);
7715 let _ = std::fs::remove_file(&path);
7716 }
7717
7718 #[test]
7719 fn caller_value_failures_are_invalid_request_and_leave_no_outbox_entry() {
7720 let path = std::env::temp_dir().join(format!(
7721 "syncular-caller-values-{}.db",
7722 uuid::Uuid::new_v4()
7723 ));
7724 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
7725 {"name":"id","type":"string","nullable":false},
7726 {"name":"project_id","type":"string","nullable":false},
7727 {"name":"title","type":"string","nullable":false},
7728 {"name":"priority","type":"integer","nullable":false},
7729 {"name":"payload","type":"bytes","nullable":false},
7730 {"name":"extra","type":"bytes","nullable":true}],"scopes":[{"pattern":"project:{project_id}"}]}]});
7731 let mut client = SyncClient::open_path(
7732 "caller-values".into(),
7733 &schema,
7734 ClientLimits::default(),
7735 path.to_str().unwrap(),
7736 )
7737 .expect("open client");
7738 client.create_synced_tables().expect("create synced tables");
7739
7740 let values = |id: &str| {
7741 Map::from_iter([
7742 ("id".to_owned(), json!(id)),
7743 ("project_id".to_owned(), json!("p1")),
7744 ("title".to_owned(), json!("first")),
7745 ("priority".to_owned(), json!(1)),
7746 ("payload".to_owned(), json!({"$bytes": "0a"})),
7747 ])
7748 };
7749 let upsert = |values: Map<String, Value>| {
7750 vec![Mutation::Upsert {
7751 table: "tasks".into(),
7752 values,
7753 base_version: None,
7754 }]
7755 };
7756 let seed = client.mutate(upsert(values("t1"))).expect("seed row");
7757 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
7758
7759 for (label, override_value, cause) in [
7764 ("unknown column", ("nope", json!(1)), "unknown column"),
7765 (
7766 "internal sync column",
7767 ("_sync_version", json!(1)),
7768 "internal sync column",
7769 ),
7770 ("wrong value type", ("title", json!(7)), "expected a string"),
7771 (
7772 "malformed bytes envelope",
7773 ("payload", json!({"$bytes": "zz"})),
7774 "expected exactly",
7775 ),
7776 (
7777 "odd-length bytes envelope",
7778 ("payload", json!({"$bytes": "abc"})),
7779 "expected exactly",
7780 ),
7781 (
7782 "plus-sign bytes envelope",
7783 ("payload", json!({"$bytes": "+a"})),
7784 "expected exactly",
7785 ),
7786 (
7787 "bytes envelope with a second key",
7788 ("payload", json!({"$bytes": "0a", "extra": 1})),
7789 "expected exactly",
7790 ),
7791 (
7792 "no bytes envelope",
7793 ("payload", json!({"nested": true})),
7794 "expected exactly",
7795 ),
7796 ] {
7797 let mut full = values("t2");
7798 full.insert(override_value.0.to_owned(), override_value.1);
7799 let error = client.mutate(upsert(full)).expect_err(label);
7800 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
7801 assert_eq!(error.message, "the authoring request is invalid", "{label}");
7802 assert!(!error.retryable, "{label}");
7803 let legacy = error.details.as_ref().unwrap()["legacyCause"]
7804 .as_str()
7805 .unwrap();
7806 assert!(
7807 legacy.starts_with("sync.invalid_request: ") && legacy.contains(cause),
7808 "{label}: {legacy}"
7809 );
7810 assert_eq!(client.pending_commit_ids(), vec![seed.clone()], "{label}");
7811 assert_eq!(client.local_revision(), 1, "{label}");
7812 assert_eq!(
7813 client.query("SELECT * FROM tasks", &[]).unwrap(),
7814 rows,
7815 "{label}"
7816 );
7817 }
7818
7819 let mut missing = values("t3");
7822 missing.remove("payload");
7823 let missing_error = client.mutate(upsert(missing)).expect_err("missing column");
7824 assert_eq!(missing_error.code, "sync.invalid_request");
7825 assert!(missing_error.details.as_ref().unwrap()["legacyCause"]
7826 .as_str()
7827 .unwrap()
7828 .contains("is not nullable"));
7829
7830 let mut bad_row_id = values("t4");
7833 bad_row_id.insert("id".to_owned(), json!({"nested": true}));
7834 let row_id_error = client.mutate(upsert(bad_row_id)).expect_err("bad rowId");
7835 assert_eq!(row_id_error.code, "sync.invalid_request");
7836
7837 for (label, partial, cause) in [
7839 (
7840 "patch wrong value type",
7841 ("priority", json!("high")),
7842 "expected an integer",
7843 ),
7844 ("patch unknown column", ("nope", json!(1)), "unknown column"),
7845 ] {
7846 let error = client
7847 .patch(
7848 "tasks",
7849 "t1",
7850 Map::from_iter([(partial.0.to_owned(), partial.1)]),
7851 None,
7852 )
7853 .expect_err(label);
7854 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
7855 assert_eq!(error.message, "the authoring request is invalid", "{label}");
7856 assert!(
7857 error.details.as_ref().unwrap()["legacyCause"]
7858 .as_str()
7859 .unwrap()
7860 .contains(cause),
7861 "{label}"
7862 );
7863 }
7864 assert_eq!(client.pending_commit_ids(), vec![seed.clone()]);
7865 assert_eq!(client.local_revision(), 1);
7866 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
7867
7868 for (label, payload) in [
7871 ("explicit null", json!(null)),
7872 ("empty envelope", json!({"$bytes": ""})),
7873 ("uppercase envelope", json!({"$bytes": "0AFF"})),
7874 ] {
7875 let mut accepted = values("t-ok");
7876 accepted.insert("extra".to_owned(), payload);
7877 let encoded = client.mutate(upsert(accepted)).unwrap_or_else(|error| {
7878 panic!("{label} must be accepted: {error:?}");
7879 });
7880 assert!(!encoded.is_empty(), "{label}");
7881 }
7882
7883 let next = client
7885 .mutate(upsert(values("t5")))
7886 .expect("valid after failures");
7887 assert_ne!(next, seed);
7888
7889 drop(client);
7890 let _ = std::fs::remove_file(&path);
7891 }
7892
7893 #[test]
7894 fn a_legacy_outbox_value_the_codec_refuses_is_dropped_instead_of_wedging() {
7895 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
7896 {"name":"id","type":"string","nullable":false},
7897 {"name":"project_id","type":"string","nullable":false},
7898 {"name":"payload","type":"bytes","nullable":true}],
7899 "scopes":[{"pattern":"project:{project_id}"}]}]});
7900 let open = |path: &std::path::Path| {
7901 SyncClient::open_path(
7902 "legacy-values".into(),
7903 &schema,
7904 ClientLimits::default(),
7905 path.to_str().unwrap(),
7906 )
7907 .expect("open client")
7908 };
7909 let row = |id: &str| {
7910 Map::from_iter([
7911 ("id".to_owned(), json!(id)),
7912 ("project_id".to_owned(), json!("p1")),
7913 ("payload".to_owned(), json!({"$bytes": "0a"})),
7914 ])
7915 };
7916 let upsert = |values| {
7917 vec![Mutation::Upsert {
7918 table: "tasks".into(),
7919 values,
7920 base_version: None,
7921 }]
7922 };
7923 for (label, stored_cell, representable) in [
7924 ("plus-sign envelope", json!({"$bytes": "+a"}), false),
7925 ("non-hex envelope", json!({"$bytes": "zz"}), false),
7926 ("odd-length envelope", json!({"$bytes": "abc"}), false),
7927 ("no envelope", json!({"nested": true}), false),
7928 ("wrong declared type", json!("high"), true),
7932 ("stored null for a non-nullable column", json!(null), true),
7933 ] {
7934 let path = std::env::temp_dir().join(format!(
7935 "syncular-legacy-values-{}-{}.db",
7936 label.replace(' ', "-"),
7937 uuid::Uuid::new_v4()
7938 ));
7939 let mut client = open(&path);
7940 client.create_synced_tables().expect("create synced tables");
7941 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
7942 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
7943 drop(client);
7944
7945 let connection = Connection::open(&path).expect("raw connection");
7947 let (column, cell) = if stored_cell.is_null() {
7948 ("project_id", stored_cell.clone())
7949 } else {
7950 ("payload", stored_cell.clone())
7951 };
7952 let mut planted_values = json!({
7953 "id": "t1", "project_id": "p1", "payload": {"$bytes": "0a"}
7954 });
7955 planted_values[column] = cell;
7956 let planted = json!([{
7957 "op": "upsert",
7958 "table": "tasks",
7959 "rowId": "t1",
7960 "baseVersion": null,
7961 "values": planted_values
7962 }]);
7963 connection
7964 .execute(
7965 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
7966 rusqlite::params![planted.to_string(), stale],
7967 )
7968 .expect("plant legacy intent");
7969 drop(connection);
7970
7971 let reopened = open(&path);
7974 assert_eq!(
7975 reopened.pending_commit_ids(),
7976 vec![successor.clone()],
7977 "{label}"
7978 );
7979 let rejection = reopened
7980 .rejections()
7981 .iter()
7982 .find(|rejection| rejection.client_commit_id == stale)
7983 .unwrap_or_else(|| panic!("{label}: rejection"));
7984 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
7985 assert_eq!(
7986 rejection.message, "the persisted commit carries values the current codec refuses",
7987 "{label}"
7988 );
7989 assert_eq!(rejection.operation.is_some(), representable, "{label}");
7990 assert_eq!(
7991 rejection
7992 .details
7993 .as_ref()
7994 .and_then(|details| details.reason.as_deref()),
7995 Some(INVALID_STORED_VALUES_REASON),
7996 "{label}"
7997 );
7998 let journal = reopened
8000 .commit_outcome(&stale)
8001 .expect("read outcome")
8002 .expect("journal entry");
8003 assert!(matches!(journal.status, CommitOutcomeStatus::Rejected));
8004 drop(reopened);
8005 let mut reopened = open(&path);
8006 let reread = reopened
8007 .commit_outcome(&stale)
8008 .expect("read outcome")
8009 .expect("journal entry");
8010 assert!(matches!(reread.status, CommitOutcomeStatus::Rejected));
8011
8012 assert_eq!(
8015 reopened
8016 .query("SELECT id FROM tasks ORDER BY id", &[])
8017 .unwrap(),
8018 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8019 "{label}"
8020 );
8021 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8022 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8023 assert!(!reopened
8024 .drop_unencodable_outbox(false, None)
8025 .expect("idempotent prepass"));
8026 let mut transport = CountingRealtimeTransport::default();
8027 let outcome = reopened.sync(&mut transport);
8028 assert!(matches!(outcome, SyncOutcome::Failed { .. }), "{outcome:?}");
8029 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8030 let next = reopened.mutate(upsert(row("t3"))).expect("after the drop");
8031 assert_eq!(reopened.pending_commit_ids(), vec![successor, next]);
8032 assert_eq!(reopened.pending_payloads().len(), 2, "{label}");
8033 drop(reopened);
8034 let _ = std::fs::remove_file(&path);
8035 }
8036 }
8037
8038 #[test]
8039 fn two_legacy_values_on_the_same_row_recover_in_fifo_order() {
8040 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8041 {"name":"id","type":"string","nullable":false},
8042 {"name":"project_id","type":"string","nullable":false},
8043 {"name":"payload","type":"bytes","nullable":true}],
8044 "scopes":[{"pattern":"project:{project_id}"}]}]});
8045 for (label, column, planted) in [
8046 ("malformed envelope", "payload", json!({"$bytes": "zz"})),
8047 (
8048 "stored null for a non-nullable column",
8049 "project_id",
8050 json!(null),
8051 ),
8052 ("wrong declared type", "payload", json!(7)),
8053 ] {
8054 let path = std::env::temp_dir().join(format!(
8055 "syncular-shared-row-{}-{}.db",
8056 label.replace(' ', "-"),
8057 uuid::Uuid::new_v4()
8058 ));
8059 let open = |path: &std::path::Path| {
8060 SyncClient::open_path(
8061 "shared-row".into(),
8062 &schema,
8063 ClientLimits::default(),
8064 path.to_str().unwrap(),
8065 )
8066 .expect("open client")
8067 };
8068 let row = |id: &str| {
8069 Map::from_iter([
8070 ("id".to_owned(), json!(id)),
8071 ("project_id".to_owned(), json!("p1")),
8072 ("payload".to_owned(), json!({"$bytes": "0a"})),
8073 ])
8074 };
8075 let upsert = |values| {
8076 vec![Mutation::Upsert {
8077 table: "tasks".into(),
8078 values,
8079 base_version: None,
8080 }]
8081 };
8082 let mut client = open(&path);
8083 client.create_synced_tables().expect("create synced tables");
8084 let older = client.mutate(upsert(row("t1"))).expect("older commit");
8085 let newer = client.mutate(upsert(row("t1"))).expect("newer commit");
8086 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8087 drop(client);
8088
8089 let connection = Connection::open(&path).expect("raw connection");
8090 for commit_id in [&older, &newer] {
8091 let mut values = row("t1");
8092 values.insert(column.to_owned(), planted.clone());
8093 let planted_ops = json!([{
8094 "op": "upsert",
8095 "table": "tasks",
8096 "rowId": "t1",
8097 "baseVersion": null,
8098 "values": values
8099 }]);
8100 connection
8101 .execute(
8102 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8103 rusqlite::params![planted_ops.to_string(), commit_id],
8104 )
8105 .expect("plant legacy intent");
8106 }
8107 drop(connection);
8108
8109 let mut reopened = open(&path);
8111 assert_eq!(
8112 reopened.pending_commit_ids(),
8113 vec![successor.clone()],
8114 "{label}"
8115 );
8116 assert_eq!(
8117 reopened
8118 .rejections()
8119 .iter()
8120 .map(|rejection| rejection.client_commit_id.clone())
8121 .collect::<Vec<_>>(),
8122 vec![older.clone(), newer.clone()],
8123 "{label}"
8124 );
8125 assert!(
8126 reopened
8127 .rejections()
8128 .iter()
8129 .all(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE
8130 && rejection.details.as_ref().and_then(|d| d.reason.as_deref())
8131 == Some(INVALID_STORED_VALUES_REASON)),
8132 "{label}"
8133 );
8134 for commit_id in [&older, &newer] {
8135 assert!(
8136 matches!(
8137 reopened
8138 .commit_outcome(commit_id)
8139 .expect("outcome")
8140 .map(|o| o.status),
8141 Some(CommitOutcomeStatus::Rejected)
8142 ),
8143 "{label}"
8144 );
8145 }
8146 assert_eq!(
8147 reopened
8148 .query("SELECT id FROM tasks ORDER BY id", &[])
8149 .unwrap(),
8150 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8151 "{label}"
8152 );
8153 let revision = reopened.local_revision();
8154 let batches = reopened.drain_change_batches();
8155 assert!(
8156 batches.iter().any(|batch| batch.rejections_changed
8157 && batch.outcomes_changed
8158 && batch.tables.iter().any(|change| change.table == "tasks")
8159 && batch.revision == revision.to_string()),
8160 "{label}: {batches:?}"
8161 );
8162 drop(reopened);
8163
8164 let mut stable = open(&path);
8167 assert_eq!(stable.local_revision(), revision, "{label}");
8168 assert!(stable.drain_change_batches().is_empty(), "{label}");
8169 assert_eq!(stable.pending_commit_ids(), vec![successor], "{label}");
8170 drop(stable);
8171 let _ = std::fs::remove_file(&path);
8172 }
8173 }
8174
8175 #[test]
8176 fn recovery_omits_a_legacy_operation_the_journal_reader_rejects() {
8177 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8178 {"name":"id","type":"string","nullable":false},
8179 {"name":"project_id","type":"string","nullable":false},
8180 {"name":"payload","type":"bytes","nullable":true}],
8181 "scopes":[{"pattern":"project:{project_id}"}]}]});
8182 for (label, second_op) in [
8183 (
8184 "delete carrying values",
8185 json!({"op": "delete", "table": "tasks", "rowId": "t9", "values": {}}),
8186 ),
8187 (
8188 "baseVersion beyond the safe bound",
8189 json!({"op": "upsert", "table": "tasks", "rowId": "t9",
8190 "baseVersion": MAX_JS_SAFE_INTEGER + 1,
8191 "values": {"id": "t9", "project_id": "p1", "payload": {"$bytes": "0a"}}}),
8192 ),
8193 ] {
8194 let path = std::env::temp_dir().join(format!(
8195 "syncular-journal-shape-{}-{}.db",
8196 label.replace(' ', "-"),
8197 uuid::Uuid::new_v4()
8198 ));
8199 let open = |path: &std::path::Path| {
8200 SyncClient::open_path(
8201 "journal-shape".into(),
8202 &schema,
8203 ClientLimits::default(),
8204 path.to_str().unwrap(),
8205 )
8206 .expect("open client")
8207 };
8208 let values = |id: &str| {
8209 Map::from_iter([
8210 ("id".to_owned(), json!(id)),
8211 ("project_id".to_owned(), json!("p1")),
8212 ("payload".to_owned(), json!({"$bytes": "0a"})),
8213 ])
8214 };
8215 let upsert = |values| {
8216 vec![Mutation::Upsert {
8217 table: "tasks".into(),
8218 values,
8219 base_version: None,
8220 }]
8221 };
8222 let mut client = open(&path);
8223 client.create_synced_tables().expect("create synced tables");
8224 let stale = client.mutate(upsert(values("t1"))).expect("stale commit");
8225 drop(client);
8226
8227 let connection = Connection::open(&path).expect("raw connection");
8228 let planted = json!([
8232 {
8233 "op": "upsert",
8234 "table": "tasks",
8235 "rowId": "t1",
8236 "baseVersion": null,
8237 "values": {"id": "t1", "project_id": null,
8238 "payload": {"$bytes": "0a"}}
8239 },
8240 second_op
8241 ]);
8242 connection
8243 .execute(
8244 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8245 rusqlite::params![planted.to_string(), stale],
8246 )
8247 .expect("plant legacy intent");
8248 drop(connection);
8249
8250 let reopened = open(&path);
8251 let journal = reopened
8252 .commit_outcome(&stale)
8253 .unwrap_or_else(|error| panic!("{label}: journal read: {error}"))
8254 .unwrap_or_else(|| panic!("{label}: journal entry"));
8255 assert!(
8256 matches!(journal.status, CommitOutcomeStatus::Rejected),
8257 "{label}"
8258 );
8259 assert!(journal.operations.is_none(), "{label}");
8260 assert!(
8261 reopened
8262 .commit_outcomes(CommitOutcomeQuery::default())
8263 .is_ok(),
8264 "{label}: outcome list query"
8265 );
8266 assert!(reopened.pending_commit_ids().is_empty(), "{label}");
8267 drop(reopened);
8268 let _ = std::fs::remove_file(&path);
8269 }
8270 }
8271
8272 #[test]
8273 fn legacy_values_recover_at_the_recreate_and_bump_reset_boundaries() {
8274 let schema = |version: i32, extra: bool| {
8275 let mut columns = vec![
8276 json!({"name":"id","type":"string","nullable":false}),
8277 json!({"name":"project_id","type":"string","nullable":false}),
8278 json!({"name":"payload","type":"bytes","nullable":true}),
8279 ];
8280 if extra {
8281 columns.push(json!({"name":"later","type":"string","nullable":true}));
8282 }
8283 json!({"version":version,"tables":[{"name":"tasks","primaryKey":"id","columns":columns,
8284 "scopes":[{"pattern":"project:{project_id}"}]}]})
8285 };
8286 for (label, bump) in [
8287 ("same-version recreation", false),
8288 ("version-bump reset", true),
8289 ("version-only bump reset", true),
8290 ] {
8291 let path = std::env::temp_dir().join(format!(
8292 "syncular-legacy-reset-{}-{}.db",
8293 label.replace(' ', "-"),
8294 uuid::Uuid::new_v4()
8295 ));
8296 let v1 = schema(1, false);
8297 let open = |schema: &Value, path: &std::path::Path| {
8298 SyncClient::open_path(
8299 "legacy-reset".into(),
8300 schema,
8301 ClientLimits::default(),
8302 path.to_str().unwrap(),
8303 )
8304 .expect("open client")
8305 };
8306 let row = |id: &str| {
8307 Map::from_iter([
8308 ("id".to_owned(), json!(id)),
8309 ("project_id".to_owned(), json!("p1")),
8310 ("payload".to_owned(), json!({"$bytes": "0a"})),
8311 ])
8312 };
8313 let upsert = |values| {
8314 vec![Mutation::Upsert {
8315 table: "tasks".into(),
8316 values,
8317 base_version: None,
8318 }]
8319 };
8320 let mut client = open(&v1, &path);
8321 client.create_synced_tables().expect("create synced tables");
8322 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8323 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8324 drop(client);
8325
8326 let connection = Connection::open(&path).expect("raw connection");
8327 let planted = json!([{
8328 "op": "upsert",
8329 "table": "tasks",
8330 "rowId": "t1",
8331 "baseVersion": null,
8332 "values": {"id": "t1", "project_id": 7, "payload": {"$bytes": "0a"}}
8333 }]);
8334 connection
8335 .execute(
8336 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8337 rusqlite::params![planted.to_string(), stale],
8338 )
8339 .expect("plant legacy intent");
8340 drop(connection);
8341
8342 let target = if bump {
8343 schema(2, label == "version-bump reset")
8344 } else {
8345 v1.clone()
8346 };
8347 let reopened = open(&target, &path);
8348 let rejection = reopened
8349 .rejections()
8350 .iter()
8351 .find(|rejection| rejection.client_commit_id == stale)
8352 .unwrap_or_else(|| panic!("{label}: rejection"));
8353 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8354 assert_eq!(
8355 rejection
8356 .details
8357 .as_ref()
8358 .and_then(|details| details.reason.as_deref()),
8359 Some(INVALID_STORED_VALUES_REASON),
8360 "{label}"
8361 );
8362 assert!(rejection.operation.is_some(), "{label}");
8363 assert_eq!(
8364 reopened.pending_commit_ids(),
8365 vec![successor.clone()],
8366 "{label}"
8367 );
8368 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8373 if label == "version-bump reset" {
8374 assert_eq!(
8375 reopened
8376 .query("SELECT id FROM tasks ORDER BY id", &[])
8377 .unwrap(),
8378 Vec::<Map<String, Value>>::new(),
8379 "{label}"
8380 );
8381 let mut next = reopened;
8385 let revision = next.local_revision();
8386 let batches = next.drain_change_batches();
8387 assert!(
8388 batches.iter().any(|batch| batch.rejections_changed
8389 && batch.outcomes_changed
8390 && batch.tables.iter().any(|change| change.table == "tasks")
8391 && batch.revision == revision.to_string()),
8392 "{label}: {batches:?}"
8393 );
8394 next.mutate(upsert(Map::from_iter([
8395 ("id".to_owned(), json!("t3")),
8396 ("project_id".to_owned(), json!("p1")),
8397 ("payload".to_owned(), json!({"$bytes": "0b"})),
8398 ("later".to_owned(), json!("v2")),
8399 ])))
8400 .expect("v2 authoring after the reset");
8401 assert_eq!(next.pending_commit_ids().len(), 2, "{label}");
8402 assert_eq!(next.pending_payloads().len(), 2, "{label}");
8403 drop(next);
8404 } else {
8405 assert_eq!(
8406 reopened
8407 .query("SELECT id FROM tasks ORDER BY id", &[])
8408 .unwrap(),
8409 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8410 "{label}"
8411 );
8412 drop(reopened);
8413 }
8414
8415 if !bump {
8418 let mut recreated = open(&v1, &path);
8419 let fresh = recreated
8420 .mutate(upsert(row("t3")))
8421 .expect("fresh commit while open");
8422 recreated.drain_change_batches();
8423 let planted_fresh = json!([{
8424 "op": "upsert",
8425 "table": "tasks",
8426 "rowId": "t3",
8427 "baseVersion": null,
8428 "values": {"id": "t3", "project_id": "p1",
8429 "payload": {"$bytes": "zz"}}
8430 }]);
8431 let connection = Connection::open(&path).expect("raw connection");
8432 let affected = connection
8433 .execute(
8434 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8435 rusqlite::params![planted_fresh.to_string(), fresh],
8436 )
8437 .expect("plant fresh legacy intent");
8438 assert_eq!(
8439 affected, 1,
8440 "{label}: the in-place plant must hit the queued commit"
8441 );
8442 drop(connection);
8443 let revision_before = recreated.local_revision();
8444 recreated
8445 .recreate_with_schema(&v1)
8446 .unwrap_or_else(|error| panic!("{label}: recreation failed: {error}"));
8447 assert_eq!(recreated.pending_commit_ids(), vec![successor], "{label}");
8448 let fresh_rejection = recreated
8449 .rejections()
8450 .iter()
8451 .find(|rejection| rejection.client_commit_id == fresh)
8452 .expect("the in-place recovery rejection");
8453 assert_eq!(fresh_rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8454 assert_eq!(
8455 fresh_rejection
8456 .details
8457 .as_ref()
8458 .and_then(|details| details.reason.as_deref()),
8459 Some(INVALID_STORED_VALUES_REASON),
8460 "{label}"
8461 );
8462 assert!(
8463 matches!(
8464 recreated
8465 .commit_outcome(&fresh)
8466 .expect("outcome read")
8467 .map(|outcome| outcome.status),
8468 Some(CommitOutcomeStatus::Rejected)
8469 ),
8470 "{label}"
8471 );
8472 assert!(recreated.local_revision() > revision_before, "{label}");
8473 let revision = recreated.local_revision();
8474 let batches = recreated.drain_change_batches();
8475 assert_eq!(batches.len(), 1, "{label}: one recovery batch");
8476 assert!(
8477 batches.iter().any(|batch| batch.rejections_changed
8478 && batch.outcomes_changed
8479 && batch.tables.iter().any(|change| change.table == "tasks")
8480 && batch.revision == revision.to_string()),
8481 "{label}: {batches:?}"
8482 );
8483 assert_eq!(recreated.pending_payloads().len(), 1, "{label}");
8484 assert_eq!(
8485 recreated
8486 .query("SELECT id FROM tasks ORDER BY id", &[])
8487 .unwrap(),
8488 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8489 "{label}: the recovered commit leaves no projection"
8490 );
8491 drop(recreated);
8492 }
8493 let _ = std::fs::remove_file(&path);
8494 }
8495 }
8496
8497 #[test]
8498 fn legacy_values_recover_over_a_synced_authoritative_row() {
8499 let path = std::env::temp_dir().join(format!(
8500 "syncular-authoritative-row-{}.db",
8501 uuid::Uuid::new_v4()
8502 ));
8503 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8504 {"name":"id","type":"string","nullable":false},
8505 {"name":"project_id","type":"string","nullable":false},
8506 {"name":"title","type":"string","nullable":false},
8507 {"name":"payload","type":"bytes","nullable":true}],
8508 "scopes":[{"pattern":"project:{project_id}"}]}]});
8509 let open = |path: &std::path::Path| {
8510 SyncClient::open_path(
8511 "authoritative-row".into(),
8512 &schema,
8513 ClientLimits::default(),
8514 path.to_str().unwrap(),
8515 )
8516 .expect("open client")
8517 };
8518 let row = |title: &str| {
8519 Map::from_iter([
8520 ("id".to_owned(), json!("t1")),
8521 ("project_id".to_owned(), json!("p1")),
8522 ("title".to_owned(), json!(title)),
8523 ("payload".to_owned(), json!({"$bytes": "0a"})),
8524 ])
8525 };
8526 let upsert = |values| {
8527 vec![Mutation::Upsert {
8528 table: "tasks".into(),
8529 values,
8530 base_version: None,
8531 }]
8532 };
8533 let mut client = open(&path);
8534 client.create_synced_tables().expect("create synced tables");
8535 client
8538 .conn
8539 .execute_batch(
8540 "INSERT INTO _syncular_base_tasks VALUES ('t1','p1','authoritative',x'0a',1);
8541 INSERT INTO tasks VALUES ('t1','p1','authoritative',x'0a',1);",
8542 )
8543 .expect("seed authoritative row");
8544 let older = client.mutate(upsert(row("older"))).expect("older");
8545 let newer = client.mutate(upsert(row("newer"))).expect("newer");
8546 let successor = client.mutate(upsert(row("successor"))).expect("successor");
8547 drop(client);
8548
8549 let connection = Connection::open(&path).expect("raw connection");
8550 for commit_id in [&older, &newer] {
8551 let planted = json!([{
8552 "op": "upsert",
8553 "table": "tasks",
8554 "rowId": "t1",
8555 "baseVersion": null,
8556 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
8557 "payload": {"$bytes": "zz"}}
8558 }]);
8559 connection
8560 .execute(
8561 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8562 rusqlite::params![planted.to_string(), commit_id],
8563 )
8564 .expect("plant legacy intent");
8565 }
8566 drop(connection);
8567
8568 let reopened = open(&path);
8569 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8570 assert_eq!(
8571 reopened
8572 .rejections()
8573 .iter()
8574 .map(|rejection| rejection.client_commit_id.clone())
8575 .collect::<Vec<_>>(),
8576 vec![older.clone(), newer.clone()]
8577 );
8578 assert_eq!(
8581 reopened.query("SELECT id,title,_syncular_version FROM tasks", &[]),
8582 Ok(vec![Map::from_iter([
8583 ("id".to_owned(), json!("t1")),
8584 ("title".to_owned(), json!("successor")),
8585 ("_syncular_version".to_owned(), json!(1)),
8586 ])])
8587 );
8588
8589 drop(reopened);
8592 let connection = Connection::open(&path).expect("raw connection");
8593 connection
8594 .execute(
8595 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8596 rusqlite::params![
8597 json!([{
8598 "op": "upsert",
8599 "table": "tasks",
8600 "rowId": "t1",
8601 "baseVersion": null,
8602 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
8603 "payload": {"$bytes": "zz"}}
8604 }])
8605 .to_string(),
8606 successor
8607 ],
8608 )
8609 .expect("plant successor intent");
8610 drop(connection);
8611 let late = open(&path);
8612 assert!(late.pending_commit_ids().is_empty());
8613 assert_eq!(
8614 late.query("SELECT id,title,_syncular_version FROM tasks", &[]),
8615 Ok(vec![Map::from_iter([
8616 ("id".to_owned(), json!("t1")),
8617 ("title".to_owned(), json!("authoritative")),
8618 ("_syncular_version".to_owned(), json!(1)),
8619 ])])
8620 );
8621 drop(late);
8622 let _ = std::fs::remove_file(&path);
8623 }
8624
8625 #[test]
8626 fn a_failed_value_recovery_commit_restores_every_side_effect() {
8627 let path = std::env::temp_dir().join(format!(
8628 "syncular-value-recovery-commit-{}.db",
8629 uuid::Uuid::new_v4()
8630 ));
8631 let path_str = path.to_str().expect("path");
8632 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8633 {"name":"id","type":"string","nullable":false},
8634 {"name":"project_id","type":"string","nullable":false},
8635 {"name":"payload","type":"bytes","nullable":true}],
8636 "scopes":[{"pattern":"project:{project_id}"}]}]});
8637 let conn = Connection::open(path_str).expect("replica connection");
8640 let mut client = SyncClient::with_connection(
8641 "value-recovery-commit".into(),
8642 &schema,
8643 ClientLimits::default(),
8644 conn,
8645 )
8646 .expect("replica");
8647 assert_eq!(
8648 client
8649 .conn
8650 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
8651 .expect("journal mode"),
8652 "delete"
8653 );
8654 client.create_synced_tables().expect("create synced tables");
8655 let stale = client
8656 .mutate(vec![Mutation::Upsert {
8657 table: "tasks".into(),
8658 values: Map::from_iter([
8659 ("id".to_owned(), json!("t1")),
8660 ("project_id".to_owned(), json!("p1")),
8661 ("payload".to_owned(), json!({"$bytes": "0a"})),
8662 ]),
8663 base_version: None,
8664 }])
8665 .expect("queued write");
8666 client.drain_change_batches();
8667 let affected = client
8668 .conn
8669 .execute(
8670 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8671 rusqlite::params![
8672 json!([{
8673 "op": "upsert",
8674 "table": "tasks",
8675 "rowId": "t1",
8676 "baseVersion": null,
8677 "values": {"id": "t1", "project_id": "p1",
8678 "payload": {"$bytes": "zz"}}
8679 }])
8680 .to_string(),
8681 stale
8682 ],
8683 )
8684 .expect("plant legacy intent");
8685 assert_eq!(affected, 1, "the planted row must exist");
8686 let revision = client.local_revision();
8687 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
8688 let prior_outbox = client.pending_commit_ids();
8689 let prior_rejections = client.rejections().len();
8690 let prior_changes = client.change_queue.len();
8691 let prior_intents = client.sync_intent_queue.len();
8692 let prior_failed = client.failed_commits.len();
8693 let reader = Connection::open(path_str).expect("reader");
8696 reader
8697 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
8698 .expect("reader snapshot");
8699 client
8700 .conn
8701 .busy_timeout(std::time::Duration::ZERO)
8702 .expect("busy timeout");
8703 let error = client
8704 .recreate_with_schema(&schema)
8705 .expect_err("a blocked COMMIT must refuse the recovery");
8706 assert!(error.starts_with("client.storage_busy:"), "{error}");
8707 assert!(client.conn.is_autocommit(), "the transaction is released");
8708 assert_eq!(client.local_revision(), revision);
8709 assert_eq!(client.pending_commit_ids(), prior_outbox);
8710 assert_eq!(client.rejections().len(), prior_rejections);
8711 assert_eq!(client.change_queue.len(), prior_changes);
8712 assert_eq!(client.sync_intent_queue.len(), prior_intents);
8713 assert_eq!(client.failed_commits.len(), prior_failed);
8714 assert_eq!(
8715 client.query("SELECT * FROM tasks", &[]).expect("rows"),
8716 rows
8717 );
8718 assert!(
8719 client
8720 .commit_outcome(&stale)
8721 .expect("outcome read")
8722 .is_none(),
8723 "the rolled-back recovery kept no outcome"
8724 );
8725 reader.execute_batch("ROLLBACK").expect("release reader");
8727 client
8728 .recreate_with_schema(&schema)
8729 .expect("retry after the blocked commit");
8730 assert!(client.pending_commit_ids().is_empty());
8731 assert_eq!(client.rejections().len(), prior_rejections + 1);
8732 assert!(client.local_revision() > revision);
8733 let batches = client.drain_change_batches();
8734 assert!(
8735 batches.iter().any(|batch| batch.rejections_changed
8736 && batch.outcomes_changed
8737 && batch.tables.iter().any(|change| change.table == "tasks")),
8738 "{batches:?}"
8739 );
8740 drop(client);
8741 let _ = std::fs::remove_file(&path);
8742 }
8743
8744 #[cfg(feature = "e2ee")]
8745 #[test]
8746 fn a_failed_recovery_restores_retained_failed_commits() {
8747 let path = std::env::temp_dir().join(format!(
8748 "syncular-value-recovery-retention-{}.db",
8749 uuid::Uuid::new_v4()
8750 ));
8751 let path_str = path.to_str().expect("path");
8752 let schema = json!({"version":1,"tables":[{"name":"secrets","primaryKey":"id","columns":[
8753 {"name":"id","type":"string","nullable":false},
8754 {"name":"project_id","type":"string","nullable":false},
8755 {"name":"note","type":"bytes","nullable":true,"encrypted":true,"declaredType":"string"}],
8756 "scopes":[{"pattern":"project:{project_id}"}]}]});
8757 let conn = Connection::open(path_str).expect("replica connection");
8758 let mut client = SyncClient::with_connection(
8759 "value-recovery-retention".into(),
8760 &schema,
8761 ClientLimits::default(),
8762 conn,
8763 )
8764 .expect("replica");
8765 client.create_synced_tables().expect("create synced tables");
8766 client.set_retain_failed_commits(true).expect("retention");
8767 client.set_encryption(crate::values::EncryptionConfig::default());
8769 client
8770 .mutate(vec![Mutation::Upsert {
8771 table: "secrets".into(),
8772 values: Map::from_iter([
8773 ("id".to_owned(), json!("s1")),
8774 ("project_id".to_owned(), json!("p1")),
8775 ("note".to_owned(), json!("secret")),
8776 ]),
8777 base_version: None,
8778 }])
8779 .expect("queued write");
8780 client.drain_change_batches();
8781 let revision = client.local_revision();
8782 let prior_outbox = client.pending_commit_ids();
8783 let prior_rejections = client.rejections().len();
8784 let prior_failed = client.failed_commits.len();
8785 let reader = Connection::open(path_str).expect("reader");
8786 reader
8787 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
8788 .expect("reader snapshot");
8789 client
8790 .conn
8791 .busy_timeout(std::time::Duration::ZERO)
8792 .expect("busy timeout");
8793 let error = client
8794 .drop_unencodable_outbox(true, None)
8795 .expect_err("a blocked COMMIT must refuse the recovery");
8796 assert!(error.starts_with("client.storage_busy:"), "{error}");
8797 assert_eq!(client.failed_commits.len(), prior_failed);
8798 assert_eq!(client.pending_commit_ids(), prior_outbox);
8799 assert_eq!(client.rejections().len(), prior_rejections);
8800 assert_eq!(client.local_revision(), revision);
8801 reader.execute_batch("ROLLBACK").expect("release reader");
8803 client
8804 .drop_unencodable_outbox(true, None)
8805 .expect("retry after the blocked commit");
8806 assert_eq!(client.failed_commits.len(), prior_failed + 1);
8807 assert!(client.pending_commit_ids().is_empty());
8808 assert_eq!(client.rejections().len(), prior_rejections + 1);
8809 drop(client);
8810 let _ = std::fs::remove_file(&path);
8811 }
8812
8813 #[test]
8814 fn client_error_maps_legacy_codes_and_static_messages() {
8815 let code_only = ClientError::from("sync.unknown_table".to_owned());
8816 assert_eq!(code_only.code, "sync.unknown_table");
8817 assert_eq!(code_only.message, "the commit targets an unknown table");
8818 assert!(code_only.details.is_none());
8819
8820 let dynamic = ClientError::from(
8821 "sync.invalid_request: table \"tasks\": patch cannot write scope column".to_owned(),
8822 );
8823 assert_eq!(dynamic.code, "sync.invalid_request");
8824 assert_eq!(dynamic.message, "the authoring request is invalid");
8825 assert!(dynamic.details.as_ref().unwrap()["legacyCause"]
8826 .as_str()
8827 .unwrap()
8828 .contains("patch cannot write scope column"));
8829
8830 let unclassified =
8831 ClientError::from("a commit must contain at least one operation".to_owned());
8832 assert_eq!(unclassified.code, "client.failed");
8833 assert_eq!(unclassified.message, "the local authoring operation failed");
8834 assert!(unclassified.details.as_ref().unwrap()["legacyCause"]
8835 .as_str()
8836 .unwrap()
8837 .contains("at least one operation"));
8838
8839 let metadata = ClientError::from(QueryReadFailure {
8841 code: None,
8842 sqlite_code: Some(1),
8843 sqlite_message: Some("generic SQLite prose".to_owned()),
8844 rollback_failure: Some(Box::new(QueryReadFailure::from(
8845 rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(1), None),
8846 ))),
8847 message: "unclassified read".to_owned(),
8848 });
8849 assert_eq!(metadata.code, "client.failed");
8850 assert_eq!(metadata.message, "the local authoring operation failed");
8851 let details = metadata.details.as_ref().unwrap();
8852 assert_eq!(details["sqliteCode"], 1);
8853 assert_eq!(details["sqliteMessage"], "generic SQLite prose");
8854 assert_eq!(details["rollbackFailure"]["sqliteCode"], 1);
8855 assert_eq!(details["legacyCause"], "unclassified read");
8856 }
8857
8858 #[test]
8859 fn schema_marker_read_keeps_storage_classification() {
8860 let busy = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
8861 rusqlite::ffi::Error::new(5),
8862 Some("database is locked".to_owned()),
8863 ));
8864 assert!(busy.starts_with("client.storage_busy:"), "{busy}");
8865 let invalid = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
8866 rusqlite::ffi::Error::new(1),
8867 Some("unreadable".to_owned()),
8868 ));
8869 assert_eq!(
8870 invalid,
8871 "sync.local_corrupt: persisted local schema marker is unreadable or invalid"
8872 );
8873
8874 let path =
8877 std::env::temp_dir().join(format!("syncular-marker-busy-{}.db", uuid::Uuid::new_v4()));
8878 let setup = Connection::open(&path).expect("setup connection");
8879 setup
8880 .execute_batch(
8881 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value TEXT);\n INSERT INTO _syncular_meta VALUES ('localSchemaVersion','1');",
8882 )
8883 .unwrap();
8884 let locker = Connection::open(&path).expect("locker connection");
8885 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
8886 let reader = Connection::open(&path).expect("reader connection");
8887 reader.busy_timeout(std::time::Duration::ZERO).unwrap();
8888 let locked = read_local_schema_version(&reader, 1).expect_err("locked marker read");
8889 assert!(locked.starts_with("client.storage_busy:"), "{locked}");
8890 locker.execute_batch("ROLLBACK").unwrap();
8891 assert_eq!(read_local_schema_version(&reader, 1).unwrap(), Some(1));
8892 let rows = setup
8893 .query_row("SELECT count(*) FROM _syncular_meta", [], |row| {
8894 row.get::<_, i64>(0)
8895 })
8896 .unwrap();
8897 assert_eq!(rows, 1, "the guard performed no writes");
8898 drop(reader);
8899 drop(locker);
8900 drop(setup);
8901 let _ = std::fs::remove_file(&path);
8902 }
8903
8904 #[cfg(feature = "crdt-yjs")]
8905 #[test]
8906 fn crdt_authoring_pre_read_preserves_storage_classification() {
8907 let path =
8908 std::env::temp_dir().join(format!("syncular-crdt-busy-{}.db", uuid::Uuid::new_v4()));
8909 let schema = json!({"version":1,"tables":[{"name":"notes","primaryKey":"id","columns":[
8910 {"name":"id","type":"string","nullable":false},
8911 {"name":"project_id","type":"string","nullable":false},
8912 {"name":"doc","type":"crdt","nullable":true,"crdtType":"yjs-doc"}],
8913 "scopes":[{"pattern":"project:{project_id}"}]}]});
8914 let mut client = SyncClient::open_path(
8915 "crdt-busy".into(),
8916 &schema,
8917 ClientLimits::default(),
8918 path.to_str().unwrap(),
8919 )
8920 .expect("open client");
8921 client.create_synced_tables().expect("create synced tables");
8922 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
8923
8924 let locker = Connection::open(&path).expect("locker connection");
8925 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
8926 let busy = client
8927 .crdt_apply_update("notes", "n1", "doc", &[])
8928 .expect_err("busy crdt pre-read");
8929 assert_eq!(busy.code, "client.storage_busy");
8930 assert!(busy.retryable);
8931 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
8932 assert!(client.pending_commit_ids().is_empty());
8933 assert_eq!(client.local_revision(), 0);
8934
8935 locker.execute_batch("ROLLBACK").unwrap();
8936 drop(locker);
8937 drop(client);
8938 let _ = std::fs::remove_file(&path);
8939 }
8940
8941 #[test]
8942 fn large_unique_import_copies_no_unrelated_tables() {
8943 let (mut client, image, table) = unique_import_fixture(10240);
8944 let pending = client.pending_commit_ids();
8945 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
8946 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
8947 assert!(matches!(
8948 client.apply_sqlite_image(
8949 &image,
8950 &table,
8951 true,
8952 &[("project_id".into(), vec!["p1".into()])],
8953 10240,
8954 7,
8955 "digest"
8956 ),
8957 Ok(10240)
8958 ));
8959 assert_eq!(client.overlay_rebuild_count.get(), 0);
8960 assert_eq!(client.pending_commit_ids(), pending);
8961 assert_eq!(
8962 client
8963 .query("SELECT count(*) AS n FROM tasks", &[])
8964 .unwrap()[0]["n"],
8965 10241
8966 );
8967 assert_eq!(
8968 client
8969 .query("SELECT count(*) AS n FROM tasks_fts", &[])
8970 .unwrap()[0]["n"],
8971 10241
8972 );
8973 assert_eq!(
8974 client
8975 .query("SELECT count(*) AS n FROM notes_fts", &[])
8976 .unwrap()[0]["n"],
8977 8192
8978 );
8979 }
8980
8981 #[test]
8982 #[ignore = "manual scaling benchmark; see bench/README.md"]
8983 fn benchmark_unique_image_import() {
8984 let mut samples = Vec::new();
8985 for rows in [10240, 51200, 102400] {
8986 for trial in 0..4 {
8987 let (mut client, image, table) = unique_import_fixture(rows);
8988 let start = std::time::Instant::now();
8989 assert!(
8990 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
8991 );
8992 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
8993 assert_eq!(
8994 client
8995 .query("SELECT count(*) AS n FROM tasks", &[])
8996 .unwrap()[0]["n"],
8997 rows + 1
8998 );
8999 assert_eq!(
9000 client
9001 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9002 .unwrap()[0]["n"],
9003 rows + 1
9004 );
9005 assert_eq!(
9006 client
9007 .query("SELECT count(*) AS n FROM notes_fts", &[])
9008 .unwrap()[0]["n"],
9009 8192
9010 );
9011 assert_eq!(client.pending_commit_ids().len(), 1);
9012 if trial > 0 {
9013 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
9014 }
9015 }
9016 }
9017 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
9018 let output = serde_json::to_string_pretty(&artifact).unwrap();
9019 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
9020 use std::io::Write;
9021 let mut file = std::fs::OpenOptions::new()
9022 .write(true)
9023 .create_new(true)
9024 .open(path)
9025 .unwrap();
9026 file.write_all(output.as_bytes()).unwrap();
9027 } else {
9028 println!("{output}");
9029 }
9030 }
9031
9032 #[test]
9033 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
9034 let schema = json!({
9035 "version": 1,
9036 "tables": [{
9037 "name": "catalogue_codes",
9038 "primaryKey": "id",
9039 "columns": [
9040 { "name": "id", "type": "string", "nullable": false },
9041 { "name": "release_id", "type": "string", "nullable": false },
9042 { "name": "code", "type": "string", "nullable": false },
9043 { "name": "title", "type": "string", "nullable": false }
9044 ],
9045 "scopes": [{ "pattern": "release:{release_id}" }],
9046 "ftsIndexes": [{
9047 "name": "catalogue_codes_fts",
9048 "columns": ["code", "title"],
9049 "tokenize": "unicode61 remove_diacritics 2"
9050 }]
9051 }]
9052 });
9053 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
9054 .expect("FTS5 client");
9055 let insert_trigger: String = client
9056 .conn
9057 .query_row(
9058 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
9059 [],
9060 |row| row.get(0),
9061 )
9062 .expect("insert trigger");
9063 let replace_guard: String = client
9064 .conn
9065 .query_row(
9066 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
9067 [],
9068 |row| row.get(0),
9069 )
9070 .expect("replace guard");
9071 assert!(!insert_trigger.contains("DELETE FROM"));
9072 assert!(replace_guard.contains("BEFORE INSERT"));
9073 assert!(replace_guard.contains("WHEN EXISTS"));
9074 let search = |client: &SyncClient, query: &str| {
9075 client
9076 .query(
9077 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
9078 &[Value::from(query)],
9079 )
9080 .expect("FTS query")
9081 };
9082
9083 client
9084 .mutate(vec![Mutation::Upsert {
9085 table: "catalogue_codes".to_owned(),
9086 values: Map::from_iter([
9087 ("id".to_owned(), Value::from("c1")),
9088 ("release_id".to_owned(), Value::from("r1")),
9089 ("code".to_owned(), Value::from("A01")),
9090 ("title".to_owned(), Value::from("Cholera")),
9091 ]),
9092 base_version: None,
9093 }])
9094 .expect("insert code");
9095 assert_eq!(search(&client, "cholera").len(), 1);
9096 let prior = client
9097 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9098 .unwrap();
9099 client
9100 .conn
9101 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
9102 .unwrap();
9103 client.create_synced_tables().unwrap();
9104 assert_eq!(
9105 client
9106 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9107 .unwrap(),
9108 prior
9109 );
9110 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
9111 let rendered = serde_json::to_string(&plan).unwrap();
9112 assert!(rendered.contains("COVERING INDEX"));
9113 assert!(rendered.contains("INDEX 0:="));
9114 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
9115 assert_eq!(search(&client, "cholera").len(), 1);
9116
9117 client
9118 .mutate(vec![Mutation::Upsert {
9119 table: "catalogue_codes".to_owned(),
9120 values: Map::from_iter([
9121 ("id".to_owned(), Value::from("c1")),
9122 ("release_id".to_owned(), Value::from("r1")),
9123 ("code".to_owned(), Value::from("A01")),
9124 ("title".to_owned(), Value::from("Enteric infection")),
9125 ]),
9126 base_version: None,
9127 }])
9128 .expect("update code");
9129 assert!(search(&client, "cholera").is_empty());
9130 assert_eq!(search(&client, "enteric").len(), 1);
9131
9132 client
9133 .mutate(vec![Mutation::Delete {
9134 table: "catalogue_codes".to_owned(),
9135 row_id: "c1".to_owned(),
9136 base_version: None,
9137 }])
9138 .expect("delete code");
9139 assert!(search(&client, "enteric").is_empty());
9140 }
9141
9142 #[test]
9143 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
9144 let schema = json!({
9145 "version": 1,
9146 "tables": [{
9147 "name": "patient_notes",
9148 "primaryKey": "id",
9149 "columns": [
9150 { "name": "id", "type": "string", "nullable": false },
9151 { "name": "practice_id", "type": "string", "nullable": false },
9152 { "name": "encryption_key_id", "type": "string", "nullable": false },
9153 { "name": "title", "type": "string", "nullable": false }
9154 ],
9155 "scopes": [{ "pattern": "practice:{practice_id}" }],
9156 "ftsIndexes": [{
9157 "name": "patient_notes_fts",
9158 "columns": ["title"],
9159 "tokenize": "unicode61 remove_diacritics 2"
9160 }]
9161 }]
9162 });
9163 let mut client = SyncClient::new(
9164 "local-purge-test".to_owned(),
9165 &schema,
9166 ClientLimits::default(),
9167 )
9168 .expect("local purge client");
9169 client
9170 .conn
9171 .execute(
9172 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
9173 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
9174 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
9175 [],
9176 )
9177 .expect("seed base rows");
9178 client.overlay_dirty.set(true);
9179 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
9180
9181 let note = |id: &str, key_id: &str, title: &str| {
9182 Map::from_iter([
9183 ("id".to_owned(), Value::from(id)),
9184 ("practice_id".to_owned(), Value::from("practice-1")),
9185 ("encryption_key_id".to_owned(), Value::from(key_id)),
9186 ("title".to_owned(), Value::from(title)),
9187 ])
9188 };
9189 let doomed = client
9190 .mutate(vec![
9191 Mutation::Upsert {
9192 table: "patient_notes".to_owned(),
9193 values: note("target", "key-revoked", "Target changed"),
9194 base_version: None,
9195 },
9196 Mutation::Upsert {
9197 table: "patient_notes".to_owned(),
9198 values: note("unrelated", "key-held", "Unrelated changed"),
9199 base_version: None,
9200 },
9201 ])
9202 .expect("doomed commit");
9203 let kept = client
9204 .mutate(vec![Mutation::Upsert {
9205 table: "patient_notes".to_owned(),
9206 values: note("kept", "key-held", "Kept optimistic"),
9207 base_version: None,
9208 }])
9209 .expect("kept commit");
9210 client.drain_change_batches();
9211
9212 let input = LocalDataPurgeInput {
9213 purge_id: "purge-001".to_owned(),
9214 targets: vec![LocalDataPurgeTarget {
9215 table: "patient_notes".to_owned(),
9216 selectors: BTreeMap::from([(
9217 "encryption_key_id".to_owned(),
9218 vec!["key-revoked".to_owned()],
9219 )]),
9220 }],
9221 };
9222 assert_eq!(
9223 client.purge_local_data(&input).expect("apply purge"),
9224 LocalDataPurgeResult {
9225 already_applied: false,
9226 purged_rows: 1,
9227 dropped_commits: 1,
9228 }
9229 );
9230 let rows = client
9231 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
9232 .expect("visible rows");
9233 assert_eq!(rows.len(), 2);
9234 assert_eq!(rows[0]["id"], "kept");
9235 assert_eq!(rows[1]["id"], "unrelated");
9236 assert_eq!(rows[1]["title"], "Unrelated original");
9237 let fts = client
9238 .query(
9239 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
9240 &[],
9241 )
9242 .expect("fts query");
9243 assert!(fts.is_empty());
9244 assert_eq!(client.outbox.len(), 1);
9245 assert_eq!(client.outbox[0].client_commit_id, kept);
9246 let outcome = client
9247 .commit_outcome(&doomed)
9248 .expect("read doomed outcome")
9249 .expect("doomed outcome");
9250 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
9251 match &outcome.results[0] {
9252 CommitOperationOutcome::Error { rejection } => {
9253 assert_eq!(rejection.code, "client.local_data_purged");
9254 assert_eq!(rejection.client_commit_id, doomed);
9255 }
9256 other => panic!("expected local purge rejection, got {other:?}"),
9257 }
9258 assert_eq!(client.drain_change_batches().len(), 1);
9259 assert_eq!(
9260 client.purge_local_data(&input).expect("retry purge"),
9261 LocalDataPurgeResult {
9262 already_applied: true,
9263 purged_rows: 0,
9264 dropped_commits: 0,
9265 }
9266 );
9267 let conflicting = LocalDataPurgeInput {
9268 purge_id: input.purge_id.clone(),
9269 targets: vec![LocalDataPurgeTarget {
9270 table: "patient_notes".to_owned(),
9271 selectors: BTreeMap::from([(
9272 "encryption_key_id".to_owned(),
9273 vec!["key-held".to_owned()],
9274 )]),
9275 }],
9276 };
9277 assert!(client
9278 .purge_local_data(&conflicting)
9279 .expect_err("id collision must fail")
9280 .contains("already used with a different plan"));
9281 }
9282}
9283
9284impl SubState {
9285 fn name(self) -> &'static str {
9286 match self {
9287 SubState::Active => "active",
9288 SubState::Revoked => "revoked",
9289 SubState::Failed => "failed",
9290 }
9291 }
9292
9293 fn parse(value: &str) -> Self {
9294 match value {
9295 "revoked" => Self::Revoked,
9296 "failed" => Self::Failed,
9297 _ => Self::Active,
9298 }
9299 }
9300}
9301
9302#[derive(Debug, Clone)]
9303struct Subscription {
9304 id: String,
9305 table: String,
9306 requested: Vec<(String, Vec<String>)>,
9307 params: Option<String>,
9308 cursor: i64,
9309 bootstrap_state: Option<String>,
9311 state: SubState,
9312 reason_code: Option<String>,
9313 effective: Option<Vec<(String, Vec<String>)>>,
9316 synced_once: bool,
9317}
9318
9319#[cfg(test)]
9320mod subscription_rearm_tests {
9321 use super::*;
9322 use serde_json::json;
9323
9324 fn client() -> SyncClient {
9325 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9326 {"name":"id","type":"string","nullable":false},
9327 {"name":"project_id","type":"string","nullable":false}],
9328 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[],"ftsIndexes":[]}]});
9329 SyncClient::new("rearm".into(), &schema, ClientLimits::default()).expect("client")
9330 }
9331
9332 fn scopes() -> Vec<(String, Vec<String>)> {
9333 vec![("project_id".to_owned(), vec!["p1".to_owned()])]
9334 }
9335
9336 fn store_revoked(client: &mut SyncClient) {
9338 client
9339 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9340 .expect("subscribe");
9341 let sub = &mut client.subs[0];
9342 sub.cursor = 41;
9343 sub.bootstrap_state = Some("resume".to_owned());
9344 sub.effective = Some(scopes());
9345 sub.state = SubState::Revoked;
9346 sub.reason_code = Some("sync.scope_revoked".to_owned());
9347 client.persist_sub(&client.subs[0]).expect("persist");
9348 client.sync_needed = false;
9349 client.sync_intent_queue.clear();
9350 }
9351
9352 #[test]
9353 fn resubscribing_the_same_intent_rearms_a_revoked_registration() {
9354 let mut client = client();
9355 store_revoked(&mut client);
9356
9357 client
9358 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9359 .expect("re-arm");
9360
9361 let view = client.subscription_state("sub").expect("state");
9362 assert_eq!(view.status, "active");
9363 assert_eq!(view.cursor, -1);
9364 assert!(!view.has_resume_token);
9365 assert_eq!(view.effective_scopes, None);
9366 assert_eq!(view.reason_code, None);
9367 assert!(client.sync_needed, "a re-arm asks for a pull");
9368 assert!(matches!(
9369 client.drain_sync_intents().as_slice(),
9370 [SyncIntent::Interactive]
9371 ));
9372 let stored: String = client
9373 .conn
9374 .query_row(
9375 "SELECT state_json FROM _syncular_subscriptions WHERE id = 'sub'",
9376 [],
9377 |row| row.get(0),
9378 )
9379 .expect("persisted row");
9380 let stored: Value = serde_json::from_str(&stored).expect("stored json");
9381 assert_eq!(stored["status"], json!("active"));
9382 assert_eq!(stored["cursor"], json!(-1));
9383 assert_eq!(stored["bootstrapState"], Value::Null);
9384 assert_eq!(stored["effectiveScopes"], Value::Null);
9385 assert_eq!(stored["reasonCode"], Value::Null);
9386 }
9387
9388 #[test]
9389 fn a_failed_registration_is_not_rearmed() {
9390 let mut client = client();
9391 client
9392 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9393 .expect("subscribe");
9394 client.subs[0].state = SubState::Failed;
9395 client.subs[0].reason_code = Some("sync.scope_revoked".to_owned());
9396 client.persist_sub(&client.subs[0]).expect("persist");
9397 client.sync_needed = false;
9398 client.sync_intent_queue.clear();
9399
9400 client
9401 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9402 .expect("subscribe");
9403
9404 let view = client.subscription_state("sub").expect("state");
9405 assert_eq!(view.status, "failed");
9406 assert_eq!(view.reason_code.as_deref(), Some("sync.scope_revoked"));
9407 assert!(!client.sync_needed);
9408 assert!(client.drain_sync_intents().is_empty());
9409 }
9410}
9411
9412#[derive(Debug, Clone)]
9413struct OutboxOp {
9414 upsert: bool,
9415 table: String,
9416 row_id: String,
9417 base_version: Option<i64>,
9418 values: Option<Map<String, Value>>,
9422}
9423
9424impl From<&OutboxOp> for CommitOperation {
9425 fn from(operation: &OutboxOp) -> Self {
9426 Self {
9427 table: operation.table.clone(),
9428 row_id: operation.row_id.clone(),
9429 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
9430 base_version: operation.base_version,
9431 values: operation.values.clone(),
9432 }
9433 }
9434}
9435
9436#[derive(Debug, Clone)]
9437struct OutboxCommit {
9438 client_commit_id: String,
9439 ops: Vec<OutboxOp>,
9440}
9441
9442fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>)> {
9446 commit
9447 .ops
9448 .iter()
9449 .filter(|operation| operation.upsert)
9450 .map(|operation| {
9451 (
9452 operation.table.as_str(),
9453 operation
9454 .values
9455 .as_ref()
9456 .map(|values| values.keys().map(String::as_str).collect())
9457 .unwrap_or_default(),
9458 )
9459 })
9460 .collect()
9461}
9462
9463type FailedCommit = (OutboxCommit, Map<String, Value>);
9464
9465#[derive(Debug, Clone)]
9466struct CompiledLocalDataPurgeTarget {
9467 table: String,
9468 selectors: Vec<(String, Vec<String>)>,
9469}
9470
9471struct StoredCommitOutcomeRow {
9472 sequence: i64,
9473 client_commit_id: String,
9474 status: String,
9475 recorded_at_ms: i64,
9476 results_json: String,
9477 operations_json: Option<String>,
9478 resolution: String,
9479 resolved_at_ms: Option<i64>,
9480 replacement_client_commit_id: Option<String>,
9481}
9482
9483impl StoredCommitOutcomeRow {
9484 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
9485 Ok(Self {
9486 sequence: row.get(0)?,
9487 client_commit_id: row.get(1)?,
9488 status: row.get(2)?,
9489 recorded_at_ms: row.get(3)?,
9490 results_json: row.get(4)?,
9491 operations_json: row.get(5)?,
9492 resolution: row.get(6)?,
9493 resolved_at_ms: row.get(7)?,
9494 replacement_client_commit_id: row.get(8)?,
9495 })
9496 }
9497}
9498
9499enum SectionError {
9502 FailClosed,
9503 Abort(String, String),
9504}
9505
9506#[derive(Debug)]
9507pub(crate) struct RequestMeta {
9508 pushed_ids: Vec<String>,
9509 fresh: Vec<(String, bool)>,
9512 pub(crate) accept: u8,
9513 deferred_commits: usize,
9517}
9518
9519#[derive(Default)]
9520struct ChangeAccumulator {
9521 tables: BTreeMap<String, Option<BTreeSet<String>>>,
9523 windows: BTreeMap<(String, String), BTreeSet<String>>,
9524 status: bool,
9525 conflicts: bool,
9526 rejections: bool,
9527 outcomes: bool,
9528}
9529
9530impl ChangeAccumulator {
9531 fn table(&mut self, table: &str) {
9532 self.tables.insert(table.to_owned(), None);
9533 }
9534
9535 fn scope(&mut self, table: &str, key: String) {
9536 match self.tables.get_mut(table) {
9537 Some(None) => {}
9538 Some(Some(keys)) => {
9539 keys.insert(key);
9540 }
9541 None => {
9542 self.tables
9543 .insert(table.to_owned(), Some(BTreeSet::from([key])));
9544 }
9545 }
9546 }
9547
9548 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
9549 self.windows
9550 .entry((base_key.to_owned(), table.to_owned()))
9551 .or_default()
9552 .insert(unit.to_owned());
9553 }
9554
9555 fn touched(&self) -> bool {
9556 !self.tables.is_empty()
9557 || !self.windows.is_empty()
9558 || self.status
9559 || self.conflicts
9560 || self.rejections
9561 || self.outcomes
9562 }
9563}
9564
9565const PUSH_OPS_PER_REQUEST: usize = 500;
9571const MAX_LOCAL_PURGE_TARGETS: usize = 64;
9572const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
9573const MAX_LOCAL_PURGE_VALUES: usize = 128;
9574const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
9575pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
9576
9577struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
9579
9580impl OverlayDirty {
9581 fn clean() -> Self {
9582 Self(RefCell::new(Some(BTreeSet::new())))
9583 }
9584 fn get(&self) -> bool {
9585 self.0
9586 .borrow()
9587 .as_ref()
9588 .is_none_or(|tables| !tables.is_empty())
9589 }
9590 fn set(&self, dirty: bool) {
9591 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
9592 }
9593 fn table(&self, table: &str) {
9594 if let Some(tables) = self.0.borrow_mut().as_mut() {
9595 tables.insert(table.to_owned());
9596 }
9597 }
9598 fn snapshot(&self) -> Option<BTreeSet<String>> {
9599 self.0.borrow().clone()
9600 }
9601 fn restore(&self, tables: Option<BTreeSet<String>>) {
9602 *self.0.borrow_mut() = tables;
9603 }
9604}
9605
9606pub struct SyncClient {
9607 #[cfg(feature = "bench-internals")]
9608 benchmark_phases: Recorder,
9609 conn: Connection,
9610 schema: ClientSchema,
9611 client_id: String,
9612 limits: ClientLimits,
9613 subs: Vec<Subscription>,
9614 outbox: Vec<OutboxCommit>,
9615 failed_commits: Vec<FailedCommit>,
9616 retain_failed_commits: bool,
9617 conflicts: Vec<ConflictRecord>,
9618 rejections: Vec<RejectionRecord>,
9619 schema_floor: Option<SchemaFloor>,
9620 lease_state: Option<LeaseState>,
9622 stopped: bool,
9624 upgrading: bool,
9626 sync_needed: bool,
9628 realtime_policy: RealtimePolicy,
9631 realtime_state: RealtimeState,
9632 realtime_reason_code: Option<String>,
9633 realtime_retry_delay_ms: Option<u64>,
9634 presence: HashMap<String, HashMap<String, PresencePeer>>,
9636 now_ms: Option<i64>,
9639 encryption: crate::values::EncryptionConfig,
9643 security_preflight: bool,
9646 transport_enabled: bool,
9647 previous_version: Option<PreviousVersionContextConfig>,
9651 insert_sql: RefCell<HashMap<String, String>>,
9657 overlay_dirty: OverlayDirty,
9660 storage_failure: RefCell<Option<QueryReadFailure>>,
9662 #[cfg(test)]
9665 overlay_rebuild_count: Cell<usize>,
9666 #[cfg(test)]
9667 acknowledged_replay_count: Cell<usize>,
9668 #[cfg(test)]
9671 outcome_prune_count: Cell<usize>,
9672 progress: ProgressObserver,
9674 change_queue: VecDeque<ClientChangeBatch>,
9675 sync_intent_queue: VecDeque<SyncIntent>,
9676 retry_delay_ms: u64,
9678 round_retry_delay_ms: Option<u64>,
9680 active_round: Option<uuid::Uuid>,
9681 last_round: Option<DiagnosticLastRound>,
9682 last_change: Option<DiagnosticLastChange>,
9683 query_failures: Vec<DiagnosticQueryFailure>,
9685 coalesced: Option<CoalescedApply>,
9687 coalesced_failure: Option<String>,
9690}
9691
9692struct CoalescedApply {
9696 rows: usize,
9697 subs: Vec<Subscription>,
9698 outbox: Vec<OutboxCommit>,
9699 failed_commits: Vec<FailedCommit>,
9700 conflicts: Vec<ConflictRecord>,
9701 rejections: Vec<RejectionRecord>,
9702 schema_floor: Option<SchemaFloor>,
9703 lease_state: Option<LeaseState>,
9704 stopped: bool,
9705 upgrading: bool,
9706 sync_needed: bool,
9707 overlay_dirty: Option<BTreeSet<String>>,
9708 change_queue: usize,
9709 sync_intent_queue: usize,
9710 last_change: Option<DiagnosticLastChange>,
9711}
9712
9713const COALESCED_APPLY_ROWS: usize = 1024;
9714
9715pub(crate) fn quote_ident(name: &str) -> String {
9716 format!("\"{}\"", name.replace('"', "\"\""))
9717}
9718pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
9722 conn.query_row(
9723 "SELECT value FROM _syncular_meta WHERE key = ?1",
9724 rusqlite::params![key],
9725 |row| row.get::<_, String>(0),
9726 )
9727 .ok()
9728}
9729
9730pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
9731 let _ = conn.execute(
9732 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
9733 rusqlite::params![key, value],
9734 );
9735}
9736
9737pub(crate) fn meta_delete(conn: &Connection, key: &str) {
9738 let _ = conn.execute(
9739 "DELETE FROM _syncular_meta WHERE key = ?1",
9740 rusqlite::params![key],
9741 );
9742}
9743
9744fn row_id_predicate(table: &TableSchema) -> String {
9746 let key = quote_ident(&table.primary_key);
9747 let text_match = format!("CAST({key} AS TEXT) = ?1");
9748 match table.columns[table.pk_index].ty {
9749 ColumnType::String | ColumnType::Json => {
9750 format!("{key} = ?1 AND {text_match}")
9751 }
9752 ColumnType::Integer | ColumnType::Boolean => {
9756 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
9757 }
9758 _ => text_match,
9761 }
9762}
9763
9764fn is_local_operation_code_like(value: &str) -> bool {
9765 let bytes = value.as_bytes();
9766 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
9767 && bytes
9768 .iter()
9769 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
9770}
9771
9772fn base_table(name: &str) -> String {
9773 quote_ident(&format!("_syncular_base_{name}"))
9774}
9775
9776type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
9781
9782fn window_base_key(base: &WindowBase) -> String {
9783 format!(
9784 "{}\0{}\0{}",
9785 base.table,
9786 base.variable,
9787 canonical_scope_json(&base.fixed_scopes)
9788 )
9789}
9790
9791fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
9793 let mut scopes = base.fixed_scopes.clone();
9794 scopes.retain(|(k, _)| k != &base.variable);
9795 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
9796 scopes
9797}
9798
9799fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
9803 let canonical = canonical_scope_json(&unit_scopes(base, unit));
9804 let digest = Sha256::digest(canonical.as_bytes());
9805 let hex = bytes_to_hex(&digest);
9806 format!("w:{}:{}", base.table, &hex[..16])
9807}
9808
9809fn visible_table(name: &str) -> String {
9810 quote_ident(name)
9811}
9812
9813const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
9814
9815fn is_synced_table_name(name: &str) -> bool {
9818 if name.starts_with("sqlite_") {
9819 return false;
9820 }
9821 if name.starts_with("_syncular_base_") {
9822 return true; }
9824 !name.starts_with("_syncular_")
9826}
9827
9828fn blob_id_for(bytes: &[u8]) -> String {
9830 let digest = Sha256::digest(bytes);
9831 format!("sha256:{}", bytes_to_hex(&digest))
9832}
9833
9834enum RowParam<'a> {
9838 Cell(&'a Option<ColumnValue>),
9839 Version(i64),
9840}
9841
9842impl rusqlite::ToSql for RowParam<'_> {
9843 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
9844 Ok(match self {
9845 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
9846 RowParam::Cell(cell) => match cell {
9847 None => ToSqlOutput::Owned(SqlValue::Null),
9848 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
9849 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
9850 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
9851 Some(ColumnValue::Boolean(b)) => {
9852 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
9853 }
9854 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
9855 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
9856 }
9857 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
9859 ToSqlOutput::Borrowed(ValueRef::Blob(b))
9860 }
9861 },
9862 })
9863 }
9864}
9865
9866fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
9868 match param.to_sql().map_err(|error| error.to_string())? {
9869 ToSqlOutput::Owned(value) => Ok(value),
9870 ToSqlOutput::Borrowed(value) => {
9871 SqlValue::try_from(value).map_err(|error| error.to_string())
9872 }
9873 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
9874 }
9875}
9876
9877fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
9885 use ssp2::segment::ColumnType;
9886 let mismatch = || {
9887 Err(format!(
9888 "image column {:?} holds a value of the wrong type",
9889 column.name
9890 ))
9891 };
9892 match value {
9893 ValueRef::Null => {
9894 if !column.nullable {
9895 return Err(format!(
9896 "image column {:?} is NULL but not nullable",
9897 column.name
9898 ));
9899 }
9900 Ok(ToSqlOutput::Owned(SqlValue::Null))
9901 }
9902 ValueRef::Integer(i) => match column.ty {
9903 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
9904 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
9905 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
9906 _ => mismatch(),
9907 },
9908 ValueRef::Real(_) => match column.ty {
9909 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
9910 _ => mismatch(),
9911 },
9912 ValueRef::Text(t) => {
9913 std::str::from_utf8(t)
9914 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
9915 match column.ty {
9916 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
9917 Ok(ToSqlOutput::Borrowed(value))
9918 }
9919 _ => mismatch(),
9920 }
9921 }
9922 ValueRef::Blob(_) => match column.ty {
9923 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
9925 _ => mismatch(),
9926 },
9927 }
9928}
9929
9930pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
9931 use rusqlite::types::ValueRef;
9932 match value {
9933 ValueRef::Null => Value::Null,
9934 ValueRef::Integer(i) => match column.ty {
9935 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
9936 ssp2::segment::ColumnType::Float => {
9937 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
9938 }
9939 _ => Value::from(i),
9940 },
9941 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
9942 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
9943 ValueRef::Blob(b) => {
9944 let mut map = Map::new();
9945 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
9946 Value::Object(map)
9947 }
9948 }
9949}
9950
9951fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
9955 Ok(match value {
9956 Value::Null => SqlValue::Null,
9957 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
9958 Value::Number(n) => {
9959 if let Some(i) = n.as_i64() {
9960 SqlValue::Integer(i)
9961 } else if let Some(f) = n.as_f64() {
9962 SqlValue::Real(f)
9963 } else {
9964 return Err(format!("query param number {n} is out of range"));
9965 }
9966 }
9967 Value::String(s) => SqlValue::Text(s.clone()),
9968 Value::Object(_) => {
9969 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
9970 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
9971 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
9972 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
9973 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
9974 })?)
9975 } else {
9976 return Err(
9977 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
9978 .to_owned(),
9979 );
9980 }
9981 }
9982 Value::Array(_) => return Err("query array params are not supported".to_owned()),
9983 })
9984}
9985
9986fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
9991 use rusqlite::types::ValueRef;
9992 match value {
9993 ValueRef::Null => Value::Null,
9994 ValueRef::Integer(i) => {
9995 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
9999 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
10000 Value::from(i)
10001 } else {
10002 let mut map = Map::new();
10003 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
10004 Value::Object(map)
10005 }
10006 }
10007 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10008 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10009 ValueRef::Blob(b) => {
10010 let mut map = Map::new();
10011 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10012 Value::Object(map)
10013 }
10014 }
10015}
10016
10017fn query_connection(
10019 conn: &Connection,
10020 sql: &str,
10021 params: &[QueryValue],
10022) -> Result<Vec<QueryRow>, QueryReadFailure> {
10023 crate::query_guard::assert_read_only_query(sql)?;
10024 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
10025 let bound: Vec<SqlValue> = params
10026 .iter()
10027 .map(json_param_to_sql)
10028 .collect::<Result<_, _>>()?;
10029 let mut stmt = conn.prepare(&lowered_sql)?;
10030 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
10031 let bound_refs: Vec<&dyn rusqlite::ToSql> =
10032 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
10033 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
10034 let mut out = Vec::new();
10035 while let Some(row) = sql_rows.next()? {
10036 let mut record = Map::new();
10037 for (i, name) in column_names.iter().enumerate() {
10038 let value = row.get_ref(i)?;
10039 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
10040 }
10041 out.push(record);
10042 }
10043 Ok(out)
10044}
10045
10046fn persisted_window_state(
10047 conn: &Connection,
10048 base: &WindowBase,
10049) -> Result<WindowState, QueryReadFailure> {
10050 let mut stmt = conn.prepare(
10051 "SELECT windows.unit, subscriptions.state_json
10052 FROM _syncular_windows AS windows
10053 JOIN _syncular_subscriptions AS subscriptions
10054 ON subscriptions.id = windows.sub_id
10055 WHERE windows.base = ?1
10056 ORDER BY windows.unit ASC",
10057 )?;
10058 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
10059 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
10060 })?;
10061 let mut units = Vec::new();
10062 let mut pending = Vec::new();
10063 for row in rows {
10064 let (unit, raw) = row?;
10065 let state: Value = serde_json::from_str(&raw)
10066 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
10067 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
10068 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
10069 || state
10070 .get("bootstrapState")
10071 .is_some_and(|value| !value.is_null());
10072 if is_pending {
10073 pending.push(unit.clone());
10074 }
10075 units.push(unit);
10076 }
10077 Ok(WindowState { units, pending })
10078}
10079
10080#[derive(Debug, Clone, Copy)]
10082pub struct SnapshotStatement<'a> {
10083 pub sql: &'a str,
10084 pub params: &'a [Value],
10085}
10086
10087#[derive(Debug, Clone, Default)]
10091pub struct SnapshotReadRequest<'a> {
10092 pub statements: Vec<SnapshotStatement<'a>>,
10093 pub coverage: &'a [WindowCoverage],
10094 pub subscriptions: Vec<String>,
10095 pub commit_ids: Vec<String>,
10096}
10097
10098#[cfg(test)]
10099thread_local! {
10100 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
10104 std::cell::RefCell::new(None);
10105}
10106
10107#[cfg(test)]
10108fn run_snapshot_read_hook() {
10109 SNAPSHOT_READ_HOOK.with(|slot| {
10110 if let Some(hook) = slot.borrow_mut().take() {
10111 hook();
10112 }
10113 });
10114}
10115
10116fn snapshot_read_connection(
10121 conn: &Connection,
10122 request: &SnapshotReadRequest<'_>,
10123) -> Result<SnapshotRead, QueryReadFailure> {
10124 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
10125 let corrupt = |message: &str| QueryReadFailure {
10126 code: Some("sync.local_corrupt"),
10127 sqlite_code: None,
10128 sqlite_message: None,
10129 rollback_failure: None,
10130 message: message.to_owned(),
10131 };
10132 let stored_failure = |error: rusqlite::Error, message: &str| match error {
10133 rusqlite::Error::Utf8Error(..)
10134 | rusqlite::Error::InvalidColumnType(..)
10135 | rusqlite::Error::FromSqlConversionFailure(..)
10136 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
10137 _ => QueryReadFailure::from(error),
10138 };
10139 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
10140 let revision: Option<String> = conn
10143 .query_row(
10144 "SELECT value FROM _syncular_meta WHERE key = ?1",
10145 rusqlite::params![LOCAL_REVISION_KEY],
10146 |row| row.get(0),
10147 )
10148 .optional()
10149 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
10150 let revision = match revision {
10151 None => "0".to_owned(),
10152 Some(value)
10153 if value == "0"
10154 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10155 && value.bytes().all(|b| b.is_ascii_digit())
10156 && value.parse::<u64>().is_ok()) =>
10157 {
10158 value
10159 }
10160 Some(_) => return Err(corrupt("persisted local revision is invalid")),
10161 };
10162 let mut queries = Vec::with_capacity(request.statements.len());
10163 for statement in &request.statements {
10164 queries.push(query_connection(conn, statement.sql, statement.params)?);
10165 #[cfg(test)]
10166 run_snapshot_read_hook();
10167 }
10168 let mut pending = Vec::new();
10169 let mut missing = Vec::new();
10170 for requested in request.coverage {
10171 let base_key = window_base_key(&requested.base);
10172 let state = persisted_window_state(conn, &requested.base)?;
10173 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
10174 let reference = WindowUnitRef {
10175 base_key: base_key.clone(),
10176 unit: unit.clone(),
10177 };
10178 if !state.units.iter().any(|held| held == &unit) {
10179 missing.push(reference);
10180 } else if state.pending.iter().any(|held| held == &unit) {
10181 pending.push(reference);
10182 }
10183 }
10184 }
10185 let mut subscriptions = Vec::with_capacity(request.subscriptions.len());
10186 for id in &request.subscriptions {
10187 let row: Option<(String, String)> = conn
10188 .query_row(
10189 "SELECT tbl, state_json FROM _syncular_subscriptions WHERE id = ?1",
10190 rusqlite::params![id],
10191 |row| Ok((row.get(0)?, row.get(1)?)),
10192 )
10193 .optional()
10194 .map_err(|error| {
10195 stored_failure(error, "persisted subscription state is invalid")
10196 })?;
10197 let Some((table, raw)) = row else {
10198 subscriptions.push(SubscriptionCatchup::Unknown { id: id.clone() });
10199 continue;
10200 };
10201 let state: Value = serde_json::from_str(&raw)
10204 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10205 let status = match state.get("status").and_then(Value::as_str) {
10206 Some(value @ ("active" | "revoked" | "failed")) => value,
10207 _ => return Err(corrupt("persisted subscription state is invalid")),
10208 };
10209 let Some(cursor) = state.get("cursor").and_then(Value::as_i64) else {
10210 return Err(corrupt("persisted subscription state is invalid"));
10211 };
10212 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
10213 return Err(corrupt("persisted subscription state is invalid"));
10214 }
10215 let effective_scopes = state
10216 .get("effectiveScopes")
10217 .filter(|value| !value.is_null());
10218 if let Some(scopes) = effective_scopes {
10219 json_to_scope_map(scopes)
10220 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10221 }
10222 let has_resume_token = match state.get("bootstrapState") {
10223 None | Some(Value::Null) => false,
10224 Some(Value::String(_)) => true,
10225 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10226 };
10227 let reason_code = match state.get("reasonCode") {
10230 None | Some(Value::Null) => None,
10231 Some(Value::String(value)) => Some(value.clone()),
10232 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10233 };
10234 let active = status == "active";
10235 subscriptions.push(SubscriptionCatchup::Known {
10236 id: id.clone(),
10237 table,
10238 status: status.to_owned(),
10239 cursor,
10240 has_resume_token,
10241 bootstrap_complete: active && cursor >= 0 && !has_resume_token,
10242 known_pending_pages: active && (has_resume_token || cursor < 0),
10243 effective_scopes: effective_scopes.cloned(),
10244 reason_code,
10245 });
10246 }
10247 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
10248 for id in &request.commit_ids {
10249 let pending: bool = conn.query_row(
10250 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
10251 rusqlite::params![id],
10252 |row| row.get(0),
10253 )?;
10254 if pending {
10255 deliveries.push(CommitDelivery::Pending {
10256 client_commit_id: id.clone(),
10257 });
10258 continue;
10259 }
10260 let row = conn
10261 .query_row(
10262 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
10263 resolution, resolved_at_ms, replacement_client_commit_id
10264 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
10265 rusqlite::params![id],
10266 StoredCommitOutcomeRow::from_row,
10267 )
10268 .optional()
10269 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
10270 deliveries.push(match row {
10271 Some(row) => CommitDelivery::Known {
10272 client_commit_id: id.clone(),
10273 outcome: SyncClient::outcome_from_row(row)
10274 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
10275 },
10276 None => CommitDelivery::Unknown {
10277 client_commit_id: id.clone(),
10278 },
10279 });
10280 }
10281 Ok(SnapshotRead {
10282 revision,
10283 queries,
10284 coverage: CoverageSnapshot {
10285 complete: pending.is_empty() && missing.is_empty(),
10286 pending,
10287 missing,
10288 },
10289 subscriptions,
10290 deliveries,
10291 })
10292 })();
10293 match result {
10294 Ok(read) => {
10295 conn.execute_batch("RELEASE syncular_snapshot_read")?;
10296 Ok(read)
10297 }
10298 Err(mut error) => {
10299 if let Err(rollback) = conn
10300 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
10301 {
10302 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
10303 }
10304 Err(error)
10305 }
10306 }
10307}
10308
10309fn snapshot_connection(
10310 conn: &Connection,
10311 sql: &str,
10312 params: &[Value],
10313 coverage: &[WindowCoverage],
10314) -> Result<QuerySnapshot, QueryReadFailure> {
10315 let read = snapshot_read_connection(
10316 conn,
10317 &SnapshotReadRequest {
10318 statements: vec![SnapshotStatement { sql, params }],
10319 coverage,
10320 subscriptions: Vec::new(),
10321 commit_ids: Vec::new(),
10322 },
10323 )?;
10324 Ok(QuerySnapshot {
10325 revision: read.revision,
10326 rows: read
10327 .queries
10328 .into_iter()
10329 .next()
10330 .expect("single-query snapshot returns one result"),
10331 coverage: read.coverage,
10332 })
10333}
10334
10335pub struct FileQuerySnapshotReader {
10340 path: String,
10341 conn: Option<Connection>,
10342}
10343
10344impl FileQuerySnapshotReader {
10345 #[must_use]
10346 pub fn new(path: impl Into<String>) -> Self {
10347 Self {
10348 path: path.into(),
10349 conn: None,
10350 }
10351 }
10352
10353 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
10354 if self.conn.is_none() {
10355 let conn = Connection::open_with_flags(
10356 &self.path,
10357 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
10358 )
10359 .map_err(QueryReadFailure::from)?;
10360 conn.busy_timeout(std::time::Duration::from_millis(250))
10361 .map_err(QueryReadFailure::from)?;
10362 self.conn = Some(conn);
10363 }
10364 self.conn
10365 .as_ref()
10366 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
10367 }
10368
10369 pub fn snapshot_read(
10372 &mut self,
10373 request: &SnapshotReadRequest<'_>,
10374 ) -> Result<SnapshotRead, QueryReadFailure> {
10375 snapshot_read_connection(self.connection()?, request)
10376 }
10377
10378 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
10381 Ok(self
10382 .snapshot_read(&SnapshotReadRequest::default())?
10383 .revision)
10384 }
10385
10386 pub fn subscription_catchup(
10388 &mut self,
10389 id: &str,
10390 ) -> Result<SubscriptionCatchup, QueryReadFailure> {
10391 let read = self.snapshot_read(&SnapshotReadRequest {
10392 subscriptions: vec![id.to_owned()],
10393 ..Default::default()
10394 })?;
10395 Ok(read
10396 .subscriptions
10397 .into_iter()
10398 .next()
10399 .expect("one requested catch-up"))
10400 }
10401
10402 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
10405 let read = self.snapshot_read(&SnapshotReadRequest {
10406 commit_ids: vec![id.to_owned()],
10407 ..Default::default()
10408 })?;
10409 Ok(read
10410 .deliveries
10411 .into_iter()
10412 .next()
10413 .expect("one requested delivery"))
10414 }
10415
10416 pub fn query_snapshot(
10419 &mut self,
10420 sql: &str,
10421 params: &[Value],
10422 coverage: &[WindowCoverage],
10423 ) -> Result<QuerySnapshot, QueryReadFailure> {
10424 snapshot_connection(self.connection()?, sql, params, coverage)
10425 }
10426}
10427
10428fn validate_authority_reads(
10429 schema: &ClientSchema,
10430 declarations: &[crate::api::AuthorityReadDeclaration],
10431) -> Result<(), String> {
10432 let mut seen = BTreeSet::new();
10433 for read in declarations {
10434 let table = schema
10435 .table(&read.table)
10436 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
10437 let plain = |name: &str| {
10438 table.columns.iter().enumerate().any(|(i, c)| {
10439 c.name == name
10440 && name != "authority_version"
10441 && !name.starts_with("_sync")
10442 && !table.encrypted_columns.iter().any(|e| e.index == i)
10443 && matches!(
10444 c.ty,
10445 ssp2::segment::ColumnType::String
10446 | ssp2::segment::ColumnType::Integer
10447 | ssp2::segment::ColumnType::Float
10448 | ssp2::segment::ColumnType::Boolean
10449 | ssp2::segment::ColumnType::Json
10450 )
10451 })
10452 };
10453 if !seen.insert(&read.table)
10454 || read.columns.is_empty()
10455 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
10456 || !read.columns.contains(&table.primary_key)
10457 || !read.columns.iter().all(|c| plain(c))
10458 || read.scopes.is_empty()
10459 || !read.scopes.iter().all(|(v, values)| {
10460 table
10461 .scope_column(v)
10462 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
10463 && !values.is_empty()
10464 && values.iter().all(|v| !v.is_empty() && v != "*")
10465 })
10466 {
10467 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
10468 }
10469 }
10470 Ok(())
10471}
10472
10473fn schema_marker_read_failure(error: rusqlite::Error) -> String {
10478 let failure = QueryReadFailure::from(error);
10479 if failure.code.is_some() {
10480 failure.to_string()
10481 } else {
10482 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
10483 }
10484}
10485
10486fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
10490 if limits.outcome_retention_max_entries == Some(0) {
10491 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
10492 }
10493 if [
10494 limits.max_push_commits_per_request,
10495 limits.max_push_operations_per_request,
10496 limits.max_push_request_bytes,
10497 ]
10498 .into_iter()
10499 .flatten()
10500 .any(|value| value == 0 || value > u32::MAX as usize)
10501 {
10502 return Err(
10503 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
10504 );
10505 }
10506 Ok(())
10507}
10508
10509fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
10510 let corrupt =
10511 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
10512 let table_type: Option<String> = conn
10513 .query_row(
10514 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
10515 [],
10516 |row| row.get(0),
10517 )
10518 .optional()
10519 .map_err(schema_marker_read_failure)?;
10520 let Some(table_type) = table_type else {
10521 return Ok(None);
10522 };
10523 if table_type != "table" {
10524 return Err(corrupt());
10525 }
10526 let mut statement = conn
10530 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
10531 .map_err(schema_marker_read_failure)?;
10532 let mut rows = statement
10533 .query([LOCAL_SCHEMA_VERSION_KEY])
10534 .map_err(schema_marker_read_failure)?;
10535 let value: Option<String> = rows
10536 .next()
10537 .map_err(schema_marker_read_failure)?
10538 .map(|row| row.get(0))
10539 .transpose()
10540 .map_err(schema_marker_read_failure)?;
10541 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
10542 return Err(corrupt());
10543 }
10544 let Some(value) = value else {
10545 let has_descriptor = conn
10546 .query_row(
10547 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
10548 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
10549 |_| Ok(()),
10550 )
10551 .optional()
10552 .map_err(schema_marker_read_failure)?
10553 .is_some();
10554 if has_descriptor {
10555 return Err(corrupt());
10556 }
10557 return Ok(None);
10558 };
10559 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10560 || !value.bytes().all(|b| b.is_ascii_digit())
10561 {
10562 return Err(corrupt());
10563 }
10564 let version = value.parse::<i32>().map_err(|_| corrupt())?;
10565 if version > requested {
10566 return Err(
10567 "client.schema_downgrade: persisted local schema is newer than the requested schema"
10568 .to_owned(),
10569 );
10570 }
10571 Ok(Some(version))
10572}
10573
10574impl SyncClient {
10575 pub fn new_with_identity(
10576 client_id: Option<String>,
10577 schema_json: &Value,
10578 limits: ClientLimits,
10579 ) -> Result<Self, String> {
10580 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
10581 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
10582 Self::with_connection(resolved, schema_json, limits, conn)
10583 }
10584
10585 pub fn new(
10586 client_id: String,
10587 schema_json: &Value,
10588 limits: ClientLimits,
10589 ) -> Result<Self, String> {
10590 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
10591 Self::with_connection(client_id, schema_json, limits, conn)
10592 }
10593
10594 pub fn open_path(
10600 client_id: String,
10601 schema_json: &Value,
10602 limits: ClientLimits,
10603 path: &str,
10604 ) -> Result<Self, String> {
10605 validate_client_limits(&limits)?;
10606 parse_schema_json(schema_json)?;
10607 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
10608 Self::with_connection(client_id, schema_json, limits, conn)
10609 }
10610
10611 pub fn open_path_with_identity(
10612 client_id: Option<String>,
10613 schema_json: &Value,
10614 limits: ClientLimits,
10615 path: &str,
10616 ) -> Result<Self, String> {
10617 validate_client_limits(&limits)?;
10618 parse_schema_json(schema_json)?;
10621 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
10622 conn.busy_timeout(std::time::Duration::from_millis(250))
10630 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
10631 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
10632 let journal_mode: String = client
10633 .conn
10634 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
10635 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
10636 if journal_mode != "wal" {
10637 return Err(
10638 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
10639 );
10640 }
10641 Ok(client)
10642 }
10643
10644 #[must_use]
10645 pub fn client_id(&self) -> &str {
10646 &self.client_id
10647 }
10648
10649 pub fn with_connection(
10656 client_id: String,
10657 schema_json: &Value,
10658 limits: ClientLimits,
10659 conn: Connection,
10660 ) -> Result<Self, String> {
10661 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
10662 }
10663
10664 fn with_connection_identity(
10667 client_id: Option<String>,
10668 schema_json: &Value,
10669 limits: ClientLimits,
10670 conn: Connection,
10671 ) -> Result<Self, String> {
10672 validate_client_limits(&limits)?;
10673 let schema = parse_schema_json(schema_json)?;
10674 validate_authority_reads(&schema, &limits.authority_reads)?;
10675 let previous_version = match limits.previous_version_context {
10678 Some(config) => {
10679 config.validate()?;
10680 Some(config)
10681 }
10682 None => None,
10683 };
10684 let mut client = SyncClient {
10685 #[cfg(feature = "bench-internals")]
10686 benchmark_phases: Recorder::default(),
10687 conn,
10688 schema,
10689 client_id: client_id
10690 .clone()
10691 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
10692 limits,
10693 subs: Vec::new(),
10694 outbox: Vec::new(),
10695 failed_commits: Vec::new(),
10696 retain_failed_commits: false,
10697 conflicts: Vec::new(),
10698 rejections: Vec::new(),
10699 schema_floor: None,
10700 lease_state: None,
10701 stopped: false,
10702 upgrading: false,
10703 sync_needed: false,
10704 realtime_policy: RealtimePolicy::Optional,
10705 realtime_state: RealtimeState::Disconnected,
10706 realtime_reason_code: None,
10707 realtime_retry_delay_ms: None,
10708 presence: HashMap::new(),
10709 now_ms: None,
10710 encryption: crate::values::EncryptionConfig::default(),
10711 security_preflight: false,
10712 transport_enabled: true,
10713 previous_version,
10714 insert_sql: RefCell::new(HashMap::new()),
10715 overlay_dirty: OverlayDirty::clean(),
10716 storage_failure: RefCell::new(None),
10717 #[cfg(test)]
10718 overlay_rebuild_count: Cell::new(0),
10719 #[cfg(test)]
10720 acknowledged_replay_count: Cell::new(0),
10721 #[cfg(test)]
10722 outcome_prune_count: Cell::new(0),
10723 progress: ProgressObserver::default(),
10724 change_queue: VecDeque::new(),
10725 sync_intent_queue: VecDeque::new(),
10726 retry_delay_ms: 250,
10727 round_retry_delay_ms: None,
10728 active_round: None,
10729 last_round: None,
10730 last_change: None,
10731 query_failures: Vec::new(),
10732 coalesced: None,
10733 coalesced_failure: None,
10734 };
10735 client
10739 .conn
10740 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
10741 client
10747 .conn
10748 .execute_batch("BEGIN IMMEDIATE")
10749 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
10750 let startup = (|| -> Result<(), String> {
10751 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
10754 client.create_bookkeeping_tables()?;
10757 match client.get_meta(CLIENT_ID_KEY) {
10758 Some(existing) => match client_id {
10759 Some(requested) if existing != requested => {
10760 return Err(format!(
10761 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
10762 ))
10763 }
10764 _ => client.client_id = existing,
10765 },
10766 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
10767 }
10768 client.restore_persisted_state()?;
10769 client.failed_commits = client.load_failed_commits()?;
10770 match marker {
10771 None => {
10772 client.create_synced_tables()?;
10773 client.persist_schema_version()?;
10774 client.save_subscription_scope_schema();
10775 }
10776 Some(version) if version == client.schema.version => {
10777 client.create_synced_tables()?;
10778 set_local_schema_descriptor(&client.conn, &client.schema)
10782 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
10783 client.save_subscription_scope_schema();
10784 }
10785 Some(_) => client.run_schema_reset()?,
10786 }
10787 client.reconcile_previous_version_at_boot()?;
10793 client.clear_satisfied_persisted_schema_floor();
10794 client.prune_unknown_subscriptions(false)?;
10795 client.drop_unencodable_outbox(false, None)?;
10804 if marker == Some(client.schema.version)
10805 && (!client.outbox.is_empty()
10806 || !client.failed_commits.is_empty()
10807 || client.has_acknowledged_rows())
10808 {
10809 client.overlay_dirty.set(true);
10813 client.rebuild_overlay()?;
10814 }
10815 client.enqueue_startup_sync_if_needed()?;
10822 Ok(())
10823 })();
10824 match startup {
10825 Ok(()) => match client.conn.execute_batch("COMMIT") {
10826 Ok(()) => Ok(client),
10827 Err(error) => {
10828 let failure = Self::sqlite_failure(&client.storage_failure, error);
10829 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
10830 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
10832 }
10833 Err(failure)
10834 }
10835 },
10836 Err(error) => {
10837 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
10838 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
10840 }
10841 Err(error)
10842 }
10843 }
10844 }
10845
10846 pub fn set_now_ms(&mut self, now_ms: i64) {
10849 self.now_ms = Some(now_ms);
10850 }
10851
10852 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
10856 self.encryption = encryption;
10857 }
10858
10859 #[must_use]
10860 pub fn security_lifecycle(&self) -> &'static str {
10861 if self.security_preflight {
10862 "preflight"
10863 } else {
10864 "active"
10865 }
10866 }
10867
10868 #[must_use]
10869 pub fn security_preflight(&self) -> bool {
10870 self.security_preflight
10871 }
10872
10873 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
10875 if self.transport_enabled != enabled {
10876 self.transport_enabled = enabled;
10877 self.sync_intent_queue.clear();
10878 self.sync_intent_queue
10879 .push_back(if enabled && !self.security_preflight {
10880 SyncIntent::Interactive
10881 } else {
10882 SyncIntent::None
10883 });
10884 }
10885 if !enabled && self.active_round.is_none() {
10887 self.disconnect_realtime(transport);
10888 }
10889 }
10890
10891 pub fn transport_enabled(&self) -> bool {
10892 self.transport_enabled
10893 }
10894
10895 pub fn begin_security_preflight(&mut self) {
10902 self.seal_security_on_teardown();
10903 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
10906 }
10907
10908 pub fn seal_security_on_teardown(&mut self) {
10918 self.cancel_sync_round();
10919 self.security_preflight = true;
10920 self.encryption = crate::values::EncryptionConfig::default();
10921 self.sync_intent_queue.clear();
10922 }
10923
10924 pub fn activate_security(
10926 &mut self,
10927 encryption: crate::values::EncryptionConfig,
10928 ) -> Result<(), String> {
10929 if !self.security_preflight {
10930 return Err(
10931 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
10932 );
10933 }
10934 self.enqueue_startup_sync_if_needed()?;
10935 self.encryption = encryption;
10936 self.security_preflight = false;
10937 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
10938 Ok(())
10939 }
10940
10941 fn clock_now_ms(&self) -> i64 {
10942 self.now_ms.unwrap_or_else(crate::wall_clock_ms)
10943 }
10944
10945 fn create_bookkeeping_tables(&self) -> Result<(), String> {
10946 self.conn
10948 .execute_batch(
10949 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
10950 seq INTEGER PRIMARY KEY AUTOINCREMENT,
10951 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
10952 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
10953 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
10954 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
10955 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
10956 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
10957 PRIMARY KEY(commit_id, idx));
10958 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
10959 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
10960 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
10961 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
10962 seq INTEGER PRIMARY KEY AUTOINCREMENT,
10963 client_commit_id TEXT NOT NULL UNIQUE,
10964 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
10965 recorded_at_ms INTEGER NOT NULL,
10966 results_json TEXT NOT NULL,
10967 operations_json TEXT,
10968 resolution TEXT NOT NULL DEFAULT 'active'
10969 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
10970 resolved_at_ms INTEGER,
10971 replacement_client_commit_id TEXT);
10972 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
10973 ON _syncular_commit_outcomes(resolution, seq);
10974 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
10975 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
10976 CREATE TABLE IF NOT EXISTS _syncular_meta (
10977 key TEXT PRIMARY KEY, value TEXT NOT NULL);
10978 CREATE TABLE IF NOT EXISTS _syncular_windows (
10979 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
10980 PRIMARY KEY (base, unit));
10981 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
10982 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
10983 effective_scopes TEXT NOT NULL);
10984 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
10985 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
10986 PRIMARY KEY(commit_id, blob_id));
10987 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
10988 ON _syncular_blob_commit_refs(blob_id);",
10989 )
10990 .map_err(|e| e.to_string())?;
10991 let _ = self
10994 .conn
10995 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
10996 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
10997 self.set_meta(LOCAL_REVISION_KEY, "0");
10998 }
10999 if self.schema_has_blobs() {
11002 let exists = self
11003 .conn
11004 .query_row(
11005 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
11006 [],
11007 |row| row.get::<_, bool>(0),
11008 )
11009 .map_err(|e| e.to_string())?;
11010 if exists {
11011 let mut statement = self
11012 .conn
11013 .prepare("PRAGMA table_info(_syncular_blobs)")
11014 .map_err(|e| e.to_string())?;
11015 let mut columns = statement
11016 .query_map([], |row| row.get::<_, String>(1))
11017 .map_err(|e| e.to_string())?
11018 .collect::<Result<Vec<_>, _>>()
11019 .map_err(|e| e.to_string())?;
11020 columns.sort();
11021 let mut expected = vec![
11022 "blob_id",
11023 "byte_length",
11024 "bytes",
11025 "created_at_ms",
11026 "media_type",
11027 ];
11028 expected.sort_unstable();
11029 if columns != expected {
11030 return Err(
11031 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
11032 );
11033 }
11034 } else {
11035 self.conn
11036 .execute_batch(
11037 "CREATE TABLE _syncular_blobs(
11038 blob_id TEXT PRIMARY KEY,
11039 bytes BLOB NOT NULL,
11040 byte_length INTEGER NOT NULL,
11041 media_type TEXT,
11042 created_at_ms INTEGER NOT NULL);",
11043 )
11044 .map_err(|e| e.to_string())?;
11045 }
11046 self.conn
11047 .execute_batch(
11048 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
11049 blob_id TEXT PRIMARY KEY,
11050 media_type TEXT,
11051 created_at_ms INTEGER NOT NULL);",
11052 )
11053 .map_err(|e| e.to_string())?;
11054 }
11055 Ok(())
11056 }
11057
11058 fn schema_has_blobs(&self) -> bool {
11060 self.schema
11061 .tables
11062 .iter()
11063 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
11064 }
11065
11066 fn get_meta(&self, key: &str) -> Option<String> {
11069 meta_get(&self.conn, key)
11070 }
11071
11072 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
11073 self.conn
11074 .query_row(
11075 "SELECT value FROM _syncular_meta WHERE key = ?1",
11076 rusqlite::params![key],
11077 |row| row.get::<_, String>(0),
11078 )
11079 .optional()
11080 .map_err(|_| {
11081 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11082 })
11083 }
11084
11085 fn persist_schema_version(&self) -> Result<(), String> {
11088 self.conn
11089 .execute(
11090 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
11091 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
11092 )
11093 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11094 set_local_schema_descriptor(&self.conn, &self.schema)
11095 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11096 }
11097
11098 fn set_meta(&self, key: &str, value: &str) {
11099 meta_set(&self.conn, key, value);
11100 }
11101
11102 fn delete_meta(&self, key: &str) {
11103 meta_delete(&self.conn, key);
11104 }
11105
11106 fn load_outbox(&self) -> Result<Vec<OutboxCommit>, String> {
11108 let mut stmt = self
11109 .conn
11110 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
11111 .map_err(|error| error.to_string())?;
11112 let rows = stmt
11113 .query_map([], |row| {
11114 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11115 })
11116 .map_err(|error| error.to_string())?;
11117 let mut commits = Vec::new();
11118 for row in rows {
11119 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
11120 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
11121 format!("invalid persisted outbox {client_commit_id:?}: {error}")
11122 })?;
11123 let mut ops = Vec::with_capacity(entries.len());
11124 for entry in entries {
11125 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
11126 ops.push(OutboxOp {
11127 upsert: op == "upsert",
11128 table: entry
11129 .get("table")
11130 .and_then(Value::as_str)
11131 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
11132 .to_owned(),
11133 row_id: entry
11134 .get("rowId")
11135 .and_then(Value::as_str)
11136 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
11137 .to_owned(),
11138 base_version: entry.get("baseVersion").and_then(Value::as_i64),
11139 values: entry.get("values").and_then(Value::as_object).cloned(),
11142 });
11143 }
11144 commits.push(OutboxCommit {
11145 client_commit_id,
11146 ops,
11147 });
11148 }
11149 Ok(commits)
11150 }
11151
11152 fn restore_persisted_state(&mut self) -> Result<(), String> {
11153 self.subs = {
11154 let mut stmt = self
11155 .conn
11156 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
11157 .map_err(|error| error.to_string())?;
11158 let rows = stmt
11159 .query_map([], |row| {
11160 Ok((
11161 row.get::<_, String>(0)?,
11162 row.get::<_, String>(1)?,
11163 row.get::<_, String>(2)?,
11164 ))
11165 })
11166 .map_err(|error| error.to_string())?;
11167 let mut subscriptions = Vec::new();
11168 for row in rows {
11169 let (id, table, raw) = row.map_err(|error| error.to_string())?;
11170 let state: Value = serde_json::from_str(&raw)
11171 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
11172 let requested = json_to_scope_map(
11173 state.get("requested").unwrap_or(&Value::Object(Map::new())),
11174 )?;
11175 let effective = state
11176 .get("effectiveScopes")
11177 .filter(|value| !value.is_null())
11178 .map(json_to_scope_map)
11179 .transpose()?;
11180 subscriptions.push(Subscription {
11181 id,
11182 table,
11183 requested,
11184 params: state
11185 .get("params")
11186 .and_then(Value::as_str)
11187 .map(str::to_owned),
11188 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
11189 bootstrap_state: state
11190 .get("bootstrapState")
11191 .and_then(Value::as_str)
11192 .map(str::to_owned),
11193 state: SubState::parse(
11194 state
11195 .get("status")
11196 .and_then(Value::as_str)
11197 .unwrap_or("active"),
11198 ),
11199 reason_code: state
11200 .get("reasonCode")
11201 .and_then(Value::as_str)
11202 .map(str::to_owned),
11203 effective,
11204 synced_once: state
11205 .get("syncedOnce")
11206 .and_then(Value::as_bool)
11207 .unwrap_or(false),
11208 });
11209 }
11210 subscriptions
11211 };
11212
11213 self.outbox = self.load_outbox()?;
11214
11215 self.prune_commit_outcomes()?;
11216 let active = self.commit_outcomes(CommitOutcomeQuery {
11217 active_only: true,
11218 ..CommitOutcomeQuery::default()
11219 })?;
11220 self.conflicts = active
11221 .iter()
11222 .flat_map(|outcome| outcome.results.iter())
11223 .filter_map(|result| match result {
11224 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
11225 _ => None,
11226 })
11227 .collect();
11228 self.rejections = active
11229 .iter()
11230 .flat_map(|outcome| outcome.results.iter())
11231 .filter_map(|result| match result {
11232 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
11233 _ => None,
11234 })
11235 .collect();
11236
11237 self.lease_state = self
11238 .get_meta(LEASE_STATE_KEY)
11239 .map(|raw| serde_json::from_str(&raw))
11240 .transpose()
11241 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
11242 self.schema_floor = self
11243 .get_meta(SCHEMA_FLOOR_KEY)
11244 .map(|raw| serde_json::from_str(&raw))
11245 .transpose()
11246 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
11247 self.stopped = self.schema_floor.is_some();
11248 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
11251 self.security_preflight = true;
11252 }
11253 Ok(())
11254 }
11255
11256 fn save_subscription_scope_schema(&self) {
11257 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
11258 .schema
11259 .tables
11260 .iter()
11261 .map(|table| {
11262 (
11263 table.name.clone(),
11264 table
11265 .scope_variables
11266 .iter()
11267 .map(|scope| {
11268 (
11269 scope.variable.clone(),
11270 [scope.prefix.clone(), scope.column.clone()],
11271 )
11272 })
11273 .collect(),
11274 )
11275 })
11276 .collect();
11277 self.set_meta(
11278 "subscriptionScopeSchema",
11279 &serde_json::to_string(&declarations).expect("scope schema serialization"),
11280 );
11281 }
11282
11283 fn prune_unknown_subscriptions(&mut self, bump: bool) -> Result<(), String> {
11285 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
11286 self.get_meta("subscriptionScopeSchema")
11287 .map(|raw| serde_json::from_str(&raw))
11288 .transpose()
11289 .map_err(|error| format!("invalid persisted scope schema: {error}"))?
11290 } else {
11291 None
11292 };
11293 let compatible = |table_name: &str, requested: &[(String, Vec<String>)]| {
11294 let Some(table) = self.schema.table(table_name) else {
11295 return false;
11296 };
11297 if requested
11298 .iter()
11299 .any(|(variable, _)| table.scope_column(variable).is_none())
11300 {
11301 return false;
11302 }
11303 if !bump {
11304 return true;
11305 }
11306 let Some(old) = previous.as_ref().and_then(|tables| tables.get(table_name)) else {
11307 return false;
11308 };
11309 let variables: BTreeSet<_> = if requested.is_empty() {
11310 old.keys()
11311 .cloned()
11312 .chain(
11313 table
11314 .scope_variables
11315 .iter()
11316 .map(|scope| scope.variable.clone()),
11317 )
11318 .collect()
11319 } else {
11320 requested
11321 .iter()
11322 .map(|(variable, _)| variable.clone())
11323 .collect()
11324 };
11325 variables.iter().all(|variable| {
11326 let current = table
11327 .scope_variables
11328 .iter()
11329 .find(|scope| &scope.variable == variable);
11330 match (old.get(variable), current) {
11331 (Some(old), Some(current)) => {
11332 old == &[current.prefix.clone(), current.column.clone()]
11333 }
11334 _ => false,
11335 }
11336 })
11337 };
11338 let mut stale_ids: BTreeSet<_> = self
11339 .subs
11340 .iter()
11341 .filter(|sub| !compatible(&sub.table, &sub.requested))
11342 .map(|sub| sub.id.clone())
11343 .collect();
11344 for (id, table, effective) in self.load_pending_evictions()? {
11345 if !compatible(&table, &effective) {
11346 stale_ids.insert(id);
11347 }
11348 }
11349 for id in &stale_ids {
11350 for sql in [
11351 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
11352 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
11353 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
11354 ] {
11355 self.conn
11356 .execute(sql, rusqlite::params![id])
11357 .map_err(|error| error.to_string())?;
11358 }
11359 }
11360 self.subs.retain(|sub| !stale_ids.contains(&sub.id));
11361 Ok(())
11362 }
11363
11364 #[must_use]
11365 pub fn local_revision(&self) -> u64 {
11366 self.get_meta(LOCAL_REVISION_KEY)
11367 .and_then(|value| value.parse().ok())
11368 .unwrap_or(0)
11369 }
11370
11371 pub fn snapshot_read(
11377 &mut self,
11378 request: &SnapshotReadRequest<'_>,
11379 owner: Option<QueryOwner<'_>>,
11380 ) -> Result<SnapshotRead, QueryReadFailure> {
11381 let result = snapshot_read_connection(&self.conn, request);
11382 if let Some(owner) = owner {
11383 self.record_query_read(owner, result.as_ref().err());
11384 }
11385 result
11386 }
11387
11388 #[must_use]
11389 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
11390 SyncStatusSnapshot {
11391 current_schema_version: self.schema.version,
11392 outbox: self.outbox.len(),
11393 upgrading: self.upgrading,
11394 lease_state: self.lease_state.clone(),
11395 schema_floor: self.schema_floor.clone(),
11396 sync_needed: self.sync_needed,
11397 previous_version_context: self.previous_version_status(),
11398 }
11399 }
11400
11401 fn previous_version_status(&self) -> PreviousVersionStatus {
11404 let Some(replica_path) = self.previous_version_replica_path() else {
11405 return PreviousVersionStatus {
11406 present: false,
11407 created_at_ms: None,
11408 };
11409 };
11410 match crate::previous_version::read_previous_version_record(&replica_path) {
11411 Ok(Some(record)) => PreviousVersionStatus {
11412 present: true,
11413 created_at_ms: Some(record.created_at_ms),
11414 },
11415 _ => PreviousVersionStatus {
11416 present: std::path::Path::new(
11417 &crate::previous_version::previous_version_container_path(&replica_path),
11418 )
11419 .exists(),
11420 created_at_ms: None,
11421 },
11422 }
11423 }
11424
11425 pub fn previous_version_snapshot(
11431 &self,
11432 spec: &PreviousVersionReadSpec,
11433 ) -> Result<PreviousVersionSnapshot, String> {
11434 let now_ms = self.clock_now_ms();
11435 crate::previous_version::previous_version_snapshot(
11436 &self.conn,
11437 self.previous_version_replica_path().as_deref(),
11438 self.previous_version.as_ref(),
11439 PreviousVersionLifecycle {
11440 lease_inactive: self.previous_version_lease_inactive(now_ms),
11441 scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
11442 coverage_complete: self.previous_version_coverage_complete(),
11443 },
11444 now_ms,
11445 self.schema.version,
11446 spec,
11447 )
11448 }
11449
11450 #[must_use]
11452 pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
11453 crate::previous_version::stored_previous_version_audit(&self.conn)
11454 }
11455
11456 pub fn previous_version_discard(
11460 &self,
11461 ) -> crate::previous_version::PreviousVersionDiscardOutcome {
11462 self.drop_previous_version()
11463 }
11464
11465 fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
11469 let absent = crate::previous_version::PreviousVersionDiscardOutcome {
11470 present: false,
11471 discarded: false,
11472 };
11473 let Some(replica_path) = self.previous_version_replica_path() else {
11474 return absent;
11475 };
11476 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
11477 .unwrap_or(absent)
11478 }
11479
11480 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
11482 let Some(lease) = &self.lease_state else {
11483 return false;
11484 };
11485 if lease.error_code.is_some() {
11486 return true;
11487 }
11488 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
11489 }
11490
11491 fn previous_version_coverage_complete(&self) -> bool {
11495 let active = self
11496 .subs
11497 .iter()
11498 .filter(|sub| sub.state == SubState::Active)
11499 .collect::<Vec<_>>();
11500 if active.is_empty() {
11501 return false;
11502 }
11503 active
11504 .iter()
11505 .all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
11506 }
11507
11508 fn previous_version_lifetime_check(&self) {
11512 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
11513 return;
11514 };
11515 let now_ms = self.clock_now_ms();
11516 if self.previous_version_lease_inactive(now_ms) {
11517 let _ = self.drop_previous_version();
11518 return;
11519 }
11520 let Some(replica_path) = self.previous_version_replica_path() else {
11521 return;
11522 };
11523 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
11524 else {
11525 return;
11526 };
11527 if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
11528 let _ = self.drop_previous_version();
11529 return;
11530 }
11531 if now_ms - record.created_at_ms > config.max_age_ms {
11532 let _ = self.drop_previous_version();
11533 return;
11534 }
11535 if self.previous_version_coverage_complete() {
11536 let _ = self.drop_previous_version();
11537 }
11538 }
11539
11540 pub fn diagnostics_snapshot(
11541 &self,
11542 request: &ClientDiagnosticsRequest,
11543 ) -> Result<ClientDiagnosticsSnapshot, String> {
11544 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
11545 return Err(format!(
11546 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
11547 ));
11548 }
11549 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
11550 for sub in &self.subs {
11551 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
11552 let complete =
11553 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
11554 let state = match sub.state {
11555 SubState::Revoked => "revoked",
11556 SubState::Failed => "failed",
11557 SubState::Active if reset => "reset",
11558 SubState::Active if complete => "complete",
11559 SubState::Active => "bootstrapping",
11560 };
11561 subscriptions.insert(
11562 sub.id.clone(),
11563 DiagnosticSubscription {
11564 id: sub.id.clone(),
11565 table: sub.table.clone(),
11566 state: state.to_owned(),
11567 complete,
11568 cursor: Some(sub.cursor),
11569 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
11570 },
11571 );
11572 }
11573 for expected in &request.expected_subscriptions {
11574 if expected.id.is_empty() || expected.table.is_empty() {
11575 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
11576 }
11577 if subscriptions
11578 .get(&expected.id)
11579 .is_some_and(|registered| registered.table != expected.table)
11580 {
11581 subscriptions.insert(
11582 expected.id.clone(),
11583 DiagnosticSubscription {
11584 id: expected.id.clone(),
11585 table: expected.table.clone(),
11586 state: "failed".to_owned(),
11587 complete: false,
11588 cursor: None,
11589 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
11590 },
11591 );
11592 } else {
11593 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
11594 DiagnosticSubscription {
11595 id: expected.id.clone(),
11596 table: expected.table.clone(),
11597 state: "unregistered".to_owned(),
11598 complete: false,
11599 cursor: None,
11600 reason_code: None,
11601 }
11602 });
11603 }
11604 }
11605 let mut ordered_subscriptions = Vec::new();
11606 let mut included = BTreeSet::new();
11607 for expected in &request.expected_subscriptions {
11608 if included.insert(expected.id.clone()) {
11609 if let Some(subscription) = subscriptions.get(&expected.id) {
11610 ordered_subscriptions.push(subscription.clone());
11611 }
11612 }
11613 }
11614 for (id, subscription) in &subscriptions {
11615 if included.insert(id.clone()) {
11616 ordered_subscriptions.push(subscription.clone());
11617 }
11618 }
11619 let subscriptions_truncated =
11620 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
11621 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
11622 let captured_at_ms = self.clock_now_ms();
11623 let lease = if let Some(error_code) = self
11624 .lease_state
11625 .as_ref()
11626 .and_then(|state| state.error_code.clone())
11627 {
11628 ClientDiagnosticsLease {
11629 state: "stopped".to_owned(),
11630 expires_at_ms: self
11631 .lease_state
11632 .as_ref()
11633 .and_then(|state| state.expires_at_ms),
11634 error_code: Some(Self::diagnostic_code(&error_code)),
11635 }
11636 } else if let Some(expires_at_ms) = self
11637 .lease_state
11638 .as_ref()
11639 .and_then(|state| state.expires_at_ms)
11640 {
11641 ClientDiagnosticsLease {
11642 state: if expires_at_ms <= captured_at_ms {
11643 "expired".to_owned()
11644 } else {
11645 "active".to_owned()
11646 },
11647 expires_at_ms: Some(expires_at_ms),
11648 error_code: None,
11649 }
11650 } else {
11651 ClientDiagnosticsLease {
11652 state: "none".to_owned(),
11653 expires_at_ms: None,
11654 error_code: None,
11655 }
11656 };
11657 let connectivity = match self.last_round.as_ref() {
11658 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
11659 Some(round) if round.status == "succeeded" => "online",
11660 Some(round)
11661 if round.status == "failed"
11662 && round
11663 .error_code
11664 .as_deref()
11665 .is_some_and(Self::retryable_transport_code) =>
11666 {
11667 "offline"
11668 }
11669 _ => "unknown",
11670 };
11671 Ok(ClientDiagnosticsSnapshot {
11672 version: CLIENT_DIAGNOSTICS_VERSION,
11673 captured_at_ms,
11674 host: ClientDiagnosticsHost {
11675 kind: "direct".to_owned(),
11676 role: "single".to_owned(),
11677 connectivity: connectivity.to_owned(),
11678 realtime: self.realtime_state,
11679 realtime_policy: self.realtime_policy,
11680 realtime_reason_code: self.realtime_reason_code.clone(),
11681 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
11682 },
11683 security_lifecycle: self.security_lifecycle().to_owned(),
11684 schema: ClientDiagnosticsSchema {
11685 current_version: self.schema.version,
11686 upgrading: self.upgrading,
11687 required_version: self
11688 .schema_floor
11689 .as_ref()
11690 .and_then(|floor| floor.required_schema_version),
11691 latest_version: self
11692 .schema_floor
11693 .as_ref()
11694 .and_then(|floor| floor.latest_schema_version),
11695 },
11696 replica: ClientDiagnosticsReplica {
11697 local_revision: self.local_revision().to_string(),
11698 sync_needed: self.sync_needed,
11699 pending_outbox: self.outbox.len(),
11700 },
11701 lease,
11702 subscriptions: ordered_subscriptions,
11703 subscriptions_truncated,
11704 last_round: self.last_round.clone(),
11705 last_change: self.last_change.clone(),
11706 storage: self.diagnostics_storage(),
11707 query_failures: self.query_failures.clone(),
11708 })
11709 }
11710
11711 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
11712 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
11713 let page_count: i64 = self
11714 .conn
11715 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
11716 let page_size: i64 = self
11717 .conn
11718 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
11719 let outbox_bytes: i64 = self
11721 .conn
11722 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
11723 .query_row([], |row| row.get(0))?;
11724 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
11725 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
11726 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
11727 let blob_bytes = if self.schema_has_blobs() {
11728 self.conn
11729 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
11730 .query_row([], |row| row.get(0))?
11731 } else {
11732 0
11733 };
11734 let pressure = self
11735 .limits
11736 .blob_cache_max_bytes
11737 .is_some_and(|limit| blob_bytes > limit);
11738 Ok(ClientDiagnosticsStorage {
11739 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
11740 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
11741 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
11742 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
11743 retained_outcome_entries: Some(outcome_entries.max(0)),
11744 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
11745 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
11746 })
11747 };
11748 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
11749 status: "unreadable".to_owned(),
11750 database_bytes_approx: None,
11751 pending_outbox_bytes_approx: None,
11752 retained_outcome_bytes_approx: None,
11753 retained_outcome_entries: None,
11754 blob_cache_bytes_approx: None,
11755 pressure_reason_code: None,
11756 })
11757 }
11758
11759 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
11760 self.change_queue.drain(..).collect()
11761 }
11762
11763 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
11764 let intents = self.sync_intent_queue.drain(..).collect();
11765 if self.transport_enabled {
11766 intents
11767 } else {
11768 vec![SyncIntent::None]
11769 }
11770 }
11771
11772 fn schedule_background_retry(&mut self) -> u64 {
11773 let delay_ms = self.retry_delay_ms;
11774 self.sync_intent_queue
11775 .push_back(SyncIntent::Background { delay_ms });
11776 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
11777 self.round_retry_delay_ms = Some(delay_ms);
11778 delay_ms
11779 }
11780
11781 fn reset_background_retry(&mut self) {
11782 self.retry_delay_ms = 250;
11783 }
11784
11785 fn retryable_transport_code(code: &str) -> bool {
11786 code == "transport.failed"
11787 || code == "transport.unavailable"
11788 || code == "sync.transport_failed"
11789 }
11790
11791 fn retryable_failure_code(code: &str) -> bool {
11796 Self::retryable_transport_code(code)
11797 || matches!(
11798 code,
11799 "sync.auth_required"
11800 | "sync.auth_lease_required"
11801 | "sync.auth_lease_revoked"
11802 | "sync.internal_error"
11803 | "sync.idempotency_cache_miss"
11804 | "sync.schema_not_ready"
11805 | "sync.segment_expired"
11806 | "sync.rate_limited"
11807 | "sync.websocket_connection_limit"
11808 )
11809 }
11810
11811 fn diagnostic_code(code: &str) -> String {
11812 let valid = !code.is_empty()
11813 && code.len() <= 96
11814 && code.contains('.')
11815 && code
11816 .bytes()
11817 .next()
11818 .is_some_and(|byte| byte.is_ascii_lowercase())
11819 && code.bytes().all(|byte| {
11820 byte.is_ascii_lowercase()
11821 || byte.is_ascii_digit()
11822 || matches!(byte, b'.' | b'_' | b'-')
11823 });
11824 if valid {
11825 code.to_owned()
11826 } else {
11827 "client.unknown_failure".to_owned()
11828 }
11829 }
11830
11831 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
11832 if self.sync_needed != value {
11833 if self.begin_observation("syncular_status").is_ok() {
11834 self.sync_needed = value;
11835 let batch = ChangeAccumulator {
11836 status: true,
11837 ..ChangeAccumulator::default()
11838 };
11839 if self.finish_observation("syncular_status", batch).is_err() {
11840 self.rollback_observation("syncular_status");
11841 }
11842 } else {
11843 self.sync_needed = value;
11844 }
11845 }
11846 if value && interactive {
11847 self.sync_intent_queue.push_back(SyncIntent::Interactive);
11848 }
11849 }
11850
11851 fn sqlite_failure(
11852 storage_failure: &RefCell<Option<QueryReadFailure>>,
11853 error: rusqlite::Error,
11854 ) -> String {
11855 let message = error.to_string();
11856 let failure = QueryReadFailure::from(error);
11857 let code = failure.code;
11858 if code.is_some() && storage_failure.borrow().is_none() {
11859 *storage_failure.borrow_mut() = Some(failure);
11860 }
11861 match code {
11862 Some(code) => format!("{code}: {message}"),
11863 None => message,
11864 }
11865 }
11866
11867 fn begin_observation(&self, name: &str) -> Result<(), String> {
11868 if let Some(failure) = &self.coalesced_failure {
11869 return Err(failure.clone());
11870 }
11871 if self.coalesced.is_some() && self.conn.is_autocommit() {
11874 return Err("the coalesced apply transaction was rolled back".to_owned());
11875 }
11876 self.conn
11877 .execute_batch(&format!("SAVEPOINT {name}"))
11878 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11879 }
11880
11881 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
11884 if self.coalesced.is_some() || !self.conn.is_autocommit() {
11885 return Ok(());
11886 }
11887 self.conn
11888 .execute_batch("BEGIN")
11889 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11890 self.coalesced = Some(CoalescedApply {
11891 rows: 0,
11892 subs: self.subs.clone(),
11893 outbox: self.outbox.clone(),
11894 failed_commits: self.failed_commits.clone(),
11895 conflicts: self.conflicts.clone(),
11896 rejections: self.rejections.clone(),
11897 schema_floor: self.schema_floor.clone(),
11898 lease_state: self.lease_state.clone(),
11899 stopped: self.stopped,
11900 upgrading: self.upgrading,
11901 sync_needed: self.sync_needed,
11902 overlay_dirty: self.overlay_dirty.snapshot(),
11903 change_queue: self.change_queue.len(),
11904 sync_intent_queue: self.sync_intent_queue.len(),
11905 last_change: self.last_change.clone(),
11906 });
11907 Ok(())
11908 }
11909
11910 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
11913 let Some(open) = &self.coalesced else {
11914 return Ok(());
11915 };
11916 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
11917 self.commit_coalesced_apply()?;
11918 self.begin_coalesced_apply()?;
11919 }
11920 if let Some(open) = &mut self.coalesced {
11921 open.rows += rows;
11922 }
11923 Ok(())
11924 }
11925
11926 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
11931 let Some(open) = self.coalesced.take() else {
11932 return Ok(());
11933 };
11934 let committed = if self.conn.is_autocommit() {
11935 Err("the coalesced apply transaction was rolled back".to_owned())
11936 } else {
11937 self.conn
11938 .execute_batch("COMMIT")
11939 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11940 };
11941 let Err(error) = committed else {
11942 return Ok(());
11943 };
11944 if !self.conn.is_autocommit() {
11945 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
11946 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
11947 }
11948 }
11949 self.subs = open.subs;
11950 self.outbox = open.outbox;
11951 self.failed_commits = open.failed_commits;
11952 self.conflicts = open.conflicts;
11953 self.rejections = open.rejections;
11954 self.schema_floor = open.schema_floor;
11955 self.lease_state = open.lease_state;
11956 self.stopped = open.stopped;
11957 self.upgrading = open.upgrading;
11958 self.sync_needed = open.sync_needed;
11959 self.overlay_dirty.restore(open.overlay_dirty);
11960 self.change_queue.truncate(open.change_queue);
11961 self.sync_intent_queue.truncate(open.sync_intent_queue);
11962 self.last_change = open.last_change;
11963 self.coalesced_failure = Some(error.clone());
11964 Err(error)
11965 }
11966
11967 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
11969 let committed = self.commit_coalesced_apply();
11970 match self.coalesced_failure.take() {
11971 Some(failure) => Err(failure),
11972 None => committed,
11973 }
11974 }
11975
11976 fn rollback_observation(&self, name: &str) {
11977 if let Err(error) = self
11978 .conn
11979 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
11980 {
11981 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
11982 if failure.rollback_failure.is_none() {
11983 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
11984 }
11985 }
11986 }
11987 }
11988
11989 fn finish_observation(
11990 &mut self,
11991 name: &str,
11992 mut batch: ChangeAccumulator,
11993 ) -> Result<(), String> {
11994 #[cfg(feature = "bench-internals")]
11995 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
11996 if self.failed_commits.iter().any(|(commit, _)| {
11997 commit
11998 .ops
11999 .iter()
12000 .any(|operation| batch.tables.contains_key(&operation.table))
12001 }) {
12002 batch.outcomes = true;
12003 }
12004 if !batch.touched() {
12005 self.conn
12006 .execute_batch(&format!("RELEASE {name}"))
12007 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12008 return Ok(());
12009 }
12010 let revision = self
12011 .local_revision()
12012 .checked_add(1)
12013 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
12014 self.conn
12015 .execute(
12016 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
12017 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
12018 )
12019 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12020 let status = batch.status.then(|| self.status_snapshot());
12021 let event = ClientChangeBatch {
12022 revision: revision.to_string(),
12023 tables: batch
12024 .tables
12025 .into_iter()
12026 .map(|(table, scope_keys)| TableChange {
12027 table,
12028 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
12029 })
12030 .collect(),
12031 windows: batch
12032 .windows
12033 .into_iter()
12034 .map(|((base_key, table), units)| WindowChange {
12035 base_key,
12036 table,
12037 units: units.into_iter().collect(),
12038 })
12039 .collect(),
12040 status,
12041 conflicts_changed: batch.conflicts,
12042 rejections_changed: batch.rejections,
12043 outcomes_changed: batch.outcomes,
12044 };
12045 self.conn
12046 .execute_batch(&format!("RELEASE {name}"))
12047 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12048 let mut diagnostic_tables = event
12049 .tables
12050 .iter()
12051 .map(|entry| entry.table.clone())
12052 .collect::<BTreeSet<_>>()
12053 .into_iter()
12054 .collect::<Vec<_>>();
12055 let mut diagnostic_windows = event
12056 .windows
12057 .iter()
12058 .map(|entry| entry.table.clone())
12059 .collect::<BTreeSet<_>>()
12060 .into_iter()
12061 .collect::<Vec<_>>();
12062 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
12063 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
12064 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
12065 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
12066 self.last_change = Some(DiagnosticLastChange {
12067 revision: event.revision.clone(),
12068 recorded_at_ms: self.clock_now_ms(),
12069 tables: diagnostic_tables,
12070 windows: diagnostic_windows,
12071 domains_truncated,
12072 status_changed: event.status.is_some(),
12073 conflicts_changed: event.conflicts_changed,
12074 rejections_changed: event.rejections_changed,
12075 outcomes_changed: event.outcomes_changed,
12076 });
12077 self.change_queue.push_back(event);
12078 Ok(())
12079 }
12080
12081 fn record_scope_map(
12082 &self,
12083 batch: &mut ChangeAccumulator,
12084 table_name: &str,
12085 scopes: &[(String, Vec<String>)],
12086 ) {
12087 let Some(table) = self.schema.table(table_name) else {
12088 return;
12089 };
12090 for (variable, values) in scopes {
12091 let Some(scope) = table
12092 .scope_variables
12093 .iter()
12094 .find(|scope| &scope.variable == variable)
12095 else {
12096 continue;
12097 };
12098 for value in values {
12099 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12100 }
12101 }
12102 }
12103
12104 fn record_row_scopes(
12106 &self,
12107 batch: &mut ChangeAccumulator,
12108 table_name: &str,
12109 row_id: &str,
12110 base: bool,
12111 ) -> bool {
12112 let Some(table) = self.schema.table(table_name) else {
12113 return false;
12114 };
12115 if table.scope_variables.is_empty() {
12116 return false;
12117 }
12118 let columns = table
12119 .scope_variables
12120 .iter()
12121 .map(|scope| quote_ident(&scope.column))
12122 .collect::<Vec<_>>()
12123 .join(", ");
12124 let full_table = if base {
12125 base_table(table_name)
12126 } else {
12127 visible_table(table_name)
12128 };
12129 let sql = format!(
12130 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
12131 row_id_predicate(table)
12132 );
12133 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
12134 return false;
12135 };
12136 let values = stmt.query_row(rusqlite::params![row_id], |row| {
12137 let mut values = Vec::with_capacity(table.scope_variables.len());
12138 for index in 0..table.scope_variables.len() {
12139 values.push(row.get::<_, Option<String>>(index)?);
12140 }
12141 Ok(values)
12142 });
12143 let Ok(values) = values else {
12144 return false;
12145 };
12146 let mut recorded = false;
12147 for (scope, value) in table.scope_variables.iter().zip(values) {
12148 if let Some(value) = value {
12149 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12150 recorded = true;
12151 }
12152 }
12153 recorded
12154 }
12155
12156 fn record_commit_changes(
12157 &self,
12158 batch: &mut ChangeAccumulator,
12159 tables: &[String],
12160 changes: &[ssp2::model::Change],
12161 ) {
12162 #[cfg(feature = "bench-internals")]
12163 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
12164 for change in changes {
12165 let Some(table_name) = tables.get(change.table_index as usize) else {
12166 continue;
12167 };
12168 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
12169 if let Some(table) = self.schema.table(table_name) {
12170 for (variable, value) in &change.scopes {
12171 if let Some(scope) = table
12172 .scope_variables
12173 .iter()
12174 .find(|scope| &scope.variable == variable)
12175 {
12176 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12177 precise = true;
12178 }
12179 }
12180 }
12181 if !precise {
12182 batch.table(table_name);
12183 }
12184 }
12185 }
12186
12187 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
12188 if effective.is_empty() {
12189 return false;
12190 }
12191 let Some(table) = self.schema.table(table_name) else {
12192 return false;
12193 };
12194 let mut clauses = Vec::new();
12195 let mut params = Vec::new();
12196 for (variable, values) in effective {
12197 let Some(column) = table.scope_column(variable) else {
12198 return false;
12199 };
12200 if values.is_empty() {
12201 return false;
12202 }
12203 let placeholders = values
12204 .iter()
12205 .map(|value| {
12206 params.push(SqlValue::Text(value.clone()));
12207 "?"
12208 })
12209 .collect::<Vec<_>>()
12210 .join(", ");
12211 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
12212 }
12213 let sql = format!(
12214 "SELECT 1 FROM {} WHERE {} LIMIT 1",
12215 base_table(table_name),
12216 clauses.join(" AND ")
12217 );
12218 self.conn
12219 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
12220 .is_ok()
12221 }
12222
12223 pub fn upgrading(&self) -> bool {
12225 self.upgrading
12226 }
12227
12228 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
12236 let new_schema = parse_schema_json(schema_json)?;
12237 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
12238 self.conn
12239 .execute_batch("BEGIN IMMEDIATE")
12240 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12241 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
12242 let prior_subs = self.subs.clone();
12248 let prior_outbox = self.outbox.clone();
12249 let prior_rejections = self.rejections.clone();
12250 let prior_upgrading = self.upgrading;
12251 let prior_stopped = self.stopped;
12252 let prior_schema_floor = self.schema_floor.clone();
12253 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12254 let prior_sync_needed = self.sync_needed;
12255 let prior_active_round = self.active_round;
12256 let prior_changes = self.change_queue.len();
12257 let prior_intents = self.sync_intent_queue.len();
12258 let prior_last_change = self.last_change.clone();
12259 let guarded = (|| -> Result<(), String> {
12260 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
12265 self.create_bookkeeping_tables()?;
12266 match marker {
12267 Some(version) if version < self.schema.version => self.run_schema_reset()?,
12268 None => {
12269 self.create_synced_tables()?;
12270 self.persist_schema_version()?;
12271 }
12272 _ => {}
12273 }
12274 self.outbox = self.load_outbox()?;
12279 self.drop_unencodable_outbox(false, None)?;
12284 if marker == Some(self.schema.version)
12285 && (!self.outbox.is_empty()
12286 || !self.failed_commits.is_empty()
12287 || self.has_acknowledged_rows())
12288 {
12289 self.overlay_dirty.set(true);
12290 self.rebuild_overlay()?;
12291 }
12292 self.prune_unknown_subscriptions(false)?;
12293 self.save_subscription_scope_schema();
12294 self.enqueue_startup_sync_if_needed()?;
12298 Ok(())
12299 })();
12300 let outcome = match guarded {
12301 Ok(()) => self
12302 .conn
12303 .execute_batch("COMMIT")
12304 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
12305 Err(error) => Err(error),
12306 };
12307 if let Err(error) = outcome {
12308 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12309 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12311 }
12312 self.schema = prior_schema;
12313 self.subs = prior_subs;
12314 self.outbox = prior_outbox;
12315 self.rejections = prior_rejections;
12316 self.upgrading = prior_upgrading;
12317 self.stopped = prior_stopped;
12318 self.schema_floor = prior_schema_floor;
12319 self.overlay_dirty.restore(prior_overlay_dirty);
12320 self.sync_needed = prior_sync_needed;
12321 self.active_round = prior_active_round;
12322 self.change_queue.truncate(prior_changes);
12323 self.sync_intent_queue.truncate(prior_intents);
12324 self.last_change = prior_last_change;
12325 self.insert_sql.borrow_mut().clear();
12327 return Err(error);
12328 }
12329 Ok(())
12330 }
12331
12332 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
12333 let startup_work = !self.stopped
12334 && (!self.outbox.is_empty()
12335 || self.has_acknowledged_rows()
12336 || !self.load_pending_evictions()?.is_empty()
12337 || self.subs.iter().any(|sub| sub.state == SubState::Active));
12338 if startup_work {
12339 self.sync_needed = true;
12340 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12341 }
12342 Ok(())
12343 }
12344
12345 fn clear_satisfied_persisted_schema_floor(&mut self) {
12352 let satisfied = self
12353 .schema_floor
12354 .as_ref()
12355 .and_then(|floor| floor.required_schema_version)
12356 .is_some_and(|required| self.schema.version >= required);
12357 if !satisfied {
12358 return;
12359 }
12360 self.schema_floor = None;
12361 self.stopped = false;
12362 self.delete_meta(SCHEMA_FLOOR_KEY);
12363 }
12364
12365 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
12370 let Some(replica_path) = self.previous_version_replica_path() else {
12371 return Ok(());
12372 };
12373 let max_age_ms = self
12374 .previous_version
12375 .filter(|config| config.enabled)
12376 .map(|config| config.max_age_ms);
12377 reconcile_previous_version_at_boot(
12378 &self.conn,
12379 &replica_path,
12380 self.schema.version,
12381 max_age_ms,
12382 self.clock_now_ms(),
12383 )
12384 }
12385
12386 fn previous_version_replica_path(&self) -> Option<String> {
12390 let path = self.conn.path()?;
12391 if path.is_empty() || path.starts_with("file:") {
12392 return None;
12393 }
12394 Some(path.to_owned())
12395 }
12396
12397 fn run_schema_reset(&mut self) -> Result<(), String> {
12403 self.begin_observation("syncular_schema_reset")?;
12404 let prior_subs = self.subs.clone();
12405 let prior_outbox = self.outbox.clone();
12406 let prior_rejections = self.rejections.clone();
12407 let prior_upgrading = self.upgrading;
12408 let prior_stopped = self.stopped;
12409 let prior_schema_floor = self.schema_floor.clone();
12410 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12411 let prior_sync_needed = self.sync_needed;
12412 let prior_active_round = self.active_round;
12413 let mut batch = ChangeAccumulator::default();
12414 let result = self
12415 .run_schema_reset_observed(&mut batch, true)
12416 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
12417 if let Err(error) = result {
12418 self.rollback_observation("syncular_schema_reset");
12419 self.subs = prior_subs;
12420 self.outbox = prior_outbox;
12421 self.rejections = prior_rejections;
12422 self.upgrading = prior_upgrading;
12423 self.stopped = prior_stopped;
12424 self.schema_floor = prior_schema_floor;
12425 self.overlay_dirty.restore(prior_overlay_dirty);
12426 self.sync_needed = prior_sync_needed;
12427 self.active_round = prior_active_round;
12428 return Err(error);
12429 }
12430 Ok(())
12431 }
12432
12433 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
12434 if self.get_meta(LOG_EPOCH_KEY).is_none() {
12435 self.begin_observation("syncular_epoch_acquisition")?;
12436 let previous_sync_needed = self.sync_needed;
12437 if let Err(error) = self.conn.execute(
12438 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12439 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12440 ) {
12441 self.rollback_observation("syncular_epoch_acquisition");
12442 return Err(Self::sqlite_failure(&self.storage_failure, error));
12443 }
12444 self.sync_needed = true;
12445 let batch = ChangeAccumulator {
12446 status: true,
12447 ..ChangeAccumulator::default()
12448 };
12449 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
12450 self.rollback_observation("syncular_epoch_acquisition");
12451 self.sync_needed = previous_sync_needed;
12452 return Err(error);
12453 }
12454 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12455 return Ok(Vec::new());
12456 }
12457 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
12458 self.begin_observation("syncular_log_epoch_reset")?;
12459 let prior_subs = self.subs.clone();
12460 let prior_outbox = self.outbox.clone();
12461 let prior_rejections = self.rejections.clone();
12462 let prior_upgrading = self.upgrading;
12463 let prior_stopped = self.stopped;
12464 let prior_schema_floor = self.schema_floor.clone();
12465 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12466 let prior_sync_needed = self.sync_needed;
12467 let prior_active_round = self.active_round;
12468 let mut batch = ChangeAccumulator::default();
12469 let result = self
12470 .run_schema_reset_observed(&mut batch, false)
12471 .and_then(|()| {
12472 self.conn
12473 .execute(
12474 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12475 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12476 )
12477 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12478 self.sync_needed = true;
12479 batch.status = true;
12480 self.finish_observation("syncular_log_epoch_reset", batch)
12481 });
12482 if let Err(error) = result {
12483 self.rollback_observation("syncular_log_epoch_reset");
12484 self.subs = prior_subs;
12485 self.outbox = prior_outbox;
12486 self.rejections = prior_rejections;
12487 self.upgrading = prior_upgrading;
12488 self.stopped = prior_stopped;
12489 self.schema_floor = prior_schema_floor;
12490 self.overlay_dirty.restore(prior_overlay_dirty);
12491 self.sync_needed = prior_sync_needed;
12492 self.active_round = prior_active_round;
12493 return Err(error);
12494 }
12495 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12496 Ok(resets)
12497 }
12498
12499 fn run_schema_reset_observed(
12500 &mut self,
12501 batch: &mut ChangeAccumulator,
12502 capture: bool,
12503 ) -> Result<(), String> {
12504 read_local_schema_version(&self.conn, self.schema.version)?;
12510 self.conn
12511 .execute_batch(
12512 "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
12513 )
12514 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12515 self.cancel_sync_round();
12516 self.upgrading = true;
12517 batch.status = true;
12518 if self.previous_version.is_some_and(|config| config.enabled) {
12526 let previous_version = self
12527 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
12528 .and_then(|value| value.parse::<i32>().ok())
12529 .unwrap_or(self.schema.version);
12530 let pending = self
12531 .outbox
12532 .iter()
12533 .map(|commit| PendingCommitAudit {
12534 commit_id: &commit.client_commit_id,
12535 operations: commit_audit_operations(commit),
12536 })
12537 .collect::<Vec<_>>();
12538 crate::previous_version::write_previous_version_audit(
12539 &self.conn,
12540 &crate::previous_version::build_previous_version_audit(
12541 &self.schema,
12542 &pending,
12543 previous_version,
12544 self.schema.version,
12545 self.clock_now_ms(),
12546 ),
12547 );
12548 }
12549 if let Some(replica_path) = self.previous_version_replica_path() {
12554 sweep_previous_version_container(&self.conn, &replica_path)?;
12555 if capture {
12556 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
12557 let previous_version = self
12558 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
12559 .and_then(|value| value.parse::<i32>().ok())
12560 .unwrap_or(self.schema.version);
12561 capture_previous_version_from_replica(
12562 &self.conn,
12563 &replica_path,
12564 previous_version,
12565 self.schema.version,
12566 &config,
12567 self.clock_now_ms(),
12568 )?;
12569 }
12570 }
12571 }
12572 for table in &self.schema.tables {
12573 batch.table(&table.name);
12574 }
12575 for (base_key, unit, table) in self.load_registered_window_units() {
12576 batch.window(&base_key, &table, &unit);
12577 }
12578 if capture {
12579 self.prune_unknown_subscriptions(true)?;
12580 }
12581 self.insert_sql.borrow_mut().clear();
12583 self.overlay_dirty.set(true);
12584 let virtual_tables: Vec<String> = {
12592 let mut stmt = self
12593 .conn
12594 .prepare(
12595 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
12596 )
12597 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12598 let rows = stmt
12599 .query_map([], |row| row.get::<_, String>(0))
12600 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12601 rows.filter_map(Result::ok)
12602 .filter(|name| is_synced_table_name(name))
12603 .collect()
12604 };
12605 for name in virtual_tables {
12606 self.conn
12607 .execute_batch(&format!(
12608 "DROP TABLE IF EXISTS {}",
12609 quote_ident(&format!("_syncular_fts_{name}"))
12610 ))
12611 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12612 self.conn
12613 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
12614 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12615 }
12616 let existing: Vec<String> = {
12617 let mut stmt = self
12618 .conn
12619 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
12620 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12621 let rows = stmt
12622 .query_map([], |row| row.get::<_, String>(0))
12623 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12624 rows.filter_map(Result::ok)
12625 .filter(|name| is_synced_table_name(name))
12626 .collect()
12627 };
12628 for name in &existing {
12629 if let Some(table) = name.strip_prefix("_syncular_base_") {
12630 batch.table(table);
12631 } else if !name.starts_with("_syncular_") {
12632 batch.table(name);
12633 }
12634 }
12635 for name in existing {
12636 self.conn
12637 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
12638 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12639 }
12640 self.create_synced_tables()?;
12642 for sub in &mut self.subs {
12644 sub.cursor = -1;
12645 sub.bootstrap_state = None;
12646 sub.effective = None;
12647 sub.state = SubState::Active;
12648 sub.reason_code = None;
12649 sub.synced_once = false;
12650 }
12651 let subs = self.subs.clone();
12652 for sub in &subs {
12653 self.persist_sub(sub)?;
12654 }
12655 self.stopped = false;
12657 self.schema_floor = None;
12658 self.delete_meta(SCHEMA_FLOOR_KEY);
12659 self.persist_schema_version()?;
12661 self.save_subscription_scope_schema();
12662 self.drop_unencodable_outbox(false, Some(batch))?;
12673 self.rebuild_overlay()?;
12675 Ok(())
12676 }
12677
12678 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
12685 let incompatible = self
12686 .outbox
12687 .iter()
12688 .filter(|commit| {
12689 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
12690 })
12691 .cloned()
12692 .collect::<Vec<_>>();
12693 if incompatible.is_empty() {
12694 return Ok(false);
12695 }
12696 let prior_outbox = self.outbox.clone();
12702 let prior_rejections = self.rejections.clone();
12703 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12704 let owns_transaction = self.conn.is_autocommit();
12705 self.begin_observation("syncular_outbox_drop")?;
12706 let mut batch = ChangeAccumulator::default();
12707 let dropped = (|| -> Result<(), String> {
12708 let mut rejections = Vec::new();
12709 for commit in &incompatible {
12710 let results = commit
12711 .ops
12712 .iter()
12713 .enumerate()
12714 .map(|(op_index, operation)| {
12715 let rejection = RejectionRecord {
12716 client_commit_id: commit.client_commit_id.clone(),
12717 op_index: op_index as i32,
12718 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
12719 message: "the persisted commit cannot encode under the current schema"
12720 .to_owned(),
12721 retryable: false,
12722 details: None,
12723 operation: Some(CommitOperation::from(operation)),
12724 };
12725 rejections.push(rejection.clone());
12726 CommitOperationOutcome::Error { rejection }
12727 })
12728 .collect::<Vec<_>>();
12729 self.persist_commit_outcome(
12730 &commit.client_commit_id,
12731 CommitOutcomeStatus::Rejected,
12732 &results,
12733 Some(&commit.ops),
12734 )?;
12735 self.delete_outbox_persisted(&commit.client_commit_id)?;
12736 for operation in &commit.ops {
12739 if self.schema.table(&operation.table).is_some() {
12740 batch.table(&operation.table);
12741 }
12742 }
12743 }
12744 self.prune_commit_outcomes()?;
12745 let incompatible_ids = incompatible
12746 .iter()
12747 .map(|commit| commit.client_commit_id.as_str())
12748 .collect::<BTreeSet<_>>();
12749 self.outbox
12750 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
12751 self.rejections.extend(rejections);
12752 self.overlay_dirty.set(true);
12755 self.rebuild_overlay()?;
12756 batch.status = true;
12757 batch.rejections = true;
12758 batch.outcomes = true;
12759 Ok(())
12760 })();
12761 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
12762 if let Err(error) = result {
12763 if owns_transaction {
12767 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12768 let mut failure = self.storage_failure.borrow_mut();
12769 let failure =
12770 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
12771 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
12772 }
12773 } else {
12774 self.rollback_observation("syncular_outbox_drop");
12775 }
12776 self.outbox = prior_outbox;
12777 self.rejections = prior_rejections;
12778 self.overlay_dirty.restore(prior_overlay_dirty);
12779 return Err(error);
12780 }
12781 Ok(true)
12782 }
12783
12784 fn create_synced_tables(&self) -> Result<(), String> {
12787 for table in &self.schema.tables {
12788 for (full, index_prefix) in [
12792 (base_table(&table.name), "_syncular_base_"),
12793 (visible_table(&table.name), ""),
12794 ] {
12795 let mut cols: Vec<String> =
12796 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
12797 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
12798 let sql = format!(
12799 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
12800 cols.join(", "),
12801 quote_ident(&table.primary_key)
12802 );
12803 self.conn
12804 .execute(&sql, [])
12805 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12806 for index in &table.indexes {
12810 let unique = if index.unique { "UNIQUE " } else { "" };
12811 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
12812 let cols_sql = index
12813 .columns
12814 .iter()
12815 .map(|c| quote_ident(c))
12816 .collect::<Vec<_>>()
12817 .join(", ");
12818 let index_sql = format!(
12819 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
12820 );
12821 self.conn
12822 .execute(&index_sql, [])
12823 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12824 }
12825 }
12826 }
12827 for table in &self.schema.tables {
12830 for index in &table.fts_indexes {
12831 self.create_fts_projection(table, index)?;
12832 }
12833 }
12834 Ok(())
12835 }
12836
12837 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
12838 let count: i64 = self
12839 .conn
12840 .query_row(
12841 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
12842 rusqlite::params![index.name],
12843 |row| row.get(0),
12844 )
12845 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12846 Ok(count > 0)
12847 }
12848
12849 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
12850 for suffix in ["ai", "ad", "au"] {
12851 self.conn
12852 .execute(
12853 &format!(
12854 "DROP TRIGGER IF EXISTS {}",
12855 quote_ident(&format!("{}_{suffix}", index.name))
12856 ),
12857 [],
12858 )
12859 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12860 }
12861 Ok(())
12862 }
12863
12864 fn create_fts_triggers(
12865 &self,
12866 table: &TableSchema,
12867 index: &FtsIndexSchema,
12868 ) -> Result<(), String> {
12869 let fts = quote_ident(&index.name);
12870 let source = visible_table(&table.name);
12871 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
12872 let pk = quote_ident(&table.primary_key);
12873 let projection_columns = std::iter::once(source_id.clone())
12874 .chain(index.columns.iter().map(|column| quote_ident(column)))
12875 .collect::<Vec<_>>()
12876 .join(", ");
12877 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
12878 .chain(
12879 index
12880 .columns
12881 .iter()
12882 .map(|column| format!("new.{}", quote_ident(column))),
12883 )
12884 .collect::<Vec<_>>()
12885 .join(", ");
12886 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
12887 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
12888 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
12889 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
12890 let bi = quote_ident(&format!("{}_bi", index.name));
12891 let ai = quote_ident(&format!("{}_ai", index.name));
12892 let ad = quote_ident(&format!("{}_ad", index.name));
12893 let au = quote_ident(&format!("{}_au", index.name));
12894 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
12895 let sql = format!(
12896 "DROP TRIGGER IF EXISTS {bi};
12897 DROP TRIGGER IF EXISTS {ai};
12898 DROP TRIGGER IF EXISTS {ad};
12899 DROP TRIGGER IF EXISTS {au};
12900 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
12901 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
12902 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
12903 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
12904 );
12905 self.conn
12906 .execute_batch(&sql)
12907 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12908 }
12909
12910 fn rebuild_fts_projection(
12911 &self,
12912 table: &TableSchema,
12913 index: &FtsIndexSchema,
12914 ) -> Result<(), String> {
12915 let fts = quote_ident(&index.name);
12916 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
12917 let pk = quote_ident(&table.primary_key);
12918 let indexed_columns = index
12919 .columns
12920 .iter()
12921 .map(|column| quote_ident(column))
12922 .collect::<Vec<_>>();
12923 let projection_columns = std::iter::once(source_id.clone())
12924 .chain(indexed_columns.iter().cloned())
12925 .collect::<Vec<_>>()
12926 .join(", ");
12927 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
12928 let sql = format!(
12929 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
12930 columns = indexed_columns.join(", "),
12931 source = visible_table(&table.name),
12932 );
12933 self.conn
12934 .execute_batch(&sql)
12935 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12936 }
12937
12938 fn create_fts_projection(
12939 &self,
12940 table: &TableSchema,
12941 index: &FtsIndexSchema,
12942 ) -> Result<(), String> {
12943 self.begin_observation("syncular_fts_schema")?;
12944 let result = (|| {
12945 let existed = self.fts_projection_exists(index)?;
12946 let tokenizer = index.tokenize.replace('\'', "''");
12947 let columns = index
12948 .columns
12949 .iter()
12950 .map(|column| quote_ident(column))
12951 .collect::<Vec<_>>()
12952 .join(", ");
12953 let sql = format!(
12954 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
12955 fts = quote_ident(&index.name),
12956 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
12957 );
12958 self.conn.execute(&sql, []).map_err(|error| {
12959 format!(
12960 "cannot create local FTS5 projection {:?}: {error}",
12961 index.name
12962 )
12963 })?;
12964 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
12965 let mapped: bool = self
12966 .conn
12967 .query_row(
12968 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
12969 [format!("_syncular_fts_{}", index.name)],
12970 |row| row.get(0),
12971 )
12972 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12973 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12974 if !mapped && existed {
12975 self.conn
12976 .execute_batch(&format!(
12977 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
12978 quote_ident(FTS_SOURCE_ID_COLUMN),
12979 quote_ident(&index.name)
12980 ))
12981 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12982 }
12983 self.create_fts_triggers(table, index)?;
12984 if !existed {
12985 self.rebuild_fts_projection(table, index)?;
12986 }
12987 Ok(())
12988 })();
12989 if let Err(error) = result {
12990 self.rollback_observation("syncular_fts_schema");
12991 return Err(error);
12992 }
12993 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
12994 self.rollback_observation("syncular_fts_schema");
12995 return Err(error.to_string());
12996 }
12997 Ok(())
12998 }
12999
13000 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
13003 let state = serde_json::json!({
13004 "requested": scope_map_to_json(&sub.requested),
13005 "params": sub.params,
13006 "cursor": sub.cursor,
13007 "bootstrapState": sub.bootstrap_state,
13008 "status": sub.state.name(),
13009 "reasonCode": sub.reason_code,
13010 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
13011 "syncedOnce": sub.synced_once,
13012 });
13013 self.conn.execute(
13014 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
13015 rusqlite::params![sub.id, sub.table, state.to_string()],
13016 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13017 }
13018
13019 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
13020 let ops: Vec<Value> = commit
13021 .ops
13022 .iter()
13023 .map(|op| {
13024 serde_json::json!({
13025 "op": if op.upsert { "upsert" } else { "delete" },
13026 "table": op.table,
13027 "rowId": op.row_id,
13028 "baseVersion": op.base_version,
13029 "values": op.values.clone().map(Value::Object),
13030 })
13031 })
13032 .collect();
13033 self.conn
13034 .execute(
13035 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
13036 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
13037 )
13038 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13039 for operation in &commit.ops {
13040 let Some(values) = &operation.values else {
13041 continue;
13042 };
13043 let Some(table) = self.schema.table(&operation.table) else {
13044 continue;
13045 };
13046 for column in table
13047 .columns
13048 .iter()
13049 .filter(|column| column.ty == ColumnType::BlobRef)
13050 {
13051 let Some(Value::String(raw)) = values.get(&column.name) else {
13052 continue;
13053 };
13054 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
13055 continue;
13056 };
13057 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
13058 continue;
13059 };
13060 self.conn
13061 .execute(
13062 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
13063 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
13064 rusqlite::params![commit.client_commit_id, blob_id],
13065 )
13066 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13067 }
13068 }
13069 Ok(())
13070 }
13071
13072 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
13073 if let Some(commit) = self
13074 .outbox
13075 .iter()
13076 .find(|commit| commit.client_commit_id == client_commit_id)
13077 {
13078 for operation in &commit.ops {
13079 self.overlay_dirty.table(&operation.table);
13080 }
13081 }
13082
13083 self.conn
13084 .execute(
13085 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
13086 rusqlite::params![client_commit_id],
13087 )
13088 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13089 self.conn
13090 .execute(
13091 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
13092 rusqlite::params![client_commit_id],
13093 )
13094 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13095 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
13096 .map(|_| ())
13097 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13098 }
13099
13100 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
13101 match status {
13102 CommitOutcomeStatus::Applied => "applied",
13103 CommitOutcomeStatus::Cached => "cached",
13104 CommitOutcomeStatus::Conflict => "conflict",
13105 CommitOutcomeStatus::Rejected => "rejected",
13106 }
13107 }
13108
13109 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
13110 match resolution {
13111 CommitOutcomeResolution::Active => "active",
13112 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
13113 CommitOutcomeResolution::Superseded => "superseded",
13114 CommitOutcomeResolution::Dismissed => "dismissed",
13115 }
13116 }
13117
13118 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
13119 match value {
13120 "applied" => Ok(CommitOutcomeStatus::Applied),
13121 "cached" => Ok(CommitOutcomeStatus::Cached),
13122 "conflict" => Ok(CommitOutcomeStatus::Conflict),
13123 "rejected" => Ok(CommitOutcomeStatus::Rejected),
13124 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
13125 }
13126 }
13127
13128 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
13129 match value {
13130 "active" => Ok(CommitOutcomeResolution::Active),
13131 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
13132 "superseded" => Ok(CommitOutcomeResolution::Superseded),
13133 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
13134 _ => Err(format!(
13135 "invalid persisted commit outcome resolution {value:?}"
13136 )),
13137 }
13138 }
13139
13140 fn persist_commit_outcome(
13141 &self,
13142 client_commit_id: &str,
13143 status: CommitOutcomeStatus,
13144 results: &[CommitOperationOutcome],
13145 operations: Option<&[OutboxOp]>,
13146 ) -> Result<(), String> {
13147 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
13148 let operations_json = operations
13149 .map(|items| {
13150 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
13151 })
13152 .transpose()
13153 .map_err(|error| error.to_string())?;
13154 self.conn
13155 .execute(
13156 "INSERT INTO _syncular_commit_outcomes (
13157 client_commit_id, status, recorded_at_ms, results_json,
13158 operations_json, resolution
13159 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
13160 rusqlite::params![
13161 client_commit_id,
13162 Self::outcome_status_name(status),
13163 self.clock_now_ms(),
13164 results_json,
13165 operations_json
13166 ],
13167 )
13168 .map(|_| ())
13169 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13170 }
13171
13172 fn stored_cell_is_representable(cell: &Value) -> bool {
13175 cell.is_null()
13176 || cell.is_string()
13177 || cell.is_boolean()
13178 || cell.is_number()
13179 || is_canonical_bytes_envelope(cell)
13180 }
13181
13182 fn stored_row_is_representable(value: &Value) -> bool {
13183 value
13184 .as_object()
13185 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
13186 }
13187
13188 fn stored_operation_is_representable(value: &Value) -> bool {
13189 value.get("table").is_some_and(Value::is_string)
13190 && value.get("rowId").is_some_and(Value::is_string)
13191 && value.get("baseVersion").is_none_or(|v| {
13192 v.as_i64()
13193 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
13194 })
13195 && match value.get("op").and_then(Value::as_str) {
13196 Some("upsert") => value
13197 .get("values")
13198 .is_some_and(Self::stored_row_is_representable),
13199 Some("delete") => value.get("values").is_none(),
13200 _ => false,
13201 }
13202 }
13203
13204 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
13208 serde_json::to_value(CommitOperation::from(operation))
13209 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
13210 }
13211
13212 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
13213 let StoredCommitOutcomeRow {
13214 sequence,
13215 client_commit_id,
13216 status,
13217 recorded_at_ms,
13218 results_json,
13219 operations_json,
13220 resolution,
13221 resolved_at_ms,
13222 replacement_client_commit_id,
13223 } = row;
13224 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
13225 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
13226 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
13227 {
13228 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
13229 }
13230 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
13231 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
13232 for result in &stored {
13233 let nested = match result.get("status").and_then(Value::as_str) {
13234 Some("conflict") => result.get("conflict"),
13235 Some("error") => result.get("rejection"),
13236 _ => None,
13237 };
13238 if nested
13239 .and_then(|record| record.get("operation"))
13240 .is_some_and(|op| !Self::stored_operation_is_representable(op))
13241 || (result.get("status").and_then(Value::as_str) == Some("conflict")
13242 && nested
13243 .and_then(|record| record.get("serverRow"))
13244 .is_some_and(|row| !Self::stored_row_is_representable(row)))
13245 {
13246 return Err(invalid());
13247 }
13248 }
13249 let results: Vec<CommitOperationOutcome> =
13250 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
13251 let has_conflict = results
13252 .iter()
13253 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
13254 let has_error = results
13255 .iter()
13256 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
13257 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
13258 let redacted = results.is_empty() && operations_json.is_none();
13259 if match status {
13260 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
13261 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
13262 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
13263 } {
13264 return Err(invalid());
13265 }
13266 let operations: Option<Vec<CommitOperation>> = operations_json
13267 .map(|raw| {
13268 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
13269 if !stored.iter().all(Self::stored_operation_is_representable) {
13270 return Err(invalid());
13271 }
13272 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
13273 })
13274 .transpose()?;
13275 for result in &results {
13276 match result {
13277 CommitOperationOutcome::Conflict { conflict }
13278 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
13279 {
13280 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
13281 }
13282 CommitOperationOutcome::Error { rejection } => {
13283 if let Some(details) = &rejection.details {
13284 let encoded = serde_json::to_string(details).map_err(|_| {
13285 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13286 })?;
13287 RejectionDetails::parse(&encoded).map_err(|_| {
13288 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13289 })?;
13290 }
13291 }
13292 _ => {}
13293 }
13294 }
13295 Ok(CommitOutcome {
13296 sequence,
13297 client_commit_id,
13298 status,
13299 recorded_at_ms,
13300 results,
13301 operations,
13302 retained_rows: None,
13303 resolution: Self::parse_outcome_resolution(&resolution)?,
13304 resolved_at_ms,
13305 replacement_client_commit_id,
13306 })
13307 }
13308
13309 fn unique_conflicts(
13311 &self,
13312 table: &TableSchema,
13313 values: &Map<String, Value>,
13314 base: bool,
13315 ) -> Result<Vec<RetainedUniqueConflict>, String> {
13316 let source = if base {
13317 base_table(&table.name)
13318 } else {
13319 visible_table(&table.name)
13320 };
13321 let mut conflicts = Vec::new();
13322 for index in table.indexes.iter().filter(|index| index.unique) {
13323 let mut params = vec![json_param_to_sql(
13324 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
13325 )?];
13326 for name in &index.columns {
13327 let column = table
13328 .columns
13329 .iter()
13330 .find(|column| column.name == *name)
13331 .ok_or("sync.local_corrupt")?;
13332 let cell = json_to_column_value(column, values.get(name))?;
13333 params.push(owned_sql_value(RowParam::Cell(&cell))?);
13334 }
13335 let sql = format!(
13336 "SELECT * FROM {source} WHERE {} != ? AND {}",
13337 quote_ident(&table.primary_key),
13338 index
13339 .columns
13340 .iter()
13341 .map(|column| format!("{} = ?", quote_ident(column)))
13342 .collect::<Vec<_>>()
13343 .join(" AND ")
13344 );
13345 let mut statement = self
13346 .conn
13347 .prepare_cached(&sql)
13348 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13349 let records = statement
13350 .query_map(rusqlite::params_from_iter(params), |record| {
13351 let values = table
13352 .columns
13353 .iter()
13354 .enumerate()
13355 .map(|(index, column)| {
13356 let value = sql_ref_to_json(column, record.get_ref(index)?);
13357 json_to_column_value(column, Some(&value)).map_err(|message| {
13358 rusqlite::Error::ToSqlConversionFailure(message.into())
13359 })
13360 })
13361 .collect::<Result<Row, _>>()?;
13362 Ok((values, record.get::<_, i64>(table.columns.len())?))
13363 })
13364 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13365 for record in records {
13366 let (row, version) =
13367 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13368 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
13369 conflicts.push(RetainedUniqueConflict {
13370 index: index.name.clone(),
13371 columns: index.columns.clone(),
13372 row_id,
13373 server_row: table
13374 .columns
13375 .iter()
13376 .zip(&row)
13377 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
13378 .collect(),
13379 server_version: version,
13380 });
13381 }
13382 }
13383 Ok(conflicts)
13384 }
13385
13386 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
13387 if let Some((commit, initial)) = self
13388 .failed_commits
13389 .iter()
13390 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
13391 {
13392 let mut rows = Vec::new();
13393 for operation in &commit.ops {
13394 let table = self
13395 .schema
13396 .table(&operation.table)
13397 .ok_or_else(|| "sync.unknown_table".to_owned())?;
13398 let key =
13399 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
13400 .map_err(|_| "sync.local_corrupt".to_owned())?;
13401 let base = self.stored_row(table, &operation.row_id, true)?;
13402 rows.push(RetainedCommitRow {
13403 table: operation.table.clone(),
13404 row_id: operation.row_id.clone(),
13405 unique_conflicts: if operation.upsert {
13406 self.unique_conflicts(
13407 table,
13408 initial
13409 .get(&key)
13410 .and_then(Value::as_object)
13411 .or(operation.values.as_ref())
13412 .ok_or("sync.local_corrupt")?,
13413 true,
13414 )?
13415 } else {
13416 Vec::new()
13417 },
13418 local_row: if operation.upsert {
13419 initial
13420 .get(&key)
13421 .and_then(Value::as_object)
13422 .cloned()
13423 .or_else(|| operation.values.clone())
13424 } else {
13425 None
13426 },
13427 server_row: base.as_ref().map(|(values, _)| {
13428 table
13429 .columns
13430 .iter()
13431 .zip(values)
13432 .map(|(column, value)| {
13433 (column.name.clone(), column_value_to_json(value))
13434 })
13435 .collect()
13436 }),
13437 server_version: base.map(|(_, version)| version),
13438 });
13439 }
13440 outcome.retained_rows = Some(rows);
13441 }
13442 Ok(outcome)
13443 }
13444
13445 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
13446 let row = self
13447 .conn
13448 .query_row(
13449 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
13450 resolution, resolved_at_ms, replacement_client_commit_id
13451 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
13452 rusqlite::params![client_commit_id],
13453 |row| {
13454 Ok(StoredCommitOutcomeRow {
13455 sequence: row.get(0)?,
13456 client_commit_id: row.get(1)?,
13457 status: row.get(2)?,
13458 recorded_at_ms: row.get(3)?,
13459 results_json: row.get(4)?,
13460 operations_json: row.get(5)?,
13461 resolution: row.get(6)?,
13462 resolved_at_ms: row.get(7)?,
13463 replacement_client_commit_id: row.get(8)?,
13464 })
13465 },
13466 )
13467 .optional()
13468 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13469 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
13470 .transpose()
13471 }
13472
13473 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
13474 if query.limit == Some(0) {
13475 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
13476 }
13477 let mut sql = String::from(
13478 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
13479 resolution, resolved_at_ms, replacement_client_commit_id
13480 FROM _syncular_commit_outcomes",
13481 );
13482 if query.active_only {
13483 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
13484 }
13485 sql.push_str(" ORDER BY seq DESC");
13486 if let Some(limit) = query.limit {
13487 sql.push_str(&format!(" LIMIT {limit}"));
13488 }
13489 let mut stmt = self
13490 .conn
13491 .prepare(&sql)
13492 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13493 let rows = stmt
13494 .query_map([], |row| {
13495 Ok(StoredCommitOutcomeRow {
13496 sequence: row.get(0)?,
13497 client_commit_id: row.get(1)?,
13498 status: row.get(2)?,
13499 recorded_at_ms: row.get(3)?,
13500 results_json: row.get(4)?,
13501 operations_json: row.get(5)?,
13502 resolution: row.get(6)?,
13503 resolved_at_ms: row.get(7)?,
13504 replacement_client_commit_id: row.get(8)?,
13505 })
13506 })
13507 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13508 let mut outcomes = Vec::new();
13509 for row in rows {
13510 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
13511 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
13512 )?)?);
13513 }
13514 Ok(outcomes)
13515 }
13516
13517 fn prune_commit_outcomes(&self) -> Result<(), String> {
13518 #[cfg(test)]
13519 self.outcome_prune_count
13520 .set(self.outcome_prune_count.get() + 1);
13521 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
13522 let count = self
13523 .conn
13524 .query_row(
13525 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
13526 [],
13527 |row| row.get::<_, i64>(0),
13528 )
13529 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
13530 as usize;
13531 let excess = count.saturating_sub(max_entries);
13532 if excess == 0 {
13533 return Ok(());
13534 }
13535 let mut stmt = self
13536 .conn
13537 .prepare(
13538 "SELECT seq FROM _syncular_commit_outcomes
13539 WHERE status IN ('applied', 'cached') OR resolution != 'active'
13540 ORDER BY seq ASC LIMIT ?1",
13541 )
13542 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13543 let rows = stmt
13544 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
13545 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13546 let sequences = rows
13547 .collect::<Result<Vec<_>, _>>()
13548 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13549 drop(stmt);
13550 for sequence in sequences {
13551 self.conn
13552 .execute(
13553 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
13554 rusqlite::params![sequence],
13555 )
13556 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13557 }
13558 Ok(())
13559 }
13560
13561 pub fn subscribe(
13564 &mut self,
13565 id: String,
13566 table: String,
13567 scopes: Vec<(String, Vec<String>)>,
13568 params: Option<String>,
13569 ) -> Result<(), String> {
13570 if self.schema.table(&table).is_none() {
13571 return Err(format!("unknown table {table:?}"));
13572 }
13573 let registered_at = self
13574 .subs
13575 .iter()
13576 .position(|subscription| subscription.id == id);
13577 if let Some(index) = registered_at {
13578 let existing = &self.subs[index];
13579 let same_intent = existing.table == table
13580 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
13581 && existing.params == params;
13582 if !same_intent {
13583 return Err(
13584 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
13585 .to_owned(),
13586 );
13587 }
13588 if existing.state != SubState::Revoked {
13593 return Ok(());
13594 }
13595 }
13596 let sub = Subscription {
13597 id: id.clone(),
13598 table,
13599 requested: scopes,
13600 params,
13601 cursor: -1,
13602 bootstrap_state: None,
13603 state: SubState::Active,
13604 reason_code: None,
13605 effective: None,
13606 synced_once: false,
13607 };
13608 self.persist_sub(&sub)?;
13609 match registered_at {
13610 Some(index) => self.subs[index] = sub,
13611 None => self.subs.push(sub),
13612 }
13613 if registered_at.is_some() {
13617 self.set_sync_needed(true, true);
13618 }
13619 Ok(())
13620 }
13621
13622 pub fn unsubscribe(&mut self, id: &str) {
13623 if self.subs.iter().any(|s| s.id == id) {
13624 self.cancel_sync_round();
13625 }
13626 self.subs.retain(|s| s.id != id);
13627 let _ = self.conn.execute(
13628 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
13629 rusqlite::params![id],
13630 );
13631 }
13632
13633 pub fn set_window(
13641 &mut self,
13642 base: &WindowBase,
13643 units: &[String],
13644 ) -> Result<CommandEffects, String> {
13645 let table = self
13646 .schema
13647 .table(&base.table)
13648 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
13649 if table.scope_column(&base.variable).is_none() {
13650 return Err(format!(
13651 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
13652 base.table, base.variable
13653 ));
13654 }
13655 let base_key = window_base_key(base);
13656 let wanted: std::collections::HashSet<&String> = units.iter().collect();
13657 let live = self.load_window_units(&base_key);
13658 self.begin_observation("syncular_window")?;
13659 let mut batch = ChangeAccumulator::default();
13660 let mut changed = false;
13661
13662 let prior_subs = self.subs.clone();
13663 let widened = (|| {
13664 for unit in units {
13666 if live.iter().any(|(u, _)| u == unit) {
13667 continue;
13668 }
13669 let sub_id = derive_sub_id(base, unit);
13670 self.delete_pending_evict(&sub_id)?;
13671 self.insert_window_unit(&base_key, unit, &sub_id)?;
13672 self.subscribe(
13673 sub_id,
13674 base.table.clone(),
13675 unit_scopes(base, unit),
13676 base.params.clone(),
13677 )?;
13678 batch.window(&base_key, &base.table, unit);
13679 changed = true;
13680 }
13681
13682 self.finish_observation("syncular_window", batch)
13683 })();
13684 if let Err(error) = widened {
13685 self.rollback_observation("syncular_window");
13686 self.subs = prior_subs;
13687 return Err(error);
13688 }
13689 for (unit, sub_id) in live {
13690 if wanted.contains(&unit) {
13691 continue;
13692 }
13693 self.rebuild_overlay_if_dirty()?;
13694 let prior_subs = self.subs.clone();
13695 self.begin_observation("syncular_window")?;
13696 let effective = self
13697 .subs
13698 .iter()
13699 .find(|sub| sub.id == sub_id)
13700 .and_then(|sub| sub.effective.clone())
13701 .unwrap_or_else(|| unit_scopes(base, &unit));
13702 let mut batch = ChangeAccumulator::default();
13703 self.record_scope_map(&mut batch, &base.table, &effective);
13704 batch.window(&base_key, &base.table, &unit);
13705 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
13706 let remaining = match result {
13707 Ok(remaining) => remaining,
13708 Err(error) => {
13709 self.rollback_observation("syncular_window");
13710 self.subs = prior_subs;
13711 return Err(error);
13712 }
13713 };
13714 if let Err(error) = self.finish_observation("syncular_window", batch) {
13715 self.rollback_observation("syncular_window");
13716 self.subs = prior_subs;
13717 return Err(error);
13718 }
13719 if remaining {
13720 self.drain_pending_evictions()?;
13721 }
13722 changed = true;
13723 }
13724 Ok(if changed {
13725 CommandEffects::interactive()
13726 } else {
13727 CommandEffects::none()
13728 })
13729 }
13730
13731 pub fn window_state(&self, base: &WindowBase) -> WindowState {
13736 let mut units = Vec::new();
13737 let mut pending = Vec::new();
13738 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
13739 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
13740 Some(sub) => {
13741 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
13742 }
13743 None => true,
13744 };
13745 if is_pending {
13746 pending.push(unit.clone());
13747 }
13748 units.push(unit);
13749 }
13750 WindowState { units, pending }
13751 }
13752
13753 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
13754 let mut stmt = match self
13755 .conn
13756 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
13757 {
13758 Ok(stmt) => stmt,
13759 Err(_) => return Vec::new(),
13760 };
13761 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
13762 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
13763 });
13764 match rows {
13765 Ok(rows) => rows.filter_map(Result::ok).collect(),
13766 Err(_) => Vec::new(),
13767 }
13768 }
13769
13770 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
13771 let mut stmt = match self.conn.prepare(
13772 "SELECT windows.base, windows.unit, subscriptions.tbl
13773 FROM _syncular_windows AS windows
13774 JOIN _syncular_subscriptions AS subscriptions
13775 ON subscriptions.id = windows.sub_id
13776 ORDER BY windows.base, windows.unit",
13777 ) {
13778 Ok(stmt) => stmt,
13779 Err(_) => return Vec::new(),
13780 };
13781 let rows = stmt.query_map([], |row| {
13782 Ok((
13783 row.get::<_, String>(0)?,
13784 row.get::<_, String>(1)?,
13785 row.get::<_, String>(2)?,
13786 ))
13787 });
13788 match rows {
13789 Ok(rows) => rows.filter_map(Result::ok).collect(),
13790 Err(_) => Vec::new(),
13791 }
13792 }
13793
13794 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
13795 self.conn
13796 .query_row(
13797 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
13798 rusqlite::params![sub_id],
13799 |row| Ok((row.get(0)?, row.get(1)?)),
13800 )
13801 .ok()
13802 }
13803
13804 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
13805 self.conn
13806 .execute(
13807 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
13808 rusqlite::params![base_key, unit, sub_id],
13809 )
13810 .map(|_| ())
13811 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13812 }
13813
13814 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
13815 self.conn
13816 .execute(
13817 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
13818 rusqlite::params![base_key, unit],
13819 )
13820 .map(|_| ())
13821 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13822 }
13823
13824 fn evict_unit(
13831 &mut self,
13832 base_key: &str,
13833 base: &WindowBase,
13834 unit: &str,
13835 sub_id: &str,
13836 ) -> Result<bool, String> {
13837 let effective = self
13838 .subs
13839 .iter()
13840 .find(|s| s.id == sub_id)
13841 .and_then(|s| s.effective.clone())
13842 .unwrap_or_else(|| unit_scopes(base, unit));
13843 let pinned = self.pinned_row_ids(&base.table);
13844 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
13845 self.delete_window_unit(base_key, unit)?;
13846 self.conn
13847 .execute(
13848 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
13849 [sub_id],
13850 )
13851 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13852 self.subs.retain(|sub| sub.id != sub_id);
13853 if deferred {
13854 self.save_pending_evict(sub_id, &base.table, &effective)?;
13855 } else {
13856 self.delete_pending_evict(sub_id)?;
13857 }
13858 Ok(remaining)
13859 }
13860
13861 fn evict_scope_rows(
13865 &self,
13866 table_name: &str,
13867 effective: &[(String, Vec<String>)],
13868 pinned: &std::collections::HashSet<String>,
13869 ) -> Result<(bool, bool), String> {
13870 if effective.is_empty() {
13871 return Ok((false, false));
13872 }
13873 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
13874 let mut clauses = Vec::new();
13875 let mut params: Vec<SqlValue> = Vec::new();
13876 for (variable, values) in effective {
13877 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
13878 if values.is_empty() {
13879 return Ok((false, false));
13880 }
13881 let holes = values
13882 .iter()
13883 .map(|v| {
13884 params.push(SqlValue::Text(v.clone()));
13885 "?"
13886 })
13887 .collect::<Vec<_>>()
13888 .join(", ");
13889 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
13890 }
13891 let scope = clauses.join(" AND ");
13892 let scope_params = params.len();
13893 let pk = quote_ident(&table.primary_key);
13894 if !pinned.is_empty() {
13895 let holes = pinned
13896 .iter()
13897 .map(|id| {
13898 params.push(SqlValue::Text(id.clone()));
13899 "?"
13900 })
13901 .collect::<Vec<_>>()
13902 .join(", ");
13903 clauses.push(format!("{pk} NOT IN ({holes})"));
13904 }
13905 let predicate = clauses.join(" AND ");
13906 let base = base_table(table_name);
13907 let visible = visible_table(table_name);
13908 let mut ids: Vec<SqlValue> = Vec::new();
13909 for source in [&base, &visible] {
13910 let mut stmt = self
13911 .conn
13912 .prepare_cached(&format!(
13913 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
13914 ))
13915 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13916 ids = stmt
13917 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
13918 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
13919 .collect::<Result<_, _>>()
13920 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13921 if !ids.is_empty() {
13922 break;
13923 }
13924 }
13925 if !ids.is_empty() {
13926 for id in &ids {
13927 self.conn
13928 .execute(
13929 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
13930 rusqlite::params![table_name, id],
13931 )
13932 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13933 }
13934 let holes = vec!["?"; ids.len()].join(", ");
13935 for source in [&base, &visible] {
13936 self.conn
13937 .execute(
13938 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
13939 rusqlite::params_from_iter(&ids),
13940 )
13941 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13942 }
13943 }
13944 let mut remaining = false;
13945 let mut deferred = false;
13946 for source in [&base, &visible] {
13947 remaining |= self
13948 .conn
13949 .query_row(
13950 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
13951 rusqlite::params_from_iter(¶ms),
13952 |row| row.get::<_, bool>(0),
13953 )
13954 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13955 deferred |= self
13956 .conn
13957 .query_row(
13958 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
13959 rusqlite::params_from_iter(¶ms[..scope_params]),
13960 |row| row.get::<_, bool>(0),
13961 )
13962 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13963 }
13964 Ok((remaining, deferred))
13965 }
13966
13967 fn drain_pending_evictions(&mut self) -> Result<(), String> {
13969 let pending = self.load_pending_evictions()?;
13970 if pending.is_empty() {
13971 return Ok(());
13972 }
13973 self.rebuild_overlay_if_dirty()?;
13974 for (sub_id, table_name, effective) in pending {
13975 if self.schema.table(&table_name).is_none() {
13976 self.delete_pending_evict(&sub_id)?;
13977 continue;
13978 }
13979 loop {
13980 std::thread::yield_now();
13981 self.begin_observation("syncular_evict")?;
13982 let pinned = self.pinned_row_ids(&table_name);
13983 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
13984 let (remaining, deferred) = match result {
13985 Ok(result) => result,
13986 Err(error) => {
13987 self.rollback_observation("syncular_evict");
13988 return Err(error);
13989 }
13990 };
13991 if !deferred {
13992 if let Err(error) = self.delete_pending_evict(&sub_id) {
13993 self.rollback_observation("syncular_evict");
13994 return Err(error);
13995 }
13996 }
13997 let mut batch = ChangeAccumulator::default();
13998 self.record_scope_map(&mut batch, &table_name, &effective);
13999 if let Err(error) = self.finish_observation("syncular_evict", batch) {
14000 self.rollback_observation("syncular_evict");
14001 return Err(error);
14002 }
14003 if !remaining {
14004 break;
14005 }
14006 }
14007 }
14008 Ok(())
14009 }
14010
14011 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
14014 let mut pinned = std::collections::HashSet::new();
14015 for commit in self
14016 .outbox
14017 .iter()
14018 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
14019 {
14020 for op in &commit.ops {
14021 if op.table == table {
14022 pinned.insert(op.row_id.clone());
14023 }
14024 }
14025 }
14026 pinned
14027 }
14028
14029 fn save_pending_evict(
14030 &self,
14031 sub_id: &str,
14032 table: &str,
14033 effective: &[(String, Vec<String>)],
14034 ) -> Result<(), String> {
14035 self.conn.execute(
14036 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
14037 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
14038 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14039 }
14040
14041 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
14042 self.conn
14043 .execute(
14044 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
14045 rusqlite::params![sub_id],
14046 )
14047 .map(|_| ())
14048 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14049 }
14050
14051 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
14052 let mut stmt = self
14053 .conn
14054 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
14055 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14056 let rows = stmt
14057 .query_map([], |row| {
14058 Ok((
14059 row.get::<_, String>(0)?,
14060 row.get::<_, String>(1)?,
14061 row.get::<_, String>(2)?,
14062 ))
14063 })
14064 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14065 rows.map(|row| {
14066 let (sub_id, table, json) =
14067 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14068 let value: Value = serde_json::from_str(&json)
14069 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14070 let effective = json_to_scope_map(&value)
14071 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14072 Ok((sub_id, table, effective))
14073 })
14074 .collect()
14075 }
14076
14077 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
14079 self.retain_failed_commits = enabled;
14080 for (commit, _) in &self.failed_commits {
14081 for operation in &commit.ops {
14082 self.overlay_dirty.table(&operation.table);
14083 }
14084 }
14085 self.failed_commits = self.load_failed_commits()?;
14086 for (commit, _) in &self.failed_commits {
14087 for operation in &commit.ops {
14088 self.overlay_dirty.table(&operation.table);
14089 }
14090 }
14091 Ok(())
14092 }
14093
14094 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
14095 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
14096 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14097 let rows = statement
14098 .query_map([], |row| {
14099 Ok((
14100 row.get::<_, String>(0)?,
14101 row.get::<_, String>(1)?,
14102 row.get::<_, String>(2)?,
14103 ))
14104 })
14105 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14106 rows.map(|row| {
14107 let (id, operations, initial) =
14108 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14109 let operations: Vec<CommitOperation> =
14110 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
14111 let initial =
14112 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
14113 Ok((
14114 OutboxCommit {
14115 client_commit_id: id,
14116 ops: operations
14117 .into_iter()
14118 .map(|op| OutboxOp {
14119 upsert: op.op == "upsert",
14120 table: op.table,
14121 row_id: op.row_id,
14122 base_version: op.base_version,
14123 values: op.values,
14124 })
14125 .collect(),
14126 },
14127 initial,
14128 ))
14129 })
14130 .collect()
14131 }
14132
14133 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
14134 let mut initial = Map::new();
14135 for operation in operations {
14136 let table = self
14137 .schema
14138 .table(&operation.table)
14139 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14140 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
14141 Some(row) => Some(row),
14142 None => self.visible_row(table, &operation.row_id)?,
14143 };
14144 if let Some((values, _)) = initial_row {
14145 let mut intended: Map<String, Value> = table
14146 .columns
14147 .iter()
14148 .zip(values.iter())
14149 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
14150 .collect();
14151 if let Some(patch) = &operation.values {
14152 intended.extend(patch.clone());
14153 }
14154 initial.insert(
14155 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14156 .map_err(|_| "sync.local_corrupt".to_owned())?,
14157 Value::Object(intended),
14158 );
14159 }
14160 }
14161 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
14162 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
14163 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
14164 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14165 self.failed_commits.push((
14166 OutboxCommit {
14167 client_commit_id: id.to_owned(),
14168 ops: operations.to_vec(),
14169 },
14170 initial,
14171 ));
14172 Ok(())
14173 }
14174
14175 fn drop_failed_commits(
14176 &mut self,
14177 matches: impl Fn(&str, &Map<String, Value>) -> bool,
14178 ) -> Result<Vec<OutboxCommit>, String> {
14179 let mut doomed = self.drop_acknowledged_rows(&matches)?;
14180 for (commit, initial) in &self.failed_commits {
14181 for operation in &commit.ops {
14182 let key =
14183 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14184 .map_err(|_| "sync.local_corrupt".to_owned())?;
14185 if initial
14186 .get(&key)
14187 .and_then(Value::as_object)
14188 .is_some_and(|values| matches(&operation.table, values))
14189 {
14190 doomed.push(commit.clone());
14191 break;
14192 }
14193 }
14194 }
14195 for commit in &doomed {
14196 self.conn
14197 .execute(
14198 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
14199 rusqlite::params![commit.client_commit_id],
14200 )
14201 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14202 }
14203 for commit in &doomed {
14204 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14205 }
14206 self.conflicts.retain(|record| {
14207 !doomed
14208 .iter()
14209 .any(|commit| commit.client_commit_id == record.client_commit_id)
14210 });
14211 self.rejections.retain(|record| {
14212 !doomed
14213 .iter()
14214 .any(|commit| commit.client_commit_id == record.client_commit_id)
14215 });
14216 self.failed_commits.retain(|(commit, _)| {
14217 !doomed
14218 .iter()
14219 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
14220 });
14221 for commit in &doomed {
14222 for operation in &commit.ops {
14223 self.overlay_dirty.table(&operation.table);
14224 }
14225 }
14226 Ok(doomed)
14227 }
14228
14229 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
14233 let prior = self.storage_failure.borrow_mut().take();
14234 let outcome = (|| -> Result<String, String> {
14235 if mutations.is_empty() {
14236 return Err(
14237 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
14238 .to_owned(),
14239 );
14240 }
14241 let mut ops = Vec::with_capacity(mutations.len());
14242 for mutation in mutations {
14243 match mutation {
14244 Mutation::Patch {
14245 table,
14246 values,
14247 base_version,
14248 } => {
14249 let schema_table = self
14250 .schema
14251 .table(&table)
14252 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14253 let values = normalize_values_casing(schema_table, values)?;
14254 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14255 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
14256 }
14257 Mutation::Upsert {
14258 table,
14259 values,
14260 base_version,
14261 } => {
14262 let schema_table = self.schema.table(&table).ok_or_else(|| {
14263 format!("sync.unknown_table: unknown table {table:?}")
14264 })?;
14265 let values = normalize_values_casing(schema_table, values)?;
14269 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14270 let values = full_row_values(schema_table, values)?;
14273 self.validate_author_encode(schema_table, &row_id, &values)?;
14279 ops.push(OutboxOp {
14280 upsert: true,
14281 table,
14282 row_id,
14283 base_version,
14284 values: Some(values),
14285 });
14286 }
14287 Mutation::Delete {
14288 table,
14289 row_id,
14290 base_version,
14291 } => {
14292 if self.schema.table(&table).is_none() {
14293 return Err(format!("sync.unknown_table: unknown table {table:?}"));
14294 }
14295 ops.push(OutboxOp {
14296 upsert: false,
14297 table,
14298 row_id,
14299 base_version,
14300 values: None,
14301 });
14302 }
14303 }
14304 }
14305 self.record_outbox_commit(ops)
14306 })();
14307 self.finish_authoring(prior, outcome)
14308 }
14309
14310 fn finish_authoring(
14316 &self,
14317 prior: Option<QueryReadFailure>,
14318 outcome: Result<String, String>,
14319 ) -> Result<String, ClientError> {
14320 let fresh = self.storage_failure.borrow_mut().take();
14321 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
14322 match outcome {
14323 Ok(id) => Ok(id),
14324 Err(message) => Err(fresh
14325 .map(ClientError::from)
14326 .unwrap_or_else(|| ClientError::from(message))),
14327 }
14328 }
14329
14330 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
14331 let commit = OutboxCommit {
14332 client_commit_id: uuid::Uuid::new_v4().to_string(),
14333 ops,
14334 };
14335 self.begin_observation("syncular_mutation")?;
14336 let mut batch = ChangeAccumulator::default();
14337 for op in &commit.ops {
14338 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
14339 if let Some(values) = &op.values {
14340 if let Some(table) = self.schema.table(&op.table) {
14341 for scope in &table.scope_variables {
14342 if let Some(Value::String(value)) = values.get(&scope.column) {
14343 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
14344 precise = true;
14345 }
14346 }
14347 }
14348 }
14349 if !precise {
14350 batch.table(&op.table);
14351 }
14352 }
14353 if let Err(error) = self.persist_outbox_insert(&commit) {
14354 self.rollback_observation("syncular_mutation");
14355 return Err(error);
14356 }
14357 let was_dirty = self.overlay_dirty.snapshot();
14358 if let Err(error) = self.rebuild_overlay_if_dirty() {
14363 self.rollback_observation("syncular_mutation");
14364 self.overlay_dirty.restore(was_dirty.clone());
14365 return Err(error);
14366 }
14367 if let Some(error) = commit
14368 .ops
14369 .iter()
14370 .find_map(|op| self.apply_outbox_op(op).err())
14371 {
14372 self.rollback_observation("syncular_mutation");
14373 self.overlay_dirty.restore(was_dirty.clone());
14374 return Err(error);
14375 }
14376 let id = commit.client_commit_id.clone();
14377 self.outbox.push(commit);
14378 batch.status = true;
14379 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
14380 self.rollback_observation("syncular_mutation");
14381 self.outbox.pop();
14382 self.overlay_dirty.restore(was_dirty.clone());
14383 return Err(error);
14384 }
14385 Ok(id)
14386 }
14387
14388 pub fn patch(
14394 &mut self,
14395 table: &str,
14396 row_id: &str,
14397 partial: Map<String, Value>,
14398 base_version: Option<i64>,
14399 ) -> Result<String, ClientError> {
14400 let prior = self.storage_failure.borrow_mut().take();
14401 let outcome = self.perform_patch(table, row_id, partial, base_version);
14402 self.finish_authoring(prior, outcome)
14403 }
14404
14405 fn perform_patch(
14408 &mut self,
14409 table: &str,
14410 row_id: &str,
14411 partial: Map<String, Value>,
14412 base_version: Option<i64>,
14413 ) -> Result<String, String> {
14414 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
14415 self.record_outbox_commit(vec![operation])
14416 }
14417
14418 fn prepare_patch(
14419 &self,
14420 table: &str,
14421 row_id: &str,
14422 partial: Map<String, Value>,
14423 base_version: Option<i64>,
14424 ) -> Result<OutboxOp, String> {
14425 let schema_table = self
14426 .schema
14427 .table(table)
14428 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
14429 let partial = normalize_values_casing(schema_table, partial)?;
14430 if let Some(pk) = partial.get(&schema_table.primary_key) {
14431 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
14432 return Err(format!(
14433 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
14434 ));
14435 }
14436 }
14437 let exists: bool = self
14440 .conn
14441 .query_row(
14442 &format!(
14443 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
14444 visible_table(table),
14445 row_id_predicate(schema_table)
14446 ),
14447 rusqlite::params![row_id],
14448 |row| row.get(0),
14449 )
14450 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14451 if !exists {
14452 return Err("sync.row_missing: a sparse patch requires a local row".to_owned());
14453 }
14454 for scope in &schema_table.scope_variables {
14464 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
14465 continue;
14466 }
14467 let Some(column) = schema_table
14468 .columns
14469 .iter()
14470 .find(|column| column.name == scope.column)
14471 else {
14472 continue;
14473 };
14474 let stored = self
14475 .conn
14476 .query_row(
14477 &format!(
14478 "SELECT {} FROM {} WHERE {}",
14479 quote_ident(&scope.column),
14480 visible_table(&schema_table.name),
14481 row_id_predicate(schema_table)
14482 ),
14483 rusqlite::params![row_id],
14484 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
14485 )
14486 .optional()
14487 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14488 let incoming = json_to_column_value(column, partial.get(&scope.column))
14493 .ok()
14494 .map(|value| column_value_to_json(&value));
14495 let equal = stored.as_ref().is_some_and(|stored| {
14499 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
14500 });
14501 if !equal {
14502 return Err(format!(
14503 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
14504 scope.column
14505 ));
14506 }
14507 }
14508 let mut values = partial;
14509 for scope in &schema_table.scope_variables {
14512 values.remove(&scope.column);
14513 }
14514 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
14515 ColumnType::Integer => match row_id.parse::<i64>() {
14517 Ok(v)
14518 if v.to_string() == row_id
14519 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
14520 .contains(&v) =>
14521 {
14522 Value::from(v)
14523 }
14524 _ => {
14525 return Err(format!(
14526 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
14527 ))
14528 }
14529 },
14530 ColumnType::Boolean => match row_id {
14531 "true" => Value::Bool(true),
14532 "false" => Value::Bool(false),
14533 _ => {
14534 return Err(format!(
14535 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
14536 ))
14537 }
14538 },
14539 _ => Value::from(row_id.to_owned()),
14540 };
14541 values.insert(schema_table.primary_key.clone(), pk_value);
14542 self.validate_author_encode(schema_table, row_id, &values)?;
14548 Ok(OutboxOp {
14549 upsert: true,
14550 table: table.to_owned(),
14551 row_id: row_id.to_owned(),
14552 base_version,
14553 values: Some(values),
14554 })
14555 }
14556
14557 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
14562 let selector = self.encryption.key_id_columns.get(&table.name)?;
14563 let index = table
14564 .columns
14565 .iter()
14566 .position(|column| &column.name == selector)?;
14567 let sql = format!(
14568 "SELECT {} FROM {} WHERE {}",
14569 quote_ident(selector),
14570 visible_table(&table.name),
14571 row_id_predicate(table)
14572 );
14573 let key: Option<String> = self
14574 .conn
14575 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
14576 .ok()?;
14577 let key = key.filter(|key| !key.is_empty())?;
14578 let mut fallback: Row = vec![None; table.columns.len()];
14579 fallback[index] = Some(ColumnValue::String(key));
14580 Some(fallback)
14581 }
14582
14583 fn validate_author_encode(
14588 &self,
14589 table: &TableSchema,
14590 row_id: &str,
14591 values: &Map<String, Value>,
14592 ) -> Result<(), String> {
14593 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
14594 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
14595 result => result.map(|_| ()),
14596 }
14597 }
14598
14599 fn drop_unencodable_outbox(
14613 &mut self,
14614 with_encryption: bool,
14615 batch: Option<&mut ChangeAccumulator>,
14616 ) -> Result<bool, String> {
14617 let failures = self
14618 .outbox
14619 .iter()
14620 .filter_map(|commit| {
14621 commit
14622 .ops
14623 .iter()
14624 .find_map(|op| {
14625 let values = op.values.as_ref().filter(|_| op.upsert)?;
14626 let table = self.schema.table(&op.table)?;
14627 let fallback = self.stored_key_fallback(table, &op.row_id);
14628 match encode_sparse_row_json(
14629 table,
14630 &op.row_id,
14631 values,
14632 &self.encryption,
14633 fallback.as_ref(),
14634 ) {
14635 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
14636 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
14637 }
14638 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
14643 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
14644 OUTBOX_INCOMPATIBLE_CODE,
14645 )),
14646 _ => None,
14647 }
14648 })
14649 .map(|(error, code)| (commit.clone(), error, code))
14650 })
14651 .collect::<Vec<_>>();
14652 if failures.is_empty() {
14653 return Ok(false);
14654 }
14655 let prior_outbox = self.outbox.clone();
14661 let prior_rejections = self.rejections.clone();
14662 let prior_failed_commits = self.failed_commits.clone();
14666 let prior_changes = self.change_queue.len();
14667 let prior_intents = self.sync_intent_queue.len();
14668 let prior_last_change = self.last_change.clone();
14669 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14670 let owns_observation = batch.is_none();
14671 let owns_transaction = self.conn.is_autocommit();
14672 let mut domains = ChangeAccumulator::default();
14673 if owns_observation {
14674 self.begin_observation("syncular_value_recovery")?;
14675 }
14676 let dropped = (|| -> Result<(), String> {
14677 for (commit, message, code) in failures {
14678 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
14679 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
14680 }
14681 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
14682 let rejection = RejectionRecord {
14683 client_commit_id: commit.client_commit_id.clone(),
14684 op_index: 0,
14685 code: code.to_owned(),
14686 message,
14687 retryable: false,
14688 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
14689 field_paths: None,
14690 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
14691 required_action: None,
14692 references: None,
14693 }),
14694 operation: representable
14695 .then(|| commit.ops.first())
14696 .flatten()
14697 .map(CommitOperation::from),
14698 };
14699 self.persist_commit_outcome(
14700 &commit.client_commit_id,
14701 CommitOutcomeStatus::Rejected,
14702 &[CommitOperationOutcome::Error {
14703 rejection: rejection.clone(),
14704 }],
14705 representable.then_some(commit.ops.as_slice()),
14706 )?;
14707 self.delete_outbox_persisted(&commit.client_commit_id)?;
14708 self.outbox
14709 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
14710 self.rejections.push(rejection);
14711 for operation in &commit.ops {
14712 domains.table(operation.table.as_str());
14713 }
14714 }
14715 domains.status = true;
14716 domains.rejections = true;
14717 domains.outcomes = true;
14718 self.prune_commit_outcomes()?;
14719 self.overlay_dirty.set(true);
14721 self.rebuild_overlay()?;
14722 Ok(())
14723 })();
14724 let result = match batch {
14725 Some(batch) => {
14726 if dropped.is_ok() {
14727 for table in domains.tables.keys() {
14728 batch.table(table.as_str());
14729 }
14730 batch.status |= domains.status;
14731 batch.rejections |= domains.rejections;
14732 batch.outcomes |= domains.outcomes;
14733 }
14734 dropped
14735 }
14736 None => {
14737 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
14738 }
14739 };
14740 if let Err(error) = result {
14741 if owns_observation {
14742 if owns_transaction {
14743 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
14744 let mut failure = self.storage_failure.borrow_mut();
14745 let failure =
14746 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
14747 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
14748 }
14749 } else {
14750 self.rollback_observation("syncular_value_recovery");
14751 }
14752 }
14753 self.outbox = prior_outbox;
14754 self.rejections = prior_rejections;
14755 self.failed_commits = prior_failed_commits;
14756 self.change_queue.truncate(prior_changes);
14757 self.sync_intent_queue.truncate(prior_intents);
14758 self.last_change = prior_last_change;
14759 self.overlay_dirty.restore(prior_overlay_dirty);
14760 return Err(error);
14761 }
14762 Ok(true)
14763 }
14764
14765 pub fn pending_commit_ids(&self) -> Vec<String> {
14766 self.outbox
14767 .iter()
14768 .map(|c| c.client_commit_id.clone())
14769 .collect()
14770 }
14771
14772 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
14776 let mut payloads = Vec::new();
14777 for commit in &self.outbox {
14778 for op in &commit.ops {
14779 let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
14780 continue;
14781 };
14782 let Some(table) = self.schema.table(&op.table) else {
14783 continue;
14784 };
14785 let fallback = self.stored_key_fallback(table, &op.row_id);
14786 if let Ok(payload) = encode_sparse_row_json(
14787 table,
14788 &op.row_id,
14789 values,
14790 &self.encryption,
14791 fallback.as_ref(),
14792 ) {
14793 payloads.push(payload);
14794 }
14795 }
14796 }
14797 payloads
14798 }
14799
14800 pub fn conflicts(&self) -> &[ConflictRecord] {
14801 &self.conflicts
14802 }
14803
14804 pub fn rejections(&self) -> &[RejectionRecord] {
14805 &self.rejections
14806 }
14807
14808 pub fn resolve_commit_outcome(
14809 &mut self,
14810 input: ResolveCommitOutcomeInput,
14811 ) -> Result<CommitOutcome, String> {
14812 let current = self
14813 .commit_outcome(&input.client_commit_id)?
14814 .ok_or_else(|| {
14815 format!(
14816 "sync.outcome_not_found: no durable outcome exists for {:?}",
14817 input.client_commit_id
14818 )
14819 })?;
14820 if current.resolution != CommitOutcomeResolution::Active {
14821 return Ok(current);
14822 }
14823 if input.resolution == CommitOutcomeResolution::Active {
14824 return Err("sync.invalid_request: resolution must leave active state".to_owned());
14825 }
14826 match input.resolution {
14827 CommitOutcomeResolution::Superseded => {
14828 let replacement = input
14829 .replacement_client_commit_id
14830 .as_deref()
14831 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
14832 .ok_or_else(|| {
14833 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
14834 .to_owned()
14835 })?;
14836 let _ = replacement;
14837 }
14838 _ if input.replacement_client_commit_id.is_some() => {
14839 return Err(
14840 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
14841 .to_owned(),
14842 );
14843 }
14844 _ => {}
14845 }
14846 let allowed = match current.status {
14847 CommitOutcomeStatus::Conflict => matches!(
14848 input.resolution,
14849 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
14850 ),
14851 CommitOutcomeStatus::Rejected => {
14852 matches!(
14853 input.resolution,
14854 CommitOutcomeResolution::Superseded
14855 | CommitOutcomeResolution::ResolvedKeepServer
14856 )
14857 }
14858 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
14859 input.resolution == CommitOutcomeResolution::Dismissed
14860 }
14861 };
14862 if !allowed {
14863 return Err(format!(
14864 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
14865 input.resolution, current.status
14866 ));
14867 }
14868
14869 self.begin_observation("syncular_outcome_resolution")?;
14870 let previous_failed = self.failed_commits.clone();
14871 let previous_conflicts = self.conflicts.clone();
14872 let previous_rejections = self.rejections.clone();
14873 let result = (|| {
14874 self.conn
14875 .execute(
14876 "UPDATE _syncular_commit_outcomes
14877 SET resolution = ?1, resolved_at_ms = ?2,
14878 replacement_client_commit_id = ?3
14879 WHERE client_commit_id = ?4 AND resolution = 'active'",
14880 rusqlite::params![
14881 Self::outcome_resolution_name(input.resolution),
14882 self.clock_now_ms(),
14883 input.replacement_client_commit_id,
14884 input.client_commit_id
14885 ],
14886 )
14887 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14888 let resolved = self
14889 .commit_outcome(¤t.client_commit_id)?
14890 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
14891 self.conn
14892 .execute(
14893 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
14894 rusqlite::params![input.client_commit_id],
14895 )
14896 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14897 for (commit, _) in &self.failed_commits {
14898 if commit.client_commit_id == input.client_commit_id {
14899 for operation in &commit.ops {
14900 self.overlay_dirty.table(&operation.table);
14901 }
14902 }
14903 }
14904 self.failed_commits
14905 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
14906 self.prune_commit_outcomes()?;
14907 Ok(resolved)
14908 })();
14909 let resolved = match result {
14910 Ok(outcome) => outcome,
14911 Err(error) => {
14912 self.rollback_observation("syncular_outcome_resolution");
14913 self.failed_commits = previous_failed;
14914 self.conflicts = previous_conflicts;
14915 self.rejections = previous_rejections;
14916 return Err(error);
14917 }
14918 };
14919 self.conflicts
14920 .retain(|record| record.client_commit_id != current.client_commit_id);
14921 self.rejections
14922 .retain(|record| record.client_commit_id != current.client_commit_id);
14923 let mut batch = ChangeAccumulator {
14924 conflicts: current.status == CommitOutcomeStatus::Conflict,
14925 rejections: current.status == CommitOutcomeStatus::Rejected,
14926 outcomes: true,
14927 ..ChangeAccumulator::default()
14928 };
14929 if let Some(operations) = ¤t.operations {
14930 for operation in operations {
14931 batch.table(&operation.table);
14932 }
14933 }
14934 if let Err(error) = self
14935 .rebuild_overlay_if_dirty()
14936 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
14937 {
14938 self.rollback_observation("syncular_outcome_resolution");
14939 self.failed_commits = previous_failed;
14940 self.conflicts = previous_conflicts;
14941 self.rejections = previous_rejections;
14942 return Err(error);
14943 }
14944 Ok(resolved)
14945 }
14946
14947 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
14948 self.schema_floor.as_ref()
14949 }
14950
14951 pub fn lease_state(&self) -> Option<&LeaseState> {
14953 self.lease_state.as_ref()
14954 }
14955
14956 fn record_lease_error(&mut self, code: &str) {
14959 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
14960 return;
14961 }
14962 let mut next = self.lease_state.clone().unwrap_or_default();
14963 next.error_code = Some(code.to_owned());
14964 self.set_lease_state(Some(next));
14965 }
14966
14967 fn set_lease_state(&mut self, next: Option<LeaseState>) {
14968 if self.lease_state == next {
14969 return;
14970 }
14971 if self.begin_observation("syncular_lease").is_err() {
14972 return;
14973 }
14974 self.lease_state = next;
14975 if let Some(lease) = &self.lease_state {
14976 if let Ok(json) = serde_json::to_string(lease) {
14977 self.set_meta(LEASE_STATE_KEY, &json);
14978 }
14979 } else {
14980 self.delete_meta(LEASE_STATE_KEY);
14981 }
14982 let batch = ChangeAccumulator {
14983 status: true,
14984 ..ChangeAccumulator::default()
14985 };
14986 if self.finish_observation("syncular_lease", batch).is_err() {
14987 self.rollback_observation("syncular_lease");
14988 }
14989 }
14990
14991 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
14992 if self.schema_floor == next {
14993 return;
14994 }
14995 if self.begin_observation("syncular_schema_floor").is_err() {
14996 return;
14997 }
14998 self.schema_floor = next;
14999 self.stopped = self.schema_floor.is_some();
15000 if let Some(floor) = &self.schema_floor {
15001 if let Ok(json) = serde_json::to_string(floor) {
15002 self.set_meta(SCHEMA_FLOOR_KEY, &json);
15003 }
15004 } else {
15005 self.delete_meta(SCHEMA_FLOOR_KEY);
15006 }
15007 let batch = ChangeAccumulator {
15008 status: true,
15009 ..ChangeAccumulator::default()
15010 };
15011 if self
15012 .finish_observation("syncular_schema_floor", batch)
15013 .is_err()
15014 {
15015 self.rollback_observation("syncular_schema_floor");
15016 }
15017 }
15018
15019 fn set_upgrading(&mut self, value: bool) {
15020 if self.upgrading == value {
15021 return;
15022 }
15023 if self.begin_observation("syncular_upgrading").is_err() {
15024 return;
15025 }
15026 self.upgrading = value;
15027 let batch = ChangeAccumulator {
15028 status: true,
15029 ..ChangeAccumulator::default()
15030 };
15031 if self
15032 .finish_observation("syncular_upgrading", batch)
15033 .is_err()
15034 {
15035 self.rollback_observation("syncular_upgrading");
15036 }
15037 }
15038
15039 pub fn sync_needed(&self) -> bool {
15040 self.sync_needed
15041 }
15042
15043 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
15044 let sub = self.subs.iter().find(|s| s.id == id)?;
15045 Some(SubscriptionStateView {
15046 id: sub.id.clone(),
15047 table: sub.table.clone(),
15048 status: sub.state.name().to_owned(),
15049 cursor: sub.cursor,
15050 has_resume_token: sub.bootstrap_state.is_some(),
15051 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
15052 reason_code: sub.reason_code.clone(),
15053 })
15054 }
15055
15056 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
15057 let schema_table = self
15058 .schema
15059 .table(table)
15060 .ok_or_else(|| format!("unknown table {table:?}"))?;
15061 let sql = format!(
15062 "SELECT * FROM {} ORDER BY {} ASC",
15063 visible_table(table),
15064 quote_ident(&schema_table.primary_key)
15065 );
15066 let mut stmt = self
15067 .conn
15068 .prepare(&sql)
15069 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15070 let mut rows = stmt
15071 .query([])
15072 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15073 let mut out = Vec::new();
15074 while let Some(row) = rows
15075 .next()
15076 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15077 {
15078 let mut values = Map::new();
15079 for (i, column) in schema_table.columns.iter().enumerate() {
15080 let value = row
15081 .get_ref(i)
15082 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15083 values.insert(column.name.clone(), sql_ref_to_json(column, value));
15084 }
15085 let version: i64 = row
15086 .get(schema_table.columns.len())
15087 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15088 let row_id = match values.get(&schema_table.primary_key) {
15089 Some(Value::String(s)) => s.clone(),
15090 Some(Value::Number(n)) => n.to_string(),
15091 Some(Value::Bool(b)) => b.to_string(),
15092 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
15093 };
15094 out.push(RowState {
15095 row_id,
15096 version,
15097 values,
15098 });
15099 }
15100 Ok(out)
15101 }
15102
15103 #[cfg(feature = "crdt-yjs")]
15119 fn crdt_column_bytes(
15120 &self,
15121 table: &str,
15122 row_id: &str,
15123 column: &str,
15124 ) -> Result<Option<Vec<u8>>, String> {
15125 let schema_table = self
15126 .schema
15127 .table(table)
15128 .ok_or_else(|| format!("unknown table {table:?}"))?;
15129 let col = schema_table
15130 .columns
15131 .iter()
15132 .find(|c| c.name == column)
15133 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
15134 if col.ty != ColumnType::Crdt {
15135 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
15136 }
15137 let sql = format!(
15138 "SELECT {} FROM {} WHERE {}",
15139 quote_ident(column),
15140 visible_table(table),
15141 row_id_predicate(schema_table)
15142 );
15143 let bytes: Option<Vec<u8>> = self
15144 .conn
15145 .prepare_cached(&sql)
15146 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15147 .query_row(rusqlite::params![row_id], |row| {
15148 row.get::<_, Option<Vec<u8>>>(0)
15149 })
15150 .map_err(|e| match e {
15151 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
15152 other => Self::sqlite_failure(&self.storage_failure, other),
15153 })?;
15154 Ok(bytes)
15155 }
15156
15157 #[cfg(feature = "crdt-yjs")]
15161 pub fn crdt_text(
15162 &self,
15163 table: &str,
15164 row_id: &str,
15165 column: &str,
15166 name: &str,
15167 ) -> Result<String, String> {
15168 let bytes = self
15169 .crdt_column_bytes(table, row_id, column)?
15170 .unwrap_or_default();
15171 crate::crdt::text(&bytes, name)
15172 }
15173
15174 #[cfg(feature = "crdt-yjs")]
15178 pub fn crdt_insert_text(
15179 &mut self,
15180 table: &str,
15181 row_id: &str,
15182 column: &str,
15183 name: &str,
15184 index: u32,
15185 value: &str,
15186 ) -> Result<String, ClientError> {
15187 let prior = self.storage_failure.borrow_mut().take();
15188 let outcome = (|| -> Result<String, String> {
15189 let current = self
15190 .crdt_column_bytes(table, row_id, column)?
15191 .unwrap_or_default();
15192 let update = crate::crdt::insert_text(¤t, name, index, value)?;
15193 self.perform_crdt_push_update(table, row_id, column, &update)
15194 })();
15195 self.finish_authoring(prior, outcome)
15196 }
15197
15198 #[cfg(feature = "crdt-yjs")]
15202 pub fn crdt_delete_text(
15203 &mut self,
15204 table: &str,
15205 row_id: &str,
15206 column: &str,
15207 name: &str,
15208 index: u32,
15209 len: u32,
15210 ) -> Result<String, ClientError> {
15211 let prior = self.storage_failure.borrow_mut().take();
15212 let outcome = (|| -> Result<String, String> {
15213 let current = self
15214 .crdt_column_bytes(table, row_id, column)?
15215 .unwrap_or_default();
15216 let update = crate::crdt::delete_text(¤t, name, index, len)?;
15217 self.perform_crdt_push_update(table, row_id, column, &update)
15218 })();
15219 self.finish_authoring(prior, outcome)
15220 }
15221
15222 #[cfg(feature = "crdt-yjs")]
15227 pub fn crdt_apply_update(
15228 &mut self,
15229 table: &str,
15230 row_id: &str,
15231 column: &str,
15232 update: &[u8],
15233 ) -> Result<String, ClientError> {
15234 let prior = self.storage_failure.borrow_mut().take();
15235 let outcome = (|| -> Result<String, String> {
15236 let current = self
15237 .crdt_column_bytes(table, row_id, column)?
15238 .unwrap_or_default();
15239 let next = crate::crdt::apply_update(¤t, update)?;
15240 self.perform_crdt_push_update(table, row_id, column, &next)
15241 })();
15242 self.finish_authoring(prior, outcome)
15243 }
15244
15245 #[cfg(feature = "crdt-yjs")]
15252 fn perform_crdt_push_update(
15253 &mut self,
15254 table: &str,
15255 row_id: &str,
15256 column: &str,
15257 crdt_bytes: &[u8],
15258 ) -> Result<String, String> {
15259 let mut bytes_obj = Map::new();
15260 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
15261 let mut partial = Map::new();
15262 partial.insert(column.to_owned(), Value::Object(bytes_obj));
15263 self.perform_patch(table, row_id, partial, None)
15264 }
15265
15266 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
15281 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
15282 }
15283
15284 #[cfg(feature = "bench-internals")]
15286 #[doc(hidden)]
15287 pub fn benchmark_connection(&mut self) -> &mut Connection {
15288 &mut self.conn
15289 }
15290
15291 #[cfg(feature = "bench-internals")]
15293 #[doc(hidden)]
15294 pub fn benchmark_phases(
15295 &self,
15296 enabled: Option<bool>,
15297 reset: bool,
15298 ) -> Result<Option<Value>, String> {
15299 self.benchmark_phases.configure(enabled, reset)?;
15300 Ok(self.benchmark_phases.snapshot())
15301 }
15302
15303 pub fn authority_snapshot(&self) -> Result<Value, String> {
15305 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
15306 if self.limits.authority_reads.is_empty() {
15307 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
15308 }
15309 self.conn
15310 .execute_batch("SAVEPOINT authority_snapshot")
15311 .map_err(|e| e.to_string())?;
15312 let result = (|| {
15313 let revision = self
15314 .get_meta(LOCAL_REVISION_KEY)
15315 .ok_or("sync.local_corrupt: authority revision is missing")?;
15316 let parsed_revision = revision
15317 .parse::<u64>()
15318 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
15319 if parsed_revision.to_string() != revision {
15320 return Err("sync.local_corrupt: authority revision is invalid".into());
15321 }
15322 let mut tables = Vec::new();
15323 for read in &self.limits.authority_reads {
15324 let table = self
15325 .schema
15326 .table(&read.table)
15327 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
15328 let scalar = |column: &str, value: &Value| -> Result<String, String> {
15329 let ty = table
15330 .columns
15331 .iter()
15332 .find(|c| c.name == column)
15333 .expect("validated authority column")
15334 .ty;
15335 if ty == ColumnType::Boolean {
15336 if let Some(value) = value.as_bool() {
15337 return Ok(value.to_string());
15338 }
15339 return match value.as_i64() {
15340 Some(0) => Ok("false".into()),
15341 Some(1) => Ok("true".into()),
15342 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
15343 };
15344 }
15345 match value {
15346 Value::String(s) => Ok(s.clone()),
15347 Value::Number(n) => Ok(n.to_string()),
15348 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
15349 .get("$bigint")
15350 .and_then(Value::as_str)
15351 .map(str::to_owned)
15352 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
15353 _ => {
15354 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
15355 }
15356 }
15357 };
15358 let matches = |row: &QueryRow| -> Result<bool, String> {
15359 for (variable, values) in &read.scopes {
15360 let column = table
15361 .scope_column(variable)
15362 .expect("validated authority scope");
15363 let value = row
15364 .get(column)
15365 .ok_or("sync.local_corrupt: authority scope column is missing")?;
15366 if value.is_null() || !values.contains(&scalar(column, value)?) {
15367 return Ok(false);
15368 }
15369 }
15370 Ok(true)
15371 };
15372 let selected = read
15373 .columns
15374 .iter()
15375 .map(|c| quote_ident(c))
15376 .collect::<Vec<_>>()
15377 .join(",");
15378 let mut intent = BTreeSet::new();
15379 let mut relevant = BTreeSet::new();
15380 for row in query_connection(&self.conn,
15381 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
15382 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
15383 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
15384 intent.insert(id.clone());
15385 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
15386 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
15387 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
15388 }
15389 }
15390 let mut rows = BTreeMap::new();
15391 for full in [base_table(&read.table), visible_table(&read.table)] {
15392 for mut values in query_connection(
15393 &self.conn,
15394 &format!(
15395 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
15396 ),
15397 &[],
15398 )
15399 .map_err(|e| e.to_string())?
15400 {
15401 if !matches(&values)? {
15402 continue;
15403 }
15404 let id = scalar(
15405 &table.primary_key,
15406 values
15407 .get(&table.primary_key)
15408 .expect("selected primary key"),
15409 )?;
15410 relevant.insert(id.clone());
15411 if full == base_table(&read.table) {
15412 let version = values
15413 .remove("authority_version")
15414 .expect("selected version");
15415 let version = version
15416 .as_i64()
15417 .ok_or("sync.local_corrupt: authority server version is invalid")?;
15418 if version >= 0 {
15419 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
15420 }
15421 }
15422 }
15423 }
15424 intent.retain(|id| relevant.contains(id));
15425 let rows = rows.into_values().collect::<Vec<_>>();
15426 let mut persisted = Vec::new();
15427 for row in query_connection(
15428 &self.conn,
15429 "SELECT state_json FROM _syncular_subscriptions WHERE tbl=? ORDER BY rowid",
15430 &[Value::String(read.table.clone())],
15431 )
15432 .map_err(|e| e.message)?
15433 {
15434 let state: Value = serde_json::from_str(
15435 row.get("state_json")
15436 .and_then(Value::as_str)
15437 .ok_or("sync.local_corrupt: authority subscription state is missing")?,
15438 )
15439 .map_err(|_| "sync.local_corrupt: authority subscription state is invalid")?;
15440 let effective = state.get("effectiveScopes").and_then(Value::as_object);
15441 let status = state
15442 .get("status")
15443 .and_then(Value::as_str)
15444 .ok_or("sync.local_corrupt: authority subscription status is missing")?;
15445 let cursor = state
15446 .get("cursor")
15447 .and_then(Value::as_i64)
15448 .ok_or("sync.local_corrupt: authority subscription cursor is missing")?;
15449 let complete = effective.is_some_and(|s| s.len() == read.scopes.len())
15450 && status == "active"
15451 && cursor >= 0
15452 && state.get("bootstrapState").is_none_or(Value::is_null)
15453 && state.get("params").is_none_or(Value::is_null);
15454 let exposed = effective.map(|s| {
15455 read.scopes
15456 .keys()
15457 .map(|v| (v.clone(), s.get(v).cloned().unwrap_or(json!([]))))
15458 .collect::<Map<_, _>>()
15459 });
15460 let requested = read
15461 .scopes
15462 .keys()
15463 .map(|v| {
15464 (
15465 v.clone(),
15466 state
15467 .get("requested")
15468 .and_then(|s| s.get(v))
15469 .cloned()
15470 .unwrap_or(json!([])),
15471 )
15472 })
15473 .collect::<Map<_, _>>();
15474 persisted.push(json!({"requestedScopes": requested, "status": status, "cursor": cursor, "effectiveScopes": exposed, "complete": complete}));
15475 }
15476 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
15477 match selectors.split_first() {
15478 None => !candidates.is_empty(),
15479 Some(((variable, values), rest)) => values.iter().all(|value| {
15480 covered(
15481 rest,
15482 &candidates
15483 .iter()
15484 .copied()
15485 .filter(|p| {
15486 p["effectiveScopes"][variable.as_str()]
15487 .as_array()
15488 .is_some_and(|held| {
15489 held.contains(&Value::String(value.clone()))
15490 || held.contains(&Value::String("*".into()))
15491 })
15492 })
15493 .collect::<Vec<_>>(),
15494 )
15495 }),
15496 }
15497 }
15498 let coverage = if covered(
15499 &read.scopes.iter().collect::<Vec<_>>(),
15500 &persisted
15501 .iter()
15502 .filter(|p| p["complete"] == true)
15503 .collect::<Vec<_>>(),
15504 ) {
15505 "complete"
15506 } else if persisted.iter().any(|p| {
15507 p["status"] == "active"
15508 && read.scopes.iter().all(|(v, values)| {
15509 values.iter().any(|value| {
15510 p["requestedScopes"][v].as_array().is_some_and(|held| {
15511 held.contains(&Value::String(value.clone()))
15512 || held.contains(&Value::String("*".into()))
15513 })
15514 })
15515 })
15516 }) {
15517 "pending"
15518 } else {
15519 "missing"
15520 };
15521 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
15522 }
15523 Ok(
15524 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
15525 )
15526 })();
15527 match result {
15528 Ok(snapshot) => {
15529 self.conn
15530 .execute_batch("RELEASE authority_snapshot")
15531 .map_err(|e| e.to_string())?;
15532 Ok(snapshot)
15533 }
15534 Err(error) => {
15535 self.conn
15536 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
15537 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
15538 Err(error)
15539 }
15540 }
15541 }
15542
15543 pub fn query_snapshot(
15547 &mut self,
15548 sql: &str,
15549 params: &[QueryValue],
15550 coverage: &[WindowCoverage],
15551 owner: Option<QueryOwner<'_>>,
15552 ) -> Result<QuerySnapshot, String> {
15553 let result = snapshot_connection(&self.conn, sql, params, coverage);
15554 if let Some(owner) = owner {
15555 self.record_query_read(owner, result.as_ref().err());
15556 }
15557 result.map_err(|failure| failure.to_string())
15558 }
15559
15560 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
15564 let position = self
15565 .query_failures
15566 .iter()
15567 .position(|entry| entry.id == owner.id);
15568 let Some(failure) = failure else {
15569 if let Some(index) = position {
15570 self.query_failures.remove(index);
15571 }
15572 return;
15573 };
15574 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
15575 .into_iter()
15576 .map(str::to_owned)
15577 .collect();
15578 let code = failure.code.unwrap_or("client.query_failed").to_owned();
15579 if position.is_some_and(|index| {
15580 let entry = &self.query_failures[index];
15581 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
15582 }) {
15583 return;
15584 }
15585 if let Some(index) = position {
15586 self.query_failures.remove(index);
15587 }
15588 self.query_failures.push(DiagnosticQueryFailure {
15589 id: owner.id.to_owned(),
15590 tables,
15591 code,
15592 sqlite_code: failure.sqlite_code,
15593 at_ms: self.clock_now_ms(),
15594 });
15595 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
15596 self.query_failures.remove(0);
15597 }
15598 }
15599
15600 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
15603 #[cfg(feature = "bench-internals")]
15604 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
15605 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
15606 let max_commits = self.limits.max_push_commits_per_request;
15607 let max_ops = self
15608 .limits
15609 .max_push_operations_per_request
15610 .unwrap_or(PUSH_OPS_PER_REQUEST);
15611 let byte_cap = self.limits.max_push_request_bytes;
15612 let accept = self.limits.accept.unwrap_or(if url_capable {
15615 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
15616 } else {
15617 DEFAULT_ACCEPT
15618 });
15619 let header = Frame::ReqHeader {
15620 client_id: self.client_id.clone(),
15621 schema_version: self.schema.version,
15622 log_epoch: log_epoch.clone(),
15623 };
15624 let pull = Frame::PullHeader {
15625 limit_commits: self.limits.limit_commits.unwrap_or(0),
15626 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
15627 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
15628 accept,
15629 };
15630 let mut fresh = Vec::new();
15631 let mut sub_frames = Vec::new();
15632 for sub in &self.subs {
15633 if sub.state != SubState::Active {
15634 continue;
15635 }
15636 let mut scopes = sub.requested.clone();
15637 sort_scope_map(&mut scopes);
15638 sub_frames.push(Frame::Subscription {
15639 id: sub.id.clone(),
15640 table: sub.table.clone(),
15641 scopes,
15642 params: sub.params.clone().map(RawJson),
15643 cursor: sub.cursor,
15644 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
15645 });
15646 fresh.push((
15647 sub.id.clone(),
15648 sub.cursor < 0 && sub.bootstrap_state.is_none(),
15649 ));
15650 }
15651 let fixed_bytes = match byte_cap {
15654 Some(_) => {
15655 let mut frames = vec![header.clone(), pull.clone()];
15656 frames.extend(sub_frames.iter().cloned());
15657 encode_message(&Message {
15658 wire_version: WIRE_VERSION,
15659 msg_kind: MsgKind::Request,
15660 frames,
15661 })
15662 .len()
15663 }
15664 None => 0,
15665 };
15666 if let Some(cap) = byte_cap {
15667 if fixed_bytes > cap {
15668 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
15669 }
15670 }
15671 let base_bytes = match byte_cap {
15672 Some(_) => encode_message(&Message {
15673 wire_version: WIRE_VERSION,
15674 msg_kind: MsgKind::Request,
15675 frames: vec![header.clone(), pull.clone()],
15676 })
15677 .len(),
15678 None => 0,
15679 };
15680 let total_outbox = if log_epoch.is_some() {
15681 self.outbox.len()
15682 } else {
15683 0
15684 };
15685 let mut push_frames: Vec<Frame> = Vec::new();
15686 let mut pushed_ids = Vec::new();
15687 let mut ops_in_request = 0usize;
15688 let mut bytes_in_request = 0usize;
15689 let mut deferred_commits = 0usize;
15690 #[cfg(feature = "bench-internals")]
15691 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
15692 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
15693 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
15697 deferred_commits = total_outbox - index;
15698 break;
15699 }
15700 if ops_in_request + commit.ops.len() > max_ops {
15701 if push_frames.is_empty() {
15702 return Err(Self::push_capacity_outcome(
15703 "operations",
15704 max_ops,
15705 commit.ops.len(),
15706 Some(commit.client_commit_id.as_str()),
15707 ));
15708 }
15709 deferred_commits = total_outbox - index;
15710 break;
15711 }
15712 let operations = commit
15713 .ops
15714 .iter()
15715 .map(|op| {
15716 let payload = op.values.as_ref().and_then(|values| {
15717 let table = self.schema.table(&op.table)?;
15718 let fallback = self.stored_key_fallback(table, &op.row_id);
15727 encode_sparse_row_json(
15728 table,
15729 &op.row_id,
15730 values,
15731 &self.encryption,
15732 fallback.as_ref(),
15733 )
15734 .ok()
15735 });
15736 ssp2::model::Operation {
15737 table: op.table.clone(),
15738 row_id: op.row_id.clone(),
15739 op: if op.upsert { Op::Upsert } else { Op::Delete },
15740 base_version: op.base_version,
15741 payload,
15742 }
15743 })
15744 .collect();
15745 let frame = Frame::PushCommit {
15746 client_commit_id: commit.client_commit_id.clone(),
15747 operations,
15748 };
15749 let frame_bytes = match byte_cap {
15750 Some(_) => {
15751 encode_message(&Message {
15752 wire_version: WIRE_VERSION,
15753 msg_kind: MsgKind::Request,
15754 frames: vec![header.clone(), frame.clone(), pull.clone()],
15755 })
15756 .len()
15757 - base_bytes
15758 }
15759 None => 0,
15760 };
15761 if let Some(cap) = byte_cap {
15765 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
15766 deferred_commits = total_outbox - index;
15767 break;
15768 }
15769 }
15770 push_frames.push(frame);
15771 pushed_ids.push(commit.client_commit_id.clone());
15772 ops_in_request += commit.ops.len();
15773 bytes_in_request += frame_bytes;
15774 }
15775 #[cfg(feature = "bench-internals")]
15776 drop(outbox_phase);
15777 if let Some(cap) = byte_cap {
15778 if fixed_bytes + bytes_in_request > cap {
15779 let blocked = if push_frames.is_empty() {
15782 None
15783 } else {
15784 self.outbox.first().map(|c| c.client_commit_id.as_str())
15785 };
15786 return Err(Self::push_capacity_outcome(
15787 "bytes",
15788 cap,
15789 fixed_bytes + bytes_in_request,
15790 blocked,
15791 ));
15792 }
15793 }
15794 let mut frames = vec![header];
15795 frames.extend(push_frames);
15796 frames.push(pull);
15797 frames.extend(sub_frames);
15798 let message = Message {
15799 wire_version: WIRE_VERSION,
15800 msg_kind: MsgKind::Request,
15801 frames,
15802 };
15803 Ok((
15804 message,
15805 RequestMeta {
15806 pushed_ids,
15807 fresh,
15808 accept,
15809 deferred_commits,
15810 },
15811 ))
15812 }
15813
15814 fn push_capacity_outcome(
15816 kind: &str,
15817 limit: usize,
15818 size: usize,
15819 client_commit_id: Option<&str>,
15820 ) -> Box<SyncOutcome> {
15821 let mut details = Map::new();
15822 details.insert("kind".to_owned(), Value::from(kind));
15823 details.insert("limit".to_owned(), Value::from(limit as u64));
15824 details.insert("size".to_owned(), Value::from(size as u64));
15825 if let Some(id) = client_commit_id {
15826 details.insert("clientCommitId".to_owned(), Value::from(id));
15827 }
15828 Box::new(SyncOutcome::Failed {
15829 error_code: "client.push_request_too_large".into(),
15830 message: "the push request exceeds a configured capacity limit".into(),
15831 details: Some(Value::Object(details)),
15832 })
15833 }
15834
15835 pub fn progress(&self) -> ProgressObserver {
15839 self.progress.clone()
15840 }
15841
15842 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
15843 let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
15844 Ok(prepared) => prepared,
15845 Err(outcome) => return *outcome,
15846 };
15847 let mut next = prepared;
15852 loop {
15853 match self.apply_sync_round(next.exchange(transport)) {
15854 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
15855 crate::AppliedSyncRound::Complete {
15856 outcome, controls, ..
15857 } => {
15858 if self.transport_enabled {
15859 for text in controls {
15860 let _ = transport.realtime_send(&text);
15861 }
15862 } else {
15863 self.disconnect_realtime(transport);
15864 }
15865 return outcome;
15866 }
15867 }
15868 }
15869 }
15870
15871 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
15872 if let SyncOutcome::Failed {
15873 error_code,
15874 message,
15875 details,
15876 } = &mut outcome
15877 {
15878 if let Some(failure) = self.storage_failure.borrow().as_ref() {
15879 let code = failure
15880 .code
15881 .expect("retained storage failures are classified");
15882 *error_code = code.to_owned();
15883 *message = match code {
15884 "client.storage_busy" => "local SQLite storage is busy",
15885 "client.storage_full" => "local SQLite storage is full",
15886 "client.storage_io" => "local SQLite storage I/O failed",
15887 "client.storage_corrupt" => "local SQLite storage is corrupt",
15888 _ => unreachable!("unrecognized classified storage failure"),
15889 }
15890 .to_owned();
15891 *details = failure.details();
15892 }
15893 }
15894 let retry_delay_ms = self.round_retry_delay_ms;
15895 self.progress.update(|p| match &outcome {
15896 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
15897 if report.failed.is_empty() =>
15898 {
15899 p.state = ProgressState::Complete
15900 }
15901 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
15902 p.state = ProgressState::Failed;
15903 p.error_code = Some("sync.scope_revoked".into());
15904 }
15905 SyncOutcome::Failed { error_code, .. } => {
15906 p.state = ProgressState::Failed;
15907 p.error_code = Some(Self::diagnostic_code(error_code));
15908 p.retry_delay_ms = retry_delay_ms;
15909 }
15910 SyncOutcome::RealtimeUnavailable { .. } => {
15911 p.state = ProgressState::Failed;
15912 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
15913 p.retry_delay_ms = retry_delay_ms;
15914 }
15915 });
15916 let completed_at_ms = self.clock_now_ms();
15917 self.last_round = Some(match &outcome {
15918 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
15919 status: "succeeded".to_owned(),
15920 started_at_ms,
15921 completed_at_ms,
15922 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
15923 counters: Some(DiagnosticRoundCounters {
15924 pushed: report.pushed,
15925 applied: report.applied.len(),
15926 rejected: report.rejected.len(),
15927 retryable: report.retryable.len(),
15928 conflicts: report.conflicts,
15929 commits_applied: report.commits_applied,
15930 segment_rows_applied: report.segment_rows_applied,
15931 bootstrapping: report.bootstrapping.len(),
15932 resets: report.resets.len(),
15933 revoked: report.revoked.len(),
15934 failed: report.failed.len(),
15935 deferred_commits: report.deferred_commits,
15936 }),
15937 error_code: None,
15938 },
15939 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
15940 status: "failed".to_owned(),
15941 started_at_ms,
15942 completed_at_ms,
15943 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
15944 counters: None,
15945 error_code: Some(Self::diagnostic_code(error_code)),
15946 },
15947 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
15948 status: "failed".to_owned(),
15949 started_at_ms,
15950 completed_at_ms,
15951 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
15952 counters: None,
15953 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
15954 },
15955 });
15956 outcome
15957 }
15958
15959 pub fn prepare_sync_round(
15962 &mut self,
15963 url_capable: bool,
15964 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
15965 if !self.transport_enabled {
15966 return Err(Box::new(SyncOutcome::Failed {
15967 error_code: "sync.offline".into(),
15968 message: "transport gate is closed".into(),
15969 details: None,
15970 }));
15971 }
15972 if self.active_round.is_some() {
15973 return Err(Box::new(SyncOutcome::Failed {
15974 error_code: "client.round_in_progress".into(),
15975 message: "a sync round is already in flight".into(),
15976 details: None,
15977 }));
15978 }
15979 self.storage_failure.borrow_mut().take();
15980 self.progress.start();
15981 self.round_retry_delay_ms = None;
15982 let started_at_ms = self.clock_now_ms();
15983 let prepared = (|| {
15984 if self.stopped {
15985 return Err(Box::new(SyncOutcome::Ok(SyncReport {
15986 schema_floor: self.schema_floor.clone(),
15987 ..SyncReport::default()
15988 })));
15989 }
15990 if self.security_preflight {
15991 return Err(Box::new(SyncOutcome::Failed {
15992 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
15993 message: "security preflight must complete before syncing".into(),
15994 details: None,
15995 }));
15996 }
15997 self.set_sync_needed(false, false);
15998 self.drain_pending_evictions()
15999 .and_then(|_| self.drop_incompatible_outbox())
16000 .and_then(|_| self.drop_unencodable_outbox(true, None))
16001 .map_err(|message| {
16002 Box::new(SyncOutcome::Failed {
16003 error_code: "storage.failed".into(),
16004 message,
16005 details: None,
16006 })
16007 })?;
16008 let uploads = if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
16009 self.pending_blob_uploads().map_err(|e| {
16010 Box::new(SyncOutcome::Failed {
16011 error_code: e.code,
16012 message: e.message,
16013 details: e.details,
16014 })
16015 })?
16016 } else {
16017 Vec::new()
16018 };
16019 if self.realtime_state != RealtimeState::Connected
16020 && self.realtime_policy == RealtimePolicy::Required
16021 {
16022 let retry_delay_ms = self.schedule_background_retry();
16023 self.realtime_retry_delay_ms = Some(retry_delay_ms);
16024 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
16025 state: self.realtime_state,
16026 reason_code: self.realtime_reason_code.clone(),
16027 retry_delay_ms,
16028 }));
16029 }
16030 let (message, meta) = self.build_request(url_capable)?;
16031 let id = uuid::Uuid::new_v4();
16032 self.active_round = Some(id);
16033 Ok(crate::PreparedSyncRound {
16034 id,
16035 started_at_ms,
16036 message,
16037 meta,
16038 uploads: uploads.into(),
16039 realtime: self.realtime_state == RealtimeState::Connected,
16040 scopes: self
16041 .subs
16042 .iter()
16043 .map(|s| (s.id.clone(), canonical_scope_json(&s.requested)))
16044 .collect(),
16045 #[cfg(feature = "bench-internals")]
16046 benchmark_phases: self.benchmark_phases.clone(),
16047 progress: self.progress.clone(),
16048 fixed_now: self.now_ms,
16049 round_deadline_at: None,
16050 })
16051 })();
16052 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
16053 }
16054
16055 pub fn cancel_sync_round(&mut self) {
16058 self.active_round = None;
16059 }
16060
16061 pub fn apply_sync_round(
16064 &mut self,
16065 completed: crate::CompletedSyncRound,
16066 ) -> crate::AppliedSyncRound {
16067 let prepared = completed.prepared;
16068 if self.active_round != Some(prepared.id) || self.security_preflight {
16069 return crate::AppliedSyncRound::Complete {
16070 outcome: SyncOutcome::Failed {
16071 error_code: "client.round_cancelled".into(),
16072 message: "sync round belongs to a released client context".into(),
16073 details: None,
16074 },
16075 controls: Vec::new(),
16076 more: false,
16077 bootstrap_advanced: false,
16078 };
16079 }
16080 self.storage_failure.borrow_mut().take();
16081 let (response, mut downloads, transport_failed) = match completed.exchange {
16082 crate::round::ExchangeResult::Upload { id, result } => {
16083 let result = result.and_then(|()| {
16084 self.conn
16085 .execute(
16086 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
16087 rusqlite::params![id],
16088 )
16089 .map(|_| ())
16090 .map_err(|error| {
16091 TransportError::new(
16092 "client.failed",
16093 Self::sqlite_failure(&self.storage_failure, error),
16094 )
16095 })
16096 });
16097 match result {
16098 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
16099 Err(TransportError {
16100 code,
16101 message,
16102 details,
16103 }) => {
16104 self.active_round = None;
16105 if Self::retryable_failure_code(&code) {
16106 self.schedule_background_retry();
16107 }
16108 let outcome = self.finish_sync_round(
16109 prepared.started_at_ms,
16110 SyncOutcome::Failed {
16111 error_code: code,
16112 message,
16113 details,
16114 },
16115 );
16116 return crate::AppliedSyncRound::Complete {
16117 outcome,
16118 controls: Vec::new(),
16119 more: false,
16120 bootstrap_advanced: false,
16121 };
16122 }
16123 }
16124 }
16125 crate::round::ExchangeResult::Reply {
16126 response,
16127 downloads,
16128 transport_failed,
16129 } => (response, downloads, transport_failed),
16130 };
16131 self.active_round = None;
16132 let before = self
16133 .subs
16134 .iter()
16135 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16136 .collect::<Vec<_>>();
16137 let outcome = (|| {
16138 let response = match response {
16139 Ok(response) => response,
16140 Err(TransportError {
16141 code,
16142 message,
16143 details,
16144 }) => {
16145 if transport_failed {
16146 if prepared.realtime {
16147 self.set_realtime_state(RealtimeState::Lost, Some(&code));
16148 }
16149 self.record_lease_error(&code);
16150 }
16151 if Self::retryable_failure_code(&code) {
16152 let delay = self.schedule_background_retry();
16153 if prepared.realtime && transport_failed {
16154 self.realtime_retry_delay_ms = Some(delay);
16155 }
16156 }
16157 return SyncOutcome::Failed {
16158 error_code: code,
16159 message,
16160 details,
16161 };
16162 }
16163 };
16164 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
16165 if let SyncOutcome::Ok(report) = &mut outcome {
16166 report.deferred_commits = prepared.meta.deferred_commits;
16167 }
16168 match &outcome {
16169 SyncOutcome::Ok(_) => self.reset_background_retry(),
16170 SyncOutcome::Failed { error_code, .. }
16171 if Self::retryable_failure_code(error_code) =>
16172 {
16173 self.schedule_background_retry();
16174 }
16175 _ => {}
16176 }
16177 if prepared.meta.deferred_commits > 0 {
16178 self.set_sync_needed(true, true);
16179 }
16180 self.previous_version_lifetime_check();
16181 outcome
16182 })();
16183 let more = self.transport_enabled
16184 && matches!(&outcome, SyncOutcome::Ok(report)
16185 if report.schema_floor.is_none()
16186 && (!report.bootstrapping.is_empty()
16187 || report.commits_applied > 0
16188 || report.segment_rows_applied > 0
16189 || !report.resets.is_empty()
16190 || report.deferred_commits > 0
16191 || self.sync_needed));
16192 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
16193 && before
16194 != self
16195 .subs
16196 .iter()
16197 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16198 .collect::<Vec<_>>();
16199 crate::AppliedSyncRound::Complete {
16200 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
16201 controls: if self.transport_enabled {
16202 downloads.controls
16203 } else {
16204 Vec::new()
16205 },
16206 more,
16207 bootstrap_advanced,
16208 }
16209 }
16210
16211 pub fn sync_until_idle(
16212 &mut self,
16213 transport: &mut dyn Transport,
16214 max_rounds: Option<u32>,
16215 ) -> SyncOutcome {
16216 if max_rounds == Some(0) {
16219 return SyncOutcome::Failed {
16220 details: None,
16221 error_code: "sync.invalid_request".into(),
16222 message: "maxRounds must be a positive integer".into(),
16223 };
16224 }
16225 let rounds = max_rounds.unwrap_or(20);
16226 let mut aggregate = SyncReport::default();
16227 let mut spent = 0;
16228 while spent < rounds {
16229 spent += 1;
16230 let before = self
16231 .subs
16232 .iter()
16233 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16234 .collect::<Vec<_>>();
16235 match self.sync(transport) {
16236 SyncOutcome::Failed {
16237 error_code,
16238 message,
16239 details,
16240 } => {
16241 return SyncOutcome::Failed {
16242 details,
16243 error_code,
16244 message,
16245 };
16246 }
16247 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
16248 SyncOutcome::BudgetExhausted(_) => {
16249 unreachable!("sync() never exhausts a round budget")
16250 }
16251 SyncOutcome::Ok(report) => {
16252 if max_rounds.is_none()
16253 && report.segment_rows_applied > 0
16254 && !report.bootstrapping.is_empty()
16255 && before
16256 != self
16257 .subs
16258 .iter()
16259 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16260 .collect::<Vec<_>>()
16261 {
16262 spent = 0;
16263 }
16264 let stop = !self.transport_enabled() || report.schema_floor.is_some();
16267 aggregate.merge(&report);
16268 if stop {
16269 return SyncOutcome::Ok(aggregate);
16270 }
16271 let more = !report.bootstrapping.is_empty()
16277 || report.commits_applied > 0
16278 || report.segment_rows_applied > 0
16279 || !report.resets.is_empty()
16280 || report.deferred_commits > 0
16281 || self.sync_needed;
16282 if !more {
16283 return SyncOutcome::Ok(aggregate);
16284 }
16285 }
16286 }
16287 }
16288 SyncOutcome::BudgetExhausted(aggregate)
16291 }
16292
16293 fn process_response(
16294 &mut self,
16295 transport: &mut dyn Transport,
16296 response: Message,
16297 meta: &RequestMeta,
16298 ) -> SyncOutcome {
16299 #[cfg(feature = "bench-internals")]
16300 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
16301 let mut report = SyncReport {
16302 pushed: meta.pushed_ids.len() as u32,
16303 ..SyncReport::default()
16304 };
16305 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
16306 HashMap::new();
16307 let pushed_ids = meta
16308 .pushed_ids
16309 .iter()
16310 .map(String::as_str)
16311 .collect::<HashSet<_>>();
16312 let mut last_final_push_result_id: Option<String> = None;
16313 for frame in &response.frames {
16314 match frame {
16315 Frame::PushResultDetails {
16316 client_commit_id,
16317 entries,
16318 } => {
16319 let details = rejection_details_by_commit
16320 .entry(client_commit_id.clone())
16321 .or_default();
16322 for entry in entries {
16323 let parsed = match RejectionDetails::parse(&entry.details.0) {
16324 Ok(value) => value,
16325 Err(message) => {
16326 return SyncOutcome::Failed {
16327 details: None,
16328 error_code: "sync.invalid_request".to_owned(),
16329 message,
16330 };
16331 }
16332 };
16333 details.insert(entry.op_index, parsed);
16334 }
16335 }
16336 Frame::PushResult {
16337 client_commit_id,
16338 status,
16339 results,
16340 ..
16341 } if pushed_ids.contains(client_commit_id.as_str())
16342 && Self::push_result_is_final(*status, results) =>
16343 {
16344 last_final_push_result_id = Some(client_commit_id.clone());
16345 }
16346 _ => {}
16347 }
16348 }
16349 let mut frames = response.frames.into_iter().peekable();
16350 if response.wire_version < 2 {
16351 return SyncOutcome::Failed {
16352 details: None,
16353 error_code: "client.invalid_host_response".to_owned(),
16354 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
16355 };
16356 }
16357 match frames.next() {
16358 Some(Frame::RespHeader {
16359 required_schema_version,
16360 latest_schema_version,
16361 log_epoch,
16362 reset_required,
16363 }) => {
16364 let Some(log_epoch) = log_epoch else {
16365 return SyncOutcome::Failed {
16366 details: None,
16367 error_code: "client.invalid_host_response".to_owned(),
16368 message: "response header omits logEpoch".to_owned(),
16369 };
16370 };
16371 let Some(reset_required) = reset_required else {
16372 return SyncOutcome::Failed {
16373 details: None,
16374 error_code: "client.invalid_host_response".to_owned(),
16375 message: "response header omits resetRequired".to_owned(),
16376 };
16377 };
16378 if let Some(required) = required_schema_version {
16379 let floor = SchemaFloor {
16382 required_schema_version: Some(required),
16383 latest_schema_version,
16384 };
16385 self.set_schema_floor(Some(floor.clone()));
16386 report.schema_floor = Some(floor);
16387 return SyncOutcome::Ok(report);
16388 }
16389 if reset_required {
16390 if frames.next().is_some() {
16391 return SyncOutcome::Failed {
16392 details: None,
16393 error_code: "client.invalid_host_response".to_owned(),
16394 message: "log-epoch reset response contains body frames".to_owned(),
16395 };
16396 }
16397 match self.run_log_epoch_reset(&log_epoch) {
16398 Ok(resets) => {
16399 report.resets = resets;
16400 return SyncOutcome::Ok(report);
16401 }
16402 Err(message) => {
16403 return SyncOutcome::Failed {
16404 details: None,
16405 error_code: "sync.local_corrupt".to_owned(),
16406 message,
16407 };
16408 }
16409 }
16410 }
16411 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
16412 return SyncOutcome::Failed {
16413 details: None,
16414 error_code: "client.invalid_host_response".to_owned(),
16415 message: "server changed logEpoch without requiring a reset".to_owned(),
16416 };
16417 }
16418 }
16419 _ => {
16420 return SyncOutcome::Failed {
16421 details: None,
16422 error_code: "sync.invalid_request".to_owned(),
16423 message: "response does not start with RESP_HEADER".to_owned(),
16424 };
16425 }
16426 }
16427
16428 if let Err(message) = self.begin_coalesced_apply() {
16429 return SyncOutcome::Failed {
16430 details: None,
16431 error_code: "storage.failed".into(),
16432 message,
16433 };
16434 }
16435 let mut failure: Option<(String, String)> = None;
16436 while let Some(frame) = frames.next() {
16437 match frame {
16438 frame @ Frame::PushResult { .. } => {
16439 let successful = matches!(
16440 &frame,
16441 Frame::PushResult {
16442 status: PushStatus::Applied | PushStatus::Cached,
16443 ..
16444 }
16445 );
16446 let mut results = vec![frame];
16447 if successful {
16448 while matches!(
16449 frames.peek(),
16450 Some(Frame::PushResult {
16451 status: PushStatus::Applied | PushStatus::Cached,
16452 ..
16453 })
16454 ) {
16455 if let Some(next) = frames.next() {
16456 results.push(next);
16457 }
16458 }
16459 }
16460 if self
16461 .handle_push_results(
16462 &results,
16463 &pushed_ids,
16464 &rejection_details_by_commit,
16465 &mut report,
16466 last_final_push_result_id.as_deref(),
16467 )
16468 .is_err()
16469 {
16470 failure = Some((
16471 "client.outcome_persistence_failed".to_owned(),
16472 "local commit outcome could not be persisted".to_owned(),
16473 ));
16474 break;
16475 }
16476 }
16477 Frame::PushResultDetails { .. } => {}
16478 Frame::SubStart {
16479 id,
16480 status,
16481 reason_code,
16482 effective_scopes,
16483 bootstrap: _,
16484 } => {
16485 let mut body = Vec::new();
16486 let mut sub_end: Option<(i64, Option<String>)> = None;
16487 for inner in frames.by_ref() {
16488 match inner {
16489 Frame::SubEnd {
16490 next_cursor,
16491 bootstrap_state,
16492 } => {
16493 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
16494 break;
16495 }
16496 Frame::Unknown { .. } => {}
16497 other => body.push(other),
16498 }
16499 }
16500 if let Err(SectionError::Abort(code, message)) = self.process_section(
16501 transport,
16502 &id,
16503 status,
16504 &reason_code,
16505 effective_scopes,
16506 body,
16507 sub_end,
16508 meta,
16509 &mut report,
16510 ) {
16511 failure = Some((code, message));
16512 break;
16513 }
16514 }
16515 Frame::Lease {
16516 lease_id,
16517 expires_at_ms,
16518 } => {
16519 self.set_lease_state(Some(LeaseState {
16522 lease_id: Some(lease_id),
16523 expires_at_ms: Some(expires_at_ms),
16524 error_code: None,
16525 }));
16526 }
16527 Frame::Error { code, message, .. } => {
16528 failure = Some((code, message));
16531 break;
16532 }
16533 Frame::Unknown { .. } => {}
16534 _ => {
16535 failure = Some((
16536 "sync.invalid_request".to_owned(),
16537 "unexpected frame in response".to_owned(),
16538 ));
16539 break;
16540 }
16541 }
16542 }
16543
16544 let applied = match failure {
16545 Some(failure) => Err(failure),
16546 None => self
16551 .rebuild_overlay_if_dirty()
16552 .and_then(|()| self.drain_pending_evictions())
16553 .map_err(|message| ("storage.failed".to_owned(), message)),
16554 };
16555 if let Err(message) = self.finish_coalesced_apply() {
16558 return SyncOutcome::Failed {
16559 details: None,
16560 error_code: "storage.failed".into(),
16561 message,
16562 };
16563 }
16564 if let Err((error_code, message)) = applied {
16565 return SyncOutcome::Failed {
16566 details: None,
16567 error_code,
16568 message,
16569 };
16570 }
16571 self.ack_after_pull(transport);
16572 if self.upgrading && report.bootstrapping.is_empty() {
16575 self.set_upgrading(false);
16576 }
16577 SyncOutcome::Ok(report)
16578 }
16579
16580 fn handle_push_results(
16583 &mut self,
16584 frames: &[Frame],
16585 pushed_ids: &HashSet<&str>,
16586 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
16587 report: &mut SyncReport,
16588 last_final_push_result_id: Option<&str>,
16589 ) -> Result<(), String> {
16590 self.begin_observation("syncular_push_results")?;
16591 let previous_failed = self.failed_commits.clone();
16592 let previous_sync_needed = self.sync_needed;
16593 let conflict_count = self.conflicts.len();
16594 let rejection_count = self.rejections.len();
16595 let applied_count = report.applied.len();
16596 let rejected_count = report.rejected.len();
16597 let retryable_count = report.retryable.len();
16598 let was_dirty = self.overlay_dirty.snapshot();
16599 let mut removed = Vec::new();
16600 let mut batch = ChangeAccumulator::default();
16601 let persisted = (|| {
16602 for frame in frames {
16603 let Frame::PushResult {
16604 client_commit_id,
16605 status,
16606 results,
16607 commit_seq,
16608 } = frame
16609 else {
16610 return Err("unexpected acknowledgement frame".to_owned());
16611 };
16612 if !pushed_ids.contains(client_commit_id.as_str()) {
16613 continue;
16614 }
16615 let Some(index) = self
16616 .outbox
16617 .iter()
16618 .position(|commit| commit.client_commit_id == *client_commit_id)
16619 else {
16620 continue;
16621 };
16622 let status = *status;
16623 let rejection_details = rejection_details_by_commit.get(client_commit_id);
16624 let operations = self.outbox[index].ops.clone();
16625 match status {
16626 PushStatus::Applied | PushStatus::Cached => {
16627 let sequence =
16630 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
16631 let affected: HashSet<_> = operations
16632 .iter()
16633 .map(|op| (op.table.as_str(), op.row_id.as_str()))
16634 .collect();
16635 let prior = self.acknowledged_operations(Some(&affected), None)?;
16636 for (op_index, operation) in operations.iter().enumerate() {
16637 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
16638 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16639 if delivered {
16640 continue;
16641 }
16642 let table = self
16643 .schema
16644 .table(&operation.table)
16645 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16646 let mut intended: Map<String, Value> = self
16647 .stored_row(table, &operation.row_id, true)?
16648 .map(|(row, _)| {
16649 table
16650 .columns
16651 .iter()
16652 .zip(&row)
16653 .map(|(column, value)| {
16654 (column.name.clone(), column_value_to_json(value))
16655 })
16656 .collect()
16657 })
16658 .unwrap_or_default();
16659 let mut scope_image = intended.clone();
16660 for op in prior
16661 .iter()
16662 .chain(
16663 self.failed_commits
16664 .iter()
16665 .flat_map(|(commit, _)| &commit.ops),
16666 )
16667 .chain(
16668 self.outbox
16669 .iter()
16670 .take(index)
16671 .flat_map(|commit| &commit.ops),
16672 )
16673 .chain(operations.iter().take(op_index + 1))
16674 {
16675 if op.table != operation.table || op.row_id != operation.row_id {
16676 continue;
16677 }
16678 if let Some(values) = &op.values {
16679 intended.extend(values.clone());
16680 } else {
16681 scope_image = intended.clone();
16682 intended.clear();
16683 }
16684 }
16685 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
16686 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
16687 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
16688 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
16689 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16690 }
16691 let journal_results = results
16692 .iter()
16693 .map(|result| {
16694 let op_index = match result {
16695 OpResult::Applied { op_index }
16696 | OpResult::Conflict { op_index, .. }
16697 | OpResult::Error { op_index, .. } => *op_index,
16698 };
16699 CommitOperationOutcome::Applied { op_index }
16700 })
16701 .collect::<Vec<_>>();
16702 let outcome_status = if status == PushStatus::Applied {
16703 CommitOutcomeStatus::Applied
16704 } else {
16705 CommitOutcomeStatus::Cached
16706 };
16707 let persisted = self
16708 .persist_commit_outcome(
16709 client_commit_id,
16710 outcome_status,
16711 &journal_results,
16712 None,
16713 )
16714 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
16715 persisted?;
16716 for operation in &operations {
16717 self.overlay_dirty.table(&operation.table);
16718 }
16719 self.sync_needed = true;
16720 batch.status = true;
16721 batch.outcomes = true;
16722 }
16723 PushStatus::Rejected => {
16724 if results.iter().any(|result| {
16725 matches!(
16726 result,
16727 OpResult::Error {
16728 code,
16729 retryable: true,
16730 ..
16731 } if code == "sync.idempotency_cache_miss"
16732 )
16733 }) {
16734 report.retryable.push(client_commit_id.to_owned());
16737 continue;
16738 }
16739
16740 let mut journal_results = Vec::with_capacity(results.len());
16741 let mut conflicts = Vec::new();
16742 let mut rejections = Vec::new();
16743 for result in results {
16744 match result {
16745 OpResult::Applied { op_index } => {
16746 journal_results.push(CommitOperationOutcome::Applied {
16747 op_index: *op_index,
16748 });
16749 }
16750 OpResult::Conflict {
16751 op_index,
16752 code,
16753 message,
16754 server_version,
16755 server_row,
16756 conflict_columns: conflict_columns_bitmap,
16757 } => {
16758 let operation = operations
16759 .get(*op_index as usize)
16760 .map(CommitOperation::from);
16761 let (table, row_id) = operation
16762 .as_ref()
16763 .map(|op| (op.table.clone(), op.row_id.clone()))
16764 .unwrap_or_default();
16765 let server_row_json = self
16766 .schema
16767 .table(&table)
16768 .and_then(|t| {
16769 decode_row_bytes(t, server_row, &self.encryption)
16770 .ok()
16771 .map(|row| (t, row))
16772 })
16773 .map(|(t, row)| {
16774 let mut map = Map::new();
16775 for (i, column) in t.columns.iter().enumerate() {
16776 map.insert(
16777 column.name.clone(),
16778 column_value_to_json(
16779 row.get(i).unwrap_or(&None),
16780 ),
16781 );
16782 }
16783 map
16784 })
16785 .unwrap_or_default();
16786 let conflict_columns = self
16789 .schema
16790 .table(&table)
16791 .map(|t| {
16792 t.columns
16793 .iter()
16794 .enumerate()
16795 .filter(|(index, _)| {
16796 conflict_columns_bitmap
16797 .get(index / 8)
16798 .is_some_and(|byte| {
16799 byte & (1 << (index % 8)) != 0
16800 })
16801 })
16802 .map(|(_, column)| column.name.clone())
16803 .collect::<Vec<_>>()
16804 })
16805 .unwrap_or_default();
16806 let conflict = ConflictRecord {
16807 client_commit_id: client_commit_id.to_owned(),
16808 op_index: *op_index,
16809 table,
16810 row_id,
16811 code: code.clone(),
16812 message: message.clone(),
16813 server_version: *server_version,
16814 server_row: server_row_json,
16815 conflict_columns,
16816 operation,
16817 };
16818 journal_results.push(CommitOperationOutcome::Conflict {
16819 conflict: conflict.clone(),
16820 });
16821 conflicts.push(conflict);
16822 }
16823 OpResult::Error {
16824 op_index,
16825 code,
16826 message,
16827 retryable,
16828 } => {
16829 let rejection = RejectionRecord {
16830 client_commit_id: client_commit_id.to_owned(),
16831 op_index: *op_index,
16832 code: code.clone(),
16833 message: message.clone(),
16834 retryable: *retryable,
16835 details: rejection_details
16836 .and_then(|details| details.get(op_index))
16837 .cloned(),
16838 operation: operations
16839 .get(*op_index as usize)
16840 .map(CommitOperation::from),
16841 };
16842 journal_results.push(CommitOperationOutcome::Error {
16843 rejection: rejection.clone(),
16844 });
16845 rejections.push(rejection);
16846 }
16847 }
16848 }
16849 if self.retain_failed_commits {
16850 self.retain_failed_commit(client_commit_id, &operations)?;
16851 }
16852 let outcome_status = if conflicts.is_empty() {
16853 CommitOutcomeStatus::Rejected
16854 } else {
16855 CommitOutcomeStatus::Conflict
16856 };
16857 let persisted = self
16858 .persist_commit_outcome(
16859 client_commit_id,
16860 outcome_status,
16861 &journal_results,
16862 Some(&operations),
16863 )
16864 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
16865 persisted?;
16866 batch.conflicts = !conflicts.is_empty();
16867 batch.rejections = !rejections.is_empty();
16868 batch.status = true;
16869 batch.outcomes = true;
16870 self.conflicts.extend(conflicts);
16871 self.rejections.extend(rejections);
16872 for operation in &operations {
16873 self.overlay_dirty.table(&operation.table);
16874 }
16875 }
16876 }
16877 removed.push((index, self.outbox.remove(index)));
16878 if batch.status {
16879 for operation in &operations {
16880 if !self.record_row_scopes(
16881 &mut batch,
16882 &operation.table,
16883 &operation.row_id,
16884 false,
16885 ) {
16886 batch.table(&operation.table);
16887 }
16888 }
16889 }
16890 match status {
16891 PushStatus::Applied | PushStatus::Cached => {
16892 report.applied.push(client_commit_id.to_owned())
16893 }
16894 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
16895 }
16896 }
16897 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
16898 self.prune_commit_outcomes()?;
16899 }
16900 self.rebuild_overlay_if_dirty()?;
16901 self.finish_observation("syncular_push_results", batch)
16902 })();
16903 if let Err(error) = persisted {
16904 self.rollback_observation("syncular_push_results");
16905 self.failed_commits = previous_failed;
16906 self.sync_needed = previous_sync_needed;
16907 for (index, commit) in removed.into_iter().rev() {
16908 self.outbox.insert(index, commit);
16909 }
16910 self.conflicts.truncate(conflict_count);
16911 self.rejections.truncate(rejection_count);
16912 self.overlay_dirty.restore(was_dirty.clone());
16913 report.applied.truncate(applied_count);
16914 report.rejected.truncate(rejected_count);
16915 report.retryable.truncate(retryable_count);
16916 return Err(error);
16917 }
16918 if !previous_sync_needed && self.sync_needed {
16919 self.sync_intent_queue.push_back(SyncIntent::Interactive);
16920 }
16921 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
16922 Ok(())
16923 }
16924
16925 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
16926 status != PushStatus::Rejected
16927 || !results.iter().any(|result| {
16928 matches!(
16929 result,
16930 OpResult::Error {
16931 code,
16932 retryable: true,
16933 ..
16934 } if code == "sync.idempotency_cache_miss"
16935 )
16936 })
16937 }
16938
16939 #[allow(clippy::too_many_arguments)]
16942 fn process_section(
16943 &mut self,
16944 transport: &mut dyn Transport,
16945 id: &str,
16946 status: SubStatus,
16947 reason_code: &str,
16948 effective_scopes: Vec<(String, Vec<String>)>,
16949 body: Vec<Frame>,
16950 sub_end: Option<(i64, Option<String>)>,
16951 meta: &RequestMeta,
16952 report: &mut SyncReport,
16953 ) -> Result<(), SectionError> {
16954 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
16955 return Ok(()); };
16957 if status != SubStatus::Active && sub_end.is_none() {
16958 return Err(SectionError::Abort(
16959 "sync.invalid_request".into(),
16960 "subscription section without SUB_END".into(),
16961 ));
16962 }
16963 match status {
16964 SubStatus::Revoked => {
16965 self.begin_observation("syncular_revocation")
16966 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
16967 let previous = self.subs[sub_index].clone();
16968 let was_dirty = self.overlay_dirty.snapshot();
16969 let previous_outbox = self.outbox.clone();
16970 let previous_failed = self.failed_commits.clone();
16971 let previous_rejections = self.rejections.clone();
16972 let previous_conflicts = self.conflicts.clone();
16973 let revoked_count = report.revoked.len();
16974 let failed_count = report.failed.len();
16975 let result = (|| {
16976 let mut batch = ChangeAccumulator::default();
16977 let registered = self.window_unit_by_sub_id(id);
16978 let (table, effective) = {
16980 let sub = &self.subs[sub_index];
16981 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
16982 };
16983 let scoped_table = self.schema.table(&table).cloned().ok_or_else(|| {
16984 SectionError::Abort(
16985 "sync.unknown_table".to_owned(),
16986 "revoked scope references an unknown table".to_owned(),
16987 )
16988 })?;
16989 let dropped = self
16990 .drop_failed_commits(|name, values| {
16991 name == table
16992 && !effective.is_empty()
16993 && effective.iter().all(|(variable, allowed)| {
16994 scoped_table
16995 .scope_column(variable)
16996 .and_then(|column| values.get(column))
16997 .and_then(Value::as_str)
16998 .is_some_and(|value| {
16999 allowed.iter().any(|allowed| allowed == value)
17000 })
17001 })
17002 })
17003 .map_err(|message| {
17004 SectionError::Abort("storage.failed".to_owned(), message)
17005 })?;
17006 for commit in dropped {
17007 batch.conflicts = true;
17008 batch.rejections = true;
17009 batch.outcomes = true;
17010 for operation in commit.ops {
17011 batch.table(&operation.table);
17012 }
17013 }
17014 let purged = self.purge_scope_rows(&table, &effective);
17015 match purged {
17016 Ok(()) => {
17017 self.record_scope_map(&mut batch, &table, &effective);
17018 let sub = &mut self.subs[sub_index];
17019 sub.state = SubState::Revoked;
17020 sub.reason_code = Some(if reason_code.is_empty() {
17021 "sync.scope_revoked".to_owned()
17022 } else {
17023 reason_code.to_owned()
17024 });
17025 report.revoked.push(id.to_owned());
17026 let doomed_effective = effective;
17027 let sub_table = table;
17028 self.persist_sub(&self.subs[sub_index].clone())
17029 .map_err(|message| {
17030 SectionError::Abort("storage.failed".into(), message)
17031 })?;
17032 let dropped = self
17033 .drop_doomed_outbox(&sub_table, &doomed_effective)
17034 .map_err(|message| {
17035 SectionError::Abort("storage.failed".to_owned(), message)
17036 })?;
17037 if dropped {
17038 batch.status = true;
17039 batch.rejections = true;
17040 batch.outcomes = true;
17041 }
17042 self.delete_unreferenced_cached_blobs().map_err(|message| {
17045 SectionError::Abort("storage.failed".into(), message)
17046 })?;
17047 }
17048 Err(()) => {
17049 let sub = &mut self.subs[sub_index];
17052 sub.state = SubState::Failed;
17053 sub.reason_code = Some("sync.scope_revoked".to_owned());
17054 report.failed.push(id.to_owned());
17055 self.persist_sub(&self.subs[sub_index].clone())
17056 .map_err(|message| {
17057 SectionError::Abort("storage.failed".into(), message)
17058 })?;
17059 }
17060 }
17061 if let Some((base_key, unit)) = registered {
17062 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17063 }
17064 self.rebuild_overlay_if_dirty()
17065 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17066 self.finish_observation("syncular_revocation", batch)
17067 .map_err(|message| {
17068 SectionError::Abort("storage.failed".to_owned(), message)
17069 })?;
17070 Ok(())
17071 })();
17072 if result.is_err() {
17073 self.rollback_observation("syncular_revocation");
17074 self.subs[sub_index] = previous;
17075 self.overlay_dirty.restore(was_dirty.clone());
17076 self.outbox = previous_outbox;
17077 self.failed_commits = previous_failed;
17078 self.rejections = previous_rejections;
17079 self.conflicts = previous_conflicts;
17080 report.revoked.truncate(revoked_count);
17081 report.failed.truncate(failed_count);
17082 }
17083 result
17084 }
17085 SubStatus::Reset => {
17086 self.begin_observation("syncular_reset")
17087 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17088 let previous = self.subs[sub_index].clone();
17089 let was_dirty = self.overlay_dirty.snapshot();
17090 let reset_count = report.resets.len();
17091 let result = (|| {
17092 let mut batch = ChangeAccumulator::default();
17093 let registered = self.window_unit_by_sub_id(id);
17094 let sub = &mut self.subs[sub_index];
17097 sub.cursor = -1;
17098 sub.bootstrap_state = None;
17099 sub.reason_code = Some(if reason_code.is_empty() {
17100 "sync.cursor_expired".to_owned()
17101 } else {
17102 reason_code.to_owned()
17103 });
17104 report.resets.push(id.to_owned());
17105 self.persist_sub(&self.subs[sub_index].clone())
17106 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17107 if let Some((base_key, unit)) = registered {
17108 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17109 }
17110 self.finish_observation("syncular_reset", batch)
17111 .map_err(|message| {
17112 SectionError::Abort("storage.failed".to_owned(), message)
17113 })?;
17114 Ok(())
17115 })();
17116 if result.is_err() {
17117 self.rollback_observation("syncular_reset");
17118 self.subs[sub_index] = previous;
17119 self.overlay_dirty.restore(was_dirty.clone());
17120 report.resets.truncate(reset_count);
17121 }
17122 result
17123 }
17124 SubStatus::Active => {
17125 let fresh = meta
17126 .fresh
17127 .iter()
17128 .find(|(fid, _)| fid == id)
17129 .map(|(_, fresh)| *fresh)
17130 .unwrap_or(false);
17131 let previous = self.subs[sub_index].clone();
17132 self.subs[sub_index].effective = Some(effective_scopes);
17133 let outcome = self
17134 .apply_section_body(transport, sub_index, body, fresh, meta, report)
17135 .and_then(|()| {
17136 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
17137 SectionError::Abort(
17138 "sync.invalid_request".into(),
17139 "subscription section without SUB_END".into(),
17140 )
17141 })?;
17142 self.apply_sub_end(sub_index, cursor, bootstrap_state)
17143 .map_err(|message| {
17144 SectionError::Abort("storage.failed".into(), message)
17145 })
17146 });
17147 match outcome {
17148 Ok(()) => {
17149 if self.subs[sub_index].bootstrap_state.is_some() {
17150 report.bootstrapping.push(id.to_owned());
17151 }
17152 Ok(())
17153 }
17154 Err(SectionError::FailClosed) => {
17155 self.subs[sub_index] = previous.clone();
17156 self.begin_observation("syncular_section_failure")
17157 .map_err(|message| {
17158 SectionError::Abort("storage.failed".into(), message)
17159 })?;
17160 let persisted = (|| {
17161 let mut batch = ChangeAccumulator::default();
17162 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
17163 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17164 }
17165 self.subs[sub_index].state = SubState::Failed;
17166 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
17167 self.persist_sub(&self.subs[sub_index])?;
17168 self.finish_observation("syncular_section_failure", batch)
17169 })();
17170 if let Err(message) = persisted {
17171 self.rollback_observation("syncular_section_failure");
17172 self.subs[sub_index] = previous;
17173 return Err(SectionError::Abort("storage.failed".into(), message));
17174 }
17175 report.failed.push(id.to_owned());
17176 Ok(())
17177 }
17178 Err(error) => {
17179 self.subs[sub_index] = previous;
17182 Err(error)
17183 }
17184 }
17185 }
17186 }
17187 }
17188
17189 fn apply_sub_end(
17190 &mut self,
17191 sub_index: usize,
17192 cursor: i64,
17193 bootstrap_state: Option<String>,
17194 ) -> Result<(), String> {
17195 #[cfg(feature = "bench-internals")]
17196 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
17197 let previous = self.subs[sub_index].clone();
17198 self.begin_observation("syncular_sub_end")?;
17199 let persisted = (|| {
17200 let mut batch = ChangeAccumulator::default();
17201 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
17202 && cursor >= 0
17203 && bootstrap_state.is_none();
17204 if completed {
17205 let table = self
17206 .schema
17207 .table(&previous.table)
17208 .ok_or_else(|| "sync.unknown_table".to_owned())?;
17209 let effective = previous.effective.as_deref().unwrap_or(&[]);
17210 let mut statement = self.conn.prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
17211 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17212 let rows = statement
17213 .query_map(rusqlite::params![previous.table, cursor], |row| {
17214 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
17215 })
17216 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17217 let mut covered = HashSet::new();
17218 for row in rows {
17219 let (id, intent) =
17220 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17221 let values: Map<String, Value> = serde_json::from_str(&intent)
17222 .map_err(|_| "sync.local_corrupt".to_owned())?;
17223 if !effective.is_empty()
17224 && effective.iter().all(|(variable, allowed)| {
17225 table
17226 .scope_column(variable)
17227 .and_then(|column| values.get(column))
17228 .is_some_and(|value| {
17229 allowed.contains(
17230 &value
17231 .as_str()
17232 .map_or_else(|| value.to_string(), str::to_owned),
17233 )
17234 })
17235 })
17236 {
17237 covered.insert(id);
17238 }
17239 }
17240 drop(statement);
17241 for id in covered {
17242 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![previous.table, id, cursor])
17243 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17244 self.overlay_dirty.table(&previous.table);
17245 batch.table(&previous.table);
17246 }
17247 self.rebuild_overlay_if_dirty()?;
17248 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
17249 batch.window(&base_key, &previous.table, &unit);
17250 }
17251 }
17252 let sub = &mut self.subs[sub_index];
17253 sub.cursor = cursor;
17254 sub.bootstrap_state = bootstrap_state;
17255 sub.reason_code = None;
17258 sub.synced_once = true;
17259 self.persist_sub(&self.subs[sub_index])?;
17260 self.finish_observation("syncular_sub_end", batch)
17261 })();
17262 if persisted.is_err() {
17263 self.rollback_observation("syncular_sub_end");
17264 self.subs[sub_index] = previous;
17265 }
17266 persisted
17267 }
17268
17269 fn apply_section_body(
17270 &mut self,
17271 transport: &mut dyn Transport,
17272 sub_index: usize,
17273 body: Vec<Frame>,
17274 fresh: bool,
17275 meta: &RequestMeta,
17276 report: &mut SyncReport,
17277 ) -> Result<(), SectionError> {
17278 let mut saw_segment = false;
17279 for frame in body {
17280 match frame {
17281 Frame::Commit {
17282 tables,
17283 changes,
17284 commit_seq,
17285 ..
17286 } => {
17287 self.apply_commit_frame(&tables, &changes, Some(commit_seq))
17288 .map_err(|(c, m)| SectionError::Abort(c, m))?;
17289 report.commits_applied += 1;
17290 }
17291 Frame::SegmentInline { payload } => {
17292 let segment = decode_rows_segment(&payload)
17293 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
17294 self.progress.update(|p| {
17295 p.phase = ProgressPhase::Import;
17296 p.subscription_id = Some(self.subs[sub_index].id.clone());
17297 p.table = Some(self.subs[sub_index].table.clone());
17298 p.segment_id = None;
17299 p.bytes_received = payload.len() as u64;
17300 p.bytes_total = Some(payload.len() as u64);
17301 p.rows_processed = 0;
17302 p.rows_total =
17303 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
17304 });
17305 let first = !saw_segment;
17306 saw_segment = true;
17307 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17308 report.segment_rows_applied += applied;
17309 }
17310 Frame::SegmentRef {
17311 segment_id,
17312 byte_length,
17313 media_type,
17314 table,
17315 row_count,
17316 as_of_commit_seq,
17317 scope_digest,
17318 row_cursor,
17319 next_row_cursor,
17320 url,
17321 url_expires_at_ms,
17322 ..
17323 } => {
17324 let advertised = match media_type {
17327 MediaType::Rows => {
17328 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
17329 || meta.accept & ACCEPT_INLINE_ROWS != 0
17330 }
17331 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
17332 };
17333 if !advertised {
17334 return Err(SectionError::Abort(
17335 "sync.invalid_request".to_owned(),
17336 format!(
17337 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
17338 media_type.name()
17339 ),
17340 ));
17341 }
17342 self.progress.update(|p| {
17343 p.phase = ProgressPhase::Download;
17344 p.subscription_id = Some(self.subs[sub_index].id.clone());
17345 p.table = Some(table.clone());
17346 p.segment_id = Some(segment_id.clone());
17347 p.bytes_received = 0;
17348 p.bytes_total = u64::try_from(byte_length).ok();
17349 p.rows_processed = 0;
17350 p.rows_total = u64::try_from(row_count).ok();
17351 });
17352 let observer = self.progress.clone();
17353 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
17354 let bytes = if let Some(url) = url {
17355 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
17360 return Err(SectionError::Abort(
17361 "sync.invalid_request".to_owned(),
17362 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
17363 .to_owned(),
17364 ));
17365 }
17366 if url_expires_at_ms.is_some_and(|exp| {
17368 exp <= transport
17369 .segment_fetch_started_at(&url)
17370 .unwrap_or_else(|| self.clock_now_ms())
17371 }) {
17372 return Err(SectionError::Abort(
17373 "sync.segment_expired".to_owned(),
17374 format!(
17375 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
17376 ),
17377 ));
17378 }
17379 transport
17380 .fetch_url(&url, &mut on_progress)
17381 .map_err(|e| SectionError::Abort(e.code, e.message))?
17382 } else {
17383 let requested_scopes_json =
17384 canonical_scope_json(&self.subs[sub_index].requested);
17385 transport
17386 .download_segment(
17387 &SegmentRequest {
17388 segment_id: segment_id.clone(),
17389 table,
17390 requested_scopes_json,
17391 },
17392 &mut on_progress,
17393 )
17394 .map_err(|e| SectionError::Abort(e.code, e.message))?
17395 };
17396 on_progress(bytes.len() as u64);
17398 let digest = Sha256::digest(&bytes);
17399 let expected = segment_id
17400 .strip_prefix("sha256:")
17401 .unwrap_or(segment_id.as_str());
17402 if bytes_to_hex(&digest) != expected {
17403 self.schedule_background_retry();
17405 return Err(SectionError::Abort(
17406 "sync.invalid_request".to_owned(),
17407 "segment bytes do not match the content address (§5.1)".to_owned(),
17408 ));
17409 }
17410 self.progress.update(|p| p.phase = ProgressPhase::Import);
17411 if media_type == MediaType::Sqlite {
17412 if row_cursor.is_some() || next_row_cursor.is_some() {
17415 return Err(SectionError::Abort(
17416 "sync.invalid_request".to_owned(),
17417 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
17418 .to_owned(),
17419 ));
17420 }
17421 let first = !saw_segment;
17422 saw_segment = true;
17423 let applied = self.apply_sqlite_segment(
17424 sub_index,
17425 &bytes,
17426 fresh && first,
17427 row_count,
17428 as_of_commit_seq,
17429 &scope_digest,
17430 )?;
17431 report.segment_rows_applied += applied;
17432 } else {
17433 let segment = decode_rows_segment(&bytes).map_err(|e| {
17434 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
17435 })?;
17436 let first = row_cursor.is_none();
17437 saw_segment = true;
17438 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17439 report.segment_rows_applied += applied;
17440 }
17441 }
17442 Frame::Unknown { .. } => {}
17443 Frame::Error { code, message, .. } => {
17444 return Err(SectionError::Abort(code, message))
17445 }
17446 _ => {
17447 return Err(SectionError::Abort(
17448 "sync.invalid_request".to_owned(),
17449 "unexpected frame inside a subscription section".to_owned(),
17450 ));
17451 }
17452 }
17453 }
17454 Ok(())
17455 }
17456
17457 fn apply_commit_frame(
17459 &mut self,
17460 tables: &[String],
17461 changes: &[ssp2::model::Change],
17462 commit_seq: Option<i64>,
17463 ) -> Result<(), (String, String)> {
17464 #[cfg(feature = "bench-internals")]
17465 let _phase = self.benchmark_phases.start(Phase::CommitApply);
17466 let was_dirty = self.overlay_dirty.snapshot();
17467 self.coalesce_unit(changes.len())
17468 .and_then(|()| self.begin_observation("syncular_commit"))
17469 .map_err(|message| ("storage.failed".into(), message))?;
17470 let applied = (|| {
17471 let mut batch = ChangeAccumulator::default();
17472 self.record_commit_changes(&mut batch, tables, changes);
17473 if let Some(sequence) = commit_seq {
17474 for change in changes {
17475 let table = tables.get(change.table_index as usize).ok_or_else(|| {
17476 (
17477 "sync.invalid_request".to_owned(),
17478 "change tableIndex out of range".to_owned(),
17479 )
17480 })?;
17481 if self
17482 .outbox
17483 .iter()
17484 .flat_map(|commit| &commit.ops)
17485 .any(|op| op.table == *table && op.row_id == change.row_id)
17486 {
17487 self.conn.execute("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)", rusqlite::params![table, change.row_id, sequence])
17488 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
17489 }
17490 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![table, change.row_id, sequence])
17491 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
17492 }
17493 }
17494 self.apply_commit_changes(tables, changes)?;
17495 self.rebuild_overlay_if_dirty()
17496 .map_err(|message| ("storage.failed".into(), message))?;
17497 self.finish_observation("syncular_commit", batch)
17498 .map_err(|message| ("storage.failed".into(), message))
17499 })();
17500 if applied.is_err() {
17501 self.rollback_observation("syncular_commit");
17502 self.overlay_dirty.restore(was_dirty.clone());
17503 }
17504 applied
17505 }
17506
17507 fn apply_commit_changes(
17508 &mut self,
17509 tables: &[String],
17510 changes: &[ssp2::model::Change],
17511 ) -> Result<(), (String, String)> {
17512 let mirror_visible = !self.overlay_dirty.get()
17517 && self.failed_commits.is_empty()
17518 && ((self.outbox.is_empty() && !self.has_acknowledged_rows())
17519 || changes.iter().all(|change| {
17520 tables
17521 .get(change.table_index as usize)
17522 .and_then(|name| self.schema.table(name))
17523 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
17524 }));
17525 for change in changes {
17526 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
17527 (
17528 "sync.invalid_request".to_owned(),
17529 "change tableIndex out of range".to_owned(),
17530 )
17531 })?;
17532 let table = self.schema.table(table_name).ok_or_else(|| {
17533 (
17534 "sync.schema_mismatch".to_owned(),
17535 format!("change targets unknown table {table_name:?}"),
17536 )
17537 })?;
17538 match change.op {
17539 Op::Upsert => {
17540 let payload = change.row.as_ref().ok_or_else(|| {
17541 (
17542 "sync.invalid_request".to_owned(),
17543 "upsert change without row payload".to_owned(),
17544 )
17545 })?;
17546 let row = {
17548 #[cfg(feature = "bench-internals")]
17549 let _phase = self.benchmark_phases.start(Phase::RowDecode);
17550 decode_row_bytes(table, payload, &self.encryption)
17551 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
17552 };
17553 let version = change.row_version.unwrap_or(0);
17554 let table_name = table.name.clone();
17555 self.write_base_row(&table_name, &row, version)
17556 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17557 if mirror_visible {
17558 self.write_row(&visible_table(&table_name), &table_name, &row, version)
17559 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17560 }
17561 }
17562 Op::Delete => {
17563 self.delete_base_row(table_name, &change.row_id)
17564 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17565 if mirror_visible {
17566 let sql = format!(
17567 "DELETE FROM {} WHERE {}",
17568 visible_table(table_name),
17569 row_id_predicate(table)
17570 );
17571 self.conn
17572 .prepare_cached(&sql)
17573 .and_then(|mut statement| {
17574 statement.execute(rusqlite::params![change.row_id])
17575 })
17576 .map_err(|error| {
17577 ("sync.invalid_request".to_owned(), error.to_string())
17578 })?;
17579 }
17580 }
17581 }
17582 }
17583 if mirror_visible {
17584 {
17585 let changed_rows: HashSet<(&str, &str)> = changes
17586 .iter()
17587 .map(|change| {
17588 (
17589 tables[change.table_index as usize].as_str(),
17590 change.row_id.as_str(),
17591 )
17592 })
17593 .collect();
17594 let acknowledged = self
17595 .acknowledged_operations(Some(&changed_rows), None)
17596 .map_err(|error| ("storage.failed".to_owned(), error))?;
17597 self.apply_outbox_ops(&acknowledged)
17598 .map_err(|message| ("storage.failed".into(), message))?;
17599 self.apply_outbox_ops(
17600 self.outbox
17601 .iter()
17602 .flat_map(|commit| &commit.ops)
17603 .filter(|op| {
17604 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
17605 }),
17606 )
17607 .map_err(|message| ("storage.failed".into(), message))?;
17608 }
17609 self.overlay_dirty.set(false);
17610 }
17611 Ok(())
17612 }
17613
17614 fn apply_segment(
17619 &mut self,
17620 sub_index: usize,
17621 segment: &RowsSegment,
17622 first_fresh_page: bool,
17623 ) -> Result<u32, SectionError> {
17624 let (sub_table, effective) = {
17625 let sub = &self.subs[sub_index];
17626 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17627 };
17628 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
17629 SectionError::Abort(
17630 "sync.schema_mismatch".to_owned(),
17631 format!("subscription table {sub_table:?} is not in the client schema"),
17632 )
17633 })?;
17634 let matches = segment.table == table.name
17639 && segment.schema_version == self.schema.version
17640 && segment.columns.len() == table.wire_columns.len()
17641 && segment
17642 .columns
17643 .iter()
17644 .zip(table.wire_columns.iter())
17645 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
17646 if !matches {
17647 return Err(SectionError::Abort(
17648 "sync.schema_mismatch".to_owned(),
17649 "segment column table does not match the generated schema (§5.2)".to_owned(),
17650 ));
17651 }
17652 let mut applied = 0u32;
17653 let empty = Vec::new();
17654 let blocks = if segment.blocks.is_empty() {
17655 std::slice::from_ref(&empty)
17656 } else {
17657 &segment.blocks
17658 };
17659 for (index, block) in blocks.iter().enumerate() {
17660 let clear = first_fresh_page && index == 0;
17661 if !clear && block.is_empty() {
17662 continue;
17663 }
17664 let was_dirty = self.overlay_dirty.snapshot();
17665 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
17666 && self
17667 .outbox
17668 .iter()
17669 .flat_map(|commit| &commit.ops)
17670 .any(|op| op.table == table.name);
17671 let mirror_visible = !reconcile_pending
17672 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
17673 self.coalesce_unit(block.len())
17674 .and_then(|()| self.begin_observation("syncular_segment_block"))
17675 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17676 let outcome = (|| {
17677 self.rebuild_overlay_if_dirty()
17678 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17679 let mut changed_keys = Vec::new();
17680 let mut batch = ChangeAccumulator::default();
17681 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
17682 batch.table(&table.name);
17683 }
17684 if clear {
17685 self.purge_scope_rows(&table.name, &effective)
17686 .map_err(|()| SectionError::FailClosed)?;
17687 }
17688 for (row_index, row) in block.iter().enumerate() {
17689 let decrypted;
17690 let values = if table.has_encrypted_columns() {
17691 let mut values = row.values.clone();
17692 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
17693 .map_err(|message| {
17694 SectionError::Abort("client.decrypt_failed".into(), message)
17695 })?;
17696 decrypted = values;
17697 &decrypted
17698 } else {
17699 &row.values
17700 };
17701 self.write_base_row(&table.name, values, row.server_version)
17702 .map_err(|message| {
17703 SectionError::Abort("sync.invalid_request".into(), message)
17704 })?;
17705 if reconcile_pending {
17706 changed_keys.push(
17707 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
17708 |message| SectionError::Abort("storage.failed".into(), message),
17709 )?,
17710 );
17711 }
17712 if mirror_visible {
17713 self.write_row(
17714 &visible_table(&table.name),
17715 &table.name,
17716 values,
17717 row.server_version,
17718 )
17719 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17720 }
17721 let processed = u64::from(applied) + row_index as u64 + 1;
17722 if processed.is_multiple_of(1024) {
17723 self.progress.update(|p| p.rows_processed = processed);
17724 }
17725 }
17726 if reconcile_pending {
17727 self.reconcile_imported_rows(&table, &mut changed_keys)
17728 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17729 }
17730 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
17731 self.apply_outbox_ops(
17732 self.outbox
17733 .iter()
17734 .flat_map(|commit| &commit.ops)
17735 .filter(|op| op.table == table.name),
17736 )
17737 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17738 self.overlay_dirty.set(false);
17739 } else {
17740 self.rebuild_overlay_if_dirty()
17741 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17742 }
17743 self.finish_observation("syncular_segment_block", batch)
17744 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
17745 })();
17746 if let Err(error) = outcome {
17747 self.rollback_observation("syncular_segment_block");
17748 self.overlay_dirty.restore(was_dirty.clone());
17749 return Err(error);
17750 }
17751 applied += block.len() as u32;
17752 self.progress
17753 .update(|p| p.rows_processed = u64::from(applied));
17754 std::thread::yield_now();
17755 }
17756 self.progress
17757 .update(|p| p.rows_processed = u64::from(applied));
17758 Ok(applied)
17759 }
17760
17761 fn apply_sqlite_segment(
17769 &mut self,
17770 sub_index: usize,
17771 bytes: &[u8],
17772 first_fresh_page: bool,
17773 row_count: i64,
17774 as_of_commit_seq: i64,
17775 scope_digest: &str,
17776 ) -> Result<u32, SectionError> {
17777 let invalid = |detail: &str| {
17778 SectionError::Abort(
17779 "sync.invalid_request".to_owned(),
17780 format!("sqlite segment rejected: {detail} (§5.3)"),
17781 )
17782 };
17783 let (sub_table, effective) = {
17784 let sub = &self.subs[sub_index];
17785 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17786 };
17787 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
17788 SectionError::Abort(
17789 "sync.schema_mismatch".to_owned(),
17790 format!("subscription table {sub_table:?} is not in the client schema"),
17791 )
17792 })?;
17793
17794 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
17795 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
17796 let img = match rusqlite::Connection::open_with_flags(
17797 &path,
17798 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
17799 ) {
17800 Ok(conn) => conn,
17801 Err(_) => {
17802 let _ = std::fs::remove_file(&path);
17803 return Err(invalid("bytes do not open as a SQLite database"));
17804 }
17805 };
17806 let outcome = self.apply_sqlite_image(
17807 &img,
17808 &table,
17809 first_fresh_page,
17810 &effective,
17811 row_count,
17812 as_of_commit_seq,
17813 scope_digest,
17814 );
17815 drop(img);
17816 let _ = std::fs::remove_file(&path);
17817 outcome
17818 }
17819
17820 #[allow(clippy::too_many_arguments)]
17821 fn apply_sqlite_image(
17822 &mut self,
17823 img: &rusqlite::Connection,
17824 table: &crate::schema::TableSchema,
17825 first_fresh_page: bool,
17826 effective: &[(String, Vec<String>)],
17827 row_count: i64,
17828 as_of_commit_seq: i64,
17829 scope_digest: &str,
17830 ) -> Result<u32, SectionError> {
17831 let invalid = |detail: String| {
17832 SectionError::Abort(
17833 "sync.invalid_request".to_owned(),
17834 format!("sqlite segment rejected: {detail} (§5.3)"),
17835 )
17836 };
17837
17838 type MetaRow = (i64, String, i64, i64, String, i64, i64);
17841 let meta: MetaRow = img
17842 .query_row(
17843 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
17844 \"scopeDigest\", \"rowCount\",
17845 (SELECT count(*) FROM _syncular_segment)
17846 FROM _syncular_segment",
17847 [],
17848 |row| {
17849 Ok((
17850 row.get(0)?,
17851 row.get(1)?,
17852 row.get(2)?,
17853 row.get(3)?,
17854 row.get(4)?,
17855 row.get(5)?,
17856 row.get(6)?,
17857 ))
17858 },
17859 )
17860 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
17861 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
17862 if meta_count != 1 {
17863 return Err(invalid(format!(
17864 "_syncular_segment must contain exactly one row, found {meta_count}"
17865 )));
17866 }
17867 if format != 1 {
17868 return Err(invalid(format!("format {format}")));
17869 }
17870 if meta_table != table.name {
17871 return Err(invalid(format!("image table {meta_table:?}")));
17872 }
17873 if schema_version != i64::from(self.schema.version) {
17874 return Err(invalid(format!("schemaVersion {schema_version}")));
17875 }
17876 if pin != as_of_commit_seq {
17877 return Err(invalid(format!("asOfCommitSeq {pin}")));
17878 }
17879 if digest != scope_digest {
17880 return Err(invalid("scopeDigest mismatch".to_owned()));
17881 }
17882 if meta_rows != row_count {
17883 return Err(invalid(format!("rowCount {meta_rows}")));
17884 }
17885
17886 let mut names: Vec<String> = Vec::new();
17888 let mut primary_keys: Vec<String> = Vec::new();
17889 {
17890 let mut stmt = img
17891 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
17892 .map_err(|_| invalid("image data table missing".to_owned()))?;
17893 let mut rows = stmt
17894 .query([])
17895 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
17896 while let Some(row) = rows
17897 .next()
17898 .map_err(|_| invalid("image data table unreadable".to_owned()))?
17899 {
17900 if row
17901 .get::<_, i64>(5)
17902 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
17903 > 0
17904 {
17905 primary_keys.push(row.get(1).map_err(|error| {
17906 invalid(Self::sqlite_failure(&self.storage_failure, error))
17907 })?);
17908 }
17909 names.push(
17910 row.get::<_, String>(1)
17911 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
17912 );
17913 }
17914 }
17915 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
17916 expected.push("_syncular_version");
17917 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
17918 return Err(SectionError::Abort(
17919 "sync.schema_mismatch".to_owned(),
17920 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
17921 ));
17922 }
17923
17924 if primary_keys != vec![table.primary_key.clone()] {
17925 return Err(invalid(
17926 "image primary key does not match generated schema".into(),
17927 ));
17928 }
17929 let actual_count: i64 = img
17930 .query_row(
17931 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
17932 [],
17933 |row| row.get(0),
17934 )
17935 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
17936 if actual_count != row_count {
17937 return Err(invalid("image row count does not match descriptor".into()));
17938 }
17939 let insert = self.insert_row_sql(&base_table(&table.name), table);
17940 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
17941 let column_list = names
17942 .iter()
17943 .map(|n| quote_ident(n))
17944 .collect::<Vec<_>>()
17945 .join(", ");
17946 let mut stmt = img
17947 .prepare(&format!(
17948 "SELECT {column_list} FROM {} ORDER BY {}",
17949 quote_ident(&table.name),
17950 quote_ident(&table.primary_key)
17951 ))
17952 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
17953 let mut rows = stmt
17954 .query([])
17955 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
17956 let mut applied = 0u32;
17957 loop {
17958 let clear = applied == 0 && first_fresh_page;
17959 let was_dirty = self.overlay_dirty.snapshot();
17960 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
17961 && self
17962 .outbox
17963 .iter()
17964 .flat_map(|commit| &commit.ops)
17965 .any(|op| op.table == table.name);
17966 let mirror_visible = !reconcile_pending
17967 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
17968 self.coalesce_unit(1024)
17970 .and_then(|()| self.begin_observation("syncular_image_chunk"))
17971 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17972 let result = (|| {
17973 self.rebuild_overlay_if_dirty()
17974 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17975 let mut changed_keys = Vec::new();
17976 let mut batch = ChangeAccumulator::default();
17977 if clear {
17978 if self.scoped_rows_exist(&table.name, effective) {
17979 batch.table(&table.name);
17980 }
17981 self.purge_scope_rows(&table.name, effective)
17982 .map_err(|()| SectionError::FailClosed)?;
17983 }
17984 let mut processed = 0u32;
17985 {
17986 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
17987 invalid(Self::sqlite_failure(&self.storage_failure, error))
17988 })?;
17989 let mut mirror = if mirror_visible {
17990 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
17991 invalid(Self::sqlite_failure(&self.storage_failure, error))
17992 })?)
17993 } else {
17994 None
17995 };
17996 for _ in 0..1024 {
17997 let Some(row) = rows.next().map_err(|error| {
17998 invalid(Self::sqlite_failure(&self.storage_failure, error))
17999 })?
18000 else {
18001 break;
18002 };
18003 for (i, column) in table.columns.iter().enumerate() {
18004 let cell = row.get_ref(i).map_err(|error| {
18005 invalid(Self::sqlite_failure(&self.storage_failure, error))
18006 })?;
18007 let param = image_cell_param(column, cell).map_err(invalid)?;
18008 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18009 invalid(Self::sqlite_failure(&self.storage_failure, error))
18010 })?;
18011 if let Some(mirror) = &mut mirror {
18012 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18013 invalid(Self::sqlite_failure(&self.storage_failure, error))
18014 })?;
18015 }
18016 }
18017 let version_index = table.columns.len();
18018 let version: i64 = row.get(version_index).map_err(|error| {
18019 invalid(Self::sqlite_failure(&self.storage_failure, error))
18020 })?;
18021 if version < 1 {
18022 return Err(invalid("image row version must be positive".into()));
18023 }
18024 ins.raw_bind_parameter(version_index + 1, version)
18025 .map_err(|error| {
18026 invalid(Self::sqlite_failure(&self.storage_failure, error))
18027 })?;
18028 ins.raw_execute().map_err(|error| {
18029 invalid(Self::sqlite_failure(&self.storage_failure, error))
18030 })?;
18031 if let Some(mirror) = &mut mirror {
18032 mirror
18033 .raw_bind_parameter(version_index + 1, version)
18034 .map_err(|error| {
18035 invalid(Self::sqlite_failure(&self.storage_failure, error))
18036 })?;
18037 mirror.raw_execute().map_err(|error| {
18038 invalid(Self::sqlite_failure(&self.storage_failure, error))
18039 })?;
18040 }
18041 if reconcile_pending {
18042 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
18043 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
18044 )?);
18045 }
18046 processed += 1;
18047 }
18048 }
18049 if processed > 0 {
18050 batch.table(&table.name);
18051 self.overlay_dirty.table(&table.name);
18052 }
18053 if reconcile_pending {
18054 self.reconcile_imported_rows(table, &mut changed_keys)
18055 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18056 }
18057 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
18058 self.apply_outbox_ops(
18059 self.outbox
18060 .iter()
18061 .flat_map(|commit| &commit.ops)
18062 .filter(|op| op.table == table.name),
18063 )
18064 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18065 self.overlay_dirty.set(false);
18066 } else {
18067 self.rebuild_overlay_if_dirty()
18068 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18069 }
18070 self.finish_observation("syncular_image_chunk", batch)
18071 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18072 Ok(processed)
18073 })();
18074 let processed = match result {
18075 Ok(processed) => processed,
18076 Err(error) => {
18077 self.rollback_observation("syncular_image_chunk");
18078 self.overlay_dirty.restore(was_dirty.clone());
18079 return Err(error);
18080 }
18081 };
18082 applied += processed;
18083 self.progress
18084 .update(|p| p.rows_processed = u64::from(applied));
18085 std::thread::yield_now();
18086 if i64::from(applied) == actual_count {
18087 break;
18088 }
18089 }
18090 Ok(applied)
18091 }
18092
18093 fn reconcile_imported_rows(
18097 &self,
18098 table: &TableSchema,
18099 keys: &mut Vec<SqlValue>,
18100 ) -> Result<(), String> {
18101 let visible = visible_table(&table.name);
18102 let base = base_table(&table.name);
18103 let pk = quote_ident(&table.primary_key);
18104 let predicate = row_id_predicate(table);
18105 let mut lookup = self.conn.prepare_cached(&format!(
18106 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
18107 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18108 let mut deleted = HashSet::new();
18109 for op in self
18110 .outbox
18111 .iter()
18112 .flat_map(|commit| &commit.ops)
18113 .filter(|op| op.table == table.name)
18114 {
18115 if op.upsert {
18116 if let Some(values) = &op.values {
18117 let key = json_to_column_value(
18118 &table.columns[table.pk_index],
18119 values.get(&table.primary_key),
18120 )?;
18121 keys.push(owned_sql_value(RowParam::Cell(&key))?);
18122 }
18123 } else if deleted.insert(&op.row_id) {
18124 keys.extend(
18125 lookup
18126 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
18127 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
18128 .collect::<Result<Vec<_>, _>>()
18129 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18130 );
18131 }
18132 }
18133 for chunk in keys.chunks(400) {
18136 let holes = vec!["?"; chunk.len()].join(",");
18137 self.conn
18138 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
18139 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18140 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18141 }
18142 let updates = table
18143 .columns
18144 .iter()
18145 .map(|column| column.name.as_str())
18146 .chain(std::iter::once("_syncular_version"))
18147 .filter(|name| *name != table.primary_key)
18148 .map(|name| {
18149 let name = quote_ident(name);
18150 format!("{name}=excluded.{name}")
18151 })
18152 .collect::<Vec<_>>()
18153 .join(",");
18154 for chunk in keys.chunks(400) {
18155 let holes = vec!["?"; chunk.len()].join(",");
18156 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
18157 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18158 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18159 }
18160 Ok(())
18161 }
18162
18163 fn compile_local_data_purge(
18166 &self,
18167 input: &LocalDataPurgeInput,
18168 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
18169 let invalid = |message: String| format!("sync.invalid_request: {message}");
18170 if input.purge_id.is_empty()
18171 || input.purge_id.len() > 128
18172 || !is_local_operation_code_like(&input.purge_id)
18173 {
18174 return Err(invalid(
18175 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
18176 ));
18177 }
18178 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
18179 return Err(invalid(format!(
18180 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
18181 )));
18182 }
18183
18184 let mut deduplicated: BTreeMap<
18185 String,
18186 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
18187 > = BTreeMap::new();
18188 for target in &input.targets {
18189 let table = self.schema.table(&target.table).ok_or_else(|| {
18190 invalid(format!(
18191 "local purge names unknown table {:?}",
18192 target.table
18193 ))
18194 })?;
18195 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
18196 return Err(invalid(format!(
18197 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
18198 target.table
18199 )));
18200 }
18201 let mut selectors = Vec::with_capacity(target.selectors.len());
18202 let mut canonical_selectors = BTreeMap::new();
18203 for (column_name, raw_values) in &target.selectors {
18204 let Some((column_index, column)) = table
18205 .columns
18206 .iter()
18207 .enumerate()
18208 .find(|(_, column)| column.name == *column_name)
18209 else {
18210 return Err(invalid(format!(
18211 "local purge target {:?} names unknown column {:?}",
18212 target.table, column_name
18213 )));
18214 };
18215 let encrypted = table
18216 .encrypted_columns
18217 .iter()
18218 .any(|candidate| candidate.index == column_index);
18219 if column.ty != ColumnType::String || encrypted {
18220 return Err(invalid(format!(
18221 "local purge selector {:?}.{:?} must be a plaintext string column",
18222 target.table, column_name
18223 )));
18224 }
18225 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
18226 return Err(invalid(format!(
18227 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
18228 target.table, column_name
18229 )));
18230 }
18231 let mut values = raw_values.clone();
18232 values.sort();
18233 values.dedup();
18234 if values.iter().any(|value| {
18235 value.is_empty()
18236 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
18237 || !is_local_operation_code_like(value)
18238 }) {
18239 return Err(invalid(format!(
18240 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
18241 )));
18242 }
18243 selectors.push((column_name.clone(), values.clone()));
18244 canonical_selectors.insert(column_name.clone(), values);
18245 }
18246 let canonical = LocalDataPurgeTarget {
18247 table: target.table.clone(),
18248 selectors: canonical_selectors,
18249 };
18250 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18251 deduplicated.insert(
18252 key,
18253 (
18254 CompiledLocalDataPurgeTarget {
18255 table: target.table.clone(),
18256 selectors,
18257 },
18258 canonical,
18259 ),
18260 );
18261 }
18262 let targets = deduplicated
18263 .values()
18264 .map(|(compiled, _)| compiled.clone())
18265 .collect::<Vec<_>>();
18266 let canonical = deduplicated
18267 .values()
18268 .map(|(_, target)| target.clone())
18269 .collect::<Vec<_>>();
18270 let canonical_plan =
18271 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18272 Ok((targets, canonical_plan))
18273 }
18274
18275 fn local_purge_base_row_ids(
18276 &self,
18277 targets: &[CompiledLocalDataPurgeTarget],
18278 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
18279 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
18280 for target in targets {
18281 let table = self
18282 .schema
18283 .table(&target.table)
18284 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
18285 let mut clauses = Vec::with_capacity(target.selectors.len());
18286 let mut params = Vec::new();
18287 for (column, values) in &target.selectors {
18288 clauses.push(format!(
18289 "{} IN ({})",
18290 quote_ident(column),
18291 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
18292 ));
18293 params.extend(values.iter().cloned().map(SqlValue::Text));
18294 }
18295 let sql = format!(
18296 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
18297 quote_ident(&table.primary_key),
18298 base_table(&target.table),
18299 clauses.join(" AND ")
18300 );
18301 let mut statement = self
18302 .conn
18303 .prepare(&sql)
18304 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18305 let rows = statement
18306 .query_map(rusqlite::params_from_iter(params), |row| {
18307 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
18308 render_row_id_json(Some(&value))
18309 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
18310 })
18311 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18312 let ids = by_table.entry(target.table.clone()).or_default();
18313 for row in rows {
18314 ids.insert(
18315 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18316 );
18317 }
18318 }
18319 Ok(by_table)
18320 }
18321
18322 fn local_purge_values_match(
18323 target: &CompiledLocalDataPurgeTarget,
18324 values: &Map<String, Value>,
18325 ) -> bool {
18326 target.selectors.iter().all(|(column, allowed)| {
18327 values
18328 .get(column)
18329 .and_then(Value::as_str)
18330 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
18331 })
18332 }
18333
18334 pub fn purge_local_data(
18338 &mut self,
18339 input: &LocalDataPurgeInput,
18340 ) -> Result<LocalDataPurgeResult, String> {
18341 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
18342 let meta_key = format!("localPurge:{}", input.purge_id);
18343 let applied_plan = self
18344 .conn
18345 .query_row(
18346 "SELECT value FROM _syncular_meta WHERE key = ?1",
18347 rusqlite::params![meta_key],
18348 |row| row.get::<_, String>(0),
18349 )
18350 .optional()
18351 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18352 if let Some(applied_plan) = applied_plan {
18353 if applied_plan != canonical_plan {
18354 return Err(format!(
18355 "sync.invalid_request: local purge id {:?} was already used with a different plan",
18356 input.purge_id
18357 ));
18358 }
18359 return Ok(LocalDataPurgeResult {
18360 already_applied: true,
18361 purged_rows: 0,
18362 dropped_commits: 0,
18363 });
18364 }
18365
18366 let _ = self.drop_previous_version();
18370
18371 let prior_outbox = self.outbox.clone();
18372 let prior_failed = self.failed_commits.clone();
18373 let prior_rejections = self.rejections.clone();
18374 let prior_conflicts = self.conflicts.clone();
18375 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18376 self.begin_observation("syncular_local_purge")?;
18377 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
18378 let row_ids = self.local_purge_base_row_ids(&targets)?;
18379 let retained_dropped = self.drop_failed_commits(|table, values| {
18380 targets.iter().any(|target| {
18381 target.table == table && Self::local_purge_values_match(target, values)
18382 })
18383 })?;
18384 let doomed = self
18385 .outbox
18386 .iter()
18387 .filter(|commit| {
18388 commit.ops.iter().any(|operation| {
18389 let matching_targets = targets
18390 .iter()
18391 .filter(|target| target.table == operation.table)
18392 .collect::<Vec<_>>();
18393 if matching_targets.is_empty() {
18394 return false;
18395 }
18396 if row_ids
18397 .get(&operation.table)
18398 .is_some_and(|ids| ids.contains(&operation.row_id))
18399 {
18400 return true;
18401 }
18402 operation.values.as_ref().is_some_and(|values| {
18403 matching_targets
18404 .iter()
18405 .any(|target| Self::local_purge_values_match(target, values))
18406 })
18407 })
18408 })
18409 .cloned()
18410 .collect::<Vec<_>>();
18411 let doomed_ids = doomed
18412 .iter()
18413 .map(|commit| commit.client_commit_id.clone())
18414 .collect::<BTreeSet<_>>();
18415 let mut batch = ChangeAccumulator::default();
18416 for commit in &retained_dropped {
18417 batch.conflicts = true;
18418 batch.rejections = true;
18419 batch.outcomes = true;
18420 for operation in &commit.ops {
18421 batch.table(&operation.table);
18422 }
18423 }
18424 let mut rejections = Vec::new();
18425 for commit in &doomed {
18426 for operation in &commit.ops {
18427 batch.table(&operation.table);
18428 }
18429 let results = commit
18430 .ops
18431 .iter()
18432 .enumerate()
18433 .map(|(op_index, operation)| {
18434 let rejection = RejectionRecord {
18435 client_commit_id: commit.client_commit_id.clone(),
18436 op_index: op_index as i32,
18437 code: "client.local_data_purged".to_owned(),
18438 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
18439 retryable: false,
18440 details: None,
18441 operation: Some(CommitOperation::from(operation)),
18442 };
18443 rejections.push(rejection.clone());
18444 CommitOperationOutcome::Error { rejection }
18445 })
18446 .collect::<Vec<_>>();
18447 self.persist_commit_outcome(
18448 &commit.client_commit_id,
18449 CommitOutcomeStatus::Rejected,
18450 &results,
18451 Some(&commit.ops),
18452 )?;
18453 self.delete_outbox_persisted(&commit.client_commit_id)?;
18454 }
18455 if !doomed.is_empty() {
18456 self.outbox
18457 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
18458 self.rejections.extend(rejections);
18459 self.prune_commit_outcomes()?;
18460 batch.status = true;
18461 batch.rejections = true;
18462 batch.outcomes = true;
18463 }
18464
18465 let mut purged_rows = 0usize;
18466 for (table, ids) in &row_ids {
18467 if ids.is_empty() {
18468 continue;
18469 }
18470 batch.table(table);
18471 for row_id in ids {
18472 self.delete_base_row(table, row_id)?;
18473 purged_rows += 1;
18474 }
18475 }
18476 self.rebuild_overlay_if_dirty()?;
18477 self.delete_unreferenced_cached_blobs()?;
18478 self.conn
18479 .execute(
18480 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
18481 rusqlite::params![meta_key, canonical_plan],
18482 )
18483 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18484 Ok((
18485 batch,
18486 LocalDataPurgeResult {
18487 already_applied: false,
18488 purged_rows,
18489 dropped_commits: doomed.len(),
18490 },
18491 ))
18492 })();
18493 let (batch, result) = match applied {
18494 Ok(value) => value,
18495 Err(error) => {
18496 self.rollback_observation("syncular_local_purge");
18497 self.outbox = prior_outbox;
18498 self.failed_commits = prior_failed;
18499 self.rejections = prior_rejections;
18500 self.conflicts = prior_conflicts;
18501 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18502 return Err(error);
18503 }
18504 };
18505 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
18506 self.rollback_observation("syncular_local_purge");
18507 self.outbox = prior_outbox;
18508 self.failed_commits = prior_failed;
18509 self.rejections = prior_rejections;
18510 self.conflicts = prior_conflicts;
18511 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18512 return Err(error);
18513 }
18514 self.cancel_sync_round();
18515 Ok(result)
18516 }
18517
18518 pub fn rebootstrap_local_data(
18524 &mut self,
18525 input: &LocalDataRebootstrapInput,
18526 ) -> Result<LocalDataRebootstrapResult, String> {
18527 if input.rebootstrap_id.is_empty()
18528 || input.rebootstrap_id.len() > 128
18529 || !is_local_operation_code_like(&input.rebootstrap_id)
18530 {
18531 return Err(
18532 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
18533 .to_owned(),
18534 );
18535 }
18536 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
18537 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
18538 let (retained_commits, reset_subscriptions) =
18539 decode_local_rebootstrap_receipt(&persisted)?;
18540 return Ok(LocalDataRebootstrapResult {
18541 already_applied: true,
18542 retained_commits,
18543 reset_subscriptions,
18544 });
18545 }
18546 if self.stopped || self.schema_floor.is_some() {
18547 return Err(
18548 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
18549 .to_owned(),
18550 );
18551 }
18552
18553 let retained_commits = self.outbox.len();
18554 let reset_subscriptions = self.subs.len();
18555 let prior_subs = self.subs.clone();
18556 let prior_upgrading = self.upgrading;
18557 let prior_stopped = self.stopped;
18558 let prior_schema_floor = self.schema_floor.clone();
18559 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18560 let prior_sync_needed = self.sync_needed;
18561 let prior_sync_intents = self.sync_intent_queue.clone();
18562 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
18563
18564 let prior_active_round = self.active_round;
18565 self.begin_observation("syncular_local_rebootstrap")?;
18566 let mut batch = ChangeAccumulator::default();
18567 let applied = (|| -> Result<(), String> {
18568 self.run_schema_reset_observed(&mut batch, false)?;
18569 self.conn
18570 .execute(
18571 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
18572 rusqlite::params![meta_key, receipt],
18573 )
18574 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18575 self.sync_needed = true;
18576 self.sync_intent_queue.push_back(SyncIntent::Interactive);
18577 batch.status = true;
18578 Ok(())
18579 })();
18580 if let Err(error) = applied {
18581 self.rollback_observation("syncular_local_rebootstrap");
18582 self.active_round = prior_active_round;
18583 self.subs = prior_subs;
18584 self.upgrading = prior_upgrading;
18585 self.stopped = prior_stopped;
18586 self.schema_floor = prior_schema_floor;
18587 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18588 self.sync_needed = prior_sync_needed;
18589 self.sync_intent_queue = prior_sync_intents;
18590 return Err(error);
18591 }
18592 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
18593 self.rollback_observation("syncular_local_rebootstrap");
18594 self.active_round = prior_active_round;
18595 self.subs = prior_subs;
18596 self.upgrading = prior_upgrading;
18597 self.stopped = prior_stopped;
18598 self.schema_floor = prior_schema_floor;
18599 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18600 self.sync_needed = prior_sync_needed;
18601 self.sync_intent_queue = prior_sync_intents;
18602 return Err(error);
18603 }
18604
18605 Ok(LocalDataRebootstrapResult {
18606 already_applied: false,
18607 retained_commits,
18608 reset_subscriptions,
18609 })
18610 }
18611
18612 fn purge_scope_rows(
18617 &mut self,
18618 table_name: &str,
18619 effective: &[(String, Vec<String>)],
18620 ) -> Result<(), ()> {
18621 if effective.is_empty() {
18622 return Ok(());
18623 }
18624 let table = self.schema.table(table_name).ok_or(())?.clone();
18625 let mut clauses = Vec::new();
18626 let mut params: Vec<SqlValue> = Vec::new();
18627 for (variable, values) in effective {
18628 let column = table.scope_column(variable).ok_or(())?;
18629 let placeholders: Vec<String> = values
18630 .iter()
18631 .map(|v| {
18632 params.push(SqlValue::Text(v.clone()));
18633 "?".to_owned()
18634 })
18635 .collect();
18636 clauses.push(format!(
18637 "{} IN ({})",
18638 quote_ident(column),
18639 placeholders.join(", ")
18640 ));
18641 }
18642 let sql = format!(
18643 "DELETE FROM {} WHERE {}",
18644 base_table(table_name),
18645 clauses.join(" AND ")
18646 );
18647 self.overlay_dirty.table(table_name);
18648 self.conn
18649 .execute(&sql, rusqlite::params_from_iter(¶ms))
18650 .map_err(|_| ())?;
18651 self.conn
18652 .execute(
18653 &format!(
18654 "DELETE FROM {} WHERE {}",
18655 visible_table(table_name),
18656 clauses.join(" AND ")
18657 ),
18658 rusqlite::params_from_iter(¶ms),
18659 )
18660 .map_err(|_| ())?;
18661 Ok(())
18662 }
18663
18664 fn drop_doomed_outbox(
18667 &mut self,
18668 table_name: &str,
18669 effective: &[(String, Vec<String>)],
18670 ) -> Result<bool, String> {
18671 if effective.is_empty() {
18672 return Ok(false);
18673 }
18674 let Some(table) = self.schema.table(table_name).cloned() else {
18675 return Ok(false);
18676 };
18677 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
18678 for (variable, values) in effective {
18679 match table.scope_column(variable) {
18680 Some(column) => mappings.push((column, values)),
18681 None => return Ok(false), }
18683 }
18684 let doomed: Vec<OutboxCommit> = self
18685 .outbox
18686 .iter()
18687 .filter(|commit| {
18688 commit.ops.iter().any(|op| {
18689 op.upsert
18690 && op.table == table_name
18691 && op.values.as_ref().is_some_and(|values| {
18692 mappings
18693 .iter()
18694 .all(|(column, allowed)| match values.get(*column) {
18695 Some(Value::String(s)) => allowed.contains(s),
18696 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
18697 Some(Value::Bool(b)) => allowed.contains(&b.to_string()),
18698 _ => false,
18699 })
18700 })
18701 })
18702 })
18703 .cloned()
18704 .collect();
18705 if doomed.is_empty() {
18706 return Ok(false);
18707 }
18708 let mut rejections = Vec::new();
18709 for commit in &doomed {
18710 let results = commit
18711 .ops
18712 .iter()
18713 .enumerate()
18714 .map(|(op_index, operation)| {
18715 let rejection = RejectionRecord {
18716 client_commit_id: commit.client_commit_id.clone(),
18717 op_index: op_index as i32,
18718 code: "sync.scope_revoked".to_owned(),
18719 message: "the commit was dropped because its effective scope was revoked"
18720 .to_owned(),
18721 retryable: false,
18722 details: None,
18723 operation: Some(CommitOperation::from(operation)),
18724 };
18725 rejections.push(rejection.clone());
18726 CommitOperationOutcome::Error { rejection }
18727 })
18728 .collect::<Vec<_>>();
18729 self.persist_commit_outcome(
18730 &commit.client_commit_id,
18731 CommitOutcomeStatus::Rejected,
18732 &results,
18733 Some(&commit.ops),
18734 )?;
18735 self.delete_outbox_persisted(&commit.client_commit_id)?;
18736 }
18737 self.prune_commit_outcomes()?;
18738 let doomed_ids = doomed
18739 .iter()
18740 .map(|commit| commit.client_commit_id.as_str())
18741 .collect::<BTreeSet<_>>();
18742 self.outbox
18743 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
18744 self.rejections.extend(rejections);
18745 Ok(true)
18746 }
18747
18748 pub fn upload_blob(
18755 &mut self,
18756 bytes: &[u8],
18757 media_type: Option<String>,
18758 name: Option<String>,
18759 ) -> Result<Value, String> {
18760 let blob_id = blob_id_for(bytes);
18761 let now = self.clock_now_ms();
18762 let tx = self
18763 .conn
18764 .transaction()
18765 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18766 tx.execute(
18767 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
18768 ON CONFLICT(blob_id) DO NOTHING",
18769 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
18770 )
18771 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18772 tx.execute(
18773 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
18774 rusqlite::params![blob_id, media_type, now],
18775 )
18776 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18777 tx.commit()
18778 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18779 self.enforce_blob_cache_cap();
18780 let mut obj = Map::new();
18781 obj.insert("blobId".to_owned(), Value::from(blob_id));
18782 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
18783 if let Some(mt) = media_type {
18784 obj.insert("mediaType".to_owned(), Value::from(mt));
18785 }
18786 if let Some(n) = name {
18787 obj.insert("name".to_owned(), Value::from(n));
18788 }
18789 Ok(Value::Object(obj))
18790 }
18791
18792 pub fn fetch_blob_bytes(
18796 &mut self,
18797 transport: &mut dyn Transport,
18798 blob_id_or_ref: &str,
18799 ) -> Result<FetchedBlob, (String, String)> {
18800 let simple = |m: String| ("client.failed".to_owned(), m);
18801 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
18802 blob_id_or_ref.to_owned()
18803 } else {
18804 let value: Value = serde_json::from_str(blob_id_or_ref)
18805 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
18806 value
18807 .get("blobId")
18808 .and_then(Value::as_str)
18809 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
18810 .to_owned()
18811 };
18812 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
18813 return Ok(cached);
18814 }
18815 if !self.transport_enabled {
18816 return Err(("sync.offline".into(), "transport gate is closed".into()));
18817 }
18818 #[cfg(feature = "bench-internals")]
18824 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
18825 let bytes = match transport
18826 .blob_download(&blob_id)
18827 .map_err(|e| (e.code, e.message))?
18828 {
18829 BlobDownload::Bytes(bytes) => bytes,
18830 BlobDownload::Url {
18831 url,
18832 url_expires_at_ms,
18833 } => {
18834 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
18836 return Err((
18837 "sync.segment_expired".to_owned(),
18838 format!(
18839 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
18840 ),
18841 ));
18842 }
18843 transport
18844 .fetch_blob_url(&url)
18845 .map_err(|e| (e.code, e.message))?
18846 }
18847 };
18848 #[cfg(feature = "bench-internals")]
18849 drop(download_phase);
18850 #[cfg(feature = "bench-internals")]
18851 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
18852 if blob_id_for(&bytes) != blob_id {
18854 return Err(simple(format!(
18855 "blob content address mismatch for {blob_id}"
18856 )));
18857 }
18858 #[cfg(feature = "bench-internals")]
18859 drop(validate_phase);
18860 #[cfg(feature = "bench-internals")]
18861 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
18862 let now = self.clock_now_ms();
18863 self.conn
18864 .execute(
18865 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
18866 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
18867 )
18868 .map_err(|e| simple(e.to_string()))?;
18869 self.conn
18870 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
18871 .map_err(|e| simple(e.to_string()))?;
18872 self.enforce_blob_cache_cap();
18873 #[cfg(feature = "bench-internals")]
18874 drop(insert_phase);
18875 self.get_cached_blob_bytes(&blob_id)
18876 .map_err(simple)?
18877 .ok_or_else(|| simple("blob cache write failed".to_owned()))
18878 }
18879
18880 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
18881 #[cfg(feature = "bench-internals")]
18882 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
18883 let mut stmt = self
18884 .conn
18885 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
18886 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18887 let mut rows = stmt
18888 .query(rusqlite::params![blob_id])
18889 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18890 if let Some(row) = rows
18891 .next()
18892 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
18893 {
18894 let bytes: Vec<u8> = row
18895 .get(0)
18896 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18897 let byte_length: i64 = row
18898 .get(1)
18899 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18900 let media_type: Option<String> = row
18901 .get(2)
18902 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18903 return Ok(Some(FetchedBlob {
18904 blob_id: blob_id.to_owned(),
18905 byte_length,
18906 bytes,
18907 media_type,
18908 }));
18909 }
18910 Ok(None)
18911 }
18912
18913 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
18915 let pending: Vec<(String, Option<String>)> = {
18916 let mut stmt = self
18917 .conn
18918 .prepare(
18919 "SELECT blob_id, media_type FROM (
18920 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
18921 UNION ALL
18922 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
18923 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
18924 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
18925 ) ORDER BY created_at_ms, blob_id",
18926 )
18927 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
18928 let rows = stmt
18929 .query_map([], |row| {
18930 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
18931 })
18932 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
18933 rows.collect::<Result<_, _>>()
18934 .map_err(|error| match error {
18935 rusqlite::Error::InvalidColumnType(..)
18936 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
18937 "sync.local_corrupt",
18938 "Pending blob upload metadata is invalid",
18939 ),
18940 _ => TransportError::new("client.failed", error.to_string()),
18941 })?
18942 };
18943 let mut uploads = Vec::new();
18944 for (blob_id, media_type) in pending {
18945 let bytes = (|| {
18946 let stored: Option<(SqlValue, SqlValue)> = self
18947 .conn
18948 .query_row(
18949 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
18950 rusqlite::params![blob_id],
18951 |row| Ok((row.get(0)?, row.get(1)?)),
18952 )
18953 .optional()
18954 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
18955 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
18956 return Err(TransportError::new(
18957 "sync.local_corrupt",
18958 "Pending blob upload body is missing or corrupt",
18959 ));
18960 };
18961 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
18962 return Err(TransportError::new(
18963 "sync.local_corrupt",
18964 "Pending blob upload body is missing or corrupt",
18965 ));
18966 }
18967 Ok(bytes)
18968 })();
18969 let failed = bytes.is_err();
18970 uploads.push(crate::round::PendingUpload {
18971 id: blob_id,
18972 bytes,
18973 media_type,
18974 });
18975 if failed {
18976 break;
18977 }
18978 }
18979 Ok(uploads)
18980 }
18981
18982 #[cfg(test)]
18983 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
18984 for upload in self.pending_blob_uploads()? {
18985 self.upload_one(
18986 transport,
18987 &upload.id,
18988 &upload.bytes?,
18989 upload.media_type.as_deref(),
18990 )?;
18991 self.conn
18992 .execute(
18993 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
18994 rusqlite::params![upload.id],
18995 )
18996 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
18997 }
18998 Ok(())
18999 }
19000
19001 #[cfg(test)]
19008 fn upload_one(
19009 &self,
19010 transport: &mut dyn Transport,
19011 blob_id: &str,
19012 bytes: &[u8],
19013 media_type: Option<&str>,
19014 ) -> Result<(), TransportError> {
19015 crate::round::upload_one(transport, blob_id, bytes, media_type, self.now_ms)
19016 }
19017
19018 fn visible_blob_ids_sql(&self) -> String {
19019 let mut selects = Vec::new();
19020 for table in &self.schema.tables {
19021 for column in table
19022 .columns
19023 .iter()
19024 .filter(|column| column.ty == ColumnType::BlobRef)
19025 {
19026 let identifier = quote_ident(&column.name);
19027 selects.push(format!(
19028 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
19029 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
19030 AND json_type({identifier}, '$.blobId') = 'text'",
19031 quote_ident(&table.name),
19032 ));
19033 }
19034 }
19035 if selects.is_empty() {
19036 "SELECT NULL AS blob_id WHERE 0".to_owned()
19037 } else {
19038 selects.join(" UNION ")
19039 }
19040 }
19041
19042 fn enforce_blob_cache_cap(&self) {
19045 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
19046 return;
19047 };
19048 let mut total: i64 = self
19049 .conn
19050 .query_row(
19051 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
19052 [],
19053 |row| row.get(0),
19054 )
19055 .unwrap_or(0);
19056 if total <= max_bytes {
19057 return;
19058 }
19059 let query = format!(
19060 "SELECT blob_id, byte_length FROM _syncular_blobs
19061 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19062 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19063 AND blob_id NOT IN ({})
19064 ORDER BY created_at_ms ASC, blob_id ASC",
19065 self.visible_blob_ids_sql(),
19066 );
19067 let candidates: Vec<(String, i64)> = {
19068 let Ok(mut stmt) = self.conn.prepare(&query) else {
19069 return;
19070 };
19071 let Ok(rows) = stmt.query_map([], |row| {
19072 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
19073 }) else {
19074 return;
19075 };
19076 rows.filter_map(Result::ok).collect()
19077 };
19078 for (blob_id, byte_length) in candidates {
19079 if total <= max_bytes {
19080 break;
19081 }
19082 let _ = self.conn.execute(
19083 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
19084 rusqlite::params![blob_id],
19085 );
19086 total -= byte_length;
19087 }
19088 }
19089
19090 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
19091 #[cfg(feature = "bench-internals")]
19092 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
19093 if !self.schema_has_blobs() {
19094 return Ok(());
19095 }
19096 self.rebuild_overlay_if_dirty()?;
19097 self.conn
19098 .execute(
19099 &format!(
19100 "DELETE FROM _syncular_blobs
19101 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19102 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19103 AND blob_id NOT IN ({})",
19104 self.visible_blob_ids_sql(),
19105 ),
19106 [],
19107 )
19108 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19109 Ok(())
19110 }
19111
19112 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
19117 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
19118 return sql.clone();
19119 }
19120 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
19121 columns.push(quote_ident("_syncular_version"));
19122 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
19123 let primary_key = quote_ident(&table.primary_key);
19124 let updates = columns
19125 .iter()
19126 .filter(|column| **column != primary_key)
19127 .map(|column| format!("{column}=excluded.{column}"))
19128 .collect::<Vec<_>>()
19129 .join(", ");
19130 let sql = format!(
19131 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
19132 columns.join(", "),
19133 placeholders.join(", ")
19134 );
19135 self.insert_sql
19136 .borrow_mut()
19137 .insert(full_table.to_owned(), sql.clone());
19138 sql
19139 }
19140
19141 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
19142 self.overlay_dirty.table(table_name);
19143 self.write_row(&base_table(table_name), table_name, row, version)
19144 }
19145
19146 fn write_row(
19147 &self,
19148 full_table: &str,
19149 table_name: &str,
19150 row: &Row,
19151 version: i64,
19152 ) -> Result<(), String> {
19153 #[cfg(feature = "bench-internals")]
19154 let _phase = self.benchmark_phases.start(Phase::RowWrite);
19155 let table = self
19156 .schema
19157 .table(table_name)
19158 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19159 let sql = self.insert_row_sql(full_table, table);
19160 let mut stmt = self
19161 .conn
19162 .prepare_cached(&sql)
19163 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19164 let params = row
19165 .iter()
19166 .map(RowParam::Cell)
19167 .chain(std::iter::once(RowParam::Version(version)));
19168 stmt.execute(rusqlite::params_from_iter(params))
19169 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19170 Ok(())
19171 }
19172
19173 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
19174 let table = self
19175 .schema
19176 .table(table_name)
19177 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19178 self.overlay_dirty.table(table_name);
19179 let sql = format!(
19180 "DELETE FROM {} WHERE {}",
19181 base_table(table_name),
19182 row_id_predicate(table)
19183 );
19184 let mut stmt = self
19185 .conn
19186 .prepare_cached(&sql)
19187 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19188 stmt.execute(rusqlite::params![row_id])
19189 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19190 Ok(())
19191 }
19192
19193 fn has_acknowledged_rows(&self) -> bool {
19194 self.conn
19195 .query_row(
19196 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
19197 [],
19198 |row| row.get(0),
19199 )
19200 .expect("read acknowledged intent")
19201 }
19202
19203 fn acknowledged_operations(
19204 &self,
19205 rows: Option<&HashSet<(&str, &str)>>,
19206 tables: Option<&BTreeSet<&str>>,
19207 ) -> Result<Vec<OutboxOp>, String> {
19208 let keys = rows.map(|rows| {
19209 serde_json::to_string(
19210 &rows
19211 .iter()
19212 .map(|(table, id)| serde_json::to_string(&(table, id)).expect("row key"))
19213 .collect::<Vec<_>>(),
19214 )
19215 .expect("row keys")
19216 });
19217 let mut statement = self
19218 .conn
19219 .prepare("SELECT op_json,intent_json FROM _syncular_acked_rows WHERE (?1 IS NULL OR json_array(tbl,id) IN(SELECT value FROM json_each(?1))) AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx")
19220 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19221 let rows = statement
19222 .query_map(
19223 rusqlite::params![
19224 keys,
19225 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
19226 ],
19227 |row| {
19228 #[cfg(test)]
19229 self.acknowledged_replay_count
19230 .set(self.acknowledged_replay_count.get() + 1);
19231 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
19232 },
19233 )
19234 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19235 rows.map(|row| {
19236 let (op, intent) =
19237 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19238 let op: CommitOperation =
19239 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
19240 let table = self
19241 .schema
19242 .table(&op.table)
19243 .ok_or_else(|| "sync.unknown_table".to_owned())?;
19244 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
19245 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
19246 } else {
19247 op.values
19248 };
19249 Ok(OutboxOp {
19250 table: op.table,
19251 row_id: op.row_id,
19252 upsert: op.op == "upsert",
19253 base_version: op.base_version,
19254 values,
19255 })
19256 })
19257 .collect()
19258 }
19259
19260 fn drop_acknowledged_rows(
19261 &self,
19262 matches: impl Fn(&str, &Map<String, Value>) -> bool,
19263 ) -> Result<Vec<OutboxCommit>, String> {
19264 let mut statement = self
19265 .conn
19266 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
19267 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19268 let rows = statement
19269 .query_map([], |row| {
19270 Ok((
19271 row.get::<_, String>(0)?,
19272 row.get::<_, String>(1)?,
19273 row.get::<_, String>(2)?,
19274 row.get::<_, String>(3)?,
19275 ))
19276 })
19277 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19278 let mut doomed = HashSet::new();
19279 for row in rows {
19280 let (id, table, intent, row_id) =
19281 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19282 let values =
19283 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
19284 let base_matches = self
19285 .schema
19286 .table(&table)
19287 .map(|schema| self.stored_row(schema, &row_id, true))
19288 .transpose()?
19289 .flatten()
19290 .is_some_and(|(base, _)| {
19291 let schema = self.schema.table(&table).expect("acknowledged table");
19292 let values = schema
19293 .columns
19294 .iter()
19295 .zip(base.iter())
19296 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
19297 .collect();
19298 matches(&table, &values)
19299 });
19300 if matches(&table, &values) || base_matches {
19301 doomed.insert(id);
19302 }
19303 }
19304 let mut dropped = Vec::new();
19305 for id in doomed {
19306 let mut statement = self
19307 .conn
19308 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
19309 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19310 let operations = statement
19311 .query_map([&id], |row| row.get::<_, String>(0))
19312 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19313 let mut ops = Vec::new();
19314 for operation in operations {
19315 let json = operation
19316 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19317 let op: CommitOperation =
19318 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
19319 self.overlay_dirty.table(&op.table);
19320 ops.push(OutboxOp {
19321 table: op.table,
19322 row_id: op.row_id,
19323 upsert: op.op == "upsert",
19324 base_version: op.base_version,
19325 values: op.values,
19326 });
19327 }
19328 dropped.push(OutboxCommit {
19329 client_commit_id: id.clone(),
19330 ops,
19331 });
19332 self.conn
19333 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
19334 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19335 }
19336 Ok(dropped)
19337 }
19338
19339 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
19342 if self.overlay_dirty.get() {
19343 self.rebuild_overlay()?;
19344 }
19345 Ok(())
19346 }
19347
19348 fn rebuild_overlay(&mut self) -> Result<(), String> {
19352 #[cfg(feature = "bench-internals")]
19353 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
19354 #[cfg(test)]
19355 self.overlay_rebuild_count
19356 .set(self.overlay_rebuild_count.get() + 1);
19357 let dirty = self.overlay_dirty.snapshot();
19358 let tables: Vec<_> = self
19359 .schema
19360 .tables
19361 .iter()
19362 .filter(|table| {
19363 dirty
19364 .as_ref()
19365 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
19366 })
19367 .cloned()
19368 .collect();
19369 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
19370 self.begin_observation("syncular_overlay")?;
19371 let rebuilt = (|| {
19372 for table in &tables {
19373 for index in &table.fts_indexes {
19374 self.drop_fts_triggers(index)?;
19375 }
19376 let visible = visible_table(&table.name);
19377 let base = base_table(&table.name);
19378 self.conn
19379 .execute_batch(&format!(
19380 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
19381 ))
19382 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19383 }
19384 let acknowledged = self.acknowledged_operations(None, Some(&names))?;
19385 self.apply_outbox_ops(
19386 acknowledged
19387 .iter()
19388 .filter(|op| names.contains(op.table.as_str())),
19389 )?;
19390 for (commit, initial) in &self.failed_commits {
19391 for operation in &commit.ops {
19392 if !names.contains(operation.table.as_str()) || !operation.upsert {
19393 continue;
19394 }
19395 if let Some(table) = self.schema.table(&operation.table) {
19396 if self.visible_row(table, &operation.row_id)?.is_none()
19397 && operation.values.as_ref().is_some_and(|values| {
19398 table
19399 .columns
19400 .iter()
19401 .all(|column| values.contains_key(&column.name))
19402 })
19403 {
19404 let key = serde_json::to_string(&(
19405 operation.table.clone(),
19406 operation.row_id.clone(),
19407 ))
19408 .expect("row key serialization");
19409 if let Some(Value::Object(values)) = initial.get(&key) {
19410 self.apply_outbox_ops(&[OutboxOp {
19411 values: Some(values.clone()),
19412 ..operation.clone()
19413 }])?;
19414 }
19415 }
19416 }
19417 }
19418 self.apply_outbox_ops(
19419 commit
19420 .ops
19421 .iter()
19422 .filter(|op| names.contains(op.table.as_str())),
19423 )?;
19424 }
19425 for commit in &self.outbox {
19426 self.apply_outbox_ops(
19427 commit
19428 .ops
19429 .iter()
19430 .filter(|op| names.contains(op.table.as_str())),
19431 )?;
19432 }
19433 for table in &tables {
19434 for index in &table.fts_indexes {
19435 self.rebuild_fts_projection(table, index)?;
19436 self.create_fts_triggers(table, index)?;
19437 }
19438 }
19439 self.conn
19440 .execute_batch("RELEASE syncular_overlay")
19441 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19442 Ok(())
19443 })();
19444 if rebuilt.is_err() {
19445 self.conn
19446 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
19447 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19448 } else {
19449 self.overlay_dirty.set(false);
19450 }
19451 rebuilt
19452 }
19453
19454 fn apply_outbox_ops<'a>(
19459 &self,
19460 ops: impl IntoIterator<Item = &'a OutboxOp>,
19461 ) -> Result<(), String> {
19462 #[cfg(feature = "bench-internals")]
19463 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
19464 for op in ops {
19465 #[cfg(feature = "bench-internals")]
19466 if let Some(phase) = &mut phase {
19467 phase.unit();
19468 }
19469 if op.upsert {
19470 if let (Some(table), Some(values)) = (
19471 self.schema
19472 .table(&op.table)
19473 .filter(|table| table.indexes.iter().any(|index| index.unique)),
19474 op.values.as_ref(),
19475 ) {
19476 let mut intended: Map<String, Value> = self
19477 .visible_row(table, &op.row_id)?
19478 .map(|(row, _)| {
19479 table
19480 .columns
19481 .iter()
19482 .zip(&row)
19483 .map(|(column, value)| {
19484 (column.name.clone(), column_value_to_json(value))
19485 })
19486 .collect()
19487 })
19488 .unwrap_or_default();
19489 intended.extend(values.clone());
19490 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
19491 continue;
19492 }
19493 }
19494 }
19495 self.apply_outbox_op(op)?;
19496 }
19497 Ok(())
19498 }
19499
19500 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
19501 let Some(table) = self.schema.table(&op.table) else {
19502 return Ok(());
19503 };
19504 let visible = visible_table(&table.name);
19505 if !op.upsert {
19506 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
19507 self.conn
19508 .prepare_cached(&sql)
19509 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
19510 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19511 return Ok(());
19512 }
19513 let Some(values) = op.values.as_ref() else {
19514 return Ok(());
19515 };
19516 let full = table
19521 .columns
19522 .iter()
19523 .all(|column| values.contains_key(&column.name));
19524 let local = self.visible_row(table, &op.row_id)?;
19525 if local.is_none() && !full {
19526 return Ok(());
19527 }
19528 let mut row: Row = Vec::with_capacity(table.columns.len());
19529 for (index, column) in table.columns.iter().enumerate() {
19530 let incoming = match values.get(&column.name) {
19531 Some(raw) => json_to_column_value(
19532 if raw.get("$bytes").is_some()
19533 && table
19534 .encrypted_columns
19535 .iter()
19536 .any(|encrypted| encrypted.index == index)
19537 {
19538 &table.wire_columns[index]
19539 } else {
19540 column
19541 },
19542 Some(raw),
19543 ),
19544 None => Ok(local
19545 .as_ref()
19546 .and_then(|(values, _)| values.get(index).cloned())
19547 .flatten()),
19548 };
19549 row.push(incoming.map_err(|_| {
19550 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
19551 })?);
19552 }
19553 let version = local.as_ref().map_or(-1, |(_, version)| *version);
19554 self.write_row(&visible, &table.name, &row, version)
19555 .map_err(|error| {
19556 if error.starts_with("UNIQUE constraint failed:") {
19557 "sync.constraint_violation: local write violates a unique constraint".to_owned()
19558 } else {
19559 error
19560 }
19561 })
19562 }
19563
19564 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
19567 self.stored_row(table, row_id, false)
19568 }
19569
19570 fn stored_row(
19571 &self,
19572 table: &TableSchema,
19573 row_id: &str,
19574 base: bool,
19575 ) -> Result<Option<(Row, i64)>, String> {
19576 let sql = format!(
19577 "SELECT * FROM {} WHERE {}",
19578 if base {
19579 base_table(&table.name)
19580 } else {
19581 visible_table(&table.name)
19582 },
19583 row_id_predicate(table)
19584 );
19585 let mut stmt = self
19586 .conn
19587 .prepare_cached(&sql)
19588 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19589 let mut rows = stmt
19590 .query(rusqlite::params![row_id])
19591 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19592 let Some(row) = rows
19593 .next()
19594 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
19595 else {
19596 return Ok(None);
19597 };
19598 let mut values = Vec::with_capacity(table.columns.len());
19599 for (index, column) in table.columns.iter().enumerate() {
19600 let raw = row
19601 .get_ref(index)
19602 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19603 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
19604 if table
19605 .encrypted_columns
19606 .iter()
19607 .any(|encrypted| encrypted.index == index)
19608 {
19609 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
19610 continue;
19611 }
19612 }
19613 let json = sql_ref_to_json(column, raw);
19614 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
19615 "sync.local_corrupt: persisted local row value is invalid".to_owned()
19616 })?);
19617 }
19618 let version: i64 = row
19619 .get(table.columns.len())
19620 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19621 Ok(Some((values, version)))
19622 }
19623
19624 #[must_use]
19628 pub fn realtime_policy(&self) -> RealtimePolicy {
19629 self.realtime_policy
19630 }
19631
19632 #[must_use]
19634 pub fn realtime_state(&self) -> RealtimeState {
19635 self.realtime_state
19636 }
19637
19638 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
19641 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
19642 return Err(
19643 "sync.invalid_request: disconnect realtime before changing the realtime policy"
19644 .to_owned(),
19645 );
19646 }
19647 self.realtime_policy = policy;
19648 match policy {
19649 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
19650 RealtimePolicy::Required | RealtimePolicy::Optional
19651 if self.realtime_state == RealtimeState::Disabled =>
19652 {
19653 self.set_realtime_state(RealtimeState::Disconnected, None);
19654 }
19655 RealtimePolicy::Required | RealtimePolicy::Optional => {}
19656 }
19657 Ok(())
19658 }
19659
19660 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
19663 self.realtime_state = state;
19664 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
19665 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
19666 self.realtime_reason_code = None;
19667 self.realtime_retry_delay_ms = None;
19668 }
19669 }
19670
19671 #[must_use]
19672 fn realtime_connected(&self) -> bool {
19673 self.realtime_state == RealtimeState::Connected
19674 }
19675
19676 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
19677 if !self.transport_enabled {
19678 return Err("sync.offline: transport gate is closed".into());
19679 }
19680 if self.realtime_policy == RealtimePolicy::Off {
19681 return Err(
19682 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
19683 );
19684 }
19685 if self.realtime_connected() {
19686 return Ok(());
19687 }
19688 self.realtime_state = RealtimeState::Connecting;
19689 match transport.realtime_connect_for_client(&self.client_id) {
19690 Ok(()) => {
19691 self.set_realtime_state(RealtimeState::Connected, None);
19692 Ok(())
19693 }
19694 Err(error) => {
19695 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
19696 Err(format!("{}: {}", error.code, error.message))
19697 }
19698 }
19699 }
19700
19701 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
19702 self.cancel_sync_round();
19703 if self.realtime_state == RealtimeState::Disconnected
19706 || self.realtime_state == RealtimeState::Disabled
19707 {
19708 return;
19709 }
19710 let _ = transport.realtime_close();
19711 self.set_realtime_state(
19712 if self.realtime_policy == RealtimePolicy::Off {
19713 RealtimeState::Disabled
19714 } else {
19715 RealtimeState::Disconnected
19716 },
19717 None,
19718 );
19719 self.presence.clear(); }
19721
19722 pub fn set_presence(
19728 &mut self,
19729 transport: &mut dyn Transport,
19730 scope_key: &str,
19731 doc: Option<&Value>,
19732 ) -> Result<(), String> {
19733 if !self.transport_enabled {
19734 return Err("sync.offline: transport gate is closed".into());
19735 }
19736 if !self.realtime_connected() {
19737 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
19738 }
19739 let text = encode_presence_publish(scope_key, doc);
19740 transport
19741 .realtime_send(&text)
19742 .map_err(|e| format!("{}: {}", e.code, e.message))
19743 }
19744
19745 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
19747 self.presence
19748 .get(scope_key)
19749 .map(|peers| peers.values().cloned().collect())
19750 .unwrap_or_default()
19751 }
19752
19753 fn apply_presence(
19755 &mut self,
19756 scope_key: String,
19757 kind: Option<PresenceKind>,
19758 actor_id: Option<String>,
19759 client_id: Option<String>,
19760 doc: Option<Value>,
19761 error: Option<String>,
19762 ) {
19763 if error.is_some() {
19766 return;
19767 }
19768 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
19769 return;
19770 };
19771 let peer_key = format!("{actor_id} {client_id}");
19772 match kind {
19773 PresenceKind::Leave => {
19774 if let Some(peers) = self.presence.get_mut(&scope_key) {
19775 peers.remove(&peer_key);
19776 if peers.is_empty() {
19777 self.presence.remove(&scope_key);
19778 }
19779 }
19780 }
19781 _ => {
19782 let doc = match doc {
19783 Some(Value::Object(_)) => doc.unwrap(),
19784 _ => return,
19785 };
19786 self.presence.entry(scope_key).or_default().insert(
19787 peer_key,
19788 PresencePeer {
19789 actor_id,
19790 client_id,
19791 doc,
19792 },
19793 );
19794 }
19795 }
19796 }
19797
19798 pub fn on_realtime_text(&mut self, text: &str) {
19800 if self.stopped || !self.transport_enabled || self.security_preflight {
19801 return;
19802 }
19803 match parse_control(text) {
19804 Ok(ControlMessage::Hello { requires_sync, .. }) => {
19805 if requires_sync {
19806 self.set_sync_needed(true, true);
19808 }
19809 }
19810 Ok(ControlMessage::Presence {
19811 scope_key,
19812 kind,
19813 actor_id,
19814 client_id,
19815 doc,
19816 error,
19817 ..
19818 }) => {
19819 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
19820 }
19821 Ok(ControlMessage::Wake { .. }) => {
19822 self.set_sync_needed(true, true);
19824 }
19825 _ => {}
19826 }
19827 }
19828
19829 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
19833 let mut transport = crate::round::DownloadResults::default();
19834 self.on_realtime_binary(&mut transport, bytes);
19835 transport.controls
19836 }
19837
19838 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
19839 if self.stopped || !self.transport_enabled || self.security_preflight {
19840 return;
19841 }
19842 #[cfg(feature = "bench-internals")]
19843 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
19844 let message = match decode_message(bytes) {
19845 Ok(m) if m.msg_kind == MsgKind::Response => m,
19846 _ => {
19847 self.set_sync_needed(true, true);
19848 return;
19849 }
19850 };
19851 #[cfg(feature = "bench-internals")]
19852 drop(decode_phase);
19853 let mut frames = message.frames.into_iter();
19854 let mut applied_cursor: Option<i64> = None;
19855 let mut any_covered = false;
19856 let mut dropped = self.begin_coalesced_apply().is_err();
19857 while let Some(frame) = frames.next() {
19858 let Frame::SubStart {
19859 id,
19860 status,
19861 effective_scopes,
19862 ..
19863 } = frame
19864 else {
19865 continue;
19866 };
19867 let mut body = Vec::new();
19868 let mut next_cursor: Option<i64> = None;
19869 for inner in frames.by_ref() {
19870 match inner {
19871 Frame::SubEnd {
19872 next_cursor: nc, ..
19873 } => {
19874 next_cursor = Some(nc);
19875 break;
19876 }
19877 Frame::Unknown { .. } => {}
19878 other => body.push(other),
19879 }
19880 }
19881 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
19882 dropped = true;
19883 continue;
19884 };
19885 let sub = &self.subs[sub_index];
19886 if status != SubStatus::Active
19889 || sub.state != SubState::Active
19890 || sub.bootstrap_state.is_some()
19891 || !sub.synced_once
19892 {
19893 dropped = true;
19894 continue;
19895 }
19896 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
19897 any_covered = true;
19899 continue;
19900 }
19901 let previous_effective = self.subs[sub_index].effective.clone();
19902 let previous_cursor = self.subs[sub_index].cursor;
19903 self.subs[sub_index].effective = Some(effective_scopes);
19904 let mut failed = false;
19905 for inner in body {
19906 match inner {
19907 Frame::Commit {
19908 tables,
19909 changes,
19910 commit_seq,
19911 ..
19912 } => {
19913 if self
19914 .apply_commit_frame(&tables, &changes, Some(commit_seq))
19915 .is_err()
19916 {
19917 failed = true;
19918 break;
19919 }
19920 }
19921 _ => {
19922 failed = true;
19923 break;
19924 }
19925 }
19926 }
19927 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
19928 self.apply_sub_end(sub_index, cursor, None)
19929 .ok()
19930 .map(|()| cursor)
19931 });
19932 let Some(next_cursor) = committed else {
19933 self.subs[sub_index].effective = previous_effective;
19934 self.subs[sub_index].cursor = previous_cursor;
19935 dropped = true;
19936 continue;
19937 };
19938 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
19939 }
19940 if self.finish_coalesced_apply().is_err() {
19943 applied_cursor = None;
19944 dropped = true;
19945 }
19946 if let Some(cursor) = applied_cursor {
19947 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
19949 let _ = transport.realtime_send(&ack);
19950 } else if !any_covered || dropped {
19951 self.set_sync_needed(true, true);
19953 }
19954 }
19955
19956 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
19960 if !self.realtime_connected() {
19961 return;
19962 }
19963 let floor = self
19964 .subs
19965 .iter()
19966 .filter(|s| {
19967 s.state == SubState::Active
19968 && s.bootstrap_state.is_none()
19969 && s.synced_once
19970 && s.cursor >= 0
19971 })
19972 .map(|s| s.cursor)
19973 .min();
19974 if let Some(cursor) = floor {
19975 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
19976 let _ = transport.realtime_send(&ack);
19977 }
19978 }
19979}
19980
19981#[cfg(test)]
19982mod previous_version_wiring_tests {
19983 use super::*;
19984 use serde_json::{json, Value};
19985
19986 fn previous_version_schema(version: i32) -> Value {
19987 json!({
19988 "version": version,
19989 "tables": [{
19990 "name": "tasks",
19991 "primaryKey": "id",
19992 "columns": [
19993 { "name": "id", "type": "string", "nullable": false },
19994 { "name": "note", "type": "string", "nullable": true }
19995 ],
19996 "scopes": []
19997 }]
19998 })
19999 }
20000
20001 fn previous_version_limits() -> crate::api::ClientLimits {
20002 crate::api::ClientLimits {
20003 previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
20004 enabled: true,
20005 ..Default::default()
20006 }),
20007 ..Default::default()
20008 }
20009 }
20010
20011 fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
20013 previous_version_limits()
20014 .previous_version_context
20015 .expect("configured")
20016 }
20017
20018 fn recapture_previous_version(client: &SyncClient) {
20021 let replica_path = client
20022 .previous_version_replica_path()
20023 .expect("file-backed replica");
20024 let outcome = crate::previous_version::capture_previous_version_from_replica(
20025 &client.conn,
20026 &replica_path,
20027 2,
20028 2,
20029 &previous_version_config(),
20030 1,
20031 )
20032 .expect("capture");
20033 assert!(matches!(
20034 outcome,
20035 crate::previous_version::CaptureOutcome::Captured(_)
20036 ));
20037 }
20038
20039 fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
20040 Subscription {
20041 id: "wiring-sub".to_owned(),
20042 table: "tasks".to_owned(),
20043 requested: Vec::new(),
20044 params: None,
20045 cursor,
20046 bootstrap_state: bootstrap_state.map(str::to_owned),
20047 state,
20048 reason_code: None,
20049 effective: None,
20050 synced_once: false,
20051 }
20052 }
20053
20054 #[test]
20055 fn schema_downgrade_refuses_all_opens_and_recreate_without_mutation() {
20056 let path =
20057 std::env::temp_dir().join(format!("syncular-downgrade-{}.db", uuid::Uuid::new_v4()));
20058 let path_str = path.to_str().expect("temp path");
20059 let schema = previous_version_schema(3);
20060 let mut older = previous_version_schema(2);
20061 older["tables"][0]["columns"] =
20062 json!([{ "name": "id", "type": "string", "nullable": false }]);
20063 let mut client = SyncClient::open_path(
20064 "downgrade".into(),
20065 &previous_version_schema(1),
20066 previous_version_limits(),
20067 path_str,
20068 )
20069 .expect("fresh");
20070 client
20071 .mutate(vec![Mutation::Upsert {
20072 table: "tasks".into(),
20073 values: Map::from_iter([
20074 ("id".into(), json!("accepted")),
20075 ("note".into(), json!("old")),
20076 ]),
20077 base_version: None,
20078 }])
20079 .expect("v1 write");
20080 client
20081 .recreate_with_schema(&schema)
20082 .expect("upgrade captures context");
20083 let commit = client
20084 .mutate(vec![Mutation::Upsert {
20085 table: "tasks".into(),
20086 values: Map::from_iter([
20087 ("id".into(), json!("queued")),
20088 ("note".into(), json!("v3-only")),
20089 ]),
20090 base_version: None,
20091 }])
20092 .expect("queued v3 write");
20093 let changes = client.conn.total_changes();
20094 let outbox = client.pending_commit_ids();
20095 assert!(outbox.contains(&commit));
20096 let descriptor = client.get_meta("localSchemaDescriptor");
20097 let container = format!(
20098 "{path_str}{}",
20099 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
20100 );
20101 let context = std::fs::read(&container).expect("captured context");
20102 assert!(client
20103 .recreate_with_schema(&older)
20104 .expect_err("refuse recreate")
20105 .starts_with("client.schema_downgrade:"));
20106 assert_eq!(client.schema.version, 3);
20107 assert_eq!(client.conn.total_changes(), changes);
20108 assert_eq!(client.pending_commit_ids(), outbox);
20109 assert_eq!(client.get_meta("localSchemaDescriptor"), descriptor);
20110 assert_eq!(
20111 client
20112 .conn
20113 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20114 row.get::<_, String>(0)
20115 })
20116 .expect("queued row"),
20117 "v3-only"
20118 );
20119 assert_eq!(std::fs::read(&container).expect("context remains"), context);
20120 drop(client);
20121 let before = std::fs::read(&path).expect("replica bytes");
20122 for entry in 0..3 {
20125 let error = match entry {
20126 0 => SyncClient::open_path(
20127 "downgrade".into(),
20128 &older,
20129 previous_version_limits(),
20130 path_str,
20131 ),
20132 1 => SyncClient::open_path_with_identity(
20133 None,
20134 &older,
20135 previous_version_limits(),
20136 path_str,
20137 ),
20138 _ => SyncClient::with_connection(
20139 "downgrade".into(),
20140 &older,
20141 previous_version_limits(),
20142 Connection::open(&path).expect("connection"),
20143 ),
20144 }
20145 .err()
20146 .expect("refuse open");
20147 assert!(error.starts_with("client.schema_downgrade:"));
20148 let conn = Connection::open(&path).expect("inspect");
20149 assert_eq!(
20150 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20151 .expect("journal mode"),
20152 "delete"
20153 );
20154 drop(conn);
20155 assert_eq!(std::fs::read(&path).expect("replica unchanged"), before);
20156 assert_eq!(
20157 std::fs::read(&container).expect("context unchanged"),
20158 context
20159 );
20160 }
20161 let reopened = SyncClient::open_path(
20162 "downgrade".into(),
20163 &schema,
20164 previous_version_limits(),
20165 path_str,
20166 )
20167 .expect("compatible reopen");
20168 assert_eq!(reopened.pending_commit_ids(), outbox);
20169 assert_eq!(
20170 reopened
20171 .conn
20172 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20173 row.get::<_, String>(0)
20174 })
20175 .expect("queued row remains"),
20176 "v3-only"
20177 );
20178 reopened.delete_meta(LOCAL_SCHEMA_VERSION_KEY);
20179 reopened.delete_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20180 drop(reopened);
20181 let legacy = SyncClient::open_path(
20182 "downgrade".into(),
20183 &schema,
20184 previous_version_limits(),
20185 path_str,
20186 )
20187 .expect("legacy no-marker reopen");
20188 assert_eq!(legacy.pending_commit_ids(), outbox);
20189 assert!(!legacy.upgrading());
20190 assert_eq!(
20191 legacy
20192 .conn
20193 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20194 row.get::<_, String>(0)
20195 })
20196 .expect("legacy row remains"),
20197 "v3-only"
20198 );
20199 drop(legacy);
20200 std::fs::remove_file(path).expect("remove replica");
20201 let _ = std::fs::remove_file(container);
20202 }
20203
20204 #[test]
20205 fn schema_marker_errors_refuse_open_and_recreate_without_mutation() {
20206 for marker in [
20207 "''",
20208 "'bad'",
20209 "'0'",
20210 "'-1'",
20211 "'2.5'",
20212 "'03'",
20213 "'3\n'",
20214 "' 3'",
20215 "'+3'",
20216 "'3e0'",
20217 "'2147483648'",
20218 "NULL",
20219 "X'33'",
20220 ] {
20221 let conn = Connection::open_in_memory().expect("connection");
20222 conn.execute_batch(&format!("CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value); INSERT INTO _syncular_meta VALUES ('localSchemaVersion', {marker});")).expect("invalid marker fixture");
20223 let error = SyncClient::with_connection(
20224 "marker".into(),
20225 &previous_version_schema(3),
20226 ClientLimits::default(),
20227 conn,
20228 )
20229 .err()
20230 .expect("refuse marker");
20231 assert!(
20232 error.starts_with("sync.local_corrupt:"),
20233 "{marker}: {error}"
20234 );
20235 }
20236 for fault in [
20237 "UPDATE _syncular_meta SET value = 'bad' WHERE key = 'localSchemaVersion'",
20238 "ALTER TABLE _syncular_meta RENAME COLUMN value TO unreadable_value",
20239 ] {
20240 let mut client = SyncClient::new(
20241 "marker".into(),
20242 &previous_version_schema(3),
20243 ClientLimits::default(),
20244 )
20245 .expect("fresh");
20246 let commit = client
20247 .mutate(vec![Mutation::Upsert {
20248 table: "tasks".into(),
20249 values: Map::from_iter([("id".into(), json!("queued"))]),
20250 base_version: None,
20251 }])
20252 .expect("queued write");
20253 client.conn.execute_batch(fault).expect("marker fault");
20254 let changes = client.conn.total_changes();
20255 let error = client
20256 .recreate_with_schema(&previous_version_schema(4))
20257 .expect_err("refuse corrupt marker");
20258 assert!(error.starts_with("sync.local_corrupt:"));
20259 assert_eq!(client.conn.total_changes(), changes);
20260 assert_eq!(client.schema.version, 3);
20261 assert_eq!(client.pending_commit_ids(), vec![commit]);
20262 }
20263 for setup in [
20264 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY)",
20265 "CREATE VIEW _syncular_meta AS SELECT 'localSchemaVersion' AS key, '3' AS value",
20266 ] {
20267 let path =
20268 std::env::temp_dir().join(format!("syncular-marker-{}.db", uuid::Uuid::new_v4()));
20269 let conn = Connection::open(&path).expect("connection");
20270 conn.execute_batch(setup)
20271 .expect("unreadable metadata fixture");
20272 drop(conn);
20273 let before = std::fs::read(&path).expect("before");
20274 let error = SyncClient::open_path_with_identity(
20275 None,
20276 &previous_version_schema(3),
20277 ClientLimits::default(),
20278 path.to_str().expect("path"),
20279 )
20280 .err()
20281 .expect("refuse unreadable marker");
20282 assert!(error.starts_with("sync.local_corrupt:"));
20283 assert_eq!(std::fs::read(&path).expect("after"), before);
20284 std::fs::remove_file(path).expect("remove fixture");
20285 }
20286 }
20287
20288 std::thread_local! {
20289 static CONCURRENT_UPGRADE: RefCell<Option<Connection>> =
20292 const { RefCell::new(None) };
20293 }
20294
20295 fn commit_concurrent_upgrade(attempts: i32) -> bool {
20296 CONCURRENT_UPGRADE.with(|slot| {
20297 if let Some(banner) = slot.borrow_mut().take() {
20298 banner
20299 .execute_batch(
20300 "UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion'; COMMIT",
20301 )
20302 .expect("commit the concurrent upgrade");
20303 }
20304 });
20305 attempts < 10
20306 }
20307
20308 fn journal_mode(conn: &Connection) -> String {
20309 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20310 .expect("journal mode")
20311 }
20312
20313 fn temp_replica(label: &str) -> std::path::PathBuf {
20314 std::env::temp_dir().join(format!("syncular-{label}-{}.db", uuid::Uuid::new_v4()))
20315 }
20316
20317 #[test]
20318 fn file_identity_constructor_requires_the_returned_wal_mode() {
20319 for path in [":memory:", ""] {
20320 let error = SyncClient::open_path_with_identity(
20321 None,
20322 &previous_version_schema(1),
20323 ClientLimits::default(),
20324 path,
20325 )
20326 .err()
20327 .expect("temporary SQLite databases cannot enter WAL mode");
20328 assert_eq!(
20329 error,
20330 "sync.invalid_request: file-backed client requires WAL journal mode"
20331 );
20332 }
20333 let path = temp_replica("confirmed-wal");
20334 let client = SyncClient::open_path_with_identity(
20335 None,
20336 &previous_version_schema(1),
20337 ClientLimits::default(),
20338 path.to_str().unwrap(),
20339 )
20340 .unwrap();
20341 assert_eq!(journal_mode(&client.conn), "wal");
20342 drop(client);
20343 std::fs::remove_file(path).unwrap();
20344 }
20345
20346 #[test]
20347 fn non_positive_generated_schema_versions_are_refused_before_storage() {
20348 for version in [json!(0), json!(-1), json!(i32::MIN), json!(2147483648i64)] {
20349 let mut schema = previous_version_schema(1);
20350 schema["version"] = version.clone();
20351 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20352 .err()
20353 .expect("refuse an out-of-range generated version");
20354 assert_eq!(
20355 error,
20356 "sync.invalid_request: generated schema version must be an integer in 1..=2147483647"
20357 );
20358 let path = temp_replica("bad-version");
20360 let path_str = path.to_str().expect("path");
20361 for identity in [false, true] {
20362 let error = if identity {
20363 SyncClient::open_path_with_identity(
20364 None,
20365 &schema,
20366 ClientLimits::default(),
20367 path_str,
20368 )
20369 } else {
20370 SyncClient::open_path(
20371 "bad-version".into(),
20372 &schema,
20373 ClientLimits::default(),
20374 path_str,
20375 )
20376 }
20377 .err()
20378 .expect("refuse before storage");
20379 assert!(
20380 error.starts_with("sync.invalid_request:"),
20381 "{version}: {error}"
20382 );
20383 assert!(
20384 !path.exists(),
20385 "{version}: a refused constructor creates no replica file"
20386 );
20387 }
20388 }
20389 let mut schema = previous_version_schema(1);
20391 schema
20392 .as_object_mut()
20393 .expect("schema object")
20394 .remove("version");
20395 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20396 .err()
20397 .expect("refuse a missing version");
20398 assert!(error.starts_with("sync.invalid_request:"), "{error}");
20399 }
20400
20401 #[test]
20402 fn a_duplicated_schema_marker_is_refused_without_a_reset() {
20403 let path = temp_replica("duplicate-marker");
20404 let path_str = path.to_str().expect("path");
20405 {
20406 let conn = Connection::open(path_str).expect("fixture");
20409 conn.execute_batch(
20410 "CREATE TABLE _syncular_meta(key TEXT, value);
20411 INSERT INTO _syncular_meta VALUES
20412 ('localSchemaVersion', '1'),
20413 ('localSchemaVersion', '2'),
20414 ('clientId', 'duplicate');",
20415 )
20416 .expect("duplicate marker fixture");
20417 }
20418 let error = SyncClient::open_path_with_identity(
20419 None,
20420 &previous_version_schema(3),
20421 ClientLimits::default(),
20422 path_str,
20423 )
20424 .err()
20425 .expect("refuse a duplicated marker");
20426 assert!(error.starts_with("sync.local_corrupt:"), "{error}");
20427 let probe = Connection::open(path_str).expect("probe");
20428 assert_eq!(
20429 probe
20430 .query_row(
20431 "SELECT COUNT(*) FROM _syncular_meta WHERE key = 'localSchemaVersion'",
20432 [],
20433 |row| row.get::<_, i64>(0),
20434 )
20435 .expect("marker rows"),
20436 2,
20437 "the duplicated rows are left for the operator, not collapsed"
20438 );
20439 assert_eq!(
20440 probe
20441 .query_row(
20442 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_syncular_%' AND name NOT LIKE 'sqlite_%'",
20443 [],
20444 |row| row.get::<_, i64>(0),
20445 )
20446 .expect("synced tables"),
20447 0,
20448 "a duplicated marker never drives the reset that recreates v3 tables"
20449 );
20450 assert_eq!(
20451 journal_mode(&probe),
20452 "delete",
20453 "the refusal leaves the journal mode alone"
20454 );
20455 drop(probe);
20456 std::fs::remove_file(path).expect("remove replica");
20457 }
20458
20459 #[test]
20460 fn a_refused_open_does_not_persist_wal_or_mutate_the_replica() {
20461 let path = temp_replica("refused-wal");
20462 let path_str = path.to_str().expect("path");
20463 drop(
20466 SyncClient::open_path(
20467 "owner".into(),
20468 &previous_version_schema(2),
20469 ClientLimits::default(),
20470 path_str,
20471 )
20472 .expect("v2 replica"),
20473 );
20474 let before = std::fs::read(&path).expect("before");
20475 let error = SyncClient::open_path_with_identity(
20476 Some("intruder".into()),
20477 &previous_version_schema(2),
20478 ClientLimits::default(),
20479 path_str,
20480 )
20481 .err()
20482 .expect("refuse the identity mismatch");
20483 assert!(error.starts_with("client.identity_mismatch:"), "{error}");
20484 let probe = Connection::open(path_str).expect("probe");
20485 assert_eq!(
20486 journal_mode(&probe),
20487 "delete",
20488 "a refused open must not persist WAL"
20489 );
20490 assert_eq!(std::fs::read(&path).expect("after"), before);
20491 assert_eq!(
20492 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20493 Some("2")
20494 );
20495 drop(probe);
20496 std::fs::remove_file(path).expect("remove replica");
20497 }
20498
20499 #[test]
20500 fn a_concurrent_marker_upgrade_refuses_a_stale_open() {
20501 let path = temp_replica("marker-race");
20502 let path_str = path.to_str().expect("path");
20503 drop(
20504 SyncClient::open_path_with_identity(
20505 Some("stale".into()),
20506 &previous_version_schema(1),
20507 ClientLimits::default(),
20508 path_str,
20509 )
20510 .expect("v1 replica"),
20511 );
20512 let banner = Connection::open(path_str).expect("banner");
20516 banner
20517 .execute_batch(
20518 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
20519 )
20520 .expect("pending upgrade");
20521 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
20522 let conn = Connection::open(path_str).expect("stale connection");
20523 conn.busy_handler(Some(commit_concurrent_upgrade))
20524 .expect("busy handler");
20525 let error = SyncClient::with_connection(
20526 "stale".into(),
20527 &previous_version_schema(1),
20528 ClientLimits::default(),
20529 conn,
20530 )
20531 .err()
20532 .expect("a concurrent upgrade must refuse the stale open");
20533 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
20534 let probe = Connection::open(path_str).expect("probe");
20535 assert_eq!(
20536 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20537 Some("2"),
20538 "the concurrent upgrade survives"
20539 );
20540 drop(probe);
20541 std::fs::remove_file(path).expect("remove replica");
20542 }
20543
20544 #[test]
20545 fn a_concurrent_marker_upgrade_refuses_a_stale_recreate() {
20546 let path = temp_replica("recreate-race");
20547 let path_str = path.to_str().expect("path");
20548 let mut client = SyncClient::open_path_with_identity(
20549 Some("stale".into()),
20550 &previous_version_schema(1),
20551 ClientLimits::default(),
20552 path_str,
20553 )
20554 .expect("v1 replica");
20555 client
20556 .mutate(vec![Mutation::Upsert {
20557 table: "tasks".into(),
20558 values: Map::from_iter([("id".into(), json!("queued"))]),
20559 base_version: None,
20560 }])
20561 .expect("queued write");
20562 let commit = client.pending_commit_ids();
20563 let banner = Connection::open(path_str).expect("banner");
20564 banner
20565 .execute_batch(
20566 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
20567 )
20568 .expect("pending upgrade");
20569 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
20570 client
20571 .conn
20572 .busy_handler(Some(commit_concurrent_upgrade))
20573 .expect("busy handler");
20574 let error = client
20575 .recreate_with_schema(&previous_version_schema(1))
20576 .expect_err("a concurrent upgrade must refuse the stale recreate");
20577 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
20578 assert_eq!(client.schema.version, 1, "the compiled schema is restored");
20579 assert_eq!(
20580 client.pending_commit_ids(),
20581 commit,
20582 "the outbox is preserved"
20583 );
20584 let probe = Connection::open(path_str).expect("probe");
20585 assert_eq!(
20586 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20587 Some("2")
20588 );
20589 drop(probe);
20590 drop(client);
20591 std::fs::remove_file(path).expect("remove replica");
20592 }
20593
20594 #[test]
20595 fn failed_schema_record_writes_roll_back_recreation() {
20596 for key in [
20597 LOCAL_SCHEMA_VERSION_KEY,
20598 crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY,
20599 ] {
20600 let mut client = SyncClient::new(
20601 "schema-record".into(),
20602 &previous_version_schema(1),
20603 ClientLimits::default(),
20604 )
20605 .expect("v1");
20606 let marker = client.get_meta(LOCAL_SCHEMA_VERSION_KEY);
20607 let descriptor = client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20608 client.conn.execute_batch(&format!("CREATE TRIGGER refuse_schema_record BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{key}' BEGIN SELECT RAISE(ABORT, 'refuse schema record'); END")).expect("fault");
20609 client
20610 .recreate_with_schema(&previous_version_schema(2))
20611 .expect_err("both record writes must propagate failure");
20612 assert_eq!(client.schema.version, 1);
20613 assert_eq!(client.get_meta(LOCAL_SCHEMA_VERSION_KEY), marker);
20614 assert_eq!(
20615 client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY),
20616 descriptor
20617 );
20618 client
20619 .conn
20620 .execute_batch("DROP TRIGGER refuse_schema_record")
20621 .expect("remove fault");
20622 client
20623 .recreate_with_schema(&previous_version_schema(2))
20624 .expect("retry");
20625 assert_eq!(client.schema.version, 2);
20626 }
20627 }
20628
20629 #[test]
20630 fn a_failed_outer_commit_restores_every_reset_side_effect() {
20631 let path = temp_replica("outer-commit");
20632 let path_str = path.to_str().expect("path");
20633 let conn = Connection::open(path_str).expect("replica connection");
20636 let mut client = SyncClient::with_connection(
20637 "outer-commit".into(),
20638 &previous_version_schema(1),
20639 previous_version_limits(),
20640 conn,
20641 )
20642 .expect("v1 replica");
20643 assert_eq!(journal_mode(&client.conn), "delete");
20644 let commit = client
20645 .mutate(vec![Mutation::Upsert {
20646 table: "tasks".into(),
20647 values: Map::from_iter([
20648 ("id".into(), json!("queued")),
20649 ("note".into(), json!("dropped-by-v2")),
20650 ]),
20651 base_version: None,
20652 }])
20653 .expect("queued write");
20654 client.drain_change_batches();
20655 let mut dropped = previous_version_schema(2);
20657 dropped["tables"][0]["columns"] =
20658 json!([{ "name": "id", "type": "string", "nullable": false }]);
20659 let marker = meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY);
20662 let revision = client.local_revision();
20663 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
20664 let columns: i64 = client
20665 .conn
20666 .query_row(
20667 "SELECT count(*) FROM pragma_table_info('tasks')",
20668 [],
20669 |row| row.get(0),
20670 )
20671 .expect("column count");
20672 let rebuilds = client.overlay_rebuild_count.get();
20673 let prior_subs = client.subs.len();
20674 let prior_changes = client.change_queue.len();
20675 let prior_intents = client.sync_intent_queue.len();
20676 let prior_last_change = client.last_change.clone();
20677 let prior_overlay_dirty = client.overlay_dirty.get();
20678 let prior_sync_needed = client.sync_needed;
20679 let prior_upgrading = client.upgrading;
20680 let prior_stopped = client.stopped;
20681 let prior_schema_floor = client.schema_floor.clone();
20682 let prior_active_round = client.active_round;
20683 let reader = Connection::open(path_str).expect("reader");
20686 reader
20687 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
20688 .expect("reader snapshot");
20689 client
20690 .conn
20691 .busy_timeout(std::time::Duration::ZERO)
20692 .expect("busy timeout");
20693 let error = client
20694 .recreate_with_schema(&dropped)
20695 .expect_err("a blocked COMMIT must refuse the recreation");
20696 assert!(error.starts_with("client.storage_busy:"), "{error}");
20697 assert!(
20698 client.overlay_rebuild_count.get() > rebuilds,
20699 "the reset ran to its overlay rebuild inside the transaction"
20700 );
20701 assert!(client.conn.is_autocommit(), "the transaction is released");
20702 assert_eq!(meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY), marker);
20704 assert_eq!(client.local_revision(), revision);
20705 assert_eq!(
20706 client.query("SELECT * FROM tasks", &[]).expect("rows"),
20707 rows
20708 );
20709 assert_eq!(
20710 client
20711 .conn
20712 .query_row(
20713 "SELECT count(*) FROM pragma_table_info('tasks')",
20714 [],
20715 |row| { row.get::<_, i64>(0) }
20716 )
20717 .expect("column count"),
20718 columns,
20719 "the previous table layout survives the rollback"
20720 );
20721 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
20722 assert_eq!(client.schema.version, 1, "compiled schema");
20725 assert_eq!(client.subs.len(), prior_subs);
20726 assert_eq!(client.change_queue.len(), prior_changes);
20727 assert_eq!(client.sync_intent_queue.len(), prior_intents);
20728 assert_eq!(client.last_change, prior_last_change);
20729 assert_eq!(client.overlay_dirty.get(), prior_overlay_dirty);
20730 assert_eq!(client.sync_needed, prior_sync_needed);
20731 assert_eq!(client.upgrading, prior_upgrading);
20732 assert_eq!(client.stopped, prior_stopped);
20733 assert_eq!(client.schema_floor, prior_schema_floor);
20734 assert_eq!(client.active_round, prior_active_round);
20735 assert!(
20736 client.rejections.is_empty(),
20737 "the dropped-column rejection is rolled back too"
20738 );
20739 let container = crate::previous_version::previous_version_container_path(path_str);
20742 assert!(std::path::Path::new(&container).exists());
20743 drop(reader);
20746 let snapshot = client
20747 .previous_version_snapshot(&PreviousVersionReadSpec {
20748 table: "tasks".into(),
20749 ..Default::default()
20750 })
20751 .expect("refuse stale capture");
20752 assert!(!snapshot.available);
20753 assert_eq!(snapshot.current_version, 1);
20754 assert!(!std::path::Path::new(&container).exists());
20755 client
20760 .conn
20761 .busy_timeout(std::time::Duration::from_secs(1))
20762 .expect("busy timeout");
20763 client
20764 .recreate_with_schema(&dropped)
20765 .expect("the recreation succeeds once the reader releases");
20766 assert_eq!(client.schema.version, 2);
20767 assert_eq!(
20768 client.pending_commit_ids(),
20769 vec![commit.clone()],
20770 "the reset kept the incompatible commit pending"
20771 );
20772 assert!(
20773 client.rejections.is_empty(),
20774 "no rejection exists before the send-time prepass"
20775 );
20776 client
20778 .prepare_sync_round(false)
20779 .expect("the send prepass runs during round preparation");
20780 assert!(client.pending_commit_ids().is_empty());
20781 assert!(
20782 client
20783 .rejections
20784 .iter()
20785 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE),
20786 "the dropped-column commit surfaced as a rejection at send time"
20787 );
20788 client.previous_version_discard();
20789 drop(client);
20790 std::fs::remove_file(path).expect("remove replica");
20791 }
20792
20793 #[test]
20794 fn the_send_time_incompatible_drop_is_atomic_and_publishes_once() {
20795 let v1 = previous_version_schema(1);
20796 let mut v2 = previous_version_schema(2);
20797 v2["tables"][0]["columns"] = json!([{ "name": "id", "type": "string", "nullable": false }]);
20799 let mut client = SyncClient::new("drop".to_string(), &v1, ClientLimits::default()).unwrap();
20800 let dropped = client
20801 .mutate(vec![Mutation::Upsert {
20802 table: "tasks".into(),
20803 values: Map::from_iter([
20804 ("id".into(), json!("dropped")),
20805 ("note".into(), json!("gone")),
20806 ]),
20807 base_version: None,
20808 }])
20809 .unwrap();
20810 let survivor = client
20814 .mutate(vec![Mutation::Delete {
20815 table: "tasks".into(),
20816 row_id: "gone".into(),
20817 base_version: None,
20818 }])
20819 .unwrap();
20820 client
20821 .recreate_with_schema(&v2)
20822 .expect("the reset preserves the outbox");
20823 assert_eq!(
20824 client.pending_commit_ids(),
20825 vec![dropped.clone(), survivor.clone()],
20826 "the reset preserves both commits"
20827 );
20828 assert!(client.rejections.is_empty(), "the reset classifies nothing");
20829 client.drain_change_batches();
20830 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
20831 assert_eq!(rows.len(), 1, "only the dropped upsert has a visible row");
20832
20833 for fault in [
20839 "outcome-insert",
20840 "outbox-delete",
20841 "overlay-rebuild",
20842 "publish-revision",
20843 ] {
20844 client.set_sync_needed(false, false);
20845 client.drain_change_batches();
20846 let revision = client.local_revision();
20847 let trigger = match fault {
20848 "outcome-insert" => {
20849 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(ABORT,'drop fault'); END"
20850 }
20851 "outbox-delete" => {
20852 "CREATE TRIGGER fail_drop BEFORE DELETE ON _syncular_outbox BEGIN SELECT RAISE(ABORT,'drop fault'); END"
20853 }
20854 "publish-revision" => {
20859 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT,'drop fault'); END"
20860 }
20861 _ => {
20862 "CREATE TRIGGER fail_drop BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT,'drop fault'); END"
20863 }
20864 };
20865 client.conn.execute_batch(trigger).unwrap();
20866 let outcome = client
20867 .prepare_sync_round(false)
20868 .err()
20869 .unwrap_or_else(|| panic!("{fault}: a fault in the drop must fail the round"));
20870 assert!(
20871 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "storage.failed"),
20872 "{fault}: {outcome:?}"
20873 );
20874 client.conn.execute_batch("DROP TRIGGER fail_drop").unwrap();
20875 assert!(client.conn.is_autocommit(), "{fault}: no leaked savepoint");
20876 assert_eq!(
20877 client.pending_commit_ids(),
20878 vec![dropped.clone(), survivor.clone()],
20879 "{fault}: the outbox is restored"
20880 );
20881 assert!(
20882 client.rejections.is_empty(),
20883 "{fault}: no in-memory rejection survives"
20884 );
20885 assert!(
20886 client.commit_outcome(&dropped).unwrap().is_none(),
20887 "{fault}: the durable outcome rolled back"
20888 );
20889 assert_eq!(client.local_revision(), revision, "{fault}: revision");
20890 assert_eq!(
20891 client.query("SELECT * FROM tasks", &[]).unwrap(),
20892 rows,
20893 "{fault}: the visible projection is unchanged"
20894 );
20895 assert!(
20896 client.drain_change_batches().is_empty(),
20897 "{fault}: nothing is published"
20898 );
20899 }
20900
20901 client
20904 .conn
20905 .execute_batch(
20906 "BEGIN; INSERT INTO _syncular_meta(key,value) VALUES ('outerProbe','kept');
20907 CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes
20908 BEGIN SELECT RAISE(ABORT,'nested drop fault'); END;",
20909 )
20910 .unwrap();
20911 assert!(client.drop_incompatible_outbox().is_err());
20912 assert!(!client.conn.is_autocommit());
20913 assert_eq!(client.get_meta("outerProbe").as_deref(), Some("kept"));
20914 assert_eq!(
20915 client.pending_commit_ids(),
20916 vec![dropped.clone(), survivor.clone()]
20917 );
20918 assert!(client.commit_outcome(&dropped).unwrap().is_none());
20919 client
20920 .conn
20921 .execute_batch("DROP TRIGGER fail_drop; COMMIT")
20922 .unwrap();
20923
20924 client.set_meta(LOG_EPOCH_KEY, "epoch");
20929 client.set_sync_needed(false, false);
20930 client.drain_change_batches();
20931 let revision = client.local_revision();
20932 let prepared = client.prepare_sync_round(false).expect("the drop succeeds");
20933 assert_eq!(client.pending_commit_ids(), vec![survivor.clone()]);
20934 assert_eq!(
20935 prepared.meta.pushed_ids,
20936 vec![survivor.clone()],
20937 "the incompatible commit is classified before any operation is sent"
20938 );
20939 let rejection = client
20940 .rejections
20941 .iter()
20942 .find(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE)
20943 .expect("a sync.outbox_incompatible rejection");
20944 assert_eq!(rejection.client_commit_id, dropped);
20945 assert!(
20946 !rejection.retryable,
20947 "a schema-incompatible commit is final"
20948 );
20949 assert_eq!(
20950 client
20951 .commit_outcome(&dropped)
20952 .unwrap()
20953 .expect("a durable rejected outcome")
20954 .status,
20955 CommitOutcomeStatus::Rejected
20956 );
20957 assert_eq!(client.local_revision(), revision + 1, "one revision");
20958 let batches = client.drain_change_batches();
20959 assert_eq!(batches.len(), 1, "one change batch");
20960 assert!(
20961 batches[0].rejections_changed
20962 && batches[0].outcomes_changed
20963 && batches[0].status.is_some(),
20964 "{:?}",
20965 batches[0]
20966 );
20967 assert!(
20968 batches[0].tables.iter().any(|entry| entry.table == "tasks"),
20969 "{:?}",
20970 batches[0]
20971 );
20972 let visible = client.query("SELECT * FROM tasks", &[]).unwrap();
20973 assert!(
20974 visible.is_empty(),
20975 "the dropped projection is undone: {visible:?}"
20976 );
20977
20978 let path = temp_replica("outbox-drop-commit");
20982 let path_str = path.to_str().expect("path");
20983 let conn = Connection::open(path_str).expect("replica connection");
20984 let mut client =
20985 SyncClient::with_connection("drop-commit".into(), &v1, ClientLimits::default(), conn)
20986 .expect("v1 replica");
20987 let dropped = client
20988 .mutate(vec![Mutation::Upsert {
20989 table: "tasks".into(),
20990 values: Map::from_iter([
20991 ("id".into(), json!("dropped")),
20992 ("note".into(), json!("gone")),
20993 ]),
20994 base_version: None,
20995 }])
20996 .unwrap();
20997 client
20998 .recreate_with_schema(&v2)
20999 .expect("the reset preserves the outbox");
21000 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21001 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21002 assert_eq!(rows.len(), 1, "the optimistic row is visible");
21003 client.set_sync_needed(false, false);
21004 client.drain_change_batches();
21005 let revision = client.local_revision();
21006 let reader = Connection::open(path_str).expect("reader");
21007 reader
21008 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
21009 .expect("reader snapshot");
21010 client
21011 .conn
21012 .busy_timeout(std::time::Duration::ZERO)
21013 .expect("busy timeout");
21014 let outcome = client
21015 .prepare_sync_round(false)
21016 .err()
21017 .expect("a blocked COMMIT must fail the round");
21018 assert!(
21019 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "client.storage_busy"),
21020 "{outcome:?}"
21021 );
21022 assert!(client.conn.is_autocommit(), "the savepoint is released");
21023 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21024 assert!(client.rejections.is_empty());
21025 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21026 assert_eq!(client.local_revision(), revision);
21027 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
21028 assert!(client.drain_change_batches().is_empty());
21029 drop(reader);
21031 client
21032 .conn
21033 .busy_timeout(std::time::Duration::from_secs(1))
21034 .expect("busy timeout");
21035 client
21036 .prepare_sync_round(false)
21037 .expect("the drop succeeds after the reader releases");
21038 assert!(client.pending_commit_ids().is_empty());
21039 assert!(client
21040 .rejections
21041 .iter()
21042 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE));
21043 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
21044 drop(client);
21045 std::fs::remove_file(path).expect("remove replica");
21046 }
21047
21048 #[test]
21049 fn previous_version_read_reasons_come_from_live_client_state() {
21050 use crate::previous_version::{
21051 PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
21052 };
21053 let path = std::env::temp_dir()
21054 .join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
21055 .to_string_lossy()
21056 .into_owned();
21057 let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
21058 {
21059 let mut setup = SyncClient::open_path(
21060 "wiring".to_owned(),
21061 &previous_version_schema(1),
21062 previous_version_limits(),
21063 &path,
21064 )
21065 .expect("v1 install");
21066 setup
21067 .mutate(vec![Mutation::Upsert {
21068 table: "tasks".to_owned(),
21069 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21070 base_version: None,
21071 }])
21072 .expect("seed a row");
21073 }
21074 let mut client = SyncClient::open_path(
21075 "wiring".to_owned(),
21076 &previous_version_schema(2),
21077 previous_version_limits(),
21078 &path,
21079 )
21080 .expect("v2 bump captures");
21081 assert!(std::path::Path::new(&container).exists());
21082 let spec = PreviousVersionReadSpec {
21083 table: "tasks".to_owned(),
21084 ..Default::default()
21085 };
21086
21087 recapture_previous_version(&client);
21089 client.lease_state = Some(LeaseState {
21090 lease_id: None,
21091 expires_at_ms: None,
21092 error_code: Some("lease.expired".to_owned()),
21093 });
21094 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21095 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21096 assert!(!snapshot.available);
21097 assert!(!std::path::Path::new(&container).exists());
21098
21099 recapture_previous_version(&client);
21101 client.set_now_ms(1_000);
21102 client.lease_state = Some(LeaseState {
21103 lease_id: Some("lease".to_owned()),
21104 expires_at_ms: Some(1_000),
21105 error_code: None,
21106 });
21107 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21108 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21109 assert!(!std::path::Path::new(&container).exists());
21110 client.lease_state = None;
21111
21112 recapture_previous_version(&client);
21114 client.subs.push(subscription(SubState::Revoked, 0, None));
21115 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21116 assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
21117 assert!(!std::path::Path::new(&container).exists());
21118 client.subs.clear();
21119
21120 recapture_previous_version(&client);
21123 client
21124 .subs
21125 .push(subscription(SubState::Active, 0, Some("resume")));
21126 assert!(!client.previous_version_coverage_complete());
21127 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21128 assert!(snapshot.available);
21129 assert_eq!(snapshot.previous_version, Some(2));
21130 assert_eq!(snapshot.rows.len(), 1);
21131
21132 recapture_previous_version(&client);
21133 client.subs[0].bootstrap_state = None;
21134 assert!(client.previous_version_coverage_complete());
21135 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21136 assert_eq!(
21137 snapshot.reason,
21138 Some(PreviousVersionReason::CoverageComplete)
21139 );
21140 assert!(!std::path::Path::new(&container).exists());
21141 client.subs.clear();
21142
21143 assert!(!client.previous_version_coverage_complete());
21145
21146 drop(client);
21147 let _ = std::fs::remove_file(&path);
21148 }
21149
21150 #[test]
21157 fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
21158 let path = std::env::temp_dir()
21159 .join(format!(
21160 "syncular-prev-lifetime-{}.db",
21161 uuid::Uuid::new_v4()
21162 ))
21163 .to_string_lossy()
21164 .into_owned();
21165 let container = format!(
21166 "{path}{}",
21167 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
21168 );
21169 let exists = || std::path::Path::new(&container).exists();
21170 {
21171 let mut setup = SyncClient::open_path(
21172 "lifetime".to_owned(),
21173 &previous_version_schema(1),
21174 previous_version_limits(),
21175 &path,
21176 )
21177 .expect("v1 install");
21178 setup
21179 .mutate(vec![Mutation::Upsert {
21180 table: "tasks".to_owned(),
21181 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21182 base_version: None,
21183 }])
21184 .expect("seed a row");
21185 }
21186 let mut client = SyncClient::open_path(
21187 "lifetime".to_owned(),
21188 &previous_version_schema(2),
21189 previous_version_limits(),
21190 &path,
21191 )
21192 .expect("v2 bump captures");
21193 client.set_now_ms(1_000);
21196 assert!(exists());
21197
21198 client.lease_state = Some(LeaseState {
21200 lease_id: None,
21201 expires_at_ms: None,
21202 error_code: Some("lease.expired".to_owned()),
21203 });
21204 client.previous_version_lifetime_check();
21205 assert!(!exists());
21206 client.lease_state = None;
21207
21208 recapture_previous_version(&client);
21211 client.subs.push(subscription(SubState::Active, 0, None));
21212 client.previous_version_lifetime_check();
21213 assert!(!exists());
21214 client.subs.clear();
21215
21216 recapture_previous_version(&client);
21218 client.subs.push(subscription(SubState::Revoked, 0, None));
21219 client.previous_version_lifetime_check();
21220 assert!(!exists());
21221 client.subs.clear();
21222
21223 recapture_previous_version(&client);
21226 client
21227 .subs
21228 .push(subscription(SubState::Active, 0, Some("resume")));
21229 client.previous_version_lifetime_check();
21230 assert!(exists());
21231 client.subs.clear();
21232
21233 drop(client);
21234 let _ = std::fs::remove_file(&path);
21235 let _ = std::fs::remove_file(&container);
21236 let _ = std::fs::remove_file(format!("{container}-wal"));
21237 let _ = std::fs::remove_file(format!("{container}-shm"));
21238 }
21239}