1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12#[cfg(test)]
13use std::cell::Cell;
14use std::cell::RefCell;
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
16
17use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
18use rusqlite::{Connection, OpenFlags, OptionalExtension};
19use serde::{Deserialize, Serialize};
20use serde_json::{json, Map, Value};
21use sha2::{Digest, Sha256};
22use ssp2::decode::WIRE_VERSION;
23use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
24use ssp2::primitives::RawJson;
25use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
26use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
27
28use crate::api::{
29 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
30 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
31 ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects, CommitDelivery,
32 CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
33 CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
34 DiagnosticLastChange, DiagnosticLastRound, DiagnosticQueryFailure, DiagnosticRoundCounters,
35 DiagnosticSubscription, FetchedBlob, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
36 LocalDataPurgeTarget, LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation,
37 PresencePeer, PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot,
38 QueryValue, RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
39 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SubscriptionCatchup,
40 SubscriptionRequest, SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport,
41 SyncStatusSnapshot, TableChange, WindowBase, WindowChange, WindowCoverage, WindowState,
42 WindowUnitRef, CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
43 MAX_DIAGNOSTIC_QUERY_FAILURES,
44};
45use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
46#[cfg(feature = "bench-internals")]
47use crate::bench::{Phase, Recorder};
48use crate::previous_version::{
49 capture_previous_version_from_replica, first_incompatibility,
50 reconcile_previous_version_at_boot, set_local_schema_descriptor,
51 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
52 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
53};
54use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
55use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
56use crate::values::{
57 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
58 encode_sparse_row_json, full_row_values, is_canonical_bytes_envelope, json_to_column_value,
59 json_to_scope_map, normalize_values_casing, render_row_id_json, scope_map_to_json,
60 sort_scope_map,
61};
62
63const DEFAULT_ACCEPT: u8 = 0b0111;
67const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
68const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
69const ACCEPT_SQLITE: u8 = 1 << 2;
70const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
71const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
72
73pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
75const LOCAL_REVISION_KEY: &str = "localRevision";
76const CLIENT_ID_KEY: &str = "clientId";
77const LEASE_STATE_KEY: &str = "leaseState";
78const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
79const LOG_EPOCH_KEY: &str = "logEpoch";
80const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
85const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
86const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
87const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
90const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
93const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
94const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
96 "the persisted commit carries values the current codec refuses";
97const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
99
100#[derive(Debug, Serialize, Deserialize)]
101#[serde(rename_all = "camelCase", deny_unknown_fields)]
102struct PersistedLocalDataRebootstrapReceipt {
103 version: u8,
104 retained_commits: u64,
105 reset_subscriptions: u64,
106}
107
108fn invalid_local_rebootstrap_receipt() -> String {
109 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
110}
111
112fn encode_local_rebootstrap_receipt(
113 retained_commits: usize,
114 reset_subscriptions: usize,
115) -> Result<String, String> {
116 let retained_commits =
117 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
118 let reset_subscriptions =
119 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
120 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
121 return Err(invalid_local_rebootstrap_receipt());
122 }
123 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
124 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
125 retained_commits,
126 reset_subscriptions,
127 })
128 .map_err(|_| invalid_local_rebootstrap_receipt())
129}
130
131fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
132 if value == "v1" {
134 return Ok((0, 0));
135 }
136 let receipt: PersistedLocalDataRebootstrapReceipt =
137 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
138 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
139 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
140 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
141 {
142 return Err(invalid_local_rebootstrap_receipt());
143 }
144 Ok((
145 usize::try_from(receipt.retained_commits)
146 .map_err(|_| invalid_local_rebootstrap_receipt())?,
147 usize::try_from(receipt.reset_subscriptions)
148 .map_err(|_| invalid_local_rebootstrap_receipt())?,
149 ))
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153enum SubState {
154 Active,
155 Revoked,
156 Failed,
157}
158
159#[cfg(test)]
160mod observation_tests {
161 use super::*;
162 use crate::native_transport::HostTransport;
163 use crate::values::encode_row_json;
164 use serde_json::json;
165
166 #[test]
167 fn authority_policy_refuses_encrypted_and_undeclared_columns_before_open() {
168 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
169 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "secret", "type": "bytes", "encrypted": true, "declaredType": "string", "nullable": true}
170 ]}]});
171 for name in [
172 "secret",
173 "missing",
174 "*",
175 "id FROM authority; DELETE FROM authority",
176 "_syncular_version",
177 ] {
178 let limits = ClientLimits {
179 authority_reads: vec![crate::api::AuthorityReadDeclaration {
180 table: "authority".into(),
181 columns: vec!["id".into(), "actor_id".into(), name.into()],
182 scopes: [("actor_id".into(), vec!["a".into()])].into(),
183 }],
184 ..Default::default()
185 };
186 let error = SyncClient::new("authority".into(), &schema, limits)
187 .err()
188 .expect("forbidden policy");
189 assert!(
190 error.starts_with("client.authority_read_forbidden:"),
191 "{error}"
192 );
193 }
194 }
195
196 #[test]
197 fn authority_snapshot_observes_one_file_transaction_during_concurrent_writes() {
198 let path = std::env::temp_dir().join(format!(
199 "syncular-authority-snapshot-{}.db",
200 uuid::Uuid::new_v4()
201 ));
202 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
203 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "revision", "type": "integer", "nullable": false}
204 ]}]});
205 let limits = ClientLimits {
206 authority_reads: vec![crate::api::AuthorityReadDeclaration {
207 table: "authority".into(),
208 columns: vec!["id".into(), "actor_id".into(), "revision".into()],
209 scopes: [("actor_id".into(), vec!["a".into()])].into(),
210 }],
211 ..Default::default()
212 };
213 let mut client =
214 SyncClient::open_path("authority".into(), &schema, limits, path.to_str().unwrap())
215 .unwrap();
216 client.begin_security_preflight();
217 let update = |conn: &Connection, revision: i64| {
218 conn.execute_batch("BEGIN IMMEDIATE").unwrap();
219 conn.execute(
220 "INSERT OR REPLACE INTO _syncular_base_authority VALUES('one','a',?1,1)",
221 [revision],
222 )
223 .unwrap();
224 conn.execute(
225 "UPDATE _syncular_meta SET value=?1 WHERE key='localRevision'",
226 [revision.to_string()],
227 )
228 .unwrap();
229 conn.execute("INSERT OR REPLACE INTO _syncular_subscriptions VALUES('s','authority',?1)", [json!({"requested": {"actor_id": ["a"]}, "effectiveScopes": {"actor_id": ["a"]}, "status": "active", "cursor": revision}).to_string()]).unwrap();
230 conn.execute_batch("COMMIT").unwrap();
231 };
232 update(&client.conn, 1);
233 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
234 let writer_barrier = barrier.clone();
235 let writer_path = path.clone();
236 let writer = std::thread::spawn(move || {
237 let conn = Connection::open(writer_path).unwrap();
238 conn.busy_timeout(std::time::Duration::from_secs(5))
239 .unwrap();
240 writer_barrier.wait();
241 for revision in 2..=100 {
242 update(&conn, revision);
243 }
244 });
245 barrier.wait();
246 for _ in 0..100 {
247 let snapshot = client.authority_snapshot().unwrap();
248 let revision = snapshot["revision"]
249 .as_str()
250 .unwrap()
251 .parse::<i64>()
252 .unwrap();
253 assert_eq!(
254 snapshot["tables"][0]["rows"][0]["values"]["revision"],
255 revision
256 );
257 assert_eq!(snapshot["tables"][0]["persisted"][0]["cursor"], revision);
258 assert_eq!(snapshot["complete"], true);
259 }
260 writer.join().unwrap();
261 assert_eq!(client.authority_snapshot().unwrap()["revision"], "100");
262 drop(client);
263 let _ = std::fs::remove_file(path);
264 }
265
266 fn client() -> SyncClient {
267 SyncClient::new(
268 "retry-test".to_owned(),
269 &json!({
270 "version": 1,
271 "tables": [{
272 "name": "tasks",
273 "primaryKey": "id",
274 "columns": [
275 { "name": "id", "type": "string", "nullable": false },
276 { "name": "project_id", "type": "string", "nullable": false }
277 ],
278 "scopes": [{ "pattern": "project:{project_id}" }]
279 }]
280 }),
281 ClientLimits::default(),
282 )
283 .expect("test client")
284 }
285
286 #[test]
287 fn subscribe_many_registers_all_or_nothing() {
288 let request = |id: &str, project: &str| SubscriptionRequest {
289 id: id.to_owned(),
290 table: "tasks".to_owned(),
291 scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
292 params: None,
293 };
294 let mut client = client();
295 client
296 .subscribe_many(vec![request("a", "p1"), request("b", "p2")])
297 .unwrap();
298 let persisted = |client: &SyncClient| -> i64 {
299 client
300 .conn
301 .query_row("SELECT count(*) FROM _syncular_subscriptions", [], |row| {
302 row.get(0)
303 })
304 .unwrap()
305 };
306 assert_eq!(persisted(&client), 2);
307 let error = client
309 .subscribe_many(vec![request("c", "p3"), request("b", "p9")])
310 .unwrap_err();
311 assert!(error.starts_with("client.subscription_intent_mismatch:"));
312 assert!(client.subscription_state("c").is_none());
313 assert_eq!(persisted(&client), 2);
314 assert!(client.conn.is_autocommit());
315 }
316
317 #[cfg(feature = "e2ee")]
318 #[test]
319 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
320 let schema = json!({
321 "version": 1,
322 "tables": [{
323 "name": "secrets",
324 "primaryKey": "id",
325 "columns": [
326 { "name": "id", "type": "string", "nullable": false },
327 { "name": "encryption_key_id", "type": "string", "nullable": true },
328 { "name": "note", "type": "bytes", "nullable": true,
329 "encrypted": true, "declaredType": "string" }
330 ],
331 "scopes": []
332 }]
333 });
334 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
335 .expect("test client");
336 let mut config = crate::values::EncryptionConfig::default();
337 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
338 config
339 .key_id_columns
340 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
341 client.set_encryption(config);
342 client
345 .write_base_row(
346 "secrets",
347 &vec![Some(ColumnValue::String("ghost".into())), None, None],
348 1,
349 )
350 .unwrap();
351 client.rebuild_overlay().expect("rebuild overlay");
352 let commit_id = client
353 .patch(
354 "secrets",
355 "ghost",
356 Map::from_iter([("note".to_owned(), json!("x"))]),
357 None,
358 )
359 .expect("patch records without an author-time encrypt failure");
360 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
361 assert!(client.drop_unencodable_outbox(true, None).unwrap());
363 assert!(client.pending_commit_ids().is_empty());
364 let rejections = client.rejections();
365 assert_eq!(rejections.len(), 1);
366 assert_eq!(rejections[0].code, "client.encrypt_failed");
367 assert_eq!(rejections[0].client_commit_id, commit_id);
368 assert_eq!(
369 client
370 .commit_outcome(&commit_id)
371 .unwrap()
372 .map(|outcome| outcome.status),
373 Some(CommitOutcomeStatus::Rejected)
374 );
375 }
376
377 #[test]
378 fn patch_scope_read_preserves_sqlite_failure() {
379 let mut client = client();
380 client
381 .conn
382 .execute_batch(
383 "DROP TABLE tasks;
384 CREATE VIEW tasks AS SELECT 't1' AS id, abs(-9223372036854775808) AS project_id;",
385 )
386 .unwrap();
387 let error = client
388 .patch(
389 "tasks",
390 "t1",
391 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
392 None,
393 )
394 .expect_err("the scope read must fail");
395 assert_ne!(error.code, "sync.invalid_request");
396 assert_eq!(error.code, "client.failed");
397 assert!(error.details.as_ref().unwrap()["legacyCause"]
398 .as_str()
399 .unwrap()
400 .contains("integer overflow"));
401 assert!(client.pending_commit_ids().is_empty());
402 assert!(client.conn.is_autocommit());
403 }
404
405 #[test]
406 fn patch_with_a_scope_column_matches_the_stored_local_row() {
407 let mut client = client();
408 client.create_synced_tables().unwrap();
409 client
410 .conn
411 .execute(
412 "INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
413 [],
414 )
415 .unwrap();
416
417 client
420 .patch(
421 "tasks",
422 "t1",
423 Map::from_iter([
424 ("project_id".to_owned(), json!("p1")),
425 ("id".to_owned(), json!("t1")),
426 ]),
427 None,
428 )
429 .expect("an equal scope column is a no-op");
430 let values = client.outbox[0].ops[0]
431 .values
432 .clone()
433 .expect("the patch records values");
434 assert_eq!(values.get("id"), Some(&json!("t1")));
435 assert!(!values.contains_key("project_id"));
436
437 let differing = client.patch(
439 "tasks",
440 "t1",
441 Map::from_iter([("project_id".to_owned(), json!("p2"))]),
442 None,
443 );
444 let differing = differing.expect_err("a differing scope column is rejected");
445 assert_eq!(differing.code, "sync.invalid_request");
446 assert_eq!(differing.message, "the authoring request is invalid");
447 assert!(
448 differing
449 .details
450 .as_ref()
451 .and_then(|details| details["legacyCause"].as_str())
452 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
453 "{differing:?}"
454 );
455
456 let absent = client.patch(
459 "tasks",
460 "ghost",
461 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
462 None,
463 );
464 let absent = absent.expect_err("an absent local row is rejected");
465 assert_eq!(absent.code, "sync.row_missing");
466 assert!(absent.message.contains("requires a local row"));
467 assert_eq!(client.pending_commit_ids().len(), 1);
468 }
469
470 #[test]
471 fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
472 let schema = json!({
473 "version": 1,
474 "tables": [
475 {
476 "name": "tenants",
477 "primaryKey": "tenant_id",
478 "columns": [
479 { "name": "tenant_id", "type": "string", "nullable": false },
480 { "name": "body", "type": "string", "nullable": false }
481 ],
482 "scopes": [{ "pattern": "tenant:{tenant_id}" }]
483 },
484 {
485 "name": "buckets",
486 "primaryKey": "id",
487 "columns": [
488 { "name": "id", "type": "string", "nullable": false },
489 { "name": "bucket", "type": "float", "nullable": false },
490 { "name": "body", "type": "string", "nullable": false }
491 ],
492 "scopes": [{ "pattern": "bucket:{bucket}" }]
493 }
494 ]
495 });
496 let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
497 .expect("test client");
498 client.create_synced_tables().unwrap();
499
500 client
502 .conn
503 .execute(
504 "INSERT INTO tenants (tenant_id, body, _syncular_version) VALUES ('t1', 'seed', 1)",
505 [],
506 )
507 .unwrap();
508 client
509 .patch(
510 "tenants",
511 "t1",
512 Map::from_iter([
513 ("tenant_id".to_owned(), json!("t1")),
514 ("body".to_owned(), json!("created")),
515 ]),
516 None,
517 )
518 .expect("the primary-key scope column is proven equal by construction");
519
520 client
522 .conn
523 .execute(
524 "INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
525 [],
526 )
527 .unwrap();
528 client
529 .patch(
530 "buckets",
531 "b1",
532 Map::from_iter([
533 ("bucket".to_owned(), json!(2)),
534 ("body".to_owned(), json!("coerced")),
535 ]),
536 None,
537 )
538 .expect("the stored REAL value coerces from a supplied integer");
539 let values = client.outbox[1].ops[0]
540 .values
541 .clone()
542 .expect("the patch records values");
543 assert!(!values.contains_key("bucket"));
544
545 let differing = client.patch(
547 "buckets",
548 "b1",
549 Map::from_iter([("bucket".to_owned(), json!(3))]),
550 None,
551 );
552 let differing = differing.expect_err("a differing scope value is rejected");
553 assert_eq!(differing.code, "sync.invalid_request");
554 assert_eq!(differing.message, "the authoring request is invalid");
555 assert!(
556 differing
557 .details
558 .as_ref()
559 .and_then(|details| details["legacyCause"].as_str())
560 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
561 "{differing:?}"
562 );
563 }
564
565 #[cfg(feature = "e2ee")]
566 #[test]
567 fn stored_key_fallback_resolves_an_integer_primary_key() {
568 let schema = json!({
569 "version": 1,
570 "tables": [{
571 "name": "patients",
572 "primaryKey": "id",
573 "columns": [
574 { "name": "id", "type": "integer", "nullable": false },
575 { "name": "encryption_key_id", "type": "string", "nullable": true },
576 { "name": "note", "type": "bytes", "nullable": true,
577 "encrypted": true, "declaredType": "string" }
578 ],
579 "scopes": []
580 }]
581 });
582 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
583 .expect("test client");
584 client.create_synced_tables().unwrap();
585 client
586 .conn
587 .execute(
588 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
589 VALUES (5, 'k1', NULL, 1)",
590 [],
591 )
592 .unwrap();
593 let mut config = crate::values::EncryptionConfig::default();
594 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
595 config
596 .key_id_columns
597 .insert("patients".to_owned(), "encryption_key_id".to_owned());
598 client.set_encryption(config);
599 let table = client.schema.table("patients").unwrap();
600 let fallback = client
601 .stored_key_fallback(table, "5")
602 .expect("the stored integer key resolves the fallback");
603 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
604 }
605
606 #[test]
607 fn patch_types_an_integer_primary_key_from_its_canonical_row_id() {
608 let schema = json!({
609 "version": 1,
610 "tables": [{
611 "name": "counters",
612 "primaryKey": "id",
613 "columns": [
614 { "name": "id", "type": "integer", "nullable": false },
615 { "name": "title", "type": "string", "nullable": false }
616 ],
617 "scopes": []
618 }]
619 });
620 let mut client = SyncClient::new("int-patch".into(), &schema, ClientLimits::default())
621 .expect("test client");
622 client
623 .write_base_row(
624 "counters",
625 &vec![
626 Some(ColumnValue::Integer(42)),
627 Some(ColumnValue::String("before".into())),
628 ],
629 1,
630 )
631 .unwrap();
632 client.rebuild_overlay().expect("rebuild overlay");
633 client
634 .patch(
635 "counters",
636 "42",
637 Map::from_iter([
638 ("id".to_owned(), json!(42)),
639 ("title".to_owned(), json!("after")),
640 ]),
641 None,
642 )
643 .expect("an integer key that renders as the rowId is not a change");
644 let rows = client.query("SELECT id, title FROM counters", &[]).unwrap();
645 assert_eq!(Value::from(rows), json!([{"id": 42, "title": "after"}]));
646 for row_id in ["042", "+42", "42.0"] {
647 let error = client
648 .patch(
649 "counters",
650 row_id,
651 Map::from_iter([("title".to_owned(), json!("x"))]),
652 None,
653 )
654 .expect_err("a non-canonical integer rowId names no row");
655 assert_eq!(error.code, "sync.row_missing", "{row_id}");
656 }
657 }
658
659 #[test]
660 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
661 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
662 {"name":"id","type":"string","nullable":false},
663 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
664 for state in ["absent", "cached", "pinned"] {
665 let path = std::env::temp_dir().join(format!(
666 "syncular-blob-stage-{}.sqlite",
667 uuid::Uuid::new_v4()
668 ));
669 let mut client = SyncClient::open_path(
670 "stage-test".into(),
671 &schema,
672 Default::default(),
673 path.to_str().unwrap(),
674 )
675 .unwrap();
676 let bytes = b"atomic stage";
677 if state != "absent" {
678 client
679 .upload_blob(bytes, Some("text/plain".into()), None)
680 .unwrap();
681 if state == "cached" {
682 client
683 .conn
684 .execute("DELETE FROM _syncular_blob_uploads", [])
685 .unwrap();
686 }
687 }
688 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
689 client
690 .conn
691 .execute_batch(
692 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
693 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
694 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
695 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
696 )
697 .unwrap();
698 let error = client
699 .upload_blob(bytes, Some("application/octet-stream".into()), None)
700 .unwrap_err();
701 assert!(error.contains("injected stage failure"), "{state}: {error}");
702 assert!(client.conn.is_autocommit());
703 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
704 drop(client);
705 let mut reopened = SyncClient::open_path(
706 "stage-test".into(),
707 &schema,
708 Default::default(),
709 path.to_str().unwrap(),
710 )
711 .unwrap();
712 let durable = reopened
713 .query("SELECT * FROM _syncular_blobs", &[])
714 .unwrap();
715 reopened
716 .conn
717 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
718 .unwrap();
719 let retried = reopened.upload_blob(bytes, None, None).unwrap();
720 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
721 assert_eq!(
722 reopened
723 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
724 .unwrap(),
725 vec![Map::from_iter([(
726 "blob_id".into(),
727 Value::from(blob_id_for(bytes)),
728 )])]
729 );
730 drop(reopened);
731 std::fs::remove_file(&path).unwrap();
732 assert_eq!(durable, bodies, "{state}: reopened body state");
733 assert_eq!(immediate, bodies, "{state}: immediate body state");
734 }
735 }
736
737 #[test]
738 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
739 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
740 {"name":"id","type":"string","nullable":false},
741 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
742 for fault in [
743 "missing",
744 "body-type",
745 "length",
746 "hash",
747 "pin-metadata",
748 "read",
749 "pin-delete",
750 ] {
751 let path = std::env::temp_dir().join(format!(
752 "syncular-blob-upload-{}.sqlite",
753 uuid::Uuid::new_v4()
754 ));
755 let mut client = SyncClient::open_path(
756 "upload-test".into(),
757 &schema,
758 Default::default(),
759 path.to_str().unwrap(),
760 )
761 .unwrap();
762 let bytes = b"durable upload";
763 let ref_value = client
764 .upload_blob(bytes, Some("text/plain".into()), None)
765 .unwrap();
766 let commit = client
767 .mutate(vec![Mutation::Upsert {
768 table: "attachments".into(),
769 values: Map::from_iter([
770 ("id".into(), Value::from("pending")),
771 (
772 "file".into(),
773 Value::from(format!(
774 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
775 blob_id_for(bytes),
776 bytes.len()
777 )),
778 ),
779 ]),
780 base_version: None,
781 }])
782 .unwrap();
783 client
784 .conn
785 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
786 .unwrap();
787 client.conn.execute_batch(match fault {
788 "missing" => "DELETE FROM _syncular_blobs",
789 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
790 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
791 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
792 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
793 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
794 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
795 }).unwrap();
796 let state = if fault == "read" {
797 Vec::new()
798 } else {
799 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
800 };
801 let mut transport = CountingRealtimeTransport::default();
802 let error = client.flush_blob_uploads(&mut transport).unwrap_err();
803 assert_eq!(
804 error.code,
805 if matches!(fault, "read" | "pin-delete") {
806 "client.failed"
807 } else {
808 "sync.local_corrupt"
809 },
810 "{fault}"
811 );
812 assert_eq!(
813 transport.blob_uploads.len(),
814 usize::from(fault == "pin-delete"),
815 "{fault}: corrupt bytes must never be uploaded"
816 );
817 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
818 if fault != "read" {
819 assert_eq!(
820 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
821 state
822 );
823 } else {
824 client
825 .conn
826 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
827 .unwrap();
828 }
829 drop(client);
830 let mut reopened = SyncClient::open_path(
831 "upload-test".into(),
832 &schema,
833 Default::default(),
834 path.to_str().unwrap(),
835 )
836 .unwrap();
837 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
838 if fault != "read" {
839 assert_eq!(
840 reopened
841 .query("SELECT * FROM _syncular_blobs", &[])
842 .unwrap(),
843 state
844 );
845 }
846 if fault == "pin-delete" {
847 reopened
848 .conn
849 .execute_batch("DROP TRIGGER fail_pin_delete")
850 .unwrap();
851 }
852 if fault == "pin-metadata" {
853 reopened
854 .conn
855 .execute(
856 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
857 [],
858 )
859 .unwrap();
860 }
861 reopened
862 .conn
863 .execute_batch(
864 "DELETE FROM _syncular_blobs;
865 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
866 )
867 .unwrap();
868 transport.blob_uploads.clear();
869 reopened.flush_blob_uploads(&mut transport).unwrap();
870 assert_eq!(
871 transport.blob_uploads,
872 [(blob_id_for(bytes), bytes.to_vec())]
873 );
874 assert_eq!(
875 reopened
876 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
877 .unwrap(),
878 Vec::<Map<String, Value>>::new()
879 );
880 assert_eq!(reopened.pending_commit_ids(), [commit]);
881 assert_eq!(
882 reopened
883 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
884 .unwrap()
885 .unwrap()
886 .bytes,
887 bytes
888 );
889 drop(reopened);
890 std::fs::remove_file(path).unwrap();
891 }
892 }
893
894 fn coalesced_fixture(sections: usize, commits: usize, rows: usize) -> (SyncClient, Message) {
898 let mut client = client();
899 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
900 let mut response = Message {
901 wire_version: WIRE_VERSION,
902 msg_kind: MsgKind::Response,
903 frames: vec![Frame::RespHeader {
904 required_schema_version: None,
905 latest_schema_version: None,
906 log_epoch: Some("epoch-1".into()),
907 reset_required: Some(false),
908 }],
909 };
910 let mut seq = 0;
911 for section in 0..sections {
912 let project = format!("p{section}");
913 let scopes = vec![("project_id".to_owned(), vec![project.clone()])];
914 client
915 .subscribe(format!("s{section}"), "tasks".into(), scopes.clone(), None)
916 .unwrap();
917 let sub = client.subs.last_mut().unwrap();
918 sub.cursor = 0;
919 sub.synced_once = true;
920 sub.effective = Some(scopes.clone());
921 let sub = sub.clone();
922 client.persist_sub(&sub).unwrap();
923 response.frames.push(Frame::SubStart {
924 id: format!("s{section}"),
925 status: SubStatus::Active,
926 reason_code: "".into(),
927 effective_scopes: scopes,
928 bootstrap: false,
929 });
930 for _ in 0..commits {
931 seq += 1;
932 let changes = (0..rows)
933 .map(|row| {
934 let id = format!("{project}-{seq}-{row}");
935 let values = Map::from_iter([
936 ("id".into(), Value::from(id.clone())),
937 ("project_id".into(), Value::from(project.clone())),
938 ]);
939 ssp2::model::Change {
940 table_index: 0,
941 row_id: id.clone(),
942 op: Op::Upsert,
943 row_version: Some(seq),
944 scopes: vec![("project_id".into(), project.clone())],
945 row: Some(
946 encode_row_json(
947 client.schema.table("tasks").unwrap(),
948 &id,
949 &values,
950 &client.encryption,
951 )
952 .unwrap(),
953 ),
954 }
955 })
956 .collect();
957 response.frames.push(Frame::Commit {
958 commit_seq: seq,
959 actor_id: "writer".into(),
960 created_at_ms: 1,
961 tables: vec!["tasks".into()],
962 changes,
963 });
964 }
965 response.frames.push(Frame::SubEnd {
966 next_cursor: seq,
967 bootstrap_state: None,
968 });
969 }
970 client.drain_change_batches();
971 (client, response)
972 }
973
974 fn count_commits(client: &SyncClient) -> std::sync::Arc<std::sync::atomic::AtomicUsize> {
975 let commits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
976 let counter = commits.clone();
977 client
978 .conn
979 .commit_hook(Some(move || {
980 counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
981 false
982 }))
983 .expect("commit hook");
984 commits
985 }
986
987 #[test]
988 fn a_response_of_many_small_sections_commits_once() {
989 for realtime in [false, true] {
990 let (mut client, response) = coalesced_fixture(70, 1, 1);
991 client.realtime_state = RealtimeState::Connected;
992 client.realtime_reason_code = None;
993 let mut transport = CountingRealtimeTransport::default();
994 let (_, meta) = client.build_request(false).unwrap();
995 let commits = count_commits(&client);
996 if realtime {
997 client.on_realtime_binary(&mut transport, &encode_message(&response));
998 } else {
999 assert!(matches!(
1000 client.process_response(&mut transport, response, &meta),
1001 SyncOutcome::Ok(_)
1002 ));
1003 }
1004 assert_eq!(
1005 commits.load(std::sync::atomic::Ordering::SeqCst),
1006 1,
1007 "{realtime}"
1008 );
1009 assert!(client.conn.is_autocommit());
1010 assert!(client.subs.iter().all(|sub| sub.cursor > 0));
1011 let revisions = client
1013 .drain_change_batches()
1014 .into_iter()
1015 .filter(|batch| !batch.tables.is_empty())
1016 .map(|batch| batch.revision)
1017 .collect::<Vec<_>>();
1018 assert_eq!(
1019 revisions,
1020 (1..=70).map(|r: u64| r.to_string()).collect::<Vec<_>>()
1021 );
1022 assert_eq!(client.query("SELECT id FROM tasks", &[]).unwrap().len(), 70);
1023 assert_eq!(
1025 transport
1026 .messages
1027 .iter()
1028 .filter(|m| m.contains("\"ack\""))
1029 .count(),
1030 1,
1031 "{realtime}"
1032 );
1033 }
1034 }
1035
1036 #[test]
1037 fn a_response_past_the_row_bound_commits_per_bound() {
1038 let (mut client, response) = coalesced_fixture(1, 3, 600);
1040 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1041 let (_, meta) = client.build_request(false).unwrap();
1042 let commits = count_commits(&client);
1043 assert!(matches!(
1044 client.process_response(&mut transport, response, &meta),
1045 SyncOutcome::Ok(_)
1046 ));
1047 assert_eq!(commits.load(std::sync::atomic::Ordering::SeqCst), 3);
1048 assert_eq!(
1049 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1050 1800
1051 );
1052 assert_eq!(client.subs[0].cursor, 3);
1053 }
1054
1055 #[test]
1056 fn a_failed_coalesced_commit_restores_memory_and_publishes_nothing() {
1057 let (mut client, response) = coalesced_fixture(2, 2, 600);
1061 client
1062 .conn
1063 .execute_batch(
1064 "PRAGMA foreign_keys = ON; CREATE TABLE fail_parent(id INTEGER PRIMARY KEY); CREATE TABLE fail_child(id INTEGER REFERENCES fail_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_commit AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'p0-1-0' BEGIN INSERT INTO fail_child VALUES(1); END",
1065 )
1066 .unwrap();
1067 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1068 let (_, meta) = client.build_request(false).unwrap();
1069 let subs = client.subs.clone();
1070 let revision = client.local_revision();
1071 let outcome = client.process_response(&mut transport, response.clone(), &meta);
1072 assert!(
1073 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == "storage.failed"),
1074 "{outcome:?}"
1075 );
1076 assert!(client.conn.is_autocommit());
1077 assert!(client.coalesced.is_none() && client.coalesced_failure.is_none());
1078 assert_eq!(client.local_revision(), revision);
1079 assert!(client
1080 .drain_change_batches()
1081 .iter()
1082 .all(|batch| batch.tables.is_empty()));
1083 assert!(client
1084 .query("SELECT id FROM tasks", &[])
1085 .unwrap()
1086 .is_empty());
1087 assert_eq!(
1088 client.subs.iter().map(|s| s.cursor).collect::<Vec<_>>(),
1089 subs.iter().map(|s| s.cursor).collect::<Vec<_>>()
1090 );
1091 client
1093 .conn
1094 .execute_batch("DROP TRIGGER fail_commit")
1095 .unwrap();
1096 let (_, meta) = client.build_request(false).unwrap();
1097 assert!(matches!(
1098 client.process_response(&mut transport, response, &meta),
1099 SyncOutcome::Ok(_)
1100 ));
1101 assert_eq!(
1102 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1103 2400
1104 );
1105 }
1106
1107 #[test]
1108 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
1109 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1110 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
1111 "scopes":[{"pattern":"project:{project_id}"}]}]});
1112 for realtime in [false, true] {
1113 for fault in [
1114 "none",
1115 "row",
1116 "revision",
1117 "commit",
1118 "cursor",
1119 "error",
1120 "missing-end",
1121 ] {
1122 if realtime && fault == "missing-end" {
1123 continue;
1124 }
1125 let path = std::env::temp_dir()
1126 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
1127 let mut client = SyncClient::open_path(
1128 "frame-test".into(),
1129 &schema,
1130 Default::default(),
1131 path.to_str().unwrap(),
1132 )
1133 .unwrap();
1134 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1135 client
1136 .subscribe(
1137 "tasks".into(),
1138 "tasks".into(),
1139 vec![("project_id".into(), vec!["p1".into()])],
1140 None,
1141 )
1142 .unwrap();
1143 client.subs[0].cursor = 0;
1144 client.subs[0].synced_once = true;
1145 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1146 client.persist_sub(&client.subs[0]).unwrap();
1147 client.realtime_state = RealtimeState::Connected;
1148 client.realtime_reason_code = None;
1149 client.drain_change_batches();
1150 let mut transport = CountingRealtimeTransport::default();
1151 let (_, meta) = client.build_request(false).unwrap();
1152 let mut response = Message {
1153 wire_version: WIRE_VERSION,
1154 msg_kind: MsgKind::Response,
1155 frames: vec![
1156 Frame::RespHeader {
1157 required_schema_version: None,
1158 latest_schema_version: None,
1159 log_epoch: Some("epoch-1".into()),
1160 reset_required: Some(false),
1161 },
1162 Frame::SubStart {
1163 id: "tasks".into(),
1164 status: SubStatus::Active,
1165 reason_code: "".into(),
1166 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1167 bootstrap: false,
1168 },
1169 ],
1170 };
1171 for (index, id) in ["first", "second"].iter().enumerate() {
1172 let values = Map::from_iter([
1173 ("id".into(), Value::from(*id)),
1174 ("project_id".into(), Value::from("p1")),
1175 ]);
1176 response.frames.push(Frame::Commit {
1177 commit_seq: index as i64 + 1,
1178 actor_id: "writer".into(),
1179 created_at_ms: 1,
1180 tables: vec!["tasks".into()],
1181 changes: vec![ssp2::model::Change {
1182 table_index: 0,
1183 row_id: (*id).into(),
1184 op: Op::Upsert,
1185 row_version: Some(index as i64 + 1),
1186 scopes: vec![("project_id".into(), "p1".into())],
1187 row: Some(
1188 encode_row_json(
1189 client.schema.table("tasks").unwrap(),
1190 id,
1191 &values,
1192 &client.encryption,
1193 )
1194 .unwrap(),
1195 ),
1196 }],
1197 });
1198 }
1199 response.frames.push(Frame::SubEnd {
1200 next_cursor: 2,
1201 bootstrap_state: None,
1202 });
1203 let valid = response.clone();
1204 match fault {
1205 "row" => {
1206 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
1207 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
1208 },
1209 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
1210 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
1211 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
1212 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
1213 "missing-end" => {response.frames.truncate(3);},
1214 _ => {}
1215 }
1216 if realtime {
1217 client.on_realtime_binary(&mut transport, &encode_message(&response));
1218 } else {
1219 assert_eq!(
1220 matches!(
1221 client.process_response(&mut transport, response, &meta),
1222 SyncOutcome::Ok(_)
1223 ),
1224 fault == "none",
1225 "{fault}"
1226 );
1227 }
1228 assert!(
1229 client.conn.is_autocommit(),
1230 "no open transaction after {realtime}/{fault}"
1231 );
1232 let count = match fault {
1236 "none" | "cursor" => 2,
1237 "commit" => 0,
1238 _ => 1,
1239 };
1240 let expected_revision = count + u64::from(realtime && fault != "none");
1241 assert_eq!(
1242 client.local_revision(),
1243 expected_revision,
1244 "{realtime}/{fault}"
1245 );
1246 let batches = client.drain_change_batches();
1247 assert_eq!(
1248 batches.len(),
1249 expected_revision as usize,
1250 "{realtime}/{fault}"
1251 );
1252 let row_batches: Vec<_> = batches
1253 .iter()
1254 .filter(|batch| !batch.tables.is_empty())
1255 .collect();
1256 assert_eq!(row_batches.len(), count as usize);
1257 assert_eq!(
1258 row_batches
1259 .iter()
1260 .map(|b| b.revision.clone())
1261 .collect::<Vec<_>>(),
1262 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
1263 );
1264 for table in ["tasks", "_syncular_base_tasks"] {
1265 assert_eq!(
1266 client
1267 .conn
1268 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
1269 .get::<_, u64>(0))
1270 .unwrap(),
1271 count,
1272 "{realtime}/{fault}/{table}"
1273 );
1274 }
1275 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1276 assert_eq!(
1277 transport
1278 .messages
1279 .iter()
1280 .filter(|m| m.contains("\"ack\""))
1281 .count(),
1282 usize::from(fault == "none")
1283 );
1284 if matches!(fault, "revision" | "commit" | "cursor") {
1285 client
1286 .conn
1287 .execute_batch("DROP TRIGGER fail_frame")
1288 .unwrap();
1289 }
1290 drop(client);
1291 let mut reopened = SyncClient::open_path(
1292 "frame-test".into(),
1293 &schema,
1294 Default::default(),
1295 path.to_str().unwrap(),
1296 )
1297 .unwrap();
1298 assert_eq!(reopened.local_revision(), expected_revision);
1299 assert_eq!(
1300 reopened
1301 .query("SELECT id FROM tasks ORDER BY id", &[])
1302 .unwrap()
1303 .len(),
1304 count as usize
1305 );
1306 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1307 let (_, meta) = reopened.build_request(false).unwrap();
1308 assert!(matches!(
1309 reopened.process_response(&mut transport, valid, &meta),
1310 SyncOutcome::Ok(_)
1311 ));
1312 assert_eq!(
1313 reopened
1314 .query("SELECT id FROM tasks ORDER BY id", &[])
1315 .unwrap()
1316 .len(),
1317 2
1318 );
1319 assert_eq!(reopened.subs[0].cursor, 2);
1320 drop(reopened);
1321 std::fs::remove_file(path).unwrap();
1322 }
1323 }
1324 }
1325
1326 #[test]
1327 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
1328 for status in [SubStatus::Reset, SubStatus::Revoked] {
1329 for fault in ["subscription", "revision"] {
1330 let mut client = client();
1331 client
1332 .subscribe(
1333 "tasks".into(),
1334 "tasks".into(),
1335 vec![("project_id".into(), vec!["p1".into()])],
1336 None,
1337 )
1338 .unwrap();
1339 client.subs[0].cursor = 41;
1340 client.subs[0].synced_once = true;
1341 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1342 client.persist_sub(&client.subs[0]).unwrap();
1343 client
1344 .mutate(vec![Mutation::Upsert {
1345 table: "tasks".into(),
1346 values: Map::from_iter([
1347 ("id".into(), Value::from("pending")),
1348 ("project_id".into(), Value::from("p1")),
1349 ]),
1350 base_version: None,
1351 }])
1352 .unwrap();
1353 let ids = client.pending_commit_ids();
1354 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
1355 let revision = client.local_revision();
1356 client.drain_change_batches();
1357 let (_, meta) = client.build_request(false).unwrap();
1358 client.conn.execute_batch(if fault == "subscription" {
1359 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
1360 } else {
1361 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
1362 }).unwrap();
1363 let mut transport = CountingRealtimeTransport::default();
1364 let mut report = SyncReport::default();
1365 if status == SubStatus::Reset {
1367 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
1368 }
1369 let result = client.process_section(
1370 &mut transport,
1371 "tasks",
1372 status,
1373 "sync.scope_revoked",
1374 vec![],
1375 vec![],
1376 Some((42, None)),
1377 &meta,
1378 &mut report,
1379 );
1380 assert!(result.is_err(), "{status:?}/{fault}");
1381 assert!(client.conn.is_autocommit());
1382 assert_eq!(client.subs[0].cursor, 41);
1383 assert_eq!(client.subs[0].state, SubState::Active);
1384 assert_eq!(client.pending_commit_ids(), ids);
1385 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
1386 assert!(client.rejections.is_empty());
1387 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
1388 assert_eq!(client.local_revision(), revision);
1389 assert!(client.drain_change_batches().is_empty());
1390 assert!(report.revoked.is_empty() && report.resets.is_empty());
1391 }
1392 }
1393 }
1394
1395 #[test]
1396 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
1397 let mut client = client();
1398 client
1399 .subscribe(
1400 "tasks".into(),
1401 "tasks".into(),
1402 vec![("project_id".into(), vec!["p1".into()])],
1403 None,
1404 )
1405 .unwrap();
1406 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1407 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
1408 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
1409 let row = |id: &str| ssp2::segment::SegmentRow {
1410 server_version: 1,
1411 values: vec![
1412 Some(ColumnValue::String(id.into())),
1413 Some(ColumnValue::String("p1".into())),
1414 ],
1415 };
1416 let segment = RowsSegment {
1417 table: "tasks".into(),
1418 schema_version: 1,
1419 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
1420 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
1421 };
1422 assert!(client.apply_segment(0, &segment, true).is_err());
1423 assert!(client.conn.is_autocommit());
1424 assert_eq!(
1425 client
1426 .query("SELECT id FROM tasks ORDER BY id", &[])
1427 .unwrap(),
1428 vec![Map::from_iter([("id".into(), Value::from("first"))])]
1429 );
1430 assert_eq!(client.local_revision(), 1);
1431 assert_eq!(client.drain_change_batches().len(), 1);
1432 assert_eq!(client.subs[0].cursor, -1);
1433 client
1434 .conn
1435 .execute_batch("DROP TRIGGER fail_block")
1436 .unwrap();
1437 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
1438 assert_eq!(client.local_revision(), 3);
1439 assert_eq!(client.drain_change_batches().len(), 2);
1440 }
1441
1442 struct SegmentBatchTransport {
1446 response: Message,
1447 bodies: BTreeMap<String, Vec<u8>>,
1448 batches: Vec<Vec<String>>,
1449 single_fetches: usize,
1450 }
1451
1452 impl Transport for SegmentBatchTransport {
1453 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1454 Ok(encode_message(&self.response))
1455 }
1456
1457 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1458 self.sync(request)
1459 }
1460
1461 fn download_segment(
1462 &mut self,
1463 _request: &SegmentRequest,
1464 _on_progress: &mut dyn FnMut(u64),
1465 ) -> Result<Vec<u8>, TransportError> {
1466 self.single_fetches += 1;
1467 Err(TransportError::new(
1468 "sync.transport_failed",
1469 "per-descriptor fetch",
1470 ))
1471 }
1472
1473 fn download_segments(
1474 &mut self,
1475 fetches: &[crate::SegmentFetch],
1476 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1477 ) -> Vec<Result<Vec<u8>, TransportError>> {
1478 self.batches.push(
1479 fetches
1480 .iter()
1481 .map(|fetch| match fetch {
1482 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
1483 crate::SegmentFetch::Url(url) => url.clone(),
1484 })
1485 .collect(),
1486 );
1487 fetches
1488 .iter()
1489 .enumerate()
1490 .map(|(index, fetch)| {
1491 let crate::SegmentFetch::Direct(request) = fetch else {
1492 panic!("the descriptors are direct-endpoint requests")
1493 };
1494 let bytes = self.bodies[&request.segment_id].clone();
1495 on_progress(index, bytes.len() as u64);
1496 Ok(bytes)
1497 })
1498 .collect()
1499 }
1500
1501 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1502 Ok(())
1503 }
1504
1505 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1506 Ok(())
1507 }
1508
1509 fn realtime_close(&mut self) -> Result<(), TransportError> {
1510 Ok(())
1511 }
1512 }
1513
1514 #[test]
1515 fn a_response_fetches_all_its_segment_pages_in_one_ordered_batch() {
1516 let mut client = client();
1517 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1518 client
1519 .subscribe(
1520 "tasks".into(),
1521 "tasks".into(),
1522 vec![("project_id".into(), vec!["p1".into()])],
1523 None,
1524 )
1525 .unwrap();
1526 client.subs[0].cursor = 0;
1527 client.subs[0].synced_once = true;
1528 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1529 client.persist_sub(&client.subs[0]).unwrap();
1530 client.drain_change_batches();
1531 let columns = client.schema.table("tasks").unwrap().wire_columns.clone();
1532
1533 let pages = ["one", "two", "three", "four"].map(|id| {
1537 let bytes = ssp2::segment::encode_rows_segment(&RowsSegment {
1538 table: "tasks".into(),
1539 schema_version: 1,
1540 columns: columns.clone(),
1541 blocks: vec![vec![ssp2::segment::SegmentRow {
1542 server_version: 1,
1543 values: vec![
1544 Some(ColumnValue::String(id.into())),
1545 Some(ColumnValue::String("p1".into())),
1546 ],
1547 }]],
1548 });
1549 (
1550 format!("sha256:{}", bytes_to_hex(&Sha256::digest(&bytes))),
1551 bytes,
1552 )
1553 });
1554
1555 let mut frames = vec![
1556 Frame::RespHeader {
1557 required_schema_version: None,
1558 latest_schema_version: None,
1559 log_epoch: Some("epoch-1".into()),
1560 reset_required: Some(false),
1561 },
1562 Frame::SubStart {
1563 id: "tasks".into(),
1564 status: SubStatus::Active,
1565 reason_code: String::new(),
1566 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1567 bootstrap: false,
1568 },
1569 ];
1570 for (segment_id, bytes) in &pages {
1571 frames.push(Frame::SegmentRef {
1572 segment_id: segment_id.clone(),
1573 media_type: MediaType::Rows,
1574 table: "tasks".into(),
1575 byte_length: bytes.len() as i64,
1576 row_count: 1,
1577 as_of_commit_seq: 1,
1578 scope_digest: "digest".into(),
1579 row_cursor: None,
1580 next_row_cursor: None,
1581 url: None,
1582 url_expires_at_ms: None,
1583 });
1584 }
1585 frames.push(Frame::SubEnd {
1586 next_cursor: 1,
1587 bootstrap_state: None,
1588 });
1589 let mut transport = SegmentBatchTransport {
1590 response: Message {
1591 wire_version: WIRE_VERSION,
1592 msg_kind: MsgKind::Response,
1593 frames,
1594 },
1595 bodies: pages
1596 .iter()
1597 .map(|(segment_id, bytes)| (segment_id.clone(), bytes.clone()))
1598 .collect(),
1599 batches: Vec::new(),
1600 single_fetches: 0,
1601 };
1602
1603 let outcome = client.sync(&mut transport);
1604 assert!(matches!(outcome, SyncOutcome::Ok(_)), "{outcome:?}");
1605 assert_eq!(
1606 transport.batches,
1607 vec![pages
1608 .iter()
1609 .map(|(segment_id, _)| segment_id.clone())
1610 .collect::<Vec<_>>()],
1611 "one batch per response, in wire order"
1612 );
1613 assert_eq!(
1614 transport.single_fetches, 0,
1615 "the batch seam replaces the per-descriptor calls"
1616 );
1617 assert_eq!(
1618 client
1619 .query("SELECT id FROM tasks ORDER BY id", &[])
1620 .unwrap(),
1621 ["four", "one", "three", "two"]
1622 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
1623 .to_vec()
1624 );
1625 assert_eq!(client.subs[0].cursor, 1);
1626 }
1627
1628 #[test]
1629 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
1630 assert_eq!(
1631 client().diagnostics_storage().blob_cache_bytes_approx,
1632 Some(0)
1633 );
1634 let client = SyncClient::new(
1635 "diagnostic-storage".to_owned(),
1636 &json!({
1637 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
1638 "columns": [{"name": "id", "type": "string", "nullable": false},
1639 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
1640 }),
1641 ClientLimits {
1642 blob_cache_max_bytes: Some(5),
1643 ..Default::default()
1644 },
1645 )
1646 .unwrap();
1647 let empty = client.diagnostics_storage();
1648 assert_eq!(empty.status, "healthy");
1649 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
1650 assert_eq!(empty.retained_outcome_entries, Some(0));
1651 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
1652 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
1653 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
1654 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
1655 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
1656 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
1657 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
1658 let written = client.diagnostics_storage();
1659 assert_eq!(written.status, "pressure");
1660 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
1661 assert_eq!(written.retained_outcome_entries, Some(2));
1662 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
1663 assert_eq!(written.blob_cache_bytes_approx, Some(8));
1664 assert_eq!(
1665 written.pressure_reason_code.as_deref(),
1666 Some("client.blob_cache_over_limit")
1667 );
1668 client
1669 .conn
1670 .execute_batch(
1671 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
1672 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
1673 )
1674 .unwrap();
1675 let staged = client.diagnostics_storage();
1676 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
1677 assert_eq!(staged.retained_outcome_entries, Some(0));
1678 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
1679 assert_eq!(staged.status, "healthy");
1680 client.conn.execute_batch("ROLLBACK").unwrap();
1681 assert_eq!(client.diagnostics_storage(), written);
1682 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
1683 let unreadable = client.diagnostics_storage();
1684 assert_eq!(unreadable.status, "unreadable");
1685 assert!(unreadable.database_bytes_approx.is_none());
1686 assert!(unreadable.pending_outbox_bytes_approx.is_none());
1687 assert!(unreadable.retained_outcome_entries.is_none());
1688 assert!(unreadable.retained_outcome_bytes_approx.is_none());
1689 assert!(unreadable.blob_cache_bytes_approx.is_none());
1690 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
1691 let restored = client.diagnostics_storage();
1692 assert_eq!(restored.status, written.status);
1693 assert_eq!(
1694 restored.retained_outcome_bytes_approx,
1695 written.retained_outcome_bytes_approx
1696 );
1697 client.conn.set_prepared_statement_cache_capacity(0);
1698 assert_eq!(client.diagnostics_storage(), restored);
1699 }
1700
1701 struct ServerErrorTransport {
1703 code: &'static str,
1704 retryable: bool,
1705 }
1706
1707 impl Transport for ServerErrorTransport {
1708 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1709 Ok(encode_message(&Message {
1710 wire_version: WIRE_VERSION,
1711 msg_kind: MsgKind::Response,
1712 frames: vec![
1713 Frame::RespHeader {
1714 required_schema_version: None,
1715 latest_schema_version: None,
1716 log_epoch: Some("epoch-1".to_owned()),
1717 reset_required: Some(false),
1718 },
1719 Frame::Error {
1720 code: self.code.to_owned(),
1721 message: "server error".to_owned(),
1722 category: "internal".to_owned(),
1723 retryable: self.retryable,
1724 recommended_action: "retryLater".to_owned(),
1725 details: None,
1726 },
1727 ],
1728 }))
1729 }
1730
1731 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1732 self.sync(request)
1733 }
1734
1735 fn download_segment(
1736 &mut self,
1737 _request: &SegmentRequest,
1738 _on_progress: &mut dyn FnMut(u64),
1739 ) -> Result<Vec<u8>, TransportError> {
1740 Err(TransportError::new("sync.transport_failed", "offline"))
1741 }
1742
1743 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1744 Ok(())
1745 }
1746
1747 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1748 Ok(())
1749 }
1750
1751 fn realtime_close(&mut self) -> Result<(), TransportError> {
1752 Ok(())
1753 }
1754 }
1755
1756 #[test]
1757 fn a_retryable_in_band_failure_keeps_the_socket_connected_and_still_asks_for_a_retry() {
1758 let mut client = client();
1759 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1760 let mut transport = ServerErrorTransport {
1761 code: "sync.schema_not_ready",
1762 retryable: true,
1763 };
1764 client.connect_realtime(&mut transport).expect("connect");
1765 assert!(matches!(
1766 client.sync(&mut transport),
1767 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.schema_not_ready"
1768 ));
1769 assert_eq!(
1770 client.realtime_state(),
1771 RealtimeState::Connected,
1772 "the socket delivered the answer, so it is not lost"
1773 );
1774 assert!(
1775 matches!(
1776 client.drain_sync_intents().as_slice(),
1777 [SyncIntent::Background { delay_ms: 250 }]
1778 ),
1779 "the retry deadline is a sync round and no reconnect satisfies it"
1780 );
1781 assert_eq!(
1782 client.progress().snapshot().unwrap().retry_delay_ms,
1783 Some(250)
1784 );
1785 }
1786
1787 #[test]
1788 fn retryable_server_error_schedules_a_background_retry() {
1789 let mut client = client();
1790 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1791 let mut transport = ServerErrorTransport {
1792 code: "sync.internal_error",
1793 retryable: true,
1794 };
1795 assert!(matches!(
1796 client.sync(&mut transport),
1797 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
1798 ));
1799 assert!(matches!(
1800 client.drain_sync_intents().as_slice(),
1801 [SyncIntent::Background { delay_ms: 250 }]
1802 ));
1803 assert_eq!(
1804 client.progress().snapshot().unwrap().retry_delay_ms,
1805 Some(250)
1806 );
1807 let mut transport = ServerErrorTransport {
1808 code: "sync.invalid_request",
1809 retryable: false,
1810 };
1811 assert!(matches!(
1812 client.sync(&mut transport),
1813 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
1814 ));
1815 assert!(client.drain_sync_intents().is_empty());
1816 let progress = client.progress().snapshot().unwrap();
1817 assert_eq!(progress.state, ProgressState::Failed);
1818 assert_eq!(progress.retry_delay_ms, None);
1819 }
1820
1821 #[test]
1822 fn sync_until_idle_reports_budget_exhaustion_or_failure() {
1823 let exhausted = SyncOutcome::BudgetExhausted(SyncReport::default()).to_json();
1826 assert_eq!(exhausted["ok"], json!(true));
1827 assert_eq!(exhausted["budgetExhausted"], json!(true));
1828 assert!(exhausted["report"].is_object());
1829
1830 let mut client = client();
1831 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1832 let mut idle_transport = HostTransport::new_from_config(&json!({})).unwrap();
1834 let zero = client.sync_until_idle(&mut idle_transport, Some(0));
1835 assert!(
1836 matches!(zero, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
1837 "{zero:?}"
1838 );
1839 let mut erroring = ServerErrorTransport {
1841 code: "sync.internal_error",
1842 retryable: false,
1843 };
1844 let failed = client.sync_until_idle(&mut erroring, Some(3));
1845 assert!(
1846 matches!(failed, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1847 "{failed:?}"
1848 );
1849 }
1850
1851 struct ScriptedSyncTransport {
1854 responses: std::collections::VecDeque<Message>,
1855 calls: usize,
1856 }
1857
1858 impl Transport for ScriptedSyncTransport {
1859 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1860 self.calls += 1;
1861 let response = self.responses.pop_front().expect("scripted response");
1862 Ok(encode_message(&response))
1863 }
1864
1865 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1866 self.sync(request)
1867 }
1868
1869 fn download_segment(
1870 &mut self,
1871 _request: &SegmentRequest,
1872 _on_progress: &mut dyn FnMut(u64),
1873 ) -> Result<Vec<u8>, TransportError> {
1874 Err(TransportError::new("sync.transport_failed", "offline"))
1875 }
1876
1877 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1878 Ok(())
1879 }
1880
1881 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1882 Ok(())
1883 }
1884
1885 fn realtime_close(&mut self) -> Result<(), TransportError> {
1886 Ok(())
1887 }
1888 }
1889
1890 #[test]
1891 fn sync_until_idle_success_then_failure_keeps_earlier_work() {
1892 let path = std::env::temp_dir().join(format!(
1893 "syncular-budget-script-{}.sqlite",
1894 uuid::Uuid::new_v4()
1895 ));
1896 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1897 {"name":"id","type":"string","nullable":false},
1898 {"name":"project_id","type":"string","nullable":false}],
1899 "scopes":[{"pattern":"project:{project_id}"}]}]});
1900 let mut client = SyncClient::open_path(
1901 "budget-script".into(),
1902 &schema,
1903 ClientLimits::default(),
1904 path.to_str().unwrap(),
1905 )
1906 .expect("open client");
1907 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1908 client
1909 .subscribe(
1910 "tasks".into(),
1911 "tasks".into(),
1912 vec![("project_id".into(), vec!["p1".into()])],
1913 None,
1914 )
1915 .unwrap();
1916 client.subs[0].cursor = 0;
1917 client.subs[0].synced_once = true;
1918 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1919 client.persist_sub(&client.subs[0]).unwrap();
1920 client.drain_change_batches();
1921
1922 let row = encode_row_json(
1923 client.schema.table("tasks").unwrap(),
1924 "kept",
1925 &Map::from_iter([
1926 ("id".into(), json!("kept")),
1927 ("project_id".into(), json!("p1")),
1928 ]),
1929 &client.encryption,
1930 )
1931 .unwrap();
1932 let success = Message {
1933 wire_version: WIRE_VERSION,
1934 msg_kind: MsgKind::Response,
1935 frames: vec![
1936 Frame::RespHeader {
1937 required_schema_version: None,
1938 latest_schema_version: None,
1939 log_epoch: Some("epoch-1".into()),
1940 reset_required: Some(false),
1941 },
1942 Frame::SubStart {
1943 id: "tasks".into(),
1944 status: SubStatus::Active,
1945 reason_code: String::new(),
1946 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1947 bootstrap: false,
1948 },
1949 Frame::Commit {
1950 commit_seq: 1,
1951 created_at_ms: 1,
1952 actor_id: "writer".into(),
1953 tables: vec!["tasks".into()],
1954 changes: vec![ssp2::model::Change {
1955 table_index: 0,
1956 row_id: "kept".into(),
1957 op: Op::Upsert,
1958 row_version: Some(1),
1959 scopes: vec![("project_id".into(), "p1".into())],
1960 row: Some(row),
1961 }],
1962 },
1963 Frame::SubEnd {
1964 next_cursor: 1,
1965 bootstrap_state: None,
1966 },
1967 ],
1968 };
1969 let failure = Message {
1970 wire_version: WIRE_VERSION,
1971 msg_kind: MsgKind::Response,
1972 frames: vec![
1973 Frame::RespHeader {
1974 required_schema_version: None,
1975 latest_schema_version: None,
1976 log_epoch: Some("epoch-1".into()),
1977 reset_required: Some(false),
1978 },
1979 Frame::Error {
1980 code: "sync.internal_error".into(),
1981 message: "second round failed".into(),
1982 category: "internal".into(),
1983 retryable: false,
1984 recommended_action: "retryLater".into(),
1985 details: None,
1986 },
1987 ],
1988 };
1989 let mut transport = ScriptedSyncTransport {
1990 responses: std::collections::VecDeque::from([success, failure]),
1991 calls: 0,
1992 };
1993 let outcome = client.sync_until_idle(&mut transport, Some(2));
1994 assert!(
1995 matches!(outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1996 "{outcome:?}"
1997 );
1998 assert_eq!(
1999 transport.calls, 2,
2000 "one successful round then the failing round"
2001 );
2002 assert_eq!(
2003 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
2004 json!("kept"),
2005 "the earlier round's durable work survives the later failure"
2006 );
2007 drop(client);
2008 let _ = std::fs::remove_file(&path);
2009 }
2010
2011 #[test]
2012 fn sync_until_idle_stops_on_a_schema_floor() {
2013 let mut client = client();
2014 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2015 let floor = Message {
2016 wire_version: WIRE_VERSION,
2017 msg_kind: MsgKind::Response,
2018 frames: vec![Frame::RespHeader {
2019 required_schema_version: Some(2),
2020 latest_schema_version: Some(2),
2021 log_epoch: Some("epoch-1".into()),
2022 reset_required: Some(false),
2023 }],
2024 };
2025 let mut transport = ScriptedSyncTransport {
2026 responses: std::collections::VecDeque::from([floor.clone(), floor]),
2027 calls: 0,
2028 };
2029 let outcome = client.sync_until_idle(&mut transport, Some(3));
2030 match outcome {
2031 SyncOutcome::Ok(report) => assert_eq!(
2032 report
2033 .schema_floor
2034 .as_ref()
2035 .and_then(|floor| floor.required_schema_version),
2036 Some(2),
2037 "the stop retains the latest schema floor"
2038 ),
2039 other => panic!("expected a schema-floor stop, got {other:?}"),
2040 }
2041 assert_eq!(
2042 transport.calls, 1,
2043 "a schema floor stops the budget loop after one round"
2044 );
2045 }
2046
2047 struct FailureScriptTransport {
2050 rounds: std::collections::VecDeque<Result<Vec<u8>, TransportError>>,
2051 requests: Vec<Message>,
2052 }
2053
2054 impl Transport for FailureScriptTransport {
2055 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2056 self.requests
2057 .push(decode_message(request).expect("the client encodes a legal request"));
2058 self.rounds.pop_front().expect("scripted round")
2059 }
2060
2061 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2062 self.sync(request)
2063 }
2064
2065 fn download_segment(
2066 &mut self,
2067 _request: &SegmentRequest,
2068 _on_progress: &mut dyn FnMut(u64),
2069 ) -> Result<Vec<u8>, TransportError> {
2070 Err(TransportError::new("transport.timeout", "segment stalled"))
2071 }
2072
2073 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2074 Ok(())
2075 }
2076
2077 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2078 Ok(())
2079 }
2080
2081 fn realtime_close(&mut self) -> Result<(), TransportError> {
2082 Ok(())
2083 }
2084 }
2085
2086 fn completed_response(frames: Vec<Frame>) -> Vec<u8> {
2088 let mut all = vec![Frame::RespHeader {
2089 required_schema_version: None,
2090 latest_schema_version: None,
2091 log_epoch: Some("epoch-1".into()),
2092 reset_required: Some(false),
2093 }];
2094 all.extend(frames);
2095 encode_message(&Message {
2096 wire_version: WIRE_VERSION,
2097 msg_kind: MsgKind::Response,
2098 frames: all,
2099 })
2100 }
2101
2102 fn carried_round_failures(request: &Message) -> Vec<String> {
2103 request
2104 .frames
2105 .iter()
2106 .filter_map(|frame| match frame {
2107 Frame::RoundFailure { code } => Some(code.clone()),
2108 _ => None,
2109 })
2110 .collect()
2111 }
2112
2113 fn failure_transport(
2114 rounds: impl IntoIterator<Item = Result<Vec<u8>, TransportError>>,
2115 ) -> FailureScriptTransport {
2116 FailureScriptTransport {
2117 rounds: rounds.into_iter().collect(),
2118 requests: Vec::new(),
2119 }
2120 }
2121
2122 #[test]
2123 fn a_round_without_a_complete_response_reports_its_code_on_every_following_request() {
2124 let mut client = client();
2125 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2126 let mut transport = failure_transport([
2127 Err(TransportError::new("transport.timeout", "round timed out")),
2128 Ok(completed_response(Vec::new())),
2129 Ok(completed_response(Vec::new())),
2130 ]);
2131 for _ in 0..3 {
2132 client.sync(&mut transport);
2133 }
2134 assert!(carried_round_failures(&transport.requests[0]).is_empty());
2135 assert_eq!(
2136 carried_round_failures(&transport.requests[1]),
2137 vec!["transport.timeout"]
2138 );
2139 assert!(
2140 matches!(transport.requests[1].frames[0], Frame::ReqHeader { .. })
2141 && matches!(transport.requests[1].frames[1], Frame::RoundFailure { .. }),
2142 "the report sits immediately after REQ_HEADER"
2143 );
2144 assert!(
2145 carried_round_failures(&transport.requests[2]).is_empty(),
2146 "a complete response ends the report"
2147 );
2148 }
2149
2150 #[test]
2151 fn a_newer_round_failure_replaces_the_kept_code() {
2152 let mut client = client();
2153 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2154 let mut transport = failure_transport([
2155 Err(TransportError::new("transport.timeout", "round timed out")),
2156 Err(TransportError::new("transport.failed", "socket closed")),
2157 Ok(completed_response(Vec::new())),
2158 Ok(completed_response(Vec::new())),
2159 ]);
2160 for _ in 0..4 {
2161 client.sync(&mut transport);
2162 }
2163 assert_eq!(
2164 carried_round_failures(&transport.requests[1]),
2165 vec!["transport.timeout"]
2166 );
2167 assert_eq!(
2168 carried_round_failures(&transport.requests[2]),
2169 vec!["transport.failed"]
2170 );
2171 assert!(carried_round_failures(&transport.requests[3]).is_empty());
2172 }
2173
2174 #[test]
2175 fn an_in_band_error_completes_the_round_and_is_never_reported() {
2176 let mut client = client();
2177 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2178 let mut transport = failure_transport([
2179 Err(TransportError::new("transport.timeout", "round timed out")),
2180 Ok(completed_response(vec![Frame::Error {
2181 code: "sync.internal_error".into(),
2182 message: "server error".into(),
2183 category: "internal".into(),
2184 retryable: false,
2185 recommended_action: "retryLater".into(),
2186 details: None,
2187 }])),
2188 Ok(completed_response(Vec::new())),
2189 ]);
2190 client.sync(&mut transport);
2191 assert!(matches!(
2192 client.sync(&mut transport),
2193 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
2194 ));
2195 client.sync(&mut transport);
2196 assert_eq!(
2197 carried_round_failures(&transport.requests[1]),
2198 vec!["transport.timeout"]
2199 );
2200 assert!(
2201 carried_round_failures(&transport.requests[2]).is_empty(),
2202 "the server already recorded an in-band ERROR"
2203 );
2204 }
2205
2206 #[test]
2207 fn a_silent_segment_fetch_is_retryable_and_reported_on_the_next_request() {
2208 let mut client = client();
2209 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2210 client
2211 .subscribe(
2212 "tasks".into(),
2213 "tasks".into(),
2214 vec![("project_id".into(), vec!["p1".into()])],
2215 None,
2216 )
2217 .unwrap();
2218 client.subs[0].cursor = 0;
2219 client.subs[0].synced_once = true;
2220 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
2221 client.persist_sub(&client.subs[0]).unwrap();
2222 client.drain_change_batches();
2223 client.drain_sync_intents();
2224 let mut transport = failure_transport([
2225 Ok(completed_response(vec![
2226 Frame::SubStart {
2227 id: "tasks".into(),
2228 status: SubStatus::Active,
2229 reason_code: String::new(),
2230 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
2231 bootstrap: false,
2232 },
2233 Frame::SegmentRef {
2234 segment_id: "sha256:stalled".into(),
2235 media_type: MediaType::Rows,
2236 table: "tasks".into(),
2237 byte_length: 1,
2238 row_count: 1,
2239 as_of_commit_seq: 1,
2240 scope_digest: "digest".into(),
2241 row_cursor: None,
2242 next_row_cursor: None,
2243 url: None,
2244 url_expires_at_ms: None,
2245 },
2246 Frame::SubEnd {
2247 next_cursor: 1,
2248 bootstrap_state: None,
2249 },
2250 ])),
2251 Ok(completed_response(Vec::new())),
2252 Ok(completed_response(Vec::new())),
2253 ]);
2254
2255 assert!(matches!(
2256 client.sync(&mut transport),
2257 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.timeout"
2258 ));
2259 assert!(
2260 matches!(
2261 client.drain_sync_intents().as_slice(),
2262 [SyncIntent::Background { delay_ms: 250 }]
2263 ),
2264 "a silent transfer asks the host for a Background retry"
2265 );
2266 client.sync(&mut transport);
2267 client.sync(&mut transport);
2268 assert!(carried_round_failures(&transport.requests[0]).is_empty());
2269 assert_eq!(
2270 carried_round_failures(&transport.requests[1]),
2271 vec!["transport.timeout"],
2272 "the round's own response was complete, so the fetch failure rides the next request"
2273 );
2274 assert!(
2275 carried_round_failures(&transport.requests[2]).is_empty(),
2276 "a complete response ends the report"
2277 );
2278 }
2279
2280 #[test]
2281 fn a_code_the_frame_grammar_rejects_is_not_attached() {
2282 let mut client = client();
2283 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2284 let mut transport = failure_transport([
2285 Err(TransportError::new("offline", "no dotted code")),
2286 Ok(completed_response(Vec::new())),
2287 ]);
2288 client.sync(&mut transport);
2289 client.sync(&mut transport);
2290 assert!(carried_round_failures(&transport.requests[1]).is_empty());
2291 }
2292
2293 #[test]
2294 fn sync_report_merge_schema_floor_is_latest_state() {
2295 let mut aggregate = SyncReport {
2296 schema_floor: Some(SchemaFloor {
2297 required_schema_version: Some(2),
2298 latest_schema_version: Some(2),
2299 }),
2300 ..SyncReport::default()
2301 };
2302 aggregate.merge(&SyncReport::default());
2303 assert!(
2304 aggregate.schema_floor.is_none(),
2305 "a later round without a floor clears it"
2306 );
2307 }
2308
2309 #[test]
2310 fn push_capacity_limits_are_hard_and_keep_intent() {
2311 let mut c = client();
2313 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2314 c.limits.max_push_operations_per_request = Some(1);
2315 let id = c
2316 .mutate(vec![
2317 Mutation::Upsert {
2318 table: "tasks".into(),
2319 values: Map::from_iter([
2320 ("id".into(), json!("a")),
2321 ("project_id".into(), json!("p1")),
2322 ]),
2323 base_version: None,
2324 },
2325 Mutation::Upsert {
2326 table: "tasks".into(),
2327 values: Map::from_iter([
2328 ("id".into(), json!("b")),
2329 ("project_id".into(), json!("p1")),
2330 ]),
2331 base_version: None,
2332 },
2333 ])
2334 .unwrap();
2335 let error = c.build_request(false).unwrap_err();
2336 let SyncOutcome::Failed {
2337 error_code,
2338 details,
2339 ..
2340 } = *error
2341 else {
2342 panic!("expected a capacity failure");
2343 };
2344 assert_eq!(error_code, "client.push_request_too_large");
2345 let details = details.expect("capacity details");
2346 assert_eq!(details["kind"], json!("operations"));
2347 assert_eq!(details["limit"], json!(1));
2348 assert_eq!(details["size"], json!(2));
2349 assert_eq!(details["clientCommitId"], json!(id));
2350 assert_eq!(c.pending_commit_ids(), vec![id]);
2351
2352 let mut c = client();
2354 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2355 c.limits.max_push_commits_per_request = Some(1);
2356 let first = c
2357 .mutate(vec![Mutation::Upsert {
2358 table: "tasks".into(),
2359 values: Map::from_iter([
2360 ("id".into(), json!("one")),
2361 ("project_id".into(), json!("p1")),
2362 ]),
2363 base_version: None,
2364 }])
2365 .unwrap();
2366 let second = c
2367 .mutate(vec![Mutation::Upsert {
2368 table: "tasks".into(),
2369 values: Map::from_iter([
2370 ("id".into(), json!("two")),
2371 ("project_id".into(), json!("p1")),
2372 ]),
2373 base_version: None,
2374 }])
2375 .unwrap();
2376 let (_, meta) = c.build_request(false).unwrap();
2377 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2378 assert_eq!(meta.deferred_commits, 1);
2379 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2380 }
2381
2382 #[test]
2383 fn push_request_byte_cap_defers_the_prefix_boundary_and_errors_the_head() {
2384 let mut c = client();
2387 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2388 let first = c
2389 .mutate(vec![Mutation::Upsert {
2390 table: "tasks".into(),
2391 values: Map::from_iter([
2392 ("id".into(), json!("one")),
2393 ("project_id".into(), json!("p1")),
2394 ]),
2395 base_version: None,
2396 }])
2397 .unwrap();
2398 let second = c
2399 .mutate(vec![Mutation::Upsert {
2400 table: "tasks".into(),
2401 values: Map::from_iter([
2402 ("id".into(), json!("two")),
2403 ("project_id".into(), json!("p1")),
2404 ]),
2405 base_version: None,
2406 }])
2407 .unwrap();
2408 let (message, meta) = c.build_request(false).unwrap();
2409 assert_eq!(meta.pushed_ids.len(), 2);
2410 let full = encode_message(&message).len();
2411 c.limits.max_push_request_bytes = Some(full - 1);
2412 let (_, meta) = c.build_request(false).unwrap();
2413 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2414 assert_eq!(meta.deferred_commits, 1);
2415 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2416
2417 let mut c = client();
2420 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2421 let huge = c
2422 .mutate(vec![Mutation::Upsert {
2423 table: "tasks".into(),
2424 values: Map::from_iter([
2425 ("id".into(), json!("x".repeat(20_000))),
2426 ("project_id".into(), json!("p1")),
2427 ]),
2428 base_version: None,
2429 }])
2430 .unwrap();
2431 let (message, meta) = c.build_request(false).unwrap();
2432 assert_eq!(meta.pushed_ids, vec![huge.clone()]);
2433 let with_huge = encode_message(&message).len();
2434 c.limits.max_push_request_bytes = Some(with_huge - 1);
2435 let error = c.build_request(false).unwrap_err();
2436 let SyncOutcome::Failed {
2437 error_code,
2438 details,
2439 ..
2440 } = *error
2441 else {
2442 panic!("expected a capacity failure");
2443 };
2444 assert_eq!(error_code, "client.push_request_too_large");
2445 let details = details.expect("capacity details");
2446 assert_eq!(details["kind"], json!("bytes"));
2447 assert_eq!(details["limit"], json!((with_huge - 1) as u64));
2448 assert_eq!(details["size"], json!(with_huge as u64));
2449 assert_eq!(details["clientCommitId"], json!(huge));
2450 assert_eq!(c.pending_commit_ids(), vec![huge]);
2451
2452 let mut c = client();
2454 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2455 c.limits.max_push_request_bytes = Some(1);
2456 let error = c.build_request(false).unwrap_err();
2457 let SyncOutcome::Failed { details, .. } = *error else {
2458 panic!("expected a capacity failure");
2459 };
2460 let details = details.expect("capacity details");
2461 assert_eq!(details["kind"], json!("bytes"));
2462 assert!(details.get("clientCommitId").is_none(), "{details}");
2463 }
2464
2465 #[test]
2466 fn push_limits_validate_at_construction() {
2467 for limits in [
2468 ClientLimits {
2469 max_push_commits_per_request: Some(0),
2470 ..Default::default()
2471 },
2472 ClientLimits {
2473 max_push_operations_per_request: Some(0),
2474 ..Default::default()
2475 },
2476 ClientLimits {
2477 max_push_request_bytes: Some(0),
2478 ..Default::default()
2479 },
2480 ClientLimits {
2481 max_push_request_bytes: Some(u32::MAX as usize + 1),
2482 ..Default::default()
2483 },
2484 ] {
2485 let schema = json!({"version":1,"tables":[]});
2486 let error = SyncClient::with_connection(
2487 "bad-limits".into(),
2488 &schema,
2489 limits,
2490 Connection::open_in_memory().unwrap(),
2491 )
2492 .err();
2493 assert_eq!(
2494 error.as_deref(),
2495 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2496 );
2497 }
2498 }
2499
2500 #[test]
2501 fn open_path_rejects_invalid_push_limits_before_storage() {
2502 for identity in [false, true] {
2503 let path = std::env::temp_dir().join(format!(
2504 "syncular-bad-limits-{}.sqlite",
2505 uuid::Uuid::new_v4()
2506 ));
2507 let schema = json!({"version":1,"tables":[]});
2508 let limits = ClientLimits {
2509 max_push_request_bytes: Some(0),
2510 ..Default::default()
2511 };
2512 let error = if identity {
2513 SyncClient::open_path_with_identity(None, &schema, limits, path.to_str().unwrap())
2514 } else {
2515 SyncClient::open_path("bad-limits".into(), &schema, limits, path.to_str().unwrap())
2516 }
2517 .err();
2518 assert_eq!(
2519 error.as_deref(),
2520 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2521 );
2522 assert!(
2523 !path.exists(),
2524 "an invalid limit is rejected before the database file is created"
2525 );
2526 }
2527 }
2528
2529 #[test]
2530 fn failed_progress_retry_delay_doubles_to_the_cap() {
2531 let mut client = client();
2532 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2533 let mut transport = ServerErrorTransport {
2534 code: "sync.internal_error",
2535 retryable: true,
2536 };
2537 let delays: Vec<Option<u64>> = (0..9)
2538 .map(|_| {
2539 client.sync(&mut transport);
2540 client.progress().snapshot().unwrap().retry_delay_ms
2541 })
2542 .collect();
2543 assert_eq!(
2544 delays,
2545 [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
2546 );
2547 }
2548
2549 #[test]
2550 fn background_retry_deadlines_back_off_and_reset() {
2551 let mut client = client();
2552 client.schedule_background_retry();
2553 assert!(matches!(
2554 client.drain_sync_intents().as_slice(),
2555 [SyncIntent::Background { delay_ms: 250 }]
2556 ));
2557 client.schedule_background_retry();
2558 assert!(matches!(
2559 client.drain_sync_intents().as_slice(),
2560 [SyncIntent::Background { delay_ms: 500 }]
2561 ));
2562 client.reset_background_retry();
2563 client.schedule_background_retry();
2564 assert!(matches!(
2565 client.drain_sync_intents().as_slice(),
2566 [SyncIntent::Background { delay_ms: 250 }]
2567 ));
2568 }
2569
2570 #[test]
2571 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
2572 let mut client = client();
2573 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2574 client
2575 .subscribe(
2576 "epoch-tasks".to_owned(),
2577 "tasks".to_owned(),
2578 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2579 None,
2580 )
2581 .expect("subscribe");
2582 client.drain_sync_intents();
2583
2584 client
2585 .run_log_epoch_reset("epoch-2")
2586 .expect("reset partition log epoch");
2587
2588 assert!(matches!(
2589 client.drain_sync_intents().as_slice(),
2590 [SyncIntent::Interactive]
2591 ));
2592 }
2593
2594 #[derive(Default)]
2595 struct CountingRealtimeTransport {
2596 connects: usize,
2597 closes: usize,
2598 messages: Vec<String>,
2599 blob_uploads: Vec<(String, Vec<u8>)>,
2600 blob_download: Option<Vec<u8>>,
2601 sync_calls: usize,
2602 realtime_sync_calls: usize,
2603 refuse_connect: bool,
2604 }
2605
2606 impl Transport for CountingRealtimeTransport {
2607 fn blob_upload(
2608 &mut self,
2609 blob_id: &str,
2610 bytes: &[u8],
2611 _media_type: Option<&str>,
2612 ) -> Result<(), TransportError> {
2613 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
2614 Ok(())
2615 }
2616
2617 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
2618 self.blob_download
2619 .clone()
2620 .map(BlobDownload::Bytes)
2621 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
2622 }
2623
2624 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2625 self.sync_calls += 1;
2626 Err(TransportError::new("sync.transport_failed", "offline"))
2627 }
2628
2629 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2630 self.realtime_sync_calls += 1;
2631 Err(TransportError::new("sync.transport_failed", "offline"))
2632 }
2633
2634 fn download_segment(
2635 &mut self,
2636 _request: &SegmentRequest,
2637 _on_progress: &mut dyn FnMut(u64),
2638 ) -> Result<Vec<u8>, TransportError> {
2639 Err(TransportError::new("sync.transport_failed", "offline"))
2640 }
2641
2642 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2643 self.connects += 1;
2644 if self.refuse_connect {
2645 return Err(TransportError::new("transport.failed", "refused"));
2646 }
2647 Ok(())
2648 }
2649
2650 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2651 self.messages.push(text.to_owned());
2652 Ok(())
2653 }
2654
2655 fn realtime_close(&mut self) -> Result<(), TransportError> {
2656 self.closes += 1;
2657 Ok(())
2658 }
2659 }
2660
2661 #[test]
2662 fn blob_results_own_downloaded_bytes() {
2663 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
2664 {"name":"id","type":"string","nullable":false},
2665 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
2666 let bytes = b"owned typed result".to_vec();
2667 let blob_id = blob_id_for(&bytes);
2668 let mut transport = CountingRealtimeTransport {
2669 blob_download: Some(bytes.clone()),
2670 ..Default::default()
2671 };
2672 let result = {
2673 let mut client =
2674 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
2675 .unwrap();
2676 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
2677 assert_eq!(result.blob_id, blob_id);
2678 assert_eq!(result.byte_length, bytes.len() as i64);
2679 assert_eq!(result.bytes, bytes);
2680 assert_eq!(result.media_type, None);
2681 assert_eq!(
2682 client
2683 .conn
2684 .query_row(
2685 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
2686 rusqlite::params![blob_id],
2687 |row| row.get::<_, i64>(0),
2688 )
2689 .unwrap(),
2690 0
2691 );
2692 result
2693 };
2694 assert_eq!(result.bytes, bytes);
2695
2696 let mut client =
2697 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
2698 assert_eq!(
2699 client
2700 .fetch_blob_bytes(&mut transport, "not json")
2701 .unwrap_err(),
2702 (
2703 "client.failed".to_owned(),
2704 "blob ref is not JSON".to_owned()
2705 )
2706 );
2707 }
2708
2709 #[test]
2710 fn visible_blob_references_survive_cache_trimming() {
2711 let schema = json!({"version":1,"tables":[
2712 {"name":"attachments","primaryKey":"id","columns":[
2713 {"name":"id","type":"string","nullable":false},
2714 {"name":"body","type":"blob_ref","nullable":true},
2715 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
2716 {"name":"avatars","primaryKey":"id","columns":[
2717 {"name":"id","type":"string","nullable":false},
2718 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
2719 ]});
2720 let bytes = b"referenced above cap".to_vec();
2721 let blob_id = blob_id_for(&bytes);
2722 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
2723 let mut client = SyncClient::new(
2724 "visible-blob-refs".to_owned(),
2725 &schema,
2726 ClientLimits {
2727 blob_cache_max_bytes: Some(1),
2728 ..Default::default()
2729 },
2730 )
2731 .unwrap();
2732 client.conn.execute(
2733 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
2734 rusqlite::params![reference, reference, reference, reference],
2735 ).unwrap();
2736 client
2737 .conn
2738 .execute(
2739 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
2740 rusqlite::params![reference],
2741 )
2742 .unwrap();
2743 client.conn.execute(
2744 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
2745 [],
2746 ).unwrap();
2747 let mut transport = CountingRealtimeTransport {
2748 blob_download: Some(bytes.clone()),
2749 ..Default::default()
2750 };
2751
2752 assert_eq!(
2753 client
2754 .fetch_blob_bytes(&mut transport, &blob_id)
2755 .unwrap()
2756 .bytes,
2757 bytes
2758 );
2759 let rows = client
2760 .conn
2761 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
2762 .unwrap()
2763 .query_map([], |row| row.get::<_, String>(0))
2764 .unwrap()
2765 .collect::<Result<Vec<_>, _>>()
2766 .unwrap();
2767 assert_eq!(rows, vec![blob_id]);
2768 }
2769
2770 #[test]
2771 fn closed_transport_never_invokes_network_and_reopen_is_host_owned() {
2772 let mut transport = CountingRealtimeTransport::default();
2773 let path = std::env::temp_dir().join(format!(
2774 "syncular-transport-gate-{}.db",
2775 uuid::Uuid::new_v4()
2776 ));
2777 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
2778 {"name":"id","type":"string","nullable":false},
2779 {"name":"project_id","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
2780 let mut client = SyncClient::open_path(
2781 "gate".into(),
2782 &schema,
2783 ClientLimits::default(),
2784 path.to_str().unwrap(),
2785 )
2786 .unwrap();
2787 client.connect_realtime(&mut transport).unwrap();
2788 client.set_transport_enabled(&mut transport, false);
2789 assert_eq!(transport.closes, 1);
2790 client.begin_security_preflight();
2791 client.activate_security(Default::default()).unwrap();
2792 client
2793 .subscribe(
2794 "tasks".into(),
2795 "tasks".into(),
2796 vec![("project_id".into(), vec!["p1".into()])],
2797 None,
2798 )
2799 .unwrap();
2800 for id in ["one", "two"] {
2801 client
2802 .mutate(vec![Mutation::Upsert {
2803 table: "tasks".into(),
2804 values: Map::from_iter([
2805 ("id".into(), json!(id)),
2806 ("project_id".into(), json!("p1")),
2807 ]),
2808 base_version: None,
2809 }])
2810 .unwrap();
2811 }
2812 assert_eq!(
2813 client
2814 .query("SELECT id FROM tasks ORDER BY id", &[])
2815 .unwrap()
2816 .len(),
2817 2
2818 );
2819 assert!(
2820 matches!(client.sync(&mut transport), SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.offline")
2821 );
2822 assert!(client
2823 .connect_realtime(&mut transport)
2824 .unwrap_err()
2825 .starts_with("sync.offline:"));
2826 assert!(client
2827 .set_presence(&mut transport, "project:p1", None)
2828 .unwrap_err()
2829 .starts_with("sync.offline:"));
2830 assert_eq!(transport.sync_calls, 0);
2831 assert_eq!(transport.realtime_sync_calls, 0);
2832 assert_eq!(transport.connects, 1);
2833 assert!(transport.messages.is_empty());
2834 assert!(matches!(
2835 client.drain_sync_intents().as_slice(),
2836 [SyncIntent::None]
2837 ));
2838 let pending = client.pending_commit_ids();
2839 drop(client);
2840 let mut reopened = SyncClient::open_path(
2841 "gate".into(),
2842 &schema,
2843 ClientLimits::default(),
2844 path.to_str().unwrap(),
2845 )
2846 .unwrap();
2847 assert!(reopened.transport_enabled());
2848 reopened.set_transport_enabled(&mut transport, false);
2849 assert_eq!(reopened.pending_commit_ids(), pending);
2850 assert_eq!(
2851 reopened.query("SELECT id FROM tasks", &[]).unwrap().len(),
2852 2
2853 );
2854 reopened.set_transport_enabled(&mut transport, true);
2855 assert!(matches!(
2856 reopened.drain_sync_intents().as_slice(),
2857 [SyncIntent::Interactive]
2858 ));
2859 reopened.set_transport_enabled(&mut transport, true);
2860 assert!(reopened.drain_sync_intents().is_empty());
2861 drop(reopened);
2862 std::fs::remove_file(path).unwrap();
2863 }
2864
2865 #[test]
2866 fn realtime_connection_ownership_is_idempotent() {
2867 let mut client = client();
2868 let mut transport = CountingRealtimeTransport {
2869 connects: 0,
2870 closes: 0,
2871 ..Default::default()
2872 };
2873 client
2874 .connect_realtime(&mut transport)
2875 .expect("first connect");
2876 client
2877 .connect_realtime(&mut transport)
2878 .expect("idempotent connect");
2879 assert_eq!(transport.connects, 1);
2880 client.disconnect_realtime(&mut transport);
2881 client.disconnect_realtime(&mut transport);
2882 assert_eq!(transport.closes, 1);
2883 client
2884 .connect_realtime(&mut transport)
2885 .expect("deliberate reconnect");
2886 assert_eq!(transport.connects, 2);
2887 }
2888
2889 #[test]
2890 fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
2891 let mut client = client();
2892 client
2893 .set_realtime_policy(RealtimePolicy::Required)
2894 .expect("set policy");
2895 let mut transport = CountingRealtimeTransport::default();
2896 match client.sync(&mut transport) {
2897 SyncOutcome::RealtimeUnavailable {
2898 state,
2899 reason_code,
2900 retry_delay_ms,
2901 } => {
2902 assert_eq!(state, RealtimeState::Disconnected);
2903 assert_eq!(reason_code, None);
2904 assert_eq!(retry_delay_ms, 250);
2905 }
2906 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2907 }
2908 assert_eq!(transport.sync_calls, 0);
2909 assert!(matches!(
2910 client.drain_sync_intents().as_slice(),
2911 [SyncIntent::Background { delay_ms: 250 }]
2912 ));
2913 let snapshot = client
2914 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2915 .expect("diagnostics");
2916 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2917 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
2918 assert_eq!(snapshot.host.realtime_reason_code, None);
2919 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2920 assert_eq!(
2921 snapshot
2922 .last_round
2923 .as_ref()
2924 .and_then(|round| round.error_code.clone()),
2925 Some(REALTIME_UNAVAILABLE_CODE.to_owned())
2926 );
2927 }
2928
2929 #[test]
2930 fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
2931 let mut client = client();
2932 client
2933 .set_realtime_policy(RealtimePolicy::Required)
2934 .expect("set policy");
2935 let mut transport = CountingRealtimeTransport {
2936 refuse_connect: true,
2937 ..Default::default()
2938 };
2939 assert!(client.connect_realtime(&mut transport).is_err());
2940 assert_eq!(client.realtime_state(), RealtimeState::Refused);
2941 let snapshot = client
2942 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2943 .expect("diagnostics");
2944 assert_eq!(
2945 snapshot.host.realtime_reason_code.as_deref(),
2946 Some("transport.failed")
2947 );
2948 match client.sync(&mut transport) {
2949 SyncOutcome::RealtimeUnavailable {
2950 state, reason_code, ..
2951 } => {
2952 assert_eq!(state, RealtimeState::Refused);
2953 assert_eq!(reason_code.as_deref(), Some("transport.failed"));
2954 }
2955 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2956 }
2957 assert_eq!(transport.sync_calls, 0);
2958 assert_eq!(transport.realtime_sync_calls, 0);
2959 }
2960
2961 #[test]
2962 fn an_unsolicited_socket_end_is_lost_with_the_transport_code_and_a_retry_intent() {
2963 let mut client = client();
2964 let mut transport = CountingRealtimeTransport::default();
2965 let ended = TransportError::new("transport.failed", "realtime connection closed");
2966 client.realtime_lost(&mut transport, &ended);
2967 assert_eq!(
2968 client.realtime_state(),
2969 RealtimeState::Disconnected,
2970 "a socket that was never connected has nothing to lose"
2971 );
2972 assert!(client.drain_sync_intents().is_empty());
2973
2974 client.connect_realtime(&mut transport).expect("connect");
2975 client.realtime_lost(&mut transport, &ended);
2976 assert_eq!(client.realtime_state(), RealtimeState::Lost);
2977 assert_eq!(transport.closes, 1);
2978 assert!(matches!(
2979 client.drain_sync_intents().as_slice(),
2980 [SyncIntent::Background { delay_ms: 250 }]
2981 ));
2982 let snapshot = client
2983 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2984 .expect("diagnostics");
2985 assert_eq!(snapshot.host.realtime, RealtimeState::Lost);
2986 assert_eq!(
2987 snapshot.host.realtime_reason_code.as_deref(),
2988 Some("transport.failed")
2989 );
2990 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2991
2992 client.realtime_lost(&mut transport, &ended);
2993 assert!(
2994 client.drain_sync_intents().is_empty(),
2995 "a second report for the same loss schedules nothing"
2996 );
2997 client.connect_realtime(&mut transport).expect("reconnect");
2998 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2999 }
3000
3001 #[test]
3002 fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
3003 let mut client = client();
3004 client
3005 .set_realtime_policy(RealtimePolicy::Required)
3006 .expect("set policy");
3007 let mut transport = CountingRealtimeTransport::default();
3008 client.connect_realtime(&mut transport).expect("connect");
3009 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3010 assert!(
3011 client.set_realtime_policy(RealtimePolicy::Off).is_err(),
3012 "a policy change cannot silently re-route a connected socket"
3013 );
3014 assert!(matches!(
3015 client.sync(&mut transport),
3016 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
3017 ));
3018 assert_eq!(client.realtime_state(), RealtimeState::Lost);
3019 assert_eq!(transport.realtime_sync_calls, 1);
3020 assert_eq!(transport.sync_calls, 0);
3021 let snapshot = client
3022 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3023 .expect("diagnostics");
3024 assert_eq!(
3025 snapshot.host.realtime_reason_code.as_deref(),
3026 Some("sync.transport_failed")
3027 );
3028 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3029 match client.sync(&mut transport) {
3030 SyncOutcome::RealtimeUnavailable {
3031 state, reason_code, ..
3032 } => {
3033 assert_eq!(state, RealtimeState::Lost);
3034 assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
3035 }
3036 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3037 }
3038 assert_eq!(transport.sync_calls, 0);
3039 client.disconnect_realtime(&mut transport);
3040 assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
3041 assert_eq!(transport.closes, 1);
3042 client.connect_realtime(&mut transport).expect("reconnect");
3043 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3044 assert_eq!(transport.connects, 2);
3045 }
3046
3047 #[test]
3048 fn off_policy_uses_http_and_refuses_an_explicit_connect() {
3049 let mut client = client();
3050 client
3051 .set_realtime_policy(RealtimePolicy::Off)
3052 .expect("set policy");
3053 let mut transport = CountingRealtimeTransport::default();
3054 assert!(client.connect_realtime(&mut transport).is_err());
3055 assert_eq!(client.realtime_state(), RealtimeState::Disabled);
3056 assert!(matches!(
3057 client.sync(&mut transport),
3058 SyncOutcome::Failed { .. }
3059 ));
3060 assert_eq!(transport.sync_calls, 1);
3061 assert_eq!(transport.realtime_sync_calls, 0);
3062 let snapshot = client
3063 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3064 .expect("diagnostics");
3065 assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
3066 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
3067 }
3068
3069 #[test]
3070 fn optional_policy_keeps_the_http_round() {
3071 let mut client = client();
3072 let mut transport = CountingRealtimeTransport::default();
3073 assert!(matches!(
3074 client.sync(&mut transport),
3075 SyncOutcome::Failed { .. }
3076 ));
3077 assert_eq!(transport.sync_calls, 1);
3078 assert_eq!(transport.realtime_sync_calls, 0);
3079 let snapshot = client
3080 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3081 .expect("diagnostics");
3082 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
3083 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
3084 }
3085
3086 #[test]
3087 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
3088 let mut client = client();
3089 client
3090 .subscribe(
3091 "repair-tasks".to_owned(),
3092 "tasks".to_owned(),
3093 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3094 None,
3095 )
3096 .expect("subscribe");
3097 {
3098 let sub = client
3099 .subs
3100 .iter_mut()
3101 .find(|sub| sub.id == "repair-tasks")
3102 .expect("subscription");
3103 sub.cursor = 42;
3104 sub.synced_once = true;
3105 let persisted = sub.clone();
3106 client.persist_sub(&persisted).unwrap();
3107 }
3108 for table in ["_syncular_base_tasks", "tasks"] {
3109 client
3110 .conn
3111 .execute(
3112 &format!(
3113 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
3114 ),
3115 rusqlite::params!["server-row", "p1"],
3116 )
3117 .expect("seed server row");
3118 }
3119 let pending = client
3120 .mutate(vec![Mutation::Upsert {
3121 table: "tasks".to_owned(),
3122 values: Map::from_iter([
3123 ("id".to_owned(), Value::from("offline-row")),
3124 ("project_id".to_owned(), Value::from("p1")),
3125 ]),
3126 base_version: None,
3127 }])
3128 .expect("queue offline work");
3129 client.drain_change_batches();
3130 client.drain_sync_intents();
3131
3132 assert_eq!(
3133 client
3134 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3135 rebootstrap_id: "support-case-001".to_owned(),
3136 })
3137 .expect("rebootstrap"),
3138 LocalDataRebootstrapResult {
3139 already_applied: false,
3140 retained_commits: 1,
3141 reset_subscriptions: 1,
3142 }
3143 );
3144 let visible_ids = client
3145 .conn
3146 .prepare("SELECT id FROM tasks ORDER BY id")
3147 .expect("prepare visible ids")
3148 .query_map([], |row| row.get::<_, String>(0))
3149 .expect("query visible ids")
3150 .collect::<Result<Vec<_>, _>>()
3151 .expect("collect visible ids");
3152 assert_eq!(visible_ids, vec!["offline-row"]);
3153 assert_eq!(client.pending_commit_ids(), vec![pending]);
3154 assert_eq!(
3155 client
3156 .subscription_state("repair-tasks")
3157 .expect("subscription")
3158 .cursor,
3159 -1
3160 );
3161 assert!(client.upgrading());
3162 assert!(client.sync_needed());
3163 assert!(matches!(
3164 client.drain_sync_intents().as_slice(),
3165 [SyncIntent::Interactive]
3166 ));
3167 assert_eq!(client.drain_change_batches().len(), 1);
3168
3169 assert_eq!(
3170 client
3171 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3172 rebootstrap_id: "support-case-001".to_owned(),
3173 })
3174 .expect("idempotent retry"),
3175 LocalDataRebootstrapResult {
3176 already_applied: true,
3177 retained_commits: 1,
3178 reset_subscriptions: 1,
3179 }
3180 );
3181 assert!(client.drain_change_batches().is_empty());
3182 }
3183
3184 #[test]
3185 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
3186 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
3187 assert_eq!(
3188 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
3189 (3, 4)
3190 );
3191 assert_eq!(
3192 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
3193 (0, 0)
3194 );
3195 for malformed in [
3196 "",
3197 "{}",
3198 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
3199 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
3200 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
3201 ] {
3202 assert_eq!(
3203 decode_local_rebootstrap_receipt(malformed)
3204 .expect_err("malformed receipt must fail"),
3205 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3206 );
3207 }
3208 }
3209
3210 #[test]
3211 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
3212 let path = std::env::temp_dir().join(format!(
3213 "syncular-rebootstrap-receipt-{}.db",
3214 uuid::Uuid::new_v4()
3215 ));
3216 let schema = json!({
3217 "version": 1,
3218 "tables": [{
3219 "name": "tasks",
3220 "primaryKey": "id",
3221 "columns": [
3222 { "name": "id", "type": "string", "nullable": false },
3223 { "name": "project_id", "type": "string", "nullable": false }
3224 ],
3225 "scopes": [{ "pattern": "project:{project_id}" }]
3226 }]
3227 });
3228 let path_string = path.to_str().expect("UTF-8 temp path");
3229
3230 {
3231 let mut first = SyncClient::open_path(
3232 "repair-restart-client".to_owned(),
3233 &schema,
3234 ClientLimits::default(),
3235 path_string,
3236 )
3237 .expect("first open");
3238 first
3239 .subscribe(
3240 "repair-tasks".to_owned(),
3241 "tasks".to_owned(),
3242 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3243 None,
3244 )
3245 .expect("subscribe");
3246 first
3247 .mutate(vec![Mutation::Upsert {
3248 table: "tasks".to_owned(),
3249 values: Map::from_iter([
3250 ("id".to_owned(), Value::from("offline-row")),
3251 ("project_id".to_owned(), Value::from("p1")),
3252 ]),
3253 base_version: None,
3254 }])
3255 .expect("queue offline work");
3256 assert_eq!(
3257 first
3258 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3259 rebootstrap_id: "restart-receipt".to_owned(),
3260 })
3261 .expect("first rebootstrap"),
3262 LocalDataRebootstrapResult {
3263 already_applied: false,
3264 retained_commits: 1,
3265 reset_subscriptions: 1,
3266 }
3267 );
3268 }
3269
3270 let mut reopened = SyncClient::open_path(
3271 "repair-restart-client".to_owned(),
3272 &schema,
3273 ClientLimits::default(),
3274 path_string,
3275 )
3276 .expect("reopen");
3277 reopened.drain_change_batches();
3278 reopened.drain_sync_intents();
3279 assert_eq!(
3280 reopened
3281 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3282 rebootstrap_id: "restart-receipt".to_owned(),
3283 })
3284 .expect("receipt replay"),
3285 LocalDataRebootstrapResult {
3286 already_applied: true,
3287 retained_commits: 1,
3288 reset_subscriptions: 1,
3289 }
3290 );
3291 assert!(reopened.drain_change_batches().is_empty());
3292 assert!(reopened.drain_sync_intents().is_empty());
3293 drop(reopened);
3294 std::fs::remove_file(path).expect("remove temp database");
3295 }
3296
3297 #[test]
3298 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
3299 let mut malformed = client();
3300 malformed
3301 .subscribe(
3302 "repair-tasks".to_owned(),
3303 "tasks".to_owned(),
3304 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3305 None,
3306 )
3307 .expect("subscribe");
3308 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
3309 malformed.drain_change_batches();
3310 malformed.drain_sync_intents();
3311 let malformed_error = malformed
3312 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3313 rebootstrap_id: "malformed".to_owned(),
3314 })
3315 .expect_err("malformed receipt must fail closed");
3316 assert_eq!(
3317 malformed_error,
3318 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3319 );
3320 assert!(!malformed.upgrading());
3321 assert_eq!(
3322 malformed
3323 .subscription_state("repair-tasks")
3324 .expect("unchanged subscription")
3325 .cursor,
3326 -1
3327 );
3328 assert!(malformed.drain_change_batches().is_empty());
3329 assert!(malformed.drain_sync_intents().is_empty());
3330
3331 let mut unreadable = client();
3332 unreadable
3333 .conn
3334 .execute(
3335 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
3336 rusqlite::params!["server-row", "p1"],
3337 )
3338 .expect("seed visible row");
3339 unreadable
3340 .conn
3341 .execute("DROP TABLE _syncular_meta", [])
3342 .expect("break marker storage");
3343 let unreadable_error = unreadable
3344 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3345 rebootstrap_id: "unreadable".to_owned(),
3346 })
3347 .expect_err("unreadable marker storage must fail closed");
3348 assert_eq!(
3349 unreadable_error,
3350 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
3351 );
3352 let visible_rows = unreadable
3353 .conn
3354 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
3355 .expect("visible projection remains");
3356 assert_eq!(visible_rows, 1);
3357 assert!(!unreadable.upgrading());
3358 assert!(unreadable.drain_change_batches().is_empty());
3359 assert!(unreadable.drain_sync_intents().is_empty());
3360 }
3361
3362 #[test]
3363 fn local_rebootstrap_cannot_bypass_schema_floor() {
3364 let mut client = client();
3365 client.set_schema_floor(Some(SchemaFloor {
3366 required_schema_version: Some(2),
3367 latest_schema_version: Some(2),
3368 }));
3369 let error = client
3370 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3371 rebootstrap_id: "blocked-floor".to_owned(),
3372 })
3373 .expect_err("schema floor must block repair");
3374 assert!(error.contains("cannot bypass an active schema-floor stop"));
3375 }
3376
3377 #[test]
3378 fn diego_applied_sparse_moves_stay_visible_until_pull_image_arrives() {
3379 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3380 "columns":[{"name":"id","type":"string","nullable":false},
3381 {"name":"project_id","type":"string","nullable":false},
3382 {"name":"starts_at_ms","type":"integer","nullable":false},
3383 {"name":"version","type":"integer","nullable":false}],
3384 "scopes":[{"pattern":"project:{project_id}"}]}]});
3385 let mut client =
3386 SyncClient::new("diego-ack-gap".into(), &schema, ClientLimits::default()).unwrap();
3387 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3388 client
3389 .write_base_row(
3390 "tasks",
3391 &vec![
3392 Some(ColumnValue::String("t1".into())),
3393 Some(ColumnValue::String("p1".into())),
3394 Some(ColumnValue::Integer(100)),
3395 Some(ColumnValue::Integer(1)),
3396 ],
3397 1,
3398 )
3399 .unwrap();
3400 client.rebuild_overlay().expect("rebuild overlay");
3401 let first = client
3402 .patch(
3403 "tasks",
3404 "t1",
3405 Map::from_iter([
3406 ("starts_at_ms".into(), json!(200)),
3407 ("version".into(), json!(2)),
3408 ]),
3409 None,
3410 )
3411 .unwrap();
3412 let (_, first_request) = client.build_request(false).unwrap();
3413 let second = client
3415 .patch(
3416 "tasks",
3417 "t1",
3418 Map::from_iter([
3419 ("starts_at_ms".into(), json!(300)),
3420 ("version".into(), json!(3)),
3421 ]),
3422 None,
3423 )
3424 .unwrap();
3425 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3426 for (id, seq, meta) in [
3427 (first, 2, first_request),
3428 (second, 3, client.build_request(false).unwrap().1),
3429 ] {
3430 let response = Message {
3431 wire_version: WIRE_VERSION,
3432 msg_kind: MsgKind::Response,
3433 frames: vec![
3434 Frame::RespHeader {
3435 required_schema_version: None,
3436 latest_schema_version: None,
3437 log_epoch: Some("epoch-1".into()),
3438 reset_required: Some(false),
3439 },
3440 Frame::PushResult {
3441 client_commit_id: id,
3442 status: PushStatus::Applied,
3443 commit_seq: Some(seq),
3444 results: vec![OpResult::Applied { op_index: 0 }],
3445 },
3446 ],
3447 };
3448 assert!(matches!(
3449 client.process_response(&mut transport, response, &meta),
3450 SyncOutcome::Ok(_)
3451 ));
3452 let row = &client
3453 .query("SELECT starts_at_ms, version FROM tasks WHERE id='t1'", &[])
3454 .unwrap()[0];
3455 assert_eq!(
3456 row["starts_at_ms"],
3457 json!(300),
3458 "an applied ACK without a pull image must not expose the old base"
3459 );
3460 assert_eq!(
3461 row["version"],
3462 json!(3),
3463 "the queued domain version must remain monotonic"
3464 );
3465 }
3466 }
3467
3468 #[test]
3469 fn first_epoch_acquisition_preserves_tables_subscriptions_and_local_intent() {
3470 for stored in [None, Some("epoch-old")] {
3471 let mut client = client();
3472 client
3473 .subscribe(
3474 "tasks".into(),
3475 "tasks".into(),
3476 vec![("project_id".into(), vec!["p1".into()])],
3477 None,
3478 )
3479 .unwrap();
3480 client
3481 .conn
3482 .execute_batch(
3483 "CREATE TRIGGER epoch_table_identity AFTER INSERT ON tasks BEGIN SELECT 1; END",
3484 )
3485 .unwrap();
3486 let id = client
3487 .mutate(vec![Mutation::Upsert {
3488 table: "tasks".into(),
3489 values: Map::from_iter([
3490 ("id".into(), json!("offline")),
3491 ("project_id".into(), json!("p1")),
3492 ]),
3493 base_version: None,
3494 }])
3495 .unwrap();
3496 if let Some(epoch) = stored {
3497 client.set_meta(LOG_EPOCH_KEY, epoch);
3498 }
3499 let before = serde_json::to_value(client.subscription_state("tasks")).unwrap();
3500 let (_, meta) = client.build_request(false).unwrap();
3501 client.drain_change_batches();
3502 client.drain_sync_intents();
3503 let response = Message {
3504 wire_version: WIRE_VERSION,
3505 msg_kind: MsgKind::Response,
3506 frames: vec![Frame::RespHeader {
3507 required_schema_version: None,
3508 latest_schema_version: None,
3509 log_epoch: Some("epoch-new".into()),
3510 reset_required: Some(true),
3511 }],
3512 };
3513 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3514 let outcome = client.process_response(&mut transport, response, &meta);
3515 let SyncOutcome::Ok(report) = outcome else {
3516 panic!("epoch round failed: {outcome:?}");
3517 };
3518 assert_eq!(
3519 report.resets,
3520 if stored.is_none() {
3521 Vec::<String>::new()
3522 } else {
3523 vec!["tasks".into()]
3524 }
3525 );
3526 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3527 assert_eq!(client.pending_commit_ids(), vec![id]);
3528 assert_eq!(
3529 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3530 json!("offline")
3531 );
3532 let trigger: i64 = client
3533 .conn
3534 .query_row(
3535 "SELECT count(*) FROM sqlite_master WHERE name='epoch_table_identity'",
3536 [],
3537 |row| row.get(0),
3538 )
3539 .unwrap();
3540 if stored.is_none() {
3541 assert!(!client.upgrading);
3542 assert!(client
3543 .drain_change_batches()
3544 .iter()
3545 .all(|batch| batch.status.as_ref().is_none_or(|status| !status.upgrading)));
3546 assert_eq!(
3547 serde_json::to_value(client.subscription_state("tasks")).unwrap(),
3548 before
3549 );
3550 assert_eq!(trigger, 1, "acquisition must not drop the synced table");
3551 } else {
3552 assert!(client.upgrading);
3553 assert_eq!(trigger, 0, "an epoch change must drop the old table");
3554 }
3555 assert!(matches!(
3556 client.drain_sync_intents().as_slice(),
3557 [SyncIntent::Interactive]
3558 ));
3559 }
3560 }
3561
3562 #[test]
3563 fn epoch_acquisition_storage_failures_preserve_ready_state_and_intent() {
3564 for key in ["logEpoch", "localRevision"] {
3565 let mut client = client();
3566 let id = client
3567 .mutate(vec![Mutation::Upsert {
3568 table: "tasks".into(),
3569 values: Map::from_iter([
3570 ("id".into(), json!("offline")),
3571 ("project_id".into(), json!("p1")),
3572 ]),
3573 base_version: None,
3574 }])
3575 .unwrap();
3576 let revision = client.local_revision();
3577 let needed = client.sync_needed;
3578 client.drain_change_batches();
3579 client.drain_sync_intents();
3580 client.conn.execute_batch(&format!("CREATE TRIGGER fail_acquisition BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected acquisition failure'); END")).unwrap();
3581 assert!(client.run_log_epoch_reset("epoch-new").is_err());
3582 assert!(client.get_meta(LOG_EPOCH_KEY).is_none());
3583 assert!(!client.upgrading);
3584 assert_eq!(client.sync_needed, needed);
3585 assert_eq!(client.local_revision(), revision);
3586 assert_eq!(client.pending_commit_ids(), [id]);
3587 assert!(client.drain_change_batches().is_empty());
3588 assert!(client.drain_sync_intents().is_empty());
3589 }
3590 }
3591
3592 #[test]
3593 fn failed_log_epoch_resets_restore_memory_and_durable_state() {
3594 for key in [
3595 "localSchemaVersion",
3596 "localSchemaDescriptor",
3597 "logEpoch",
3598 "localRevision",
3599 ] {
3600 let mut client = client();
3601 client.set_meta(LOG_EPOCH_KEY, "epoch-old");
3602 let commit = client
3603 .mutate(vec![Mutation::Upsert {
3604 table: "tasks".into(),
3605 values: Map::from_iter([
3606 ("id".into(), json!("offline")),
3607 ("project_id".into(), json!("p1")),
3608 ]),
3609 base_version: None,
3610 }])
3611 .unwrap();
3612 client.active_round = Some(uuid::Uuid::new_v4());
3613 client.overlay_dirty.set(false);
3614 let round = client.active_round;
3615 let needed = client.sync_needed;
3616 let revision = client.local_revision();
3617 let rows = serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap();
3618 client.drain_change_batches();
3619 client.drain_sync_intents();
3620 client.conn.execute_batch(&format!("CREATE TRIGGER fail_reset BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected reset failure'); END")).unwrap();
3621 assert!(client.run_log_epoch_reset("epoch-new").is_err(), "{key}");
3622 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-old"));
3623 assert!(!client.upgrading);
3624 assert_eq!(client.active_round, round);
3625 assert!(!client.overlay_dirty.get());
3626 assert_eq!(client.sync_needed, needed);
3627 assert_eq!(client.local_revision(), revision);
3628 assert_eq!(
3629 client.pending_commit_ids().as_slice(),
3630 std::slice::from_ref(&commit)
3631 );
3632 assert_eq!(
3633 serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap(),
3634 rows
3635 );
3636 assert!(client.drain_change_batches().is_empty());
3637 assert!(client.drain_sync_intents().is_empty());
3638 assert!(client.conn.is_autocommit());
3639 client
3640 .conn
3641 .execute_batch("DROP TRIGGER fail_reset")
3642 .unwrap();
3643 client.run_log_epoch_reset("epoch-new").unwrap();
3644 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3645 assert_eq!(client.pending_commit_ids(), [commit]);
3646 }
3647 }
3648
3649 #[test]
3650 fn acknowledged_restore_and_replay_ignore_unrelated_subscription_count() {
3651 for subscriptions in [0, 2, 6, 12] {
3652 let mut client = SyncClient::new("ack-scope".into(), &json!({ "version": 1, "tables": [
3653 { "name": "tasks", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] },
3654 { "name": "docs", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] }
3655 ] }), ClientLimits::default()).unwrap();
3656 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3657 client.conn.execute_batch("CREATE TABLE ack_writes(kind TEXT); CREATE TRIGGER count_ack_restore AFTER DELETE ON tasks BEGIN INSERT INTO ack_writes VALUES('restore'); END; CREATE TRIGGER count_ack_replay AFTER INSERT ON tasks BEGIN INSERT INTO ack_writes VALUES('replay'); END;").unwrap();
3658 for index in 0..32 {
3659 client
3660 .mutate(vec![Mutation::Upsert {
3661 table: "tasks".into(),
3662 values: Map::from_iter([
3663 ("id".into(), json!(format!("task-{index}"))),
3664 ("project_id".into(), json!("p1")),
3665 ]),
3666 base_version: None,
3667 }])
3668 .unwrap();
3669 }
3670 let (_, meta) = client.build_request(false).unwrap();
3671 let mut frames = vec![Frame::RespHeader {
3672 required_schema_version: None,
3673 latest_schema_version: None,
3674 log_epoch: Some("epoch-1".into()),
3675 reset_required: Some(false),
3676 }];
3677 frames.extend(meta.pushed_ids.iter().enumerate().map(|(index, id)| {
3678 Frame::PushResult {
3679 client_commit_id: id.clone(),
3680 status: PushStatus::Applied,
3681 commit_seq: Some(index as i64 + 1),
3682 results: vec![OpResult::Applied { op_index: 0 }],
3683 }
3684 }));
3685 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3686 client.acknowledged_replay_count.set(0);
3687 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3688 assert!(matches!(
3689 client.process_response(
3690 &mut transport,
3691 Message {
3692 wire_version: WIRE_VERSION,
3693 msg_kind: MsgKind::Response,
3694 frames
3695 },
3696 &meta
3697 ),
3698 SyncOutcome::Ok(_)
3699 ));
3700 assert_eq!(client.acknowledged_replay_count.get(), 32);
3701 assert_eq!(
3702 client
3703 .query(
3704 "SELECT count(*) AS n FROM ack_writes WHERE kind='restore'",
3705 &[]
3706 )
3707 .unwrap()[0]["n"],
3708 json!(32)
3709 );
3710 assert_eq!(
3711 client
3712 .query(
3713 "SELECT count(*) AS n FROM ack_writes WHERE kind='replay'",
3714 &[]
3715 )
3716 .unwrap()[0]["n"],
3717 json!(32)
3718 );
3719 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3720 client.acknowledged_replay_count.set(0);
3721 for index in 0..subscriptions {
3722 client
3723 .subscribe(
3724 format!("docs-{index}"),
3725 "docs".into(),
3726 vec![("project_id".into(), vec![format!("scope-{index}")])],
3727 None,
3728 )
3729 .unwrap();
3730 let sub = client.subs.len() - 1;
3731 client.subs[sub].effective =
3732 Some(vec![("project_id".into(), vec![format!("scope-{index}")])]);
3733 let segment = RowsSegment {
3734 table: "docs".into(),
3735 schema_version: 1,
3736 columns: client.schema.table("docs").unwrap().wire_columns.clone(),
3737 blocks: if index % 2 == 0 {
3738 vec![vec![ssp2::segment::SegmentRow {
3739 values: vec![
3740 Some(ColumnValue::String(format!("doc-{index}"))),
3741 Some(ColumnValue::String(format!("scope-{index}"))),
3742 ],
3743 server_version: 1,
3744 }]]
3745 } else {
3746 vec![]
3747 },
3748 };
3749 assert!(client.apply_segment(sub, &segment, true).is_ok());
3750 }
3751 assert_eq!(
3752 client
3753 .query("SELECT count(*) AS n FROM ack_writes", &[])
3754 .unwrap()[0]["n"],
3755 json!(0),
3756 "unrelated bootstrap must not restore/replay tasks"
3757 );
3758 assert_eq!(
3759 client.acknowledged_replay_count.get(),
3760 0,
3761 "unrelated bootstrap must not decode task ACKs"
3762 );
3763 assert_eq!(
3764 client
3765 .query("SELECT count(*) AS n FROM tasks", &[])
3766 .unwrap()[0]["n"],
3767 json!(32)
3768 );
3769 assert_eq!(
3770 client
3771 .query("SELECT count(*) AS n FROM _syncular_acked_rows", &[])
3772 .unwrap()[0]["n"],
3773 json!(32)
3774 );
3775 }
3776 }
3777
3778 #[test]
3779 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
3780 let mut client = client();
3781 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3782 const COMMIT_COUNT: usize = 32;
3783
3784 for index in 0..COMMIT_COUNT {
3785 client
3786 .mutate(vec![Mutation::Upsert {
3787 table: "tasks".to_owned(),
3788 values: Map::from_iter([
3789 ("id".to_owned(), Value::from(format!("task-{index}"))),
3790 ("project_id".to_owned(), Value::from("project-1")),
3791 ]),
3792 base_version: None,
3793 }])
3794 .expect("queue commit");
3795 }
3796
3797 let (_, request_meta) = client.build_request(false).unwrap();
3798 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
3799 let mut frames = vec![Frame::RespHeader {
3800 required_schema_version: None,
3801 latest_schema_version: None,
3802 log_epoch: Some("epoch-1".to_owned()),
3803 reset_required: Some(false),
3804 }];
3805 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
3806 |(index, client_commit_id)| Frame::PushResult {
3807 client_commit_id: client_commit_id.clone(),
3808 status: PushStatus::Applied,
3809 commit_seq: Some(index as i64 + 1),
3810 results: vec![OpResult::Applied { op_index: 0 }],
3811 },
3812 ));
3813 let response = Message {
3814 wire_version: WIRE_VERSION,
3815 msg_kind: MsgKind::Response,
3816 frames,
3817 };
3818 let mut transport =
3819 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
3820
3821 client.overlay_rebuild_count.set(0);
3822 client.outcome_prune_count.set(0);
3823 let outcome = client.process_response(&mut transport, response, &request_meta);
3824 assert!(matches!(outcome, SyncOutcome::Ok(_)));
3825 assert!(client.pending_commit_ids().is_empty());
3826 assert_eq!(
3827 client.overlay_rebuild_count.get(),
3828 1,
3829 "one response must reconcile its acknowledged commits with one overlay rebuild"
3830 );
3831 assert_eq!(
3832 client.outcome_prune_count.get(),
3833 1,
3834 "one response must enforce outcome retention once"
3835 );
3836 }
3837
3838 #[test]
3839 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
3840 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
3841 "columns": [
3842 {"name":"id", "type":"string", "nullable":false},
3843 {"name":"project_id", "type":"string", "nullable":false},
3844 {"name":"title", "type":"string", "nullable":false}
3845 ], "scopes": [{"pattern":"project:{project_id}"}],
3846 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
3847 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
3848 }] });
3849 let mut incremental =
3850 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3851 let mut reference =
3852 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3853 let tables = vec!["tasks".to_owned()];
3854 for index in 0..32 {
3855 let id = format!("task-{}", index % 8);
3856 let values = Map::from_iter([
3857 ("id".to_owned(), Value::from(id.clone())),
3858 (
3859 "project_id".to_owned(),
3860 Value::from(format!("p{}", index % 2)),
3861 ),
3862 ("title".to_owned(), Value::from(format!("needle{index}"))),
3863 ]);
3864 let payload = encode_row_json(
3865 incremental.schema.table("tasks").unwrap(),
3866 &id,
3867 &values,
3868 &incremental.encryption,
3869 )
3870 .unwrap();
3871 let change = ssp2::model::Change {
3872 table_index: 0,
3873 row_id: id,
3874 op: if index % 5 == 0 {
3875 Op::Delete
3876 } else {
3877 Op::Upsert
3878 },
3879 row_version: Some(index + 1),
3880 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
3881 row: Some(payload),
3882 };
3883 reference.overlay_dirty.set(true);
3884 for instance in [&mut incremental, &mut reference] {
3885 instance.begin_observation("mirror_test").unwrap();
3886 let mut batch = ChangeAccumulator::default();
3887 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
3888 instance
3889 .apply_commit_changes(&tables, std::slice::from_ref(&change))
3890 .unwrap();
3891 instance
3892 .rebuild_overlay_if_dirty()
3893 .expect("rebuild overlay");
3894 instance.finish_observation("mirror_test", batch).unwrap();
3895 }
3896 for sql in [
3897 "SELECT * FROM tasks ORDER BY id",
3898 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
3899 ] {
3900 assert_eq!(
3901 incremental.query(sql, &[]).unwrap(),
3902 reference.query(sql, &[]).unwrap()
3903 );
3904 }
3905 assert_eq!(incremental.local_revision(), reference.local_revision());
3906 assert_eq!(
3907 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3908 serde_json::to_value(reference.drain_change_batches()).unwrap()
3909 );
3910 }
3911 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
3912 assert_eq!(reference.overlay_rebuild_count.get(), 32);
3913 }
3914
3915 #[test]
3916 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
3917 let cases = [
3918 (
3919 "string",
3920 vec![
3921 json!("1"),
3922 json!("01"),
3923 json!(""),
3924 json!("nul\u{0}id"),
3925 json!("é"),
3926 ],
3927 ),
3928 (
3929 "integer",
3930 vec![
3931 json!(i64::MIN),
3932 json!(-1),
3933 json!(0),
3934 json!(1),
3935 json!(i64::MAX),
3936 ],
3937 ),
3938 ("boolean", vec![json!(false), json!(true)]),
3939 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
3940 ];
3941 for (kind, ids) in cases {
3942 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3943 "columns":[{"name":"id","type":kind,"nullable":false},
3944 {"name":"project_id","type":"string","nullable":false}],
3945 "scopes":[{"pattern":"project:{project_id}"}]}]});
3946 let mut client =
3947 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
3948 for (index, id) in ids.iter().enumerate() {
3949 let table = client.schema.table("tasks").unwrap();
3950 client
3951 .write_base_row(
3952 "tasks",
3953 &vec![
3954 if kind == "integer" {
3955 Some(ColumnValue::Integer(id.as_i64().unwrap()))
3958 } else {
3959 json_to_column_value(&table.columns[0], Some(id)).unwrap()
3960 },
3961 Some(ColumnValue::String(format!("p{index}"))),
3962 ],
3963 1,
3964 )
3965 .unwrap();
3966 }
3967 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3968 for base in [true, false] {
3969 let full = if base {
3970 base_table("tasks")
3971 } else {
3972 visible_table("tasks")
3973 };
3974 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
3975 for row_id in [
3976 "1",
3977 "01",
3978 "1.0",
3979 "1.0000000000000002",
3980 "+1",
3981 "1e0",
3982 " 1",
3983 "-1",
3984 "0",
3985 "9223372036854775807",
3986 "-9223372036854775808",
3987 "true",
3988 "false",
3989 "",
3990 "nul\u{0}id",
3991 "é",
3992 "null",
3993 "{\"a\":1}",
3994 ] {
3995 let old_sql =
3996 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
3997 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
3998 let old = client
3999 .conn
4000 .prepare(&old_sql)
4001 .unwrap()
4002 .query_map([row_id], |row| row.get::<_, String>(0))
4003 .unwrap()
4004 .collect::<Result<Vec<_>, _>>()
4005 .unwrap();
4006 let new = client
4007 .conn
4008 .prepare_cached(&new_sql)
4009 .unwrap()
4010 .query_map([row_id], |row| row.get::<_, String>(0))
4011 .unwrap()
4012 .collect::<Result<Vec<_>, _>>()
4013 .unwrap();
4014 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
4015 let mut batch = ChangeAccumulator::default();
4016 assert_eq!(
4017 client.record_row_scopes(&mut batch, "tasks", row_id, base),
4018 !old.is_empty()
4019 );
4020 let expected = old.first().map(|scope| {
4021 BTreeMap::from([(
4022 "tasks".to_owned(),
4023 Some(BTreeSet::from([format!("project:{scope}")])),
4024 )])
4025 });
4026 assert_eq!(batch.tables, expected.unwrap_or_default());
4027 client
4028 .conn
4029 .execute_batch("SAVEPOINT lookup_delete")
4030 .unwrap();
4031 if base {
4032 client.delete_base_row("tasks", row_id).unwrap();
4033 } else {
4034 client
4035 .apply_outbox_ops(&[OutboxOp {
4036 upsert: false,
4037 table: "tasks".into(),
4038 row_id: row_id.into(),
4039 base_version: None,
4040 values: None,
4041 }])
4042 .expect("replay delete");
4043 }
4044 let remaining: i64 = client
4045 .conn
4046 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
4047 row.get(0)
4048 })
4049 .unwrap();
4050 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
4051 client
4052 .conn
4053 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
4054 .unwrap();
4055 client.overlay_dirty.set(false);
4056 }
4057 for (predicate, expected) in [
4058 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
4059 (predicate, "SEARCH"),
4060 ] {
4061 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
4062 let details = client
4063 .conn
4064 .prepare(&sql)
4065 .unwrap()
4066 .query_map(["1"], |row| row.get::<_, String>(3))
4067 .unwrap()
4068 .collect::<Result<Vec<_>, _>>()
4069 .unwrap();
4070 assert!(
4071 details.iter().any(|detail| detail.contains(expected)),
4072 "{kind}: {details:?}"
4073 );
4074 }
4075 }
4076 }
4077 }
4078
4079 #[test]
4080 fn primary_keys_reject_every_ineligible_column_type() {
4081 for ty in ["float", "bytes", "crdt", "blob_ref"] {
4085 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4086 "columns":[{"name":"id","type":ty,"nullable":false},
4087 {"name":"project_id","type":"string","nullable":false}],
4088 "scopes":[{"pattern":"project:{project_id}"}]}]});
4089 let error =
4090 match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
4091 Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
4092 Err(error) => error,
4093 };
4094 assert!(
4095 error.contains("has an ineligible column type (§2.4)"),
4096 "{ty}: {error}"
4097 );
4098 }
4099 for ty in ["string", "integer", "boolean", "json"] {
4100 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4101 "columns":[{"name":"id","type":ty,"nullable":false},
4102 {"name":"project_id","type":"string","nullable":false}],
4103 "scopes":[{"pattern":"project:{project_id}"}]}]});
4104 SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
4105 .unwrap_or_else(|error| panic!("{ty}: {error}"));
4106 }
4107 }
4108
4109 #[test]
4110 fn pending_row_reconciliation_matches_full_fifo_replay() {
4111 for unique in [false, true] {
4112 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
4113 "name": name, "primaryKey": "id", "columns": [
4114 {"name":"id", "type":"string", "nullable":false},
4115 {"name":"project_id", "type":"string", "nullable":false},
4116 {"name":"title", "type":"string", "nullable":false}],
4117 "scopes": [{"pattern":"project:{project_id}"}],
4118 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
4119 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
4120 })))});
4121 let mut incremental =
4122 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
4123 let mut reference =
4124 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
4125 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
4126 for instance in [&mut incremental, &mut reference] {
4127 for table in &tables {
4128 for index in 0..12 {
4129 instance
4130 .write_base_row(
4131 table,
4132 &vec![
4133 Some(ColumnValue::String(format!("row{index}"))),
4134 Some(ColumnValue::String("p0".into())),
4135 Some(ColumnValue::String(format!("base{index}"))),
4136 ],
4137 1,
4138 )
4139 .unwrap();
4140 }
4141 }
4142 instance
4143 .rebuild_overlay_if_dirty()
4144 .expect("rebuild overlay");
4145 }
4146 for index in 0..80 {
4147 let mutations = (0..3)
4148 .map(|offset| {
4149 let table = tables[(index + offset) % 2].clone();
4150 let id = format!("row{}", (index + offset) % 16);
4151 if (index + offset) % 7 == 0 {
4152 Mutation::Delete {
4153 table,
4154 row_id: id,
4155 base_version: None,
4156 }
4157 } else {
4158 Mutation::Upsert {
4159 table,
4160 values: Map::from_iter([
4161 ("id".into(), Value::from(id)),
4162 ("project_id".into(), Value::from(format!("p{}", index % 3))),
4163 (
4164 "title".into(),
4165 Value::from(format!("needle{}", (index + offset) % 16)),
4166 ),
4167 ]),
4168 base_version: None,
4169 }
4170 }
4171 })
4172 .collect::<Vec<_>>();
4173 for instance in [&mut incremental, &mut reference] {
4174 instance.mutate(mutations.clone()).unwrap();
4175 }
4176 let changes = (0..3)
4179 .map(|offset| {
4180 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
4181 let id = format!("row{}", (index + offset) % 16);
4182 let values = Map::from_iter([
4183 ("id".into(), Value::from(id.clone())),
4184 (
4185 "project_id".into(),
4186 Value::from(format!("p{}", (index + 1) % 3)),
4187 ),
4188 (
4189 "title".into(),
4190 Value::from(format!("remote{index}_{offset}")),
4191 ),
4192 ]);
4193 ssp2::model::Change {
4194 table_index: table_index as u16,
4195 row_id: id.clone(),
4196 op: if (index + offset) % 5 == 0 {
4197 Op::Delete
4198 } else {
4199 Op::Upsert
4200 },
4201 row_version: Some(index as i64 + 2),
4202 scopes: vec![],
4203 row: Some(
4204 encode_row_json(
4205 incremental.schema.table(&tables[table_index]).unwrap(),
4206 &id,
4207 &values,
4208 &incremental.encryption,
4209 )
4210 .unwrap(),
4211 ),
4212 }
4213 })
4214 .collect::<Vec<_>>();
4215 reference.overlay_dirty.set(true);
4216 incremental
4217 .apply_commit_frame(&tables, &changes, None)
4218 .unwrap();
4219 reference
4220 .apply_commit_frame(&tables, &changes, None)
4221 .unwrap();
4222 for table in &tables {
4223 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
4224 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
4225 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
4226 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
4227 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
4228 }
4229 }
4230 assert_eq!(incremental.local_revision(), reference.local_revision());
4231 assert_eq!(
4232 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4233 serde_json::to_value(reference.drain_change_batches()).unwrap()
4234 );
4235 }
4236 assert_eq!(
4237 incremental.overlay_rebuild_count.get(),
4238 if unique { 81 } else { 21 }
4239 );
4240 assert_eq!(reference.overlay_rebuild_count.get(), 81);
4241 }
4242 }
4243
4244 #[test]
4245 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
4246 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
4247 "columns":[{"name":"id", "type":"string", "nullable":false},
4248 {"name":"title", "type":"string", "nullable":false}],
4249 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
4250 let mut client =
4251 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
4252 for (id, title) in [("a", "original"), ("b", "occupied")] {
4253 client
4254 .write_base_row(
4255 "tasks",
4256 &vec![
4257 Some(ColumnValue::String(id.into())),
4258 Some(ColumnValue::String(title.into())),
4259 ],
4260 1,
4261 )
4262 .unwrap();
4263 }
4264 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4265 let pending = client
4266 .mutate(vec![Mutation::Upsert {
4267 table: "tasks".into(),
4268 values: Map::from_iter([
4269 ("id".into(), Value::from("a")),
4270 ("title".into(), Value::from("available")),
4271 ]),
4272 base_version: None,
4273 }])
4274 .unwrap();
4275 assert_eq!(
4276 client
4277 .conn
4278 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4279 .get::<_, String>(
4280 0
4281 ))
4282 .unwrap(),
4283 "available"
4284 );
4285 for (title, expected) in [("available", "original"), ("occupied", "available")] {
4286 let change = ssp2::model::Change {
4287 table_index: 0,
4288 row_id: "b".into(),
4289 op: Op::Upsert,
4290 row_version: Some(2),
4291 scopes: vec![],
4292 row: Some(
4293 encode_row_json(
4294 client.schema.table("tasks").unwrap(),
4295 "b",
4296 &Map::from_iter([
4297 ("id".into(), Value::from("b")),
4298 ("title".into(), Value::from(title)),
4299 ]),
4300 &client.encryption,
4301 )
4302 .unwrap(),
4303 ),
4304 };
4305 client
4306 .apply_commit_frame(&["tasks".into()], &[change], None)
4307 .unwrap();
4308 assert_eq!(
4309 client
4310 .conn
4311 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4312 .get::<_, String>(
4313 0
4314 ))
4315 .unwrap(),
4316 expected
4317 );
4318 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
4319 }
4320 assert_eq!(client.overlay_rebuild_count.get(), 3);
4321 }
4322
4323 fn measure_scoped_overlay(unchanged_rows: i64, rounds: i64) -> Vec<u128> {
4324 let schema = json!({ "version": 1, "tables": [
4325 { "name":"tasks", "primaryKey":"id", "columns":[
4326 {"name":"id","type":"string","nullable":false},
4327 {"name":"project_id","type":"string","nullable":false}],
4328 "scopes":[{"pattern":"project:{project_id}"}] },
4329 { "name":"catalogue", "primaryKey":"id", "columns":[
4330 {"name":"id","type":"string","nullable":false},
4331 {"name":"project_id","type":"string","nullable":false},
4332 {"name":"title","type":"string","nullable":false}],
4333 "scopes":[{"pattern":"project:{project_id}"}],
4334 "ftsIndexes":[{"name":"catalogue_fts","columns":["title"],"tokenize":"unicode61"}] }
4335 ] });
4336 let mut client =
4337 SyncClient::new("overlay-scope".into(), &schema, ClientLimits::default()).unwrap();
4338 if unchanged_rows > 0 {
4339 client.conn.execute_batch(&format!("WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i<{unchanged_rows}) INSERT INTO _syncular_base_catalogue SELECT 'c'||i,'p1','needle '||i,1 FROM n")).unwrap();
4340 }
4341 client.rebuild_overlay().expect("rebuild overlay");
4342 client.conn.execute_batch("CREATE TABLE audit(tbl TEXT); CREATE TRIGGER catalogue_changed AFTER DELETE ON catalogue BEGIN INSERT INTO audit VALUES ('catalogue'); END;").unwrap();
4343 let mut elapsed = Vec::new();
4344 for round in 1..=rounds {
4345 let pending = client
4346 .mutate(vec![Mutation::Upsert {
4347 table: "tasks".into(),
4348 values: Map::from_iter([
4349 ("id".into(), json!("t1")),
4350 ("project_id".into(), json!("p1")),
4351 ]),
4352 base_version: None,
4353 }])
4354 .unwrap();
4355 let started = std::time::Instant::now();
4356 client
4357 .write_base_row(
4358 "tasks",
4359 &vec![
4360 Some(ColumnValue::String("t1".into())),
4361 Some(ColumnValue::String("p1".into())),
4362 ],
4363 round,
4364 )
4365 .unwrap();
4366 let mut report = SyncReport::default();
4367 client
4368 .handle_push_results(
4369 &[Frame::PushResult {
4370 client_commit_id: pending.clone(),
4371 status: PushStatus::Applied,
4372 commit_seq: Some(round),
4373 results: vec![OpResult::Applied { op_index: 0 }],
4374 }],
4375 &HashSet::from([pending.as_str()]),
4376 &HashMap::new(),
4377 &mut report,
4378 Some(&pending),
4379 )
4380 .unwrap();
4381 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4382 elapsed.push(started.elapsed().as_nanos());
4383 }
4384 assert_eq!(
4385 client
4386 .conn
4387 .query_row("SELECT count(*) FROM audit", [], |r| r.get::<_, i64>(0))
4388 .unwrap(),
4389 0,
4390 "unchanged catalogue must never be copied"
4391 );
4392 assert_eq!(
4393 client
4394 .conn
4395 .query_row(
4396 "SELECT count(*) FROM catalogue_fts WHERE catalogue_fts MATCH 'needle'",
4397 [],
4398 |r| r.get::<_, i64>(0)
4399 )
4400 .unwrap(),
4401 unchanged_rows
4402 );
4403 assert_eq!(
4404 client.query("SELECT * FROM tasks", &[]).unwrap()[0]["_syncular_version"],
4405 json!(rounds)
4406 );
4407 elapsed
4408 }
4409
4410 #[test]
4411 fn changed_overlay_leaves_large_unchanged_table_and_fts_untouched() {
4412 let elapsed = measure_scoped_overlay(100_000, 1);
4413 eprintln!(
4414 "table-scoped overlay with 100k unchanged rows: {} ns",
4415 elapsed[0]
4416 );
4417 }
4418
4419 #[test]
4420 #[ignore = "isolated performance measurement; run with the shared heavy-check lock"]
4421 fn benchmark_scoped_overlay() {
4422 for rows in [0, 100_000, 200_000] {
4423 let mut elapsed = measure_scoped_overlay(rows, 30);
4424 elapsed.sort_unstable();
4425 eprintln!(
4426 "{}",
4427 json!({"unchangedRows": rows, "rounds": elapsed.len(),
4428 "medianNs": elapsed[elapsed.len()/2], "minNs": elapsed[0],
4429 "maxNs": elapsed[elapsed.len()-1], "unchangedDeletes": 0})
4430 );
4431 }
4432 }
4433
4434 #[test]
4435 fn released_context_discards_in_flight_ack_and_server_rows() {
4436 for barrier in [
4437 "preflight",
4438 "purge",
4439 "rebootstrap",
4440 "subscription",
4441 "failed-purge-replay",
4442 "failed-purge-publish",
4443 "failed-rebootstrap",
4444 ] {
4445 let mut client = client();
4446 client.set_meta(LOG_EPOCH_KEY, "epoch");
4447 client
4448 .subscribe(
4449 "tasks".into(),
4450 "tasks".into(),
4451 vec![("project_id".into(), vec!["p1".into()])],
4452 None,
4453 )
4454 .unwrap();
4455 let commit = client
4456 .mutate(vec![Mutation::Upsert {
4457 table: "tasks".into(),
4458 values: Map::from_iter([
4459 ("id".into(), json!("local")),
4460 ("project_id".into(), json!("p1")),
4461 ]),
4462 base_version: None,
4463 }])
4464 .unwrap();
4465 let prepared = client.prepare_sync_round(false).unwrap();
4466 client.set_transport_enabled(&mut CountingRealtimeTransport::default(), false);
4467 let payload = encode_row_json(
4468 &client.schema.tables[0],
4469 "server",
4470 &Map::from_iter([
4471 ("id".into(), json!("server")),
4472 ("project_id".into(), json!("p1")),
4473 ]),
4474 &client.encryption,
4475 )
4476 .unwrap();
4477 let completed = crate::CompletedSyncRound {
4478 prepared,
4479 exchange: crate::round::ExchangeResult::Reply {
4480 transport_failed: false,
4481 downloads: Default::default(),
4482 response: Ok(Message {
4483 wire_version: WIRE_VERSION,
4484 msg_kind: MsgKind::Response,
4485 frames: vec![
4486 Frame::RespHeader {
4487 required_schema_version: None,
4488 latest_schema_version: None,
4489 log_epoch: Some("epoch".into()),
4490 reset_required: Some(false),
4491 },
4492 Frame::PushResult {
4493 client_commit_id: commit.clone(),
4494 status: PushStatus::Applied,
4495 commit_seq: Some(1),
4496 results: vec![OpResult::Applied { op_index: 0 }],
4497 },
4498 Frame::SubStart {
4499 id: "tasks".into(),
4500 status: SubStatus::Active,
4501 reason_code: String::new(),
4502 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
4503 bootstrap: true,
4504 },
4505 Frame::Commit {
4506 commit_seq: 1,
4507 created_at_ms: 0,
4508 actor_id: "actor".into(),
4509 tables: vec!["tasks".into()],
4510 changes: vec![ssp2::model::Change {
4511 table_index: 0,
4512 row_id: "server".into(),
4513 op: Op::Upsert,
4514 row_version: Some(1),
4515 scopes: vec![("project_id".into(), "p1".into())],
4516 row: Some(payload),
4517 }],
4518 },
4519 Frame::SubEnd {
4520 next_cursor: 1,
4521 bootstrap_state: None,
4522 },
4523 ],
4524 }),
4525 },
4526 };
4527 match barrier {
4528 "preflight" => client.begin_security_preflight(),
4529 "purge" => {
4530 client
4531 .purge_local_data(&LocalDataPurgeInput {
4532 purge_id: "pending-round".into(),
4533 targets: vec![LocalDataPurgeTarget {
4534 table: "tasks".into(),
4535 selectors: BTreeMap::from([(
4536 "project_id".into(),
4537 vec!["p1".into()],
4538 )]),
4539 }],
4540 })
4541 .unwrap();
4542 }
4543 "rebootstrap" => {
4544 client
4545 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4546 rebootstrap_id: "pending-round".into(),
4547 })
4548 .unwrap();
4549 }
4550 "failed-purge-replay" | "failed-purge-publish" | "failed-rebootstrap" => {
4551 let trigger = if barrier == "failed-purge-replay" {
4552 "CREATE TRIGGER fail_local BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4553 } else {
4554 "CREATE TRIGGER fail_local BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4555 };
4556 client.conn.execute_batch(trigger).unwrap();
4557 let revision = client.local_revision();
4558 if barrier == "failed-rebootstrap" {
4559 assert!(client
4560 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4561 rebootstrap_id: "failed-round".into(),
4562 })
4563 .is_err());
4564 } else {
4565 assert!(client
4566 .purge_local_data(&LocalDataPurgeInput {
4567 purge_id: "failed-round".into(),
4568 targets: vec![LocalDataPurgeTarget {
4569 table: "tasks".into(),
4570 selectors: BTreeMap::from([(
4571 "project_id".into(),
4572 vec!["p1".into()]
4573 )]),
4574 }],
4575 })
4576 .is_err());
4577 }
4578 assert_eq!(client.local_revision(), revision);
4579 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
4580 client
4581 .conn
4582 .execute_batch("DROP TRIGGER fail_local")
4583 .unwrap();
4584 let applied = client.apply_sync_round(completed);
4585 assert!(
4586 matches!(
4587 applied,
4588 crate::AppliedSyncRound::Complete {
4589 outcome: SyncOutcome::Ok(_),
4590 ..
4591 }
4592 ),
4593 "{barrier}"
4594 );
4595 assert!(client.pending_commit_ids().is_empty());
4596 assert_eq!(
4597 client
4598 .query("SELECT id FROM tasks WHERE id = 'server'", &[])
4599 .unwrap()
4600 .len(),
4601 1
4602 );
4603 continue;
4604 }
4605 "subscription" => {
4606 client.unsubscribe("tasks");
4607 client
4608 .subscribe(
4609 "tasks".into(),
4610 "tasks".into(),
4611 vec![("project_id".into(), vec!["p2".into()])],
4612 None,
4613 )
4614 .unwrap();
4615 }
4616 _ => unreachable!(),
4617 }
4618 let pending_before_reply = client.pending_commit_ids();
4619 let revision = client.local_revision();
4620 let applied = client.apply_sync_round(completed);
4621 assert!(
4622 matches!(applied, crate::AppliedSyncRound::Complete { outcome: SyncOutcome::Failed { ref error_code, .. }, ref controls, .. } if error_code == "client.round_cancelled" && controls.is_empty())
4623 );
4624 assert_eq!(client.local_revision(), revision);
4625 assert!(client
4626 .conn
4627 .query_row(
4628 "SELECT NOT EXISTS(SELECT 1 FROM tasks WHERE id='server')",
4629 [],
4630 |r| r.get::<_, bool>(0)
4631 )
4632 .unwrap());
4633 assert_eq!(client.pending_commit_ids(), pending_before_reply);
4634 }
4635 }
4636
4637 #[test]
4638 fn sync_round_deadline_is_anchored_once_per_round_and_carried_across_continues() {
4639 struct DeadlineRecording {
4640 budget: std::time::Duration,
4641 seen: Vec<Option<std::time::Instant>>,
4642 }
4643 impl Transport for DeadlineRecording {
4644 fn round_deadline(&self) -> Option<std::time::Duration> {
4645 Some(self.budget)
4646 }
4647 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4648 self.seen.push(deadline);
4649 }
4650 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4651 Err(TransportError::new("transport.failed", "injected"))
4652 }
4653 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4654 self.sync(request)
4655 }
4656 fn download_segment(
4657 &mut self,
4658 _: &SegmentRequest,
4659 _: &mut dyn FnMut(u64),
4660 ) -> Result<Vec<u8>, TransportError> {
4661 Err(TransportError::new("transport.failed", "injected"))
4662 }
4663 fn blob_upload_grant(
4664 &mut self,
4665 _: &str,
4666 _: u64,
4667 _: Option<&str>,
4668 ) -> Result<crate::BlobUploadGrant, TransportError> {
4669 Ok(crate::BlobUploadGrant::Present)
4670 }
4671 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4672 Ok(())
4673 }
4674 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4675 Ok(())
4676 }
4677 fn realtime_close(&mut self) -> Result<(), TransportError> {
4678 Ok(())
4679 }
4680 }
4681
4682 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4683 {"name":"id","type":"string","nullable":false},
4684 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4685 let mut client =
4686 SyncClient::new("deadline-round".into(), &schema, Default::default()).unwrap();
4687 client.set_meta(LOG_EPOCH_KEY, "epoch");
4688 client
4689 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4690 .unwrap();
4691
4692 let mut transport = DeadlineRecording {
4693 budget: std::time::Duration::from_secs(30),
4694 seen: Vec::new(),
4695 };
4696 let prepared = client.prepare_sync_round(false).unwrap();
4697 assert_eq!(prepared.uploads.len(), 1, "one staged upload");
4698 let prepared = match client.apply_sync_round(prepared.exchange(&mut transport)) {
4700 crate::AppliedSyncRound::Continue(prepared) => prepared,
4701 crate::AppliedSyncRound::Complete { .. } => panic!("the upload must continue"),
4702 };
4703 let completed = prepared.exchange(&mut transport);
4705 let anchor = completed
4706 .prepared
4707 .round_deadline_at
4708 .expect("the upload round anchors a deadline");
4709 assert!(matches!(
4710 client.apply_sync_round(completed),
4711 crate::AppliedSyncRound::Complete { .. }
4712 ));
4713 assert_eq!(transport.seen.len(), 4, "one scope per exchange");
4714 assert_eq!(
4715 transport.seen[0],
4716 Some(anchor),
4717 "the upload anchors the round"
4718 );
4719 assert_eq!(
4720 transport.seen[1], None,
4721 "the upload exchange clears its scope"
4722 );
4723 assert_eq!(
4724 transport.seen[2],
4725 Some(anchor),
4726 "a continuation reuses the carried anchor instead of refreshing it"
4727 );
4728 assert_eq!(transport.seen[3], None, "the continuation clears its scope");
4729
4730 transport.budget = std::time::Duration::from_secs(60);
4733 let prepared = client.prepare_sync_round(false).unwrap();
4734 let completed = prepared.exchange(&mut transport);
4735 let fresh = completed
4736 .prepared
4737 .round_deadline_at
4738 .expect("the next round anchors a deadline");
4739 client.apply_sync_round(completed);
4740 assert!(
4741 fresh >= anchor + std::time::Duration::from_secs(30),
4742 "the next round anchors a fresh deadline from the new budget"
4743 );
4744 assert_eq!(transport.seen[4], Some(fresh));
4745 assert_eq!(
4746 transport.seen[5], None,
4747 "the new round clears its scope too"
4748 );
4749 }
4750
4751 #[derive(Default)]
4754 struct RoundDeadlineProbe {
4755 budget: Option<std::time::Duration>,
4756 budget_queries: std::cell::Cell<usize>,
4757 scoped: Vec<Option<std::time::Instant>>,
4758 }
4759
4760 impl Transport for RoundDeadlineProbe {
4761 fn round_deadline(&self) -> Option<std::time::Duration> {
4762 self.budget_queries.set(self.budget_queries.get() + 1);
4763 self.budget
4764 }
4765 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4766 self.scoped.push(deadline);
4767 }
4768 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4769 Err(TransportError::new("transport.failed", "injected"))
4770 }
4771 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4772 self.sync(request)
4773 }
4774 fn download_segment(
4775 &mut self,
4776 _: &SegmentRequest,
4777 _: &mut dyn FnMut(u64),
4778 ) -> Result<Vec<u8>, TransportError> {
4779 Err(TransportError::new("transport.failed", "injected"))
4780 }
4781 fn blob_upload_grant(
4782 &mut self,
4783 _: &str,
4784 _: u64,
4785 _: Option<&str>,
4786 ) -> Result<crate::BlobUploadGrant, TransportError> {
4787 Ok(crate::BlobUploadGrant::Present)
4788 }
4789 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4790 Ok(())
4791 }
4792 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4793 Ok(())
4794 }
4795 fn realtime_close(&mut self) -> Result<(), TransportError> {
4796 Ok(())
4797 }
4798 }
4799
4800 #[test]
4801 fn an_unusable_upload_body_does_not_anchor_the_round_deadline() {
4802 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4803 {"name":"id","type":"string","nullable":false},
4804 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4805 let mut client =
4806 SyncClient::new("deadline-unusable".into(), &schema, ClientLimits::default()).unwrap();
4807 client.set_meta(LOG_EPOCH_KEY, "epoch");
4808 client
4809 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4810 .unwrap();
4811 client
4812 .conn
4813 .execute_batch("DELETE FROM _syncular_blobs")
4814 .unwrap();
4815 let prepared = client.prepare_sync_round(false).unwrap();
4816 assert!(
4817 prepared.uploads.front().is_some_and(|u| u.bytes.is_err()),
4818 "the staged body must fail locally"
4819 );
4820 let mut transport = RoundDeadlineProbe {
4821 budget: Some(std::time::Duration::from_secs(30)),
4822 ..Default::default()
4823 };
4824 let completed = prepared.exchange(&mut transport);
4825 assert!(matches!(
4826 completed.exchange,
4827 crate::round::ExchangeResult::Upload { result: Err(_), .. }
4828 ));
4829 assert_eq!(
4830 transport.budget_queries.get(),
4831 0,
4832 "a round with no network work never asks for a budget"
4833 );
4834 assert_eq!(transport.scoped, vec![None]);
4835 }
4836
4837 #[test]
4838 fn a_transport_budget_the_clock_cannot_represent_fails_before_network() {
4839 struct HugeBudget;
4840 impl Transport for HugeBudget {
4841 fn round_deadline(&self) -> Option<std::time::Duration> {
4842 Some(std::time::Duration::MAX)
4843 }
4844 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4845 panic!("no network for an invalid budget")
4846 }
4847 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4848 self.sync(request)
4849 }
4850 fn download_segment(
4851 &mut self,
4852 _: &SegmentRequest,
4853 _: &mut dyn FnMut(u64),
4854 ) -> Result<Vec<u8>, TransportError> {
4855 panic!("no network for an invalid budget")
4856 }
4857 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4858 Ok(())
4859 }
4860 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4861 Ok(())
4862 }
4863 fn realtime_close(&mut self) -> Result<(), TransportError> {
4864 Ok(())
4865 }
4866 }
4867
4868 let mut client = client();
4869 client.set_meta(LOG_EPOCH_KEY, "epoch");
4870 let prepared = client.prepare_sync_round(false).unwrap();
4871 let outcome = match client.apply_sync_round(prepared.exchange(&mut HugeBudget)) {
4872 crate::AppliedSyncRound::Complete { outcome, .. } => outcome,
4873 crate::AppliedSyncRound::Continue(_) => {
4874 panic!("an unrepresentable budget must not continue")
4875 }
4876 };
4877 match outcome {
4878 SyncOutcome::Failed {
4879 error_code,
4880 message,
4881 ..
4882 } => {
4883 assert_eq!(error_code, "sync.invalid_request");
4884 assert_eq!(message, "transport round deadline is out of range");
4885 }
4886 _ => panic!("an unrepresentable budget must fail synchronously"),
4887 }
4888 }
4889
4890 #[test]
4891 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
4892 for failure in ["visible", "revision", "commit"] {
4893 let mut client = client();
4894 let id = client
4895 .mutate(vec![Mutation::Upsert {
4896 table: "tasks".into(),
4897 values: Map::from_iter([
4898 ("id".into(), Value::from("t1")),
4899 ("project_id".into(), Value::from("pending")),
4900 ]),
4901 base_version: None,
4902 }])
4903 .unwrap();
4904 client.drain_change_batches();
4905 let revision = client.local_revision();
4906 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
4907 let trigger = match failure {
4908 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
4909 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
4910 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
4911 };
4912 client.conn.execute_batch(&trigger).unwrap();
4913 let change = ssp2::model::Change {
4914 table_index: 0,
4915 row_id: "t1".into(),
4916 op: Op::Upsert,
4917 row_version: Some(2),
4918 scopes: vec![],
4919 row: Some(
4920 encode_row_json(
4921 client.schema.table("tasks").unwrap(),
4922 "t1",
4923 &Map::from_iter([
4924 ("id".into(), Value::from("t1")),
4925 ("project_id".into(), Value::from("remote")),
4926 ]),
4927 &client.encryption,
4928 )
4929 .unwrap(),
4930 ),
4931 };
4932 assert!(client
4933 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), None)
4934 .is_err());
4935 assert!(client.conn.is_autocommit());
4936 assert!(!client.overlay_dirty.get());
4937 assert_eq!(client.local_revision(), revision);
4938 assert!(client.drain_change_batches().is_empty());
4939 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
4940 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
4941 assert!(client
4942 .query("SELECT * FROM _syncular_base_tasks", &[])
4943 .unwrap()
4944 .is_empty());
4945 client
4946 .conn
4947 .execute_batch("DROP TRIGGER fail_pending")
4948 .unwrap();
4949 client
4950 .apply_commit_frame(&["tasks".into()], &[change], None)
4951 .unwrap();
4952 assert_eq!(client.local_revision(), revision + 1);
4953 assert_eq!(client.pending_commit_ids(), vec![id]);
4954 let mut expected = before[0].clone();
4957 expected.insert("_syncular_version".to_owned(), Value::from(2));
4958 assert_eq!(
4959 client.query("SELECT * FROM tasks", &[]).unwrap(),
4960 vec![expected]
4961 );
4962 assert_eq!(
4963 client
4964 .conn
4965 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
4966 .get::<_, String>(0))
4967 .unwrap(),
4968 "remote"
4969 );
4970 }
4971 }
4972
4973 #[test]
4974 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
4975 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
4976 "columns": [
4977 {"name":"id", "type":"string", "nullable":false},
4978 {"name":"project_id", "type":"string", "nullable":false},
4979 {"name":"title", "type":"string", "nullable":false}],
4980 "scopes": [{"pattern":"project:{project_id}"}],
4981 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
4982 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
4983 let mut incremental =
4984 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4985 let mut reference =
4986 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4987 for instance in [&mut incremental, &mut reference] {
4988 for index in 0..16 {
4989 instance
4990 .write_base_row(
4991 "tasks",
4992 &vec![
4993 Some(ColumnValue::String(format!("task{index}"))),
4994 Some(ColumnValue::String("p0".to_owned())),
4995 Some(ColumnValue::String(format!("base{index}"))),
4996 ],
4997 1,
4998 )
4999 .unwrap();
5000 }
5001 instance
5002 .rebuild_overlay_if_dirty()
5003 .expect("rebuild overlay");
5004 instance.overlay_rebuild_count.set(0);
5005 }
5006 for index in 0..100 {
5007 let mutations = (0..1 + index % 4)
5008 .map(|offset| {
5009 let id = format!("task{}", (index + offset) % 24);
5010 if (index + offset) % 7 == 0 {
5011 Mutation::Delete {
5012 table: "tasks".to_owned(),
5013 row_id: id,
5014 base_version: None,
5015 }
5016 } else {
5017 Mutation::Upsert {
5018 table: "tasks".to_owned(),
5019 values: Map::from_iter([
5020 ("id".to_owned(), Value::from(id)),
5021 (
5022 "project_id".to_owned(),
5023 Value::from(format!("p{}", index % 2)),
5024 ),
5025 (
5026 "title".to_owned(),
5027 Value::from(format!("needle{}", (index + offset) % 24)),
5028 ),
5029 ]),
5030 base_version: None,
5031 }
5032 }
5033 })
5034 .collect::<Vec<_>>();
5035 reference.overlay_dirty.set(true);
5036 incremental.mutate(mutations.clone()).unwrap();
5037 reference.mutate(mutations).unwrap();
5038 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
5039 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
5040 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
5041 }
5042 assert_eq!(incremental.local_revision(), reference.local_revision());
5043 assert_eq!(
5044 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
5045 serde_json::to_value(reference.drain_change_batches()).unwrap()
5046 );
5047 }
5048 assert_eq!(incremental.outbox.len(), 100);
5049 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
5050 assert_eq!(reference.overlay_rebuild_count.get(), 100);
5051 }
5052
5053 #[test]
5054 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
5055 for fail_at in ["outbox", "revision"] {
5056 let mut client = client();
5057 let values = Map::from_iter([
5058 ("id".to_owned(), Value::from("t1")),
5059 ("project_id".to_owned(), Value::from("p1")),
5060 ]);
5061 let mutation = Mutation::Upsert {
5062 table: "tasks".to_owned(),
5063 values,
5064 base_version: None,
5065 };
5066 let trigger = if fail_at == "outbox" {
5067 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
5068 } else {
5069 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
5070 };
5071 client.conn.execute_batch(&trigger).unwrap();
5072 let revision = client.local_revision();
5073 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
5074 assert_eq!(client.local_revision(), revision);
5075 assert!(client.outbox.is_empty());
5076 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
5077 assert_eq!(
5078 client
5079 .conn
5080 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5081 .get::<_, i64>(0))
5082 .unwrap(),
5083 0
5084 );
5085 assert!(client.drain_change_batches().is_empty());
5086 assert!(!client.overlay_dirty.get());
5087 client
5088 .conn
5089 .execute_batch("DROP TRIGGER fail_append")
5090 .unwrap();
5091 client.mutate(vec![mutation]).unwrap();
5092 assert_eq!(client.outbox.len(), 1);
5093 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
5094 }
5095 }
5096
5097 #[test]
5098 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
5099 let mut client = client();
5100 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
5101 let values = Map::from_iter([
5102 ("id".to_owned(), Value::from("t1")),
5103 ("project_id".to_owned(), Value::from("p1")),
5104 ]);
5105 let payload = encode_row_json(
5106 client.schema.table("tasks").unwrap(),
5107 "t1",
5108 &values,
5109 &client.encryption,
5110 )
5111 .unwrap();
5112 let first = ssp2::model::Change {
5113 table_index: 0,
5114 row_id: "t1".to_owned(),
5115 op: Op::Upsert,
5116 row_version: Some(1),
5117 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
5118 row: Some(payload),
5119 };
5120 let mut invalid = first.clone();
5121 invalid.table_index = 1;
5122 let revision = client.local_revision();
5123 client.begin_observation("failed_mirror").unwrap();
5124 assert!(client
5125 .apply_commit_changes(&tables, &[first, invalid])
5126 .is_err());
5127 client.rollback_observation("failed_mirror");
5128 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
5129 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
5130 assert_eq!(
5131 client
5132 .conn
5133 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
5134 .get::<_, i64>(
5135 0
5136 ))
5137 .unwrap(),
5138 0
5139 );
5140 assert_eq!(client.local_revision(), revision);
5141 assert!(client.drain_change_batches().is_empty());
5142 }
5143
5144 #[test]
5145 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
5146 for failure in ["journal", "revision", "commit", "later-frame"] {
5147 let mut client = client();
5148 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5149 for id in ["first", "second"] {
5150 client
5151 .mutate(vec![Mutation::Upsert {
5152 table: "tasks".to_owned(),
5153 values: Map::from_iter([
5154 ("id".to_owned(), Value::from(id)),
5155 ("project_id".to_owned(), Value::from("p1")),
5156 ]),
5157 base_version: None,
5158 }])
5159 .unwrap();
5160 }
5161 let (_, meta) = client.build_request(false).unwrap();
5162 client
5164 .mutate(vec![Mutation::Upsert {
5165 table: "tasks".to_owned(),
5166 values: Map::from_iter([
5167 ("id".to_owned(), Value::from("later")),
5168 ("project_id".to_owned(), Value::from("p1")),
5169 ]),
5170 base_version: None,
5171 }])
5172 .unwrap();
5173 let ids = client.pending_commit_ids();
5174 let rows = client
5175 .query("SELECT * FROM tasks ORDER BY id", &[])
5176 .unwrap();
5177 let revision = client.local_revision();
5178 client.drain_change_batches();
5179 if failure != "later-frame" {
5180 client.conn.execute_batch(match failure {
5181 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
5182 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
5183 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
5184 }).unwrap();
5185 }
5186 let mut response = Message {
5187 wire_version: WIRE_VERSION,
5188 msg_kind: MsgKind::Response,
5189 frames: vec![Frame::RespHeader {
5190 required_schema_version: None,
5191 latest_schema_version: None,
5192 log_epoch: Some("epoch-1".to_owned()),
5193 reset_required: Some(false),
5194 }],
5195 };
5196 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
5197 response.frames.push(Frame::PushResult {
5198 client_commit_id: id.to_owned(),
5199 status: PushStatus::Applied,
5200 commit_seq: Some(1),
5201 results: vec![OpResult::Applied { op_index: 0 }],
5202 });
5203 }
5204 if failure == "later-frame" {
5205 response.frames.push(Frame::Error {
5206 code: "sync.invalid_request".to_owned(),
5207 message: "later error".to_owned(),
5208 category: "protocol".to_owned(),
5209 retryable: false,
5210 recommended_action: "retry".to_owned(),
5211 details: None,
5212 });
5213 }
5214 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5215 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5216 let expected = match failure {
5219 "later-frame" => "sync.invalid_request",
5220 "commit" => "storage.failed",
5221 _ => "client.outcome_persistence_failed",
5222 };
5223 assert!(
5224 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5225 "{failure}: {outcome:?}"
5226 );
5227 if failure == "later-frame" {
5228 assert_eq!(client.pending_commit_ids(), ids[2..]);
5229 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5230 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
5231 let batches = client.drain_change_batches();
5232 assert_eq!(batches.len(), 1);
5233 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
5234 assert_eq!(client.local_revision(), revision + 1);
5235 response.frames.pop();
5236 } else {
5237 assert_eq!(client.pending_commit_ids(), ids);
5238 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5239 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
5240 assert_eq!(client.local_revision(), revision);
5241 assert!(client.drain_change_batches().is_empty());
5242 assert_eq!(
5243 client
5244 .query("SELECT * FROM tasks ORDER BY id", &[])
5245 .unwrap(),
5246 rows
5247 );
5248 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
5249 }
5250 for frame in &mut response.frames {
5251 if let Frame::PushResult { status, .. } = frame {
5252 *status = PushStatus::Cached;
5253 }
5254 }
5255 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
5256 else {
5257 panic!("retry failed");
5258 };
5259 assert_eq!(
5260 report.applied,
5261 if failure == "later-frame" {
5262 vec![]
5263 } else {
5264 ids[..2].to_vec()
5265 }
5266 );
5267 assert_eq!(client.pending_commit_ids(), ids[2..]);
5268 let batches = client.drain_change_batches();
5269 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
5270 if let Some(batch) = batches.first() {
5271 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
5272 }
5273 assert_eq!(client.local_revision(), revision + 1);
5274 }
5275 }
5276
5277 #[test]
5278 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
5279 for failure in ["revision", "journal", "commit"] {
5280 for status in ["applied", "cached", "rejected", "conflict"] {
5281 let mut client = client();
5282 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5283 for id in ["first", "later"] {
5284 client
5285 .mutate(vec![Mutation::Upsert {
5286 table: "tasks".to_owned(),
5287 values: Map::from_iter([
5288 ("id".to_owned(), Value::from(id)),
5289 ("project_id".to_owned(), Value::from("p1")),
5290 ]),
5291 base_version: None,
5292 }])
5293 .unwrap();
5294 }
5295 let (_, meta) = client.build_request(false).unwrap();
5296 let ids = client.pending_commit_ids();
5297 let rows = client
5298 .query("SELECT * FROM tasks ORDER BY id", &[])
5299 .unwrap();
5300 let revision = client.local_revision();
5301 client.drain_change_batches();
5302 let (push_status, results) = match status {
5303 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
5304 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
5305 "conflict" => (
5306 PushStatus::Rejected,
5307 vec![OpResult::Conflict {
5308 op_index: 0,
5309 code: "sync.version_conflict".to_owned(),
5310 message: "conflict".to_owned(),
5311 server_version: 1,
5312 server_row: encode_row_json(
5313 client.schema.table("tasks").unwrap(),
5314 "first",
5315 &Map::from_iter([
5316 ("id".to_owned(), Value::from("first")),
5317 ("project_id".to_owned(), Value::from("p1")),
5318 ]),
5319 &client.encryption,
5320 )
5321 .unwrap(),
5322 conflict_columns: Vec::new(),
5323 }],
5324 ),
5325 _ => (
5326 PushStatus::Rejected,
5327 vec![OpResult::Error {
5328 op_index: 0,
5329 code: "sync.validation_failed".to_owned(),
5330 message: "rejected".to_owned(),
5331 retryable: false,
5332 }],
5333 ),
5334 };
5335 let response = Message {
5336 wire_version: WIRE_VERSION,
5337 msg_kind: MsgKind::Response,
5338 frames: vec![
5339 Frame::RespHeader {
5340 required_schema_version: None,
5341 latest_schema_version: None,
5342 log_epoch: Some("epoch-1".to_owned()),
5343 reset_required: Some(false),
5344 },
5345 Frame::PushResult {
5346 client_commit_id: ids[0].clone(),
5347 status: push_status,
5348 commit_seq: Some(1),
5349 results,
5350 },
5351 ],
5352 };
5353 client.conn.execute_batch(match failure {
5354 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
5355 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
5356 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
5357 }).unwrap();
5358 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5359 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5360 assert_eq!(
5361 client.pending_commit_ids(),
5362 ids,
5363 "{failure}/{status}: pending identity"
5364 );
5365 assert_eq!(
5366 client.local_revision(),
5367 revision,
5368 "{failure}/{status}: revision"
5369 );
5370 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5371 assert!(client.conflicts().is_empty());
5372 assert!(client.rejections().is_empty());
5373 assert!(client.drain_change_batches().is_empty());
5374 assert_eq!(
5375 client
5376 .query("SELECT * FROM tasks ORDER BY id", &[])
5377 .unwrap(),
5378 rows
5379 );
5380 let expected = if failure == "commit" {
5383 "storage.failed"
5384 } else {
5385 "client.outcome_persistence_failed"
5386 };
5387 assert!(
5388 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5389 "{failure}/{status}: failed round"
5390 );
5391 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
5392 assert!(matches!(
5393 client.process_response(&mut transport, response, &meta),
5394 SyncOutcome::Ok(_)
5395 ));
5396 assert_eq!(client.pending_commit_ids(), ids[1..]);
5397 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5398 assert_eq!(client.drain_change_batches().len(), 1);
5399 }
5400 }
5401 }
5402
5403 #[test]
5404 fn mixed_push_results_reconcile_each_boundary_and_prune_once_per_response() {
5405 let mut client = client();
5406 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5407 for index in 0..4 {
5408 client
5409 .mutate(vec![Mutation::Upsert {
5410 table: "tasks".to_owned(),
5411 values: Map::from_iter([
5412 ("id".to_owned(), Value::from(format!("task-{index}"))),
5413 ("project_id".to_owned(), Value::from("project-1")),
5414 ]),
5415 base_version: None,
5416 }])
5417 .expect("queue commit");
5418 }
5419
5420 let (_, request_meta) = client.build_request(false).unwrap();
5421 let ids = &request_meta.pushed_ids;
5422 assert_eq!(ids.len(), 4);
5423 let response = Message {
5424 wire_version: WIRE_VERSION,
5425 msg_kind: MsgKind::Response,
5426 frames: vec![
5427 Frame::RespHeader {
5428 required_schema_version: None,
5429 latest_schema_version: None,
5430 log_epoch: Some("epoch-1".to_owned()),
5431 reset_required: Some(false),
5432 },
5433 Frame::PushResult {
5434 client_commit_id: ids[0].clone(),
5435 status: PushStatus::Applied,
5436 commit_seq: Some(1),
5437 results: vec![OpResult::Applied { op_index: 0 }],
5438 },
5439 Frame::PushResult {
5440 client_commit_id: ids[1].clone(),
5441 status: PushStatus::Cached,
5442 commit_seq: Some(2),
5443 results: vec![OpResult::Applied { op_index: 0 }],
5444 },
5445 Frame::PushResult {
5446 client_commit_id: ids[2].clone(),
5447 status: PushStatus::Rejected,
5448 commit_seq: None,
5449 results: vec![OpResult::Error {
5450 op_index: 0,
5451 code: "sync.validation_failed".to_owned(),
5452 message: "rejected".to_owned(),
5453 retryable: false,
5454 }],
5455 },
5456 Frame::PushResult {
5457 client_commit_id: ids[3].clone(),
5458 status: PushStatus::Rejected,
5459 commit_seq: None,
5460 results: vec![OpResult::Error {
5461 op_index: 0,
5462 code: "sync.idempotency_cache_miss".to_owned(),
5463 message: "retry".to_owned(),
5464 retryable: true,
5465 }],
5466 },
5467 ],
5468 };
5469 let mut transport =
5470 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
5471
5472 client.overlay_rebuild_count.set(0);
5473 client.outcome_prune_count.set(0);
5474 let outcome = client.process_response(&mut transport, response, &request_meta);
5475 let SyncOutcome::Ok(report) = outcome else {
5476 panic!("mixed push-result response failed");
5477 };
5478
5479 assert_eq!(report.applied, ids[..2]);
5480 assert_eq!(report.rejected, ids[2..3]);
5481 assert_eq!(report.retryable, ids[3..4]);
5482 assert_eq!(client.pending_commit_ids(), ids[3..4]);
5483 assert_eq!(
5484 client
5485 .query("SELECT id FROM tasks ORDER BY id", &[])
5486 .expect("query visible overlay"),
5487 ["task-0", "task-1", "task-3"]
5488 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
5489 .to_vec()
5490 );
5491 assert_eq!(client.overlay_rebuild_count.get(), 2);
5493 assert_eq!(client.outcome_prune_count.get(), 1);
5494 }
5495
5496 fn replay_schema(fts: bool) -> Value {
5497 json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
5498 {"name":"id","type":"string","nullable":false},
5499 {"name":"project_id","type":"string","nullable":false},
5500 {"name":"title","type":"string","nullable":false},
5501 {"name":"done","type":"boolean","nullable":false}
5502 ],"scopes":[{"pattern":"project:{project_id}"}], "ftsIndexes": if fts { json!([{"name":"tasks_fts","columns":["title"],"tokenize":"unicode61"}]) } else { json!([]) }}]})
5503 }
5504
5505 #[test]
5506 fn aborted_schema_and_epoch_resets_restore_memory_and_durable_intent() {
5507 for schema_reset in [false, true] {
5508 let mut schema = replay_schema(false);
5509 let mut client =
5510 SyncClient::new("reset-failure".into(), &schema, ClientLimits::default()).unwrap();
5511 let id = client
5512 .mutate(vec![Mutation::Upsert {
5513 table: "tasks".into(),
5514 values: Map::from_iter([
5515 ("id".into(), json!("queued")),
5516 ("project_id".into(), json!("p1")),
5517 ("title".into(), json!("old")),
5518 ("done".into(), json!(false)),
5519 ]),
5520 base_version: None,
5521 }])
5522 .unwrap();
5523 client.set_meta(LOG_EPOCH_KEY, "old");
5524 let revision = client.local_revision();
5525 let needed = client.sync_needed;
5526 client.drain_change_batches();
5527 client.drain_sync_intents();
5528 client.conn.execute_batch("CREATE TRIGGER refuse_revision BEFORE INSERT ON _syncular_meta WHEN NEW.key='localRevision' BEGIN SELECT RAISE(ABORT,'reset completion failed'); END;").unwrap();
5529 if schema_reset {
5530 schema["version"] = json!(2);
5531 schema["tables"][0]["columns"]
5532 .as_array_mut()
5533 .unwrap()
5534 .retain(|column| column["name"] != "title");
5535 client.schema = parse_schema_json(&schema).unwrap();
5536 assert!(client.run_schema_reset().is_err());
5537 } else {
5538 assert!(client.run_log_epoch_reset("new").is_err());
5539 }
5540 assert!(!client.upgrading);
5541 assert_eq!(client.sync_needed, needed);
5542 assert_eq!(client.local_revision(), revision);
5543 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5544 assert!(client.rejections.is_empty());
5545 assert!(client.commit_outcome(&id).unwrap().is_none());
5546 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("old"));
5547 assert_eq!(
5548 client
5549 .query("SELECT title FROM tasks WHERE id='queued'", &[])
5550 .unwrap()[0]["title"],
5551 "old"
5552 );
5553 assert_eq!(
5554 client
5555 .conn
5556 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5557 .get::<_, i64>(0))
5558 .unwrap(),
5559 1
5560 );
5561 assert!(client.drain_change_batches().is_empty());
5562 assert!(client.drain_sync_intents().is_empty());
5563 assert!(client.conn.is_autocommit());
5564 }
5565 }
5566
5567 #[test]
5568 fn cached_blob_cleanup_propagates_delete_failures() {
5569 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5570 {"name":"id","type":"string","nullable":false},
5571 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5572 let mut client =
5573 SyncClient::new("blob-cleanup".into(), &schema, ClientLimits::default()).unwrap();
5574 client.conn.execute_batch("CREATE TRIGGER refuse_blob_cleanup BEFORE DELETE ON _syncular_blobs BEGIN SELECT RAISE(ABORT,'blob cleanup failed'); END;").unwrap();
5575 client.conn.execute("INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,created_at_ms) VALUES ('test',X'01',1,0)", []).unwrap();
5576 client
5577 .subscribe("attachments".into(), "attachments".into(), vec![], None)
5578 .unwrap();
5579 client.subs[0].effective = Some(vec![]);
5580 client.persist_sub(&client.subs[0]).unwrap();
5581 let revision = client.local_revision();
5582 let (_, meta) = client.build_request(false).unwrap();
5583 let mut transport = CountingRealtimeTransport::default();
5584 let mut report = SyncReport::default();
5585 assert!(client
5586 .process_section(
5587 &mut transport,
5588 "attachments",
5589 SubStatus::Revoked,
5590 "sync.scope_revoked",
5591 vec![],
5592 vec![],
5593 Some((0, None)),
5594 &meta,
5595 &mut report
5596 )
5597 .is_err());
5598 assert_eq!(client.subs[0].state, SubState::Active);
5599 assert_eq!(client.local_revision(), revision);
5600 assert!(report.revoked.is_empty());
5601 assert!(client.conn.is_autocommit());
5602 client
5603 .conn
5604 .execute_batch("DROP TRIGGER refuse_blob_cleanup")
5605 .unwrap();
5606 assert!(client
5607 .process_section(
5608 &mut transport,
5609 "attachments",
5610 SubStatus::Revoked,
5611 "sync.scope_revoked",
5612 vec![],
5613 vec![],
5614 Some((0, None)),
5615 &meta,
5616 &mut report
5617 )
5618 .is_ok());
5619 assert_eq!(client.subs[0].state, SubState::Revoked);
5620 assert_eq!(
5621 client
5622 .conn
5623 .query_row("SELECT count(*) FROM _syncular_blobs", [], |row| row
5624 .get::<_, i64>(0))
5625 .unwrap(),
5626 0
5627 );
5628 }
5629
5630 #[test]
5631 fn overlay_replay_trigger_refuses_reopen_and_preserves_durable_intent() {
5632 let path =
5633 std::env::temp_dir().join(format!("syncular-replay-{}.db", uuid::Uuid::new_v4()));
5634 let schema = replay_schema(false);
5635 let mut client = SyncClient::open_path(
5636 "replay".into(),
5637 &schema,
5638 ClientLimits::default(),
5639 path.to_str().unwrap(),
5640 )
5641 .unwrap();
5642 let ids = ["early", "bad"].map(|id| {
5643 client
5644 .mutate(vec![Mutation::Upsert {
5645 table: "tasks".into(),
5646 values: Map::from_iter([
5647 ("id".into(), json!(id)),
5648 ("project_id".into(), json!("p1")),
5649 ("title".into(), json!(id)),
5650 ("done".into(), json!(false)),
5651 ]),
5652 base_version: None,
5653 }])
5654 .unwrap()
5655 });
5656 client.conn.execute_batch("DELETE FROM tasks; CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.id='bad' BEGIN SELECT RAISE(ABORT,'replay failed'); END;").unwrap();
5657 let revision = client.local_revision();
5658 drop(client);
5659 let error = SyncClient::open_path(
5660 "replay".into(),
5661 &schema,
5662 ClientLimits::default(),
5663 path.to_str().unwrap(),
5664 )
5665 .err()
5666 .expect("reopen must fail");
5667 assert!(error.contains("replay failed"));
5668 let conn = Connection::open(&path).unwrap();
5669 assert_eq!(
5670 conn.query_row("SELECT count(*) FROM tasks", [], |row| row.get::<_, i64>(0))
5671 .unwrap(),
5672 0
5673 );
5674 assert_eq!(
5675 conn.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5676 .get::<_, i64>(0))
5677 .unwrap(),
5678 2
5679 );
5680 assert_eq!(
5681 meta_get(&conn, LOCAL_REVISION_KEY),
5682 Some(revision.to_string())
5683 );
5684 conn.execute_batch("DROP TRIGGER fail_replay").unwrap();
5685 drop(conn);
5686 let reopened = SyncClient::open_path(
5687 "replay".into(),
5688 &schema,
5689 ClientLimits::default(),
5690 path.to_str().unwrap(),
5691 )
5692 .unwrap();
5693 assert_eq!(reopened.pending_commit_ids(), ids);
5694 assert_eq!(
5695 reopened
5696 .query("SELECT id FROM tasks ORDER BY id", &[])
5697 .unwrap()
5698 .len(),
5699 2
5700 );
5701 drop(reopened);
5702 std::fs::remove_file(path).unwrap();
5703 }
5704
5705 #[test]
5706 fn overlay_replay_failure_rolls_back_push_and_pull_apply_boundaries() {
5707 let schema = replay_schema(false);
5708 let mut client =
5709 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5710 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5711 client
5712 .write_base_row(
5713 "tasks",
5714 &vec![
5715 Some(ColumnValue::String("queued".into())),
5716 Some(ColumnValue::String("p1".into())),
5717 Some(ColumnValue::String("server".into())),
5718 Some(ColumnValue::Boolean(false)),
5719 ],
5720 1,
5721 )
5722 .unwrap();
5723 client.rebuild_overlay().unwrap();
5724 let id = client
5725 .patch(
5726 "tasks",
5727 "queued",
5728 Map::from_iter([("title".into(), json!("local"))]),
5729 Some(1),
5730 )
5731 .unwrap();
5732 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
5733 let base = client
5734 .query("SELECT * FROM _syncular_base_tasks", &[])
5735 .unwrap();
5736 let revision = client.local_revision();
5737 client.drain_change_batches();
5738 client.conn.execute_batch("CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.title='local' BEGIN SELECT RAISE(ABORT,'replay failed'); END").unwrap();
5739 let remote = vec![
5740 Some(ColumnValue::String("queued".into())),
5741 Some(ColumnValue::String("p1".into())),
5742 Some(ColumnValue::String("remote".into())),
5743 Some(ColumnValue::Boolean(false)),
5744 ];
5745 let bytes = crate::values::encode_row_json(
5746 client.schema.table("tasks").unwrap(),
5747 "queued",
5748 &Map::from_iter(
5749 client
5750 .schema
5751 .table("tasks")
5752 .unwrap()
5753 .columns
5754 .iter()
5755 .zip(&remote)
5756 .map(|(column, value)| (column.name.clone(), column_value_to_json(value))),
5757 ),
5758 &client.encryption,
5759 )
5760 .unwrap();
5761 let change = ssp2::model::Change {
5762 table_index: 0,
5763 op: Op::Upsert,
5764 row_id: "queued".into(),
5765 row_version: Some(2),
5766 scopes: vec![("project_id".into(), "p1".into())],
5767 row: Some(bytes),
5768 };
5769 assert!(client
5770 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), Some(2))
5771 .is_err());
5772 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5773 assert_eq!(
5774 client
5775 .query("SELECT * FROM _syncular_base_tasks", &[])
5776 .unwrap(),
5777 base
5778 );
5779 assert_eq!(client.local_revision(), revision);
5780 assert!(client.drain_change_batches().is_empty());
5781 let response = Message {
5782 wire_version: 2,
5783 msg_kind: MsgKind::Response,
5784 frames: vec![
5785 Frame::RespHeader {
5786 required_schema_version: None,
5787 latest_schema_version: None,
5788 log_epoch: Some("epoch-1".into()),
5789 reset_required: Some(false),
5790 },
5791 Frame::PushResult {
5792 client_commit_id: id.clone(),
5793 status: PushStatus::Applied,
5794 commit_seq: Some(2),
5795 results: vec![OpResult::Applied { op_index: 0 }],
5796 },
5797 ],
5798 };
5799 let (_, meta) = client.build_request(false).unwrap();
5800 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5801 assert!(
5802 matches!(client.process_response(&mut transport,response.clone(),&meta),SyncOutcome::Failed { error_code,.. } if error_code=="client.outcome_persistence_failed")
5803 );
5804 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5805 assert!(client.commit_outcome(&id).unwrap().is_none());
5806 assert_eq!(
5807 client
5808 .conn
5809 .query_row("SELECT count(*) FROM _syncular_acked_rows", [], |row| row
5810 .get::<_, i64>(
5811 0
5812 ))
5813 .unwrap(),
5814 0
5815 );
5816 assert_eq!(
5817 client
5818 .conn
5819 .query_row("SELECT count(*) FROM _syncular_row_deliveries", [], |row| {
5820 row.get::<_, i64>(0)
5821 })
5822 .unwrap(),
5823 0
5824 );
5825 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5826 assert_eq!(client.local_revision(), revision);
5827 assert!(client.drain_change_batches().is_empty());
5828 client
5829 .conn
5830 .execute_batch("DROP TRIGGER fail_replay")
5831 .unwrap();
5832 assert!(matches!(
5833 client.process_response(&mut transport, response, &meta),
5834 SyncOutcome::Ok(_)
5835 ));
5836 client
5837 .apply_commit_frame(&["tasks".into()], &[change], Some(2))
5838 .unwrap();
5839 assert!(client.pending_commit_ids().is_empty());
5840 assert_eq!(
5841 client.query("SELECT title FROM tasks", &[]).unwrap()[0]["title"],
5842 json!("remote")
5843 );
5844 }
5845
5846 #[test]
5847 fn overlay_replay_distinguishes_absence_from_sql_and_value_decode_errors() {
5848 let client = SyncClient::new(
5849 "replay".into(),
5850 &replay_schema(false),
5851 ClientLimits::default(),
5852 )
5853 .unwrap();
5854 let table = client.schema.table("tasks").unwrap();
5855 let patch = OutboxOp {
5856 table: "tasks".into(),
5857 row_id: "absent".into(),
5858 upsert: true,
5859 base_version: Some(1),
5860 values: Some(Map::from_iter([
5861 ("id".into(), json!("absent")),
5862 ("title".into(), json!("patch")),
5863 ])),
5864 };
5865 assert!(client.visible_row(table, "absent").unwrap().is_none());
5866 client
5867 .apply_outbox_ops([&patch])
5868 .expect("sparse replay over truly absent row");
5869 client
5870 .conn
5871 .execute_batch("INSERT INTO tasks VALUES ('bad','p1','title','invalid',1)")
5872 .unwrap();
5873 assert!(client
5874 .visible_row(table, "bad")
5875 .unwrap_err()
5876 .starts_with("sync.local_corrupt:"));
5877 let mut invalid = patch.clone();
5878 invalid.values = Some(Map::from_iter([
5879 ("id".into(), json!("bad-value")),
5880 ("project_id".into(), json!("p1")),
5881 ("title".into(), json!("title")),
5882 ("done".into(), json!({"bad":true})),
5883 ]));
5884 assert!(client
5885 .apply_outbox_ops([&invalid])
5886 .unwrap_err()
5887 .starts_with("sync.local_corrupt:"));
5888 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5889 assert!(
5890 client.apply_outbox_ops([&patch]).is_err(),
5891 "SQL failure must not count as absence"
5892 );
5893 }
5894
5895 #[test]
5896 fn overlay_replay_keeps_secondary_unique_deferral_and_lazy_base_precedence() {
5897 let mut schema = replay_schema(false);
5898 schema["tables"][0]["indexes"] =
5899 json!([{"name":"tasks_project_title", "columns":["project_id","title"],"unique":true}]);
5900 let mut client =
5901 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5902 let id = client
5903 .mutate(vec![Mutation::Upsert {
5904 table: "tasks".into(),
5905 values: Map::from_iter([
5906 ("id".into(), json!("queued")),
5907 ("project_id".into(), json!("p1")),
5908 ("title".into(), json!("occupied")),
5909 ("done".into(), json!(false)),
5910 ]),
5911 base_version: None,
5912 }])
5913 .unwrap();
5914 client
5915 .write_base_row(
5916 "tasks",
5917 &vec![
5918 Some(ColumnValue::String("server".into())),
5919 Some(ColumnValue::String("p1".into())),
5920 Some(ColumnValue::String("occupied".into())),
5921 Some(ColumnValue::Boolean(false)),
5922 ],
5923 1,
5924 )
5925 .unwrap();
5926 client.rebuild_overlay_if_dirty().unwrap();
5927 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5928 assert!(client
5929 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5930 .unwrap()
5931 .is_none());
5932 assert!(client.rejections().is_empty());
5933 client.delete_base_row("tasks", "server").unwrap();
5934 client.rebuild_overlay_if_dirty().unwrap();
5935 assert!(client
5936 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5937 .unwrap()
5938 .is_some());
5939 client
5940 .write_base_row(
5941 "tasks",
5942 &vec![
5943 Some(ColumnValue::String("base".into())),
5944 Some(ColumnValue::String("p1".into())),
5945 Some(ColumnValue::String("base-title".into())),
5946 Some(ColumnValue::Boolean(false)),
5947 ],
5948 1,
5949 )
5950 .unwrap();
5951 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5952 client
5953 .retain_failed_commit(
5954 "retained",
5955 &[OutboxOp {
5956 table: "tasks".into(),
5957 row_id: "base".into(),
5958 upsert: true,
5959 base_version: Some(1),
5960 values: Some(Map::from_iter([
5961 ("id".into(), json!("base")),
5962 ("title".into(), json!("patch")),
5963 ])),
5964 }],
5965 )
5966 .expect("an existing base must not read the unavailable visible table");
5967 }
5968
5969 #[test]
5970 fn overlay_replay_fts_and_savepoint_release_failures_roll_back_all_visible_changes() {
5971 for failure in ["fts", "release"] {
5972 let mut client = SyncClient::new(
5973 "replay".into(),
5974 &replay_schema(true),
5975 ClientLimits::default(),
5976 )
5977 .unwrap();
5978 client
5979 .mutate(vec![Mutation::Upsert {
5980 table: "tasks".into(),
5981 values: Map::from_iter([
5982 ("id".into(), json!("queued")),
5983 ("project_id".into(), json!("p1")),
5984 ("title".into(), json!("needle")),
5985 ("done".into(), json!(false)),
5986 ]),
5987 base_version: None,
5988 }])
5989 .unwrap();
5990 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
5991 let fts_before = client
5992 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5993 .unwrap();
5994 let triggers_before: Vec<String> = client
5995 .conn
5996 .prepare("SELECT sql FROM sqlite_master WHERE type='trigger' ORDER BY name")
5997 .unwrap()
5998 .query_map([], |row| row.get(0))
5999 .unwrap()
6000 .collect::<Result<_, _>>()
6001 .unwrap();
6002 if failure == "fts" {
6003 client.conn.execute_batch("CREATE TRIGGER fail_fts BEFORE INSERT ON _syncular_fts_tasks_fts BEGIN SELECT RAISE(ABORT,'fts replay failed'); END;").unwrap();
6004 } else {
6005 client.conn.execute_batch("PRAGMA foreign_keys=ON; CREATE TABLE parent(id TEXT PRIMARY KEY); CREATE TABLE deferred(id TEXT REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_release AFTER INSERT ON tasks BEGIN INSERT INTO deferred VALUES ('missing'); END;").unwrap();
6006 }
6007 client.overlay_dirty.set(true);
6008 assert!(client.rebuild_overlay().is_err());
6009 assert!(client.overlay_dirty.get());
6010 assert!(client.conn.is_autocommit());
6011 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
6012 assert_eq!(
6013 client
6014 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
6015 .unwrap(),
6016 fts_before
6017 );
6018 let original_triggers: Vec<String> = client.conn.prepare("SELECT sql FROM sqlite_master WHERE type='trigger' AND name NOT LIKE 'fail_%' ORDER BY name").unwrap().query_map([], |row| row.get(0)).unwrap().collect::<Result<_,_>>().unwrap();
6019 assert_eq!(original_triggers, triggers_before);
6020 assert_eq!(client.pending_commit_ids().len(), 1);
6021 client
6022 .conn
6023 .execute_batch(if failure == "fts" {
6024 "DROP TRIGGER fail_fts"
6025 } else {
6026 "DROP TRIGGER fail_release"
6027 })
6028 .unwrap();
6029 client
6030 .rebuild_overlay()
6031 .expect("retry after removing the fault");
6032 }
6033 }
6034
6035 #[test]
6036 fn secondary_unique_collision_preserves_existing_synced_row() {
6037 let client = SyncClient::new(
6038 "unique-upsert-test".to_owned(),
6039 &json!({
6040 "version": 1,
6041 "tables": [{
6042 "name": "tasks",
6043 "primaryKey": "id",
6044 "columns": [
6045 { "name": "id", "type": "string", "nullable": false },
6046 { "name": "project_id", "type": "string", "nullable": false },
6047 { "name": "title", "type": "string", "nullable": false }
6048 ],
6049 "scopes": [{ "pattern": "project:{project_id}" }],
6050 "indexes": [{
6051 "name": "tasks_by_project_title",
6052 "columns": ["project_id", "title"],
6053 "unique": true
6054 }]
6055 }]
6056 }),
6057 ClientLimits::default(),
6058 )
6059 .expect("test client");
6060 let table = client.schema.table("tasks").expect("tasks table");
6061 let sql = client.insert_row_sql(&base_table("tasks"), table);
6062
6063 client
6064 .conn
6065 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
6066 .expect("insert first row");
6067 client
6068 .conn
6069 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
6070 .expect("update same primary key");
6071 client
6072 .conn
6073 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
6074 .expect("insert second row");
6075 assert!(client
6076 .conn
6077 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
6078 .is_err());
6079
6080 let rows = client
6081 .conn
6082 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
6083 .expect("prepare rows")
6084 .query_map([], |row| {
6085 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
6086 })
6087 .expect("query rows")
6088 .collect::<Result<Vec<_>, _>>()
6089 .expect("collect rows");
6090 assert_eq!(
6091 rows,
6092 vec![
6093 ("t1".to_owned(), "updated".to_owned()),
6094 ("t2".to_owned(), "original".to_owned())
6095 ]
6096 );
6097 }
6098
6099 #[test]
6100 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
6101 let path = std::env::temp_dir().join(format!(
6102 "syncular-startup-intent-{}.db",
6103 uuid::Uuid::new_v4()
6104 ));
6105 let schema = json!({
6106 "version": 1,
6107 "tables": [{
6108 "name": "tasks",
6109 "primaryKey": "id",
6110 "columns": [
6111 { "name": "id", "type": "string", "nullable": false },
6112 { "name": "project_id", "type": "string", "nullable": false }
6113 ],
6114 "scopes": [{ "pattern": "project:{project_id}" }]
6115 }]
6116 });
6117
6118 {
6119 let mut first = SyncClient::open_path_with_identity(
6120 None,
6121 &schema,
6122 ClientLimits::default(),
6123 path.to_str().expect("UTF-8 temp path"),
6124 )
6125 .expect("first open");
6126 first
6127 .set_window(
6128 &WindowBase {
6129 table: "tasks".to_owned(),
6130 variable: "project_id".to_owned(),
6131 fixed_scopes: Vec::new(),
6132 params: None,
6133 },
6134 &["persisted".to_owned()],
6135 )
6136 .expect("persist window");
6137 }
6138
6139 let mut reopened = SyncClient::open_path_with_identity(
6140 None,
6141 &schema,
6142 ClientLimits::default(),
6143 path.to_str().expect("UTF-8 temp path"),
6144 )
6145 .expect("reopen");
6146 assert!(reopened.sync_needed());
6147 assert!(matches!(
6148 reopened.drain_sync_intents().as_slice(),
6149 [SyncIntent::Interactive]
6150 ));
6151 drop(reopened);
6152 std::fs::remove_file(path).expect("remove temp database");
6153 }
6154
6155 #[test]
6156 fn reopening_preserves_immutable_subscription_identity_and_progress() {
6157 let path = std::env::temp_dir().join(format!(
6158 "syncular-subscription-identity-{}.db",
6159 uuid::Uuid::new_v4()
6160 ));
6161 let schema = json!({
6162 "version": 1,
6163 "tables": [
6164 {
6165 "name": "tasks",
6166 "primaryKey": "id",
6167 "columns": [
6168 { "name": "id", "type": "string", "nullable": false },
6169 { "name": "project_id", "type": "string", "nullable": false }
6170 ],
6171 "scopes": [{ "pattern": "project:{project_id}" }]
6172 },
6173 {
6174 "name": "docs",
6175 "primaryKey": "id",
6176 "columns": [
6177 { "name": "id", "type": "string", "nullable": false },
6178 { "name": "org_id", "type": "string", "nullable": false },
6179 { "name": "project_id", "type": "string", "nullable": false }
6180 ],
6181 "scopes": [
6182 { "pattern": "org:{org_id}" },
6183 { "pattern": "project:{projectId}", "column": "project_id" }
6184 ]
6185 }
6186 ]
6187 });
6188
6189 {
6190 let mut first = SyncClient::open_path_with_identity(
6191 None,
6192 &schema,
6193 ClientLimits::default(),
6194 path.to_str().expect("UTF-8 temp path"),
6195 )
6196 .expect("first open");
6197 first
6198 .subscribe(
6199 "stable-subscription".to_owned(),
6200 "tasks".to_owned(),
6201 vec![(
6202 "project_id".to_owned(),
6203 vec!["p2".to_owned(), "p1".to_owned()],
6204 )],
6205 Some(r#"{"view":"v1"}"#.to_owned()),
6206 )
6207 .expect("persist subscription");
6208 let persisted = {
6209 let subscription = first
6210 .subs
6211 .iter_mut()
6212 .find(|subscription| subscription.id == "stable-subscription")
6213 .expect("subscription");
6214 subscription.cursor = 41;
6215 subscription.bootstrap_state = Some("resume-token".to_owned());
6216 subscription.effective = Some(vec![(
6217 "project_id".to_owned(),
6218 vec!["p1".to_owned(), "p2".to_owned()],
6219 )]);
6220 subscription.synced_once = true;
6221 subscription.clone()
6222 };
6223 first.persist_sub(&persisted).unwrap();
6224 }
6225
6226 let mut reopened = SyncClient::open_path_with_identity(
6227 None,
6228 &schema,
6229 ClientLimits::default(),
6230 path.to_str().expect("UTF-8 temp path"),
6231 )
6232 .expect("reopen");
6233 let progress = reopened
6234 .subscription_state("stable-subscription")
6235 .expect("persisted state");
6236 assert_eq!(progress.cursor, 41);
6237 assert!(progress.has_resume_token);
6238
6239 reopened
6240 .subscribe(
6241 "stable-subscription".to_owned(),
6242 "tasks".to_owned(),
6243 vec![(
6244 "project_id".to_owned(),
6245 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
6246 )],
6247 Some(r#"{"view":"v1"}"#.to_owned()),
6248 )
6249 .expect("canonical intent is idempotent");
6250 assert_eq!(
6251 reopened
6252 .subscription_state("stable-subscription")
6253 .expect("unchanged state")
6254 .cursor,
6255 progress.cursor
6256 );
6257
6258 for (table, scopes, params) in [
6259 (
6260 "tasks",
6261 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6262 Some(r#"{"view":"v1"}"#.to_owned()),
6263 ),
6264 (
6265 "tasks",
6266 vec![(
6267 "project_id".to_owned(),
6268 vec!["p2".to_owned(), "p1".to_owned()],
6269 )],
6270 Some(r#"{"view":"v2"}"#.to_owned()),
6271 ),
6272 (
6273 "docs",
6274 vec![
6275 ("org_id".to_owned(), vec!["o1".to_owned()]),
6276 ("projectId".to_owned(), vec!["p1".to_owned()]),
6277 ],
6278 Some(r#"{"view":"v1"}"#.to_owned()),
6279 ),
6280 ] {
6281 let error = reopened
6282 .subscribe(
6283 "stable-subscription".to_owned(),
6284 table.to_owned(),
6285 scopes,
6286 params,
6287 )
6288 .expect_err("identity rebind must fail");
6289 assert!(error.starts_with("client.subscription_intent_mismatch:"));
6290 assert_eq!(
6291 reopened
6292 .subscription_state("stable-subscription")
6293 .expect("unchanged state")
6294 .cursor,
6295 progress.cursor
6296 );
6297 }
6298
6299 drop(reopened);
6300 std::fs::remove_file(path).expect("remove temp database");
6301 }
6302
6303 #[test]
6304 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
6305 let path = std::env::temp_dir().join(format!(
6306 "syncular-satisfied-schema-floor-{}.db",
6307 uuid::Uuid::new_v4()
6308 ));
6309 let schema = json!({
6310 "version": 23,
6311 "tables": [{
6312 "name": "tasks",
6313 "primaryKey": "id",
6314 "columns": [
6315 { "name": "id", "type": "string", "nullable": false },
6316 { "name": "project_id", "type": "string", "nullable": false }
6317 ],
6318 "scopes": [{ "pattern": "project:{project_id}" }]
6319 }]
6320 });
6321
6322 {
6323 let mut first = SyncClient::open_path_with_identity(
6324 None,
6325 &schema,
6326 ClientLimits::default(),
6327 path.to_str().expect("UTF-8 temp path"),
6328 )
6329 .expect("first open");
6330 first
6331 .subscribe(
6332 "tasks".to_owned(),
6333 "tasks".to_owned(),
6334 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6335 None,
6336 )
6337 .expect("persist subscription");
6338 first.set_schema_floor(Some(SchemaFloor {
6339 required_schema_version: Some(22),
6340 latest_schema_version: Some(22),
6341 }));
6342 }
6343
6344 let mut reopened = SyncClient::open_path_with_identity(
6345 None,
6346 &schema,
6347 ClientLimits::default(),
6348 path.to_str().expect("UTF-8 temp path"),
6349 )
6350 .expect("reopen");
6351 assert!(reopened.schema_floor().is_none());
6352 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
6353 assert!(reopened.sync_needed());
6354 assert!(matches!(
6355 reopened.drain_sync_intents().as_slice(),
6356 [SyncIntent::Interactive]
6357 ));
6358 drop(reopened);
6359 std::fs::remove_file(path).expect("remove temp database");
6360 }
6361
6362 #[test]
6363 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
6364 let path = std::env::temp_dir().join(format!(
6365 "syncular-unsatisfied-schema-floor-{}.db",
6366 uuid::Uuid::new_v4()
6367 ));
6368 let schema = json!({
6369 "version": 1,
6370 "tables": [{
6371 "name": "tasks",
6372 "primaryKey": "id",
6373 "columns": [
6374 { "name": "id", "type": "string", "nullable": false },
6375 { "name": "project_id", "type": "string", "nullable": false }
6376 ],
6377 "scopes": [{ "pattern": "project:{project_id}" }]
6378 }]
6379 });
6380
6381 {
6382 let mut first = SyncClient::open_path_with_identity(
6383 None,
6384 &schema,
6385 ClientLimits::default(),
6386 path.to_str().expect("UTF-8 temp path"),
6387 )
6388 .expect("first open");
6389 first
6390 .subscribe(
6391 "tasks".to_owned(),
6392 "tasks".to_owned(),
6393 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6394 None,
6395 )
6396 .expect("persist subscription");
6397 first.set_schema_floor(Some(SchemaFloor {
6398 required_schema_version: Some(2),
6399 latest_schema_version: Some(2),
6400 }));
6401 }
6402
6403 let reopened = SyncClient::open_path_with_identity(
6404 None,
6405 &schema,
6406 ClientLimits::default(),
6407 path.to_str().expect("UTF-8 temp path"),
6408 )
6409 .expect("reopen");
6410 assert_eq!(
6411 reopened.schema_floor(),
6412 Some(&SchemaFloor {
6413 required_schema_version: Some(2),
6414 latest_schema_version: Some(2),
6415 })
6416 );
6417 assert!(!reopened.sync_needed());
6418 drop(reopened);
6419 std::fs::remove_file(path).expect("remove temp database");
6420 }
6421
6422 #[test]
6423 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
6424 let path = std::env::temp_dir().join(format!(
6425 "syncular-indexed-column-bump-{}.db",
6426 uuid::Uuid::new_v4()
6427 ));
6428 let old_schema = json!({
6429 "version": 1,
6430 "tables": [
6431 {
6432 "name": "tasks",
6433 "primaryKey": "id",
6434 "columns": [
6435 { "name": "id", "type": "string", "nullable": false },
6436 { "name": "project_id", "type": "string", "nullable": false }
6437 ],
6438 "scopes": [{ "pattern": "project:{project_id}" }]
6439 },
6440 {
6441 "name": "legacy",
6442 "primaryKey": "id",
6443 "columns": [
6444 { "name": "id", "type": "string", "nullable": false },
6445 { "name": "project_id", "type": "string", "nullable": false }
6446 ],
6447 "scopes": [{ "pattern": "project:{project_id}" }]
6448 }
6449 ]
6450 });
6451 {
6452 let mut first = SyncClient::open_path_with_identity(
6453 None,
6454 &old_schema,
6455 ClientLimits::default(),
6456 path.to_str().expect("UTF-8 temp path"),
6457 )
6458 .expect("open old schema");
6459 first
6460 .subscribe(
6461 "legacy-sub".to_owned(),
6462 "legacy".to_owned(),
6463 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6464 None,
6465 )
6466 .expect("persist legacy subscription");
6467 }
6468
6469 let new_schema = json!({
6470 "version": 2,
6471 "tables": [{
6472 "name": "tasks",
6473 "primaryKey": "id",
6474 "columns": [
6475 { "name": "id", "type": "string", "nullable": false },
6476 { "name": "project_id", "type": "string", "nullable": false },
6477 { "name": "facility_membership_id", "type": "string", "nullable": true }
6478 ],
6479 "scopes": [{ "pattern": "project:{project_id}" }],
6480 "indexes": [{
6481 "name": "tasks_by_membership",
6482 "columns": ["project_id", "facility_membership_id"],
6483 "unique": false
6484 }]
6485 }]
6486 });
6487 let upgraded = SyncClient::open_path_with_identity(
6488 None,
6489 &new_schema,
6490 ClientLimits::default(),
6491 path.to_str().expect("UTF-8 temp path"),
6492 )
6493 .expect("open upgraded schema");
6494 let columns = upgraded
6495 .conn
6496 .prepare("PRAGMA table_info(tasks)")
6497 .expect("prepare columns")
6498 .query_map([], |row| row.get::<_, String>(1))
6499 .expect("query columns")
6500 .collect::<Result<Vec<_>, _>>()
6501 .expect("collect columns");
6502 assert!(columns.contains(&"facility_membership_id".to_owned()));
6503 let index_count: i64 = upgraded
6504 .conn
6505 .query_row(
6506 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
6507 [],
6508 |row| row.get(0),
6509 )
6510 .expect("query index");
6511 assert_eq!(index_count, 1);
6512 assert!(upgraded.subscription_state("legacy-sub").is_none());
6513 drop(upgraded);
6514 std::fs::remove_file(path).expect("remove temp database");
6515 }
6516
6517 #[test]
6518 fn migrates_pre_envelope_outcome_journal_additively() {
6519 let path = std::env::temp_dir().join(format!(
6520 "syncular-outcome-migration-{}.db",
6521 uuid::Uuid::new_v4()
6522 ));
6523 let conn = Connection::open(&path).expect("open legacy database");
6524 conn.execute_batch(
6525 "CREATE TABLE _syncular_commit_outcomes (
6526 seq INTEGER PRIMARY KEY AUTOINCREMENT,
6527 client_commit_id TEXT NOT NULL UNIQUE,
6528 status TEXT NOT NULL,
6529 recorded_at_ms INTEGER NOT NULL,
6530 results_json TEXT NOT NULL,
6531 resolution TEXT NOT NULL DEFAULT 'active',
6532 resolved_at_ms INTEGER,
6533 replacement_client_commit_id TEXT);",
6534 )
6535 .expect("create legacy outcome journal");
6536 drop(conn);
6537 let schema = json!({
6538 "version": 1,
6539 "tables": [{
6540 "name": "tasks",
6541 "primaryKey": "id",
6542 "columns": [
6543 { "name": "id", "type": "string", "nullable": false },
6544 { "name": "project_id", "type": "string", "nullable": false }
6545 ],
6546 "scopes": [{ "pattern": "project:{project_id}" }]
6547 }]
6548 });
6549 let client = SyncClient::open_path(
6550 "migration-native".to_owned(),
6551 &schema,
6552 ClientLimits::default(),
6553 path.to_str().expect("UTF-8 temp path"),
6554 )
6555 .expect("migrate database");
6556 let has_operations = client
6557 .conn
6558 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
6559 .expect("prepare table info")
6560 .query_map([], |row| row.get::<_, String>(1))
6561 .expect("query table info")
6562 .filter_map(Result::ok)
6563 .any(|column| column == "operations_json");
6564 assert!(has_operations);
6565 drop(client);
6566 std::fs::remove_file(path).expect("remove temp database");
6567 }
6568
6569 #[test]
6570 fn retained_rejection_survives_file_reopen_and_security_purge() {
6571 let path =
6572 std::env::temp_dir().join(format!("syncular-retained-{}.db", uuid::Uuid::new_v4()));
6573 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id", "columns": [
6574 { "name": "id", "type": "string", "nullable": false },
6575 { "name": "project_id", "type": "string", "nullable": false },
6576 { "name": "title", "type": "string", "nullable": false }
6577 ], "scopes": [{ "pattern": "project:{project_id}" }] }] });
6578 {
6579 let mut first = SyncClient::open_path(
6580 "retained-native".to_owned(),
6581 &schema,
6582 ClientLimits::default(),
6583 path.to_str().unwrap(),
6584 )
6585 .unwrap();
6586 first
6587 .write_base_row(
6588 "tasks",
6589 &vec![
6590 Some(ColumnValue::String("t1".into())),
6591 Some(ColumnValue::String("p1".into())),
6592 Some(ColumnValue::String("server".into())),
6593 ],
6594 1,
6595 )
6596 .unwrap();
6597 first.rebuild_overlay().expect("rebuild overlay");
6598 let operation = OutboxOp {
6599 table: "tasks".into(),
6600 row_id: "t1".into(),
6601 upsert: true,
6602 base_version: Some(1),
6603 values: Some(Map::from_iter([
6604 ("id".into(), json!("t1")),
6605 ("title".into(), json!("mine")),
6606 ])),
6607 };
6608 first
6609 .retain_failed_commit("retained", std::slice::from_ref(&operation))
6610 .unwrap();
6611 first
6612 .persist_commit_outcome(
6613 "retained",
6614 CommitOutcomeStatus::Rejected,
6615 &[CommitOperationOutcome::Error {
6616 rejection: RejectionRecord {
6617 client_commit_id: "retained".into(),
6618 op_index: 0,
6619 code: "validation.denied".into(),
6620 message: "denied".into(),
6621 retryable: false,
6622 details: None,
6623 operation: None,
6624 },
6625 }],
6626 Some(&[operation]),
6627 )
6628 .unwrap();
6629 first.rebuild_overlay().expect("rebuild overlay");
6630 }
6631 {
6632 let mut reopened = SyncClient::open_path(
6633 "retained-native".to_owned(),
6634 &schema,
6635 ClientLimits::default(),
6636 path.to_str().unwrap(),
6637 )
6638 .unwrap();
6639 let row = reopened
6640 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6641 .expect("read visible row")
6642 .unwrap();
6643 assert_eq!(row.0[2], Some(ColumnValue::String("mine".into())));
6644 assert_eq!(
6645 reopened
6646 .commit_outcome("retained")
6647 .unwrap()
6648 .unwrap()
6649 .retained_rows
6650 .unwrap()[0]
6651 .server_row
6652 .as_ref()
6653 .unwrap()["title"],
6654 json!("server")
6655 );
6656 reopened
6657 .conn
6658 .execute("DELETE FROM _syncular_base_tasks WHERE id = 't1'", [])
6659 .unwrap();
6660 reopened.rebuild_overlay().expect("rebuild overlay");
6661 assert!(reopened
6662 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6663 .expect("read visible row")
6664 .is_none());
6665 let retained = reopened.commit_outcome("retained").unwrap().unwrap();
6666 assert!(retained.retained_rows.is_some());
6667 assert_eq!(
6668 retained.operations.unwrap()[0].values.as_ref().unwrap()["title"],
6669 json!("mine")
6670 );
6671 reopened
6672 .purge_local_data(&LocalDataPurgeInput {
6673 purge_id: "protected-purge".into(),
6674 targets: vec![LocalDataPurgeTarget {
6675 table: "tasks".into(),
6676 selectors: BTreeMap::from_iter([("project_id".into(), vec!["p1".into()])]),
6677 }],
6678 })
6679 .unwrap();
6680 assert!(reopened
6681 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6682 .expect("read visible row")
6683 .is_none());
6684 assert!(reopened
6685 .commit_outcome("retained")
6686 .unwrap()
6687 .unwrap()
6688 .retained_rows
6689 .is_none());
6690 }
6691 std::fs::remove_file(path).unwrap();
6692 }
6693
6694 #[test]
6695 fn durable_conflict_outcome_and_resolution_survive_reopen() {
6696 let path = std::env::temp_dir().join(format!(
6697 "syncular-durable-outcome-{}.db",
6698 uuid::Uuid::new_v4()
6699 ));
6700 let schema = json!({
6701 "version": 1,
6702 "tables": [{
6703 "name": "tasks",
6704 "primaryKey": "id",
6705 "columns": [
6706 { "name": "id", "type": "string", "nullable": false },
6707 { "name": "project_id", "type": "string", "nullable": false }
6708 ],
6709 "scopes": [{ "pattern": "project:{project_id}" }]
6710 }]
6711 });
6712 let conflict = ConflictRecord {
6713 client_commit_id: "losing-commit".to_owned(),
6714 op_index: 0,
6715 table: "tasks".to_owned(),
6716 row_id: "t1".to_owned(),
6717 code: "sync.version_conflict".to_owned(),
6718 message: "stale base version".to_owned(),
6719 server_version: 2,
6720 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6721 conflict_columns: vec!["project_id".to_owned()],
6722 operation: Some(CommitOperation {
6723 table: "tasks".to_owned(),
6724 row_id: "t1".to_owned(),
6725 op: "upsert".to_owned(),
6726 base_version: Some(1),
6727 values: Some(Map::from_iter([
6728 ("id".into(), json!("t1")),
6729 ("project_id".into(), json!("p1")),
6730 ])),
6731 }),
6732 };
6733 let failed_operations = vec![
6734 OutboxOp {
6735 upsert: true,
6736 table: "tasks".to_owned(),
6737 row_id: "t1".to_owned(),
6738 base_version: Some(1),
6739 values: Some(Map::from_iter([
6740 ("id".into(), json!("t1")),
6741 ("project_id".into(), json!("p1")),
6742 ])),
6743 },
6744 OutboxOp {
6745 upsert: true,
6746 table: "tasks".to_owned(),
6747 row_id: "status-event-1".to_owned(),
6748 base_version: Some(0),
6749 values: Some(Map::from_iter([
6750 ("id".into(), json!("status-event-1")),
6751 ("project_id".into(), json!("p1")),
6752 ])),
6753 },
6754 ];
6755
6756 {
6757 let mut first = SyncClient::open_path(
6758 "durable-native".to_owned(),
6759 &schema,
6760 ClientLimits::default(),
6761 path.to_str().expect("UTF-8 temp path"),
6762 )
6763 .expect("first open");
6764 first
6765 .begin_observation("test_outcome")
6766 .expect("begin outcome");
6767 first
6768 .persist_commit_outcome(
6769 "losing-commit",
6770 CommitOutcomeStatus::Conflict,
6771 &[CommitOperationOutcome::Conflict {
6772 conflict: conflict.clone(),
6773 }],
6774 Some(&failed_operations),
6775 )
6776 .expect("persist outcome");
6777 first.conflicts.push(conflict);
6778 first
6779 .finish_observation(
6780 "test_outcome",
6781 ChangeAccumulator {
6782 conflicts: true,
6783 outcomes: true,
6784 ..ChangeAccumulator::default()
6785 },
6786 )
6787 .expect("commit outcome");
6788 }
6789
6790 {
6791 let mut reopened = SyncClient::open_path(
6792 "durable-native".to_owned(),
6793 &schema,
6794 ClientLimits::default(),
6795 path.to_str().expect("UTF-8 temp path"),
6796 )
6797 .expect("reopen");
6798 assert_eq!(reopened.conflicts().len(), 1);
6799 assert_eq!(
6800 reopened.conflicts()[0].conflict_columns,
6801 vec!["project_id".to_owned()],
6802 "conflictColumns survives the journal round trip"
6803 );
6804 let outcome = reopened
6805 .commit_outcome("losing-commit")
6806 .expect("read outcome")
6807 .expect("outcome");
6808 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6809 let operations = outcome.operations.expect("aggregate envelope");
6810 assert_eq!(operations.len(), 2);
6811 assert_eq!(operations[1].row_id, "status-event-1");
6812 let resolved = reopened
6813 .resolve_commit_outcome(ResolveCommitOutcomeInput {
6814 client_commit_id: "losing-commit".to_owned(),
6815 resolution: CommitOutcomeResolution::ResolvedKeepServer,
6816 replacement_client_commit_id: None,
6817 })
6818 .expect("resolve");
6819 assert_eq!(
6820 resolved.resolution,
6821 CommitOutcomeResolution::ResolvedKeepServer
6822 );
6823 assert!(reopened.conflicts().is_empty());
6824 }
6825
6826 let reopened = SyncClient::open_path(
6827 "durable-native".to_owned(),
6828 &schema,
6829 ClientLimits::default(),
6830 path.to_str().expect("UTF-8 temp path"),
6831 )
6832 .expect("second reopen");
6833 assert!(reopened.conflicts().is_empty());
6834 assert_eq!(
6835 reopened
6836 .commit_outcome("losing-commit")
6837 .expect("read outcome")
6838 .expect("outcome")
6839 .resolution,
6840 CommitOutcomeResolution::ResolvedKeepServer
6841 );
6842 drop(reopened);
6843 std::fs::remove_file(path).expect("remove temp database");
6844 }
6845
6846 #[test]
6847 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
6848 let path =
6849 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6850 let schema = json!({
6851 "version": 1,
6852 "tables": [{
6853 "name": "tasks",
6854 "primaryKey": "id",
6855 "columns": [
6856 { "name": "id", "type": "string", "nullable": false },
6857 { "name": "project_id", "type": "string", "nullable": false }
6858 ],
6859 "scopes": [{ "pattern": "project:{project_id}" }]
6860 }]
6861 });
6862 let mut client = SyncClient::open_path_with_identity(
6863 Some("sidecar-client".to_owned()),
6864 &schema,
6865 ClientLimits::default(),
6866 path.to_str().expect("UTF-8 temp path"),
6867 )
6868 .expect("open owner");
6869 let base = WindowBase {
6870 table: "tasks".to_owned(),
6871 variable: "project_id".to_owned(),
6872 fixed_scopes: Vec::new(),
6873 params: None,
6874 };
6875 client
6876 .set_window(&base, &["one".to_owned()])
6877 .expect("set window");
6878 client
6879 .mutate(vec![Mutation::Upsert {
6880 table: "tasks".to_owned(),
6881 values: Map::from_iter([
6882 ("id".to_owned(), Value::from("t1")),
6883 ("project_id".to_owned(), Value::from("one")),
6884 ]),
6885 base_version: None,
6886 }])
6887 .expect("local mutate");
6888
6889 let coverage = [WindowCoverage {
6890 base,
6891 units: vec!["one".to_owned(), "missing".to_owned()],
6892 }];
6893 let owner = client
6894 .query_snapshot(
6895 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6896 &[],
6897 &coverage,
6898 None,
6899 )
6900 .expect("owner snapshot");
6901 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6902 let sidecar = reader
6903 .query_snapshot(
6904 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6905 &[],
6906 &coverage,
6907 )
6908 .expect("sidecar snapshot");
6909
6910 assert_eq!(sidecar.revision, owner.revision);
6911 assert_eq!(sidecar.rows, owner.rows);
6912 assert_eq!(
6913 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
6914 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
6915 );
6916 assert_eq!(sidecar.revision, "2");
6917 assert_eq!(sidecar.rows[0]["id"], "t1");
6918 assert_eq!(sidecar.rows[0]["server_version"], -1);
6919 assert!(!sidecar.coverage.complete);
6920 assert_eq!(sidecar.coverage.pending.len(), 1);
6921 assert_eq!(sidecar.coverage.missing.len(), 1);
6922
6923 drop(reader);
6924 drop(client);
6925 std::fs::remove_file(path).expect("remove temp database");
6926 }
6927
6928 #[test]
6929 fn file_snapshot_reader_reports_catchup_delivery_and_multi_statement() {
6930 let path =
6931 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6932 let schema = json!({
6933 "version": 1,
6934 "tables": [{
6935 "name": "tasks",
6936 "primaryKey": "id",
6937 "columns": [
6938 { "name": "id", "type": "string", "nullable": false },
6939 { "name": "project_id", "type": "string", "nullable": false }
6940 ],
6941 "scopes": [{ "pattern": "project:{project_id}" }]
6942 }]
6943 });
6944 let mut client = SyncClient::open_path_with_identity(
6945 Some("sidecar-catchup".to_owned()),
6946 &schema,
6947 ClientLimits::default(),
6948 path.to_str().expect("UTF-8 temp path"),
6949 )
6950 .expect("open owner");
6951 client
6952 .subscribe(
6953 "s1".into(),
6954 "tasks".into(),
6955 vec![("project_id".into(), vec!["one".into()])],
6956 None,
6957 )
6958 .expect("subscribe");
6959 let commit_id = client
6960 .mutate(vec![Mutation::Upsert {
6961 table: "tasks".to_owned(),
6962 values: Map::from_iter([
6963 ("id".to_owned(), Value::from("t1")),
6964 ("project_id".to_owned(), Value::from("one")),
6965 ]),
6966 base_version: None,
6967 }])
6968 .expect("mutate");
6969 let revision = client.local_revision().to_string();
6970
6971 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6972 assert_eq!(reader.local_revision().expect("revision"), revision);
6973
6974 let read = reader
6975 .snapshot_read(&SnapshotReadRequest {
6976 statements: vec![
6977 SnapshotStatement {
6978 sql: "SELECT 1 AS n",
6979 params: &[],
6980 },
6981 SnapshotStatement {
6982 sql: "SELECT id FROM tasks ORDER BY id",
6983 params: &[],
6984 },
6985 ],
6986 coverage: &[],
6987 subscriptions: vec!["s1".to_owned(), "absent".to_owned()],
6988 commit_ids: vec![commit_id.clone(), "never-known".to_owned()],
6989 })
6990 .expect("snapshot read");
6991 assert_eq!(read.revision, revision);
6992 assert_eq!(read.queries.len(), 2);
6993 assert_eq!(read.queries[0][0]["n"], 1);
6994 assert_eq!(read.queries[1][0]["id"], "t1");
6995 match &read.subscriptions[0] {
6996 SubscriptionCatchup::Known {
6997 status,
6998 cursor,
6999 bootstrap_complete,
7000 known_pending_pages,
7001 ..
7002 } => {
7003 assert_eq!(status, "active");
7004 assert_eq!(*cursor, -1);
7005 assert!(!bootstrap_complete);
7006 assert!(known_pending_pages);
7007 }
7008 other => panic!("expected a known subscription: {other:?}"),
7009 }
7010 assert!(matches!(
7011 &read.subscriptions[1],
7012 SubscriptionCatchup::Unknown { id } if id.as_str() == "absent"
7013 ));
7014 assert!(matches!(
7015 &read.deliveries[0],
7016 CommitDelivery::Pending { client_commit_id } if client_commit_id == &commit_id
7017 ));
7018 assert!(matches!(
7019 &read.deliveries[1],
7020 CommitDelivery::Unknown { client_commit_id } if client_commit_id.as_str() == "never-known"
7021 ));
7022
7023 assert!(reader
7025 .query_snapshot("SELECT * FROM does_not_exist", &[], &[])
7026 .is_err());
7027 assert_eq!(
7028 reader.local_revision().expect("revision after rollback"),
7029 revision
7030 );
7031 assert!(reader
7033 .query_snapshot("DELETE FROM tasks", &[], &[])
7034 .is_err());
7035
7036 client
7038 .conn
7039 .execute(
7040 "UPDATE _syncular_meta SET value = '+1' WHERE key = 'localRevision'",
7041 [],
7042 )
7043 .expect("corrupt revision");
7044 let failure = reader.local_revision().expect_err("corrupt revision fails");
7045 assert_eq!(failure.code, Some("sync.local_corrupt"));
7046
7047 drop(reader);
7048 drop(client);
7049 std::fs::remove_file(path).expect("remove temp database");
7050 }
7051
7052 #[test]
7053 fn snapshot_delivery_is_the_persisted_journal_view_without_retained_rows() {
7054 let path =
7055 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7056 let schema = json!({
7057 "version": 1,
7058 "tables": [{
7059 "name": "tasks",
7060 "primaryKey": "id",
7061 "columns": [
7062 { "name": "id", "type": "string", "nullable": false },
7063 { "name": "project_id", "type": "string", "nullable": false }
7064 ],
7065 "scopes": [{ "pattern": "project:{project_id}" }]
7066 }]
7067 });
7068 let mut client = SyncClient::open_path_with_identity(
7069 Some("sidecar-delivery".to_owned()),
7070 &schema,
7071 ClientLimits::default(),
7072 path.to_str().expect("UTF-8 temp path"),
7073 )
7074 .expect("open owner");
7075 client.begin_observation("test_delivery").expect("observe");
7076 let operations = [OutboxOp {
7077 upsert: true,
7078 table: "tasks".to_owned(),
7079 row_id: "t1".to_owned(),
7080 base_version: Some(1),
7081 values: Some(Map::from_iter([
7082 ("id".into(), json!("t1")),
7083 ("project_id".into(), json!("p1")),
7084 ])),
7085 }];
7086 client
7087 .persist_commit_outcome(
7088 "conflict-commit",
7089 CommitOutcomeStatus::Conflict,
7090 &[CommitOperationOutcome::Conflict {
7091 conflict: ConflictRecord {
7092 client_commit_id: "conflict-commit".to_owned(),
7093 op_index: 0,
7094 table: "tasks".to_owned(),
7095 row_id: "t1".to_owned(),
7096 code: "sync.version_conflict".to_owned(),
7097 message: "stale base version".to_owned(),
7098 server_version: 2,
7099 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
7100 conflict_columns: vec!["project_id".to_owned()],
7101 operation: None,
7102 },
7103 }],
7104 Some(&operations),
7105 )
7106 .expect("persist conflict");
7107 client
7108 .persist_commit_outcome(
7109 "rejected-commit",
7110 CommitOutcomeStatus::Rejected,
7111 &[CommitOperationOutcome::Error {
7112 rejection: RejectionRecord {
7113 client_commit_id: "rejected-commit".to_owned(),
7114 op_index: 0,
7115 code: "sync.unknown_table".to_owned(),
7116 message: "unknown table".to_owned(),
7117 retryable: false,
7118 details: None,
7119 operation: None,
7120 },
7121 }],
7122 Some(&operations),
7123 )
7124 .expect("persist rejection");
7125 client
7126 .finish_observation(
7127 "test_delivery",
7128 ChangeAccumulator {
7129 conflicts: true,
7130 outcomes: true,
7131 ..ChangeAccumulator::default()
7132 },
7133 )
7134 .expect("commit outcomes");
7135
7136 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7137 match reader.commit_delivery("conflict-commit").expect("conflict") {
7138 CommitDelivery::Known { outcome, .. } => {
7139 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
7140 assert!(matches!(
7141 outcome.results.as_slice(),
7142 [CommitOperationOutcome::Conflict { conflict }]
7143 if conflict.code == "sync.version_conflict"
7144 && conflict.server_version == 2
7145 ));
7146 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
7147 assert!(
7148 outcome.retained_rows.is_none(),
7149 "the sidecar reports the persisted view, not owner-derived retained rows"
7150 );
7151 }
7152 other => panic!("expected a known conflict: {other:?}"),
7153 }
7154 match reader
7155 .commit_delivery("rejected-commit")
7156 .expect("rejection")
7157 {
7158 CommitDelivery::Known { outcome, .. } => {
7159 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
7160 assert!(matches!(
7161 outcome.results.as_slice(),
7162 [CommitOperationOutcome::Error { rejection }]
7163 if rejection.code == "sync.unknown_table"
7164 ));
7165 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
7166 assert!(outcome.retained_rows.is_none());
7167 }
7168 other => panic!("expected a known rejection: {other:?}"),
7169 }
7170 drop(reader);
7171 drop(client);
7172 std::fs::remove_file(path).expect("remove temp database");
7173 }
7174
7175 #[test]
7176 fn snapshot_read_isolates_a_concurrent_writer_commit() {
7177 use std::sync::mpsc::channel;
7178 let path = std::env::temp_dir().join(format!(
7179 "syncular-read-concurrent-{}.db",
7180 uuid::Uuid::new_v4()
7181 ));
7182 let schema = json!({
7183 "version": 1,
7184 "tables": [{
7185 "name": "tasks",
7186 "primaryKey": "id",
7187 "columns": [
7188 { "name": "id", "type": "string", "nullable": false },
7189 { "name": "project_id", "type": "string", "nullable": false }
7190 ],
7191 "scopes": [{ "pattern": "project:{project_id}" }]
7192 }]
7193 });
7194 let mut client = SyncClient::open_path_with_identity(
7195 Some("sidecar-concurrent".to_owned()),
7196 &schema,
7197 ClientLimits::default(),
7198 path.to_str().expect("UTF-8 temp path"),
7199 )
7200 .expect("open owner");
7201 client
7202 .mutate(vec![Mutation::Upsert {
7203 table: "tasks".to_owned(),
7204 values: Map::from_iter([
7205 ("id".to_owned(), Value::from("t1")),
7206 ("project_id".to_owned(), Value::from("one")),
7207 ]),
7208 base_version: None,
7209 }])
7210 .expect("seed row");
7211 let revision_before = client.local_revision().to_string();
7212
7213 let (ready_tx, ready_rx) = channel::<()>();
7216 let (done_tx, done_rx) = channel::<()>();
7217 let writer_path = path.to_string_lossy().into_owned();
7218 let writer_schema = schema.clone();
7219 let writer = std::thread::spawn(move || {
7220 let mut writer = SyncClient::open_path_with_identity(
7221 Some("sidecar-concurrent".to_owned()),
7222 &writer_schema,
7223 ClientLimits::default(),
7224 &writer_path,
7225 )
7226 .expect("open writer");
7227 ready_rx.recv().expect("reader signalled");
7228 writer
7229 .mutate(vec![Mutation::Upsert {
7230 table: "tasks".to_owned(),
7231 values: Map::from_iter([
7232 ("id".to_owned(), Value::from("t2")),
7233 ("project_id".to_owned(), Value::from("one")),
7234 ]),
7235 base_version: None,
7236 }])
7237 .expect("writer commit");
7238 let _ = done_tx.send(());
7239 });
7240 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7241 SNAPSHOT_READ_HOOK.with(|slot| {
7242 *slot.borrow_mut() = Some(Box::new(move || {
7243 ready_tx.send(()).expect("signal writer");
7244 done_rx.recv().expect("writer finished");
7245 }));
7246 });
7247 let read = reader
7248 .snapshot_read(&SnapshotReadRequest {
7249 statements: vec![
7250 SnapshotStatement {
7251 sql: "SELECT count(*) AS n FROM tasks",
7252 params: &[],
7253 },
7254 SnapshotStatement {
7255 sql: "SELECT count(*) AS n FROM tasks",
7256 params: &[],
7257 },
7258 ],
7259 coverage: &[],
7260 subscriptions: Vec::new(),
7261 commit_ids: Vec::new(),
7262 })
7263 .expect("snapshot read");
7264 assert_eq!(
7265 read.queries[0][0]["n"], 1,
7266 "first statement sees the pre-commit row"
7267 );
7268 assert_eq!(
7269 read.queries[1][0]["n"], 1,
7270 "a later statement keeps the one read snapshot"
7271 );
7272 assert_eq!(
7273 read.revision, revision_before,
7274 "the read reports the snapshot revision, not the writer's"
7275 );
7276 writer.join().expect("join writer");
7277
7278 assert_ne!(
7279 reader.local_revision().expect("revision after commit"),
7280 revision_before,
7281 "the next read observes the committed revision"
7282 );
7283 let after = reader
7284 .query_snapshot("SELECT count(*) AS n FROM tasks", &[], &[])
7285 .expect("count after commit");
7286 assert_eq!(
7287 after.rows[0]["n"], 2,
7288 "the committed row is visible afterwards"
7289 );
7290
7291 drop(reader);
7292 drop(client);
7293 std::fs::remove_file(path).expect("remove temp database");
7294 }
7295
7296 #[test]
7297 fn snapshot_read_rejects_every_non_canonical_revision() {
7298 let path = std::env::temp_dir().join(format!(
7299 "syncular-read-revision-{}.db",
7300 uuid::Uuid::new_v4()
7301 ));
7302 let schema = json!({
7303 "version": 1,
7304 "tables": [{
7305 "name": "tasks",
7306 "primaryKey": "id",
7307 "columns": [
7308 { "name": "id", "type": "string", "nullable": false },
7309 { "name": "project_id", "type": "string", "nullable": false }
7310 ],
7311 "scopes": [{ "pattern": "project:{project_id}" }]
7312 }]
7313 });
7314 let client = SyncClient::open_path_with_identity(
7315 Some("sidecar-revision".to_owned()),
7316 &schema,
7317 ClientLimits::default(),
7318 path.to_str().expect("UTF-8 temp path"),
7319 )
7320 .expect("open owner");
7321 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7322 for bad in [
7323 "+1",
7324 "01",
7325 "-1",
7326 " 1",
7327 "1 ",
7328 "18446744073709551616",
7329 "1.0",
7330 "abc",
7331 "",
7332 ] {
7333 client
7334 .conn
7335 .execute(
7336 "UPDATE _syncular_meta SET value = ?1 WHERE key = 'localRevision'",
7337 rusqlite::params![bad],
7338 )
7339 .expect("corrupt revision");
7340 let failure = reader.local_revision().expect_err("corrupt marker fails");
7341 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad:?}");
7342 }
7343 for value in ["X'31'", "CAST(X'FF' AS TEXT)"] {
7344 client
7345 .conn
7346 .execute_batch(&format!(
7347 "UPDATE _syncular_meta SET value={value} WHERE key='localRevision'"
7348 ))
7349 .expect("corrupt stored revision type");
7350 assert_eq!(
7351 reader.local_revision().expect_err("invalid SQL field").code,
7352 Some("sync.local_corrupt")
7353 );
7354 }
7355 client
7357 .conn
7358 .execute("DELETE FROM _syncular_meta WHERE key = 'localRevision'", [])
7359 .expect("delete marker");
7360 assert_eq!(reader.local_revision().expect("legacy zero"), "0");
7361
7362 drop(reader);
7363 drop(client);
7364 std::fs::remove_file(path).expect("remove temp database");
7365 }
7366
7367 #[test]
7368 fn snapshot_read_rejects_corrupt_subscription_state() {
7369 let path = std::env::temp_dir().join(format!(
7370 "syncular-read-substate-{}.db",
7371 uuid::Uuid::new_v4()
7372 ));
7373 let schema = json!({
7374 "version": 1,
7375 "tables": [{
7376 "name": "tasks",
7377 "primaryKey": "id",
7378 "columns": [
7379 { "name": "id", "type": "string", "nullable": false },
7380 { "name": "project_id", "type": "string", "nullable": false }
7381 ],
7382 "scopes": [{ "pattern": "project:{project_id}" }]
7383 }]
7384 });
7385 let mut client = SyncClient::open_path_with_identity(
7386 Some("sidecar-substate".to_owned()),
7387 &schema,
7388 ClientLimits::default(),
7389 path.to_str().expect("UTF-8 temp path"),
7390 )
7391 .expect("open owner");
7392 client
7393 .subscribe(
7394 "s1".into(),
7395 "tasks".into(),
7396 vec![("project_id".into(), vec!["one".into()])],
7397 None,
7398 )
7399 .expect("subscribe");
7400 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7401 for bad in [
7402 "not json".to_owned(),
7403 "{}".to_owned(),
7404 json!({ "status": "bogus", "cursor": 0 }).to_string(),
7405 json!({ "status": "active", "cursor": "x" }).to_string(),
7406 json!({ "status": "active", "cursor": 0, "bootstrapState": 7 }).to_string(),
7407 json!({ "status": "active", "cursor": 0, "reasonCode": 7 }).to_string(),
7408 ] {
7409 client
7410 .conn
7411 .execute(
7412 "UPDATE _syncular_subscriptions SET state_json = ?1 WHERE id = 's1'",
7413 rusqlite::params![bad],
7414 )
7415 .expect("corrupt subscription state");
7416 let failure = reader
7417 .subscription_catchup("s1")
7418 .expect_err("corrupt state fails");
7419 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad}");
7420 }
7421
7422 for assignment in [
7423 "tbl=X'7461736b73'",
7424 "state_json=X'7b7d'",
7425 "state_json=CAST(X'FF' AS TEXT)",
7426 ] {
7427 client.conn.execute_batch(&format!(
7428 "UPDATE _syncular_subscriptions SET tbl='tasks',state_json='{{\"status\":\"active\",\"cursor\":0}}'; UPDATE _syncular_subscriptions SET {assignment} WHERE id='s1'"
7429 )).expect("corrupt stored subscription type");
7430 assert_eq!(
7431 reader
7432 .subscription_catchup("s1")
7433 .expect_err("invalid SQL field")
7434 .code,
7435 Some("sync.local_corrupt")
7436 );
7437 }
7438
7439 drop(reader);
7440 drop(client);
7441 std::fs::remove_file(path).expect("remove temp database");
7442 }
7443
7444 #[test]
7445 fn snapshot_read_rolls_back_a_later_invalid_statement_and_guards_every_write() {
7446 let path = std::env::temp_dir().join(format!(
7447 "syncular-read-rollback-{}.db",
7448 uuid::Uuid::new_v4()
7449 ));
7450 let schema = json!({
7451 "version": 1,
7452 "tables": [{
7453 "name": "tasks",
7454 "primaryKey": "id",
7455 "columns": [
7456 { "name": "id", "type": "string", "nullable": false },
7457 { "name": "project_id", "type": "string", "nullable": false }
7458 ],
7459 "scopes": [{ "pattern": "project:{project_id}" }]
7460 }]
7461 });
7462 let mut client = SyncClient::open_path_with_identity(
7463 Some("sidecar-rollback".to_owned()),
7464 &schema,
7465 ClientLimits::default(),
7466 path.to_str().expect("UTF-8 temp path"),
7467 )
7468 .expect("open owner");
7469 let commit_id = client
7470 .mutate(vec![Mutation::Upsert {
7471 table: "tasks".to_owned(),
7472 values: Map::from_iter([
7473 ("id".to_owned(), Value::from("t1")),
7474 ("project_id".to_owned(), Value::from("one")),
7475 ]),
7476 base_version: None,
7477 }])
7478 .expect("seed row");
7479 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7480 let revision = reader.local_revision().expect("revision");
7481
7482 let failure = reader
7485 .snapshot_read(&SnapshotReadRequest {
7486 statements: vec![
7487 SnapshotStatement {
7488 sql: "SELECT id FROM tasks",
7489 params: &[],
7490 },
7491 SnapshotStatement {
7492 sql: "SELECT * FROM does_not_exist",
7493 params: &[],
7494 },
7495 ],
7496 coverage: &[],
7497 subscriptions: Vec::new(),
7498 commit_ids: vec![commit_id.clone()],
7499 })
7500 .expect_err("second statement fails");
7501 assert_eq!(failure.code, None);
7502 assert_eq!(
7503 reader.local_revision().expect("revision after rollback"),
7504 revision,
7505 "a rolled-back read consumes no revision"
7506 );
7507 assert!(
7508 !matches!(
7509 reader
7510 .commit_delivery(&commit_id)
7511 .expect("delivery after rollback"),
7512 CommitDelivery::Unknown { .. }
7513 ),
7514 "the same reader serves the next read after releasing the connection"
7515 );
7516
7517 let guard = reader
7519 .snapshot_read(&SnapshotReadRequest {
7520 statements: vec![SnapshotStatement {
7521 sql: "DELETE FROM tasks",
7522 params: &[],
7523 }],
7524 coverage: &[],
7525 subscriptions: Vec::new(),
7526 commit_ids: Vec::new(),
7527 })
7528 .expect_err("write guard");
7529 assert!(guard.message.contains("read-only"), "{}", guard.message);
7530
7531 drop(reader);
7532 drop(client);
7533 std::fs::remove_file(path).expect("remove temp database");
7534 }
7535
7536 #[test]
7537 fn owned_query_failures_classify_order_bound_and_clear() {
7538 let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7539 rusqlite::ffi::Error::new(266),
7540 Some("private SQLite prose".to_owned()),
7541 ));
7542 assert_eq!(io.code, Some("client.storage_io"));
7543 assert_eq!(io.sqlite_code, Some(266));
7544 assert_eq!(io.message, "local SQLite storage I/O failed");
7545 let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7546 rusqlite::ffi::Error::new(26),
7547 None,
7548 ));
7549 assert_eq!(corrupt.code, Some("client.storage_corrupt"));
7550 assert!(!io.retryable());
7551 assert!(!corrupt.retryable());
7552
7553 for (raw, extended) in [(5, 5), (5 | (1 << 8), 261), (6, 6), (6 | (1 << 8), 262)] {
7556 let busy = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7557 rusqlite::ffi::Error::new(raw),
7558 Some("database is locked".to_owned()),
7559 ));
7560 assert_eq!(busy.code, Some("client.storage_busy"));
7561 assert_eq!(busy.sqlite_code, Some(extended));
7562 assert_eq!(busy.message, "local SQLite storage is busy");
7563 assert!(busy.retryable());
7564 }
7565 let unrelated = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7566 rusqlite::ffi::Error::new(1),
7567 Some("generic SQLite prose".to_owned()),
7568 ));
7569 assert_eq!(unrelated.code, None);
7570 assert!(!unrelated.retryable());
7571
7572 let mut client = client();
7573 let generic = QueryReadFailure::from("private query prose".to_owned());
7574 client.record_query_read(
7575 QueryOwner {
7576 id: "first",
7577 tables: &["tasks", "tasks"],
7578 },
7579 Some(&generic),
7580 );
7581 client.record_query_read(
7582 QueryOwner {
7583 id: "second",
7584 tables: &["tasks"],
7585 },
7586 Some(&generic),
7587 );
7588 client.record_query_read(
7589 QueryOwner {
7590 id: "first",
7591 tables: &["docs"],
7592 },
7593 Some(&io),
7594 );
7595 let failures = &client
7596 .diagnostics_snapshot(&Default::default())
7597 .expect("diagnostics")
7598 .query_failures;
7599 assert_eq!(
7600 failures
7601 .iter()
7602 .map(|entry| entry.id.as_str())
7603 .collect::<Vec<_>>(),
7604 vec!["second", "first"]
7605 );
7606 assert_eq!(failures[1].tables, vec!["docs"]);
7607 assert_eq!(failures[1].code, "client.storage_io");
7608
7609 for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
7610 client.record_query_read(
7611 QueryOwner {
7612 id: &format!("query-{index}"),
7613 tables: &["tasks"],
7614 },
7615 Some(&generic),
7616 );
7617 }
7618 let failures = &client
7619 .diagnostics_snapshot(&Default::default())
7620 .expect("bounded diagnostics")
7621 .query_failures;
7622 assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
7623 assert_eq!(failures.last().expect("last failure").id, "query-256");
7624
7625 client
7626 .query_snapshot(
7627 "SELECT id FROM tasks",
7628 &[],
7629 &[],
7630 Some(QueryOwner {
7631 id: "query-256",
7632 tables: &["tasks"],
7633 }),
7634 )
7635 .expect("successful owned read");
7636 assert!(client
7637 .diagnostics_snapshot(&Default::default())
7638 .expect("cleared diagnostics")
7639 .query_failures
7640 .iter()
7641 .all(|entry| entry.id != "query-256"));
7642 }
7643
7644 #[test]
7645 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
7646 let mut client = client();
7647 let image = Connection::open_in_memory().unwrap();
7648 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
7649 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7650 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
7651 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7652 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
7653 let table = client.schema.table("tasks").unwrap().clone();
7654 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7655 assert!(client
7656 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
7657 .is_err());
7658 assert!(client.conn.is_autocommit());
7659 assert_eq!(
7660 client
7661 .query("SELECT count(*) AS n FROM tasks", &[])
7662 .unwrap()[0]["n"],
7663 1024
7664 );
7665 assert_eq!(client.drain_change_batches().len(), 1);
7666 image
7667 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
7668 .unwrap();
7669 assert!(matches!(
7670 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
7671 Ok(2050)
7672 ));
7673 assert_eq!(
7674 client
7675 .query("SELECT count(*) AS n FROM tasks", &[])
7676 .unwrap()[0]["n"],
7677 2050
7678 );
7679 assert_eq!(client.drain_change_batches().len(), 3);
7680 }
7681
7682 #[test]
7683 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
7684 let mut client = client();
7685 let base = WindowBase {
7686 table: "tasks".into(),
7687 variable: "project_id".into(),
7688 fixed_scopes: Vec::new(),
7689 params: None,
7690 };
7691 client
7692 .set_window(&base, &["p1".into(), "p2".into()])
7693 .unwrap();
7694 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7695 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7696 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
7697 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7698 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7699 client.drain_change_batches();
7700 assert!(client.set_window(&base, &["p2".into()]).is_err());
7701 assert!(client.conn.is_autocommit());
7702 assert_eq!(
7703 client
7704 .query("SELECT count(*) AS n FROM tasks", &[])
7705 .unwrap()[0]["n"],
7706 1027
7707 );
7708 assert_eq!(client.window_state(&base).units, vec!["p2"]);
7709 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
7710 assert_eq!(client.drain_change_batches().len(), 1);
7711 client
7712 .conn
7713 .execute_batch("DROP TRIGGER interrupt_evict")
7714 .unwrap();
7715 client.drain_pending_evictions().unwrap();
7716 assert!(client.load_pending_evictions().unwrap().is_empty());
7717 assert_eq!(
7718 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
7719 "held"
7720 );
7721 assert_eq!(client.drain_change_batches().len(), 2);
7722 }
7723
7724 #[test]
7725 fn eviction_marker_failures_roll_back_window_changes() {
7726 let mut client = client();
7727 let base = WindowBase {
7728 table: "tasks".into(),
7729 variable: "project_id".into(),
7730 fixed_scopes: Vec::new(),
7731 params: None,
7732 };
7733 client.set_window(&base, &["p1".into()]).unwrap();
7734 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7735 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7736 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7737 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7738 let revision = client.local_revision();
7739 assert!(client.set_window(&base, &[]).is_err());
7740 assert!(client.conn.is_autocommit());
7741 assert_eq!(client.local_revision(), revision);
7742 assert_eq!(
7743 client
7744 .query("SELECT count(*) AS n FROM tasks", &[])
7745 .unwrap()[0]["n"],
7746 2050
7747 );
7748 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7749 client.conn.execute_batch("DROP TRIGGER fail_marker;
7750 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7751 assert!(client.set_window(&base, &[]).is_err());
7752 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
7753 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7754 assert!(client.drain_pending_evictions().is_err());
7755 assert!(client.conn.is_autocommit());
7756 assert_eq!(
7757 client
7758 .query("SELECT count(*) AS n FROM tasks", &[])
7759 .unwrap()[0]["n"],
7760 2
7761 );
7762 assert!(client.set_window(&base, &["p1".into()]).is_err());
7763 assert!(client.conn.is_autocommit());
7764 assert!(client.window_state(&base).units.is_empty());
7765 assert!(client.subs.is_empty());
7766 assert_eq!(
7767 client
7768 .query(
7769 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
7770 &[]
7771 )
7772 .unwrap()[0]["n"],
7773 1
7774 );
7775 client
7776 .conn
7777 .execute_batch("DROP TRIGGER fail_marker")
7778 .unwrap();
7779 client.set_window(&base, &["p1".into()]).unwrap();
7780 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7781 }
7782
7783 #[test]
7784 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
7785 let mut client = client();
7786 client
7787 .conn
7788 .execute(
7789 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
7790 [],
7791 )
7792 .unwrap();
7793 assert_eq!(
7794 client.drain_pending_evictions().unwrap_err(),
7795 "sync.local_corrupt: invalid pending eviction scopes"
7796 );
7797 assert!(client.conn.is_autocommit());
7798 assert!(client.enqueue_startup_sync_if_needed().is_err());
7799 }
7800
7801 #[test]
7802 fn image_chunks_reject_null_primary_keys() {
7803 let mut client = client();
7804 let image = Connection::open_in_memory().unwrap();
7805 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7806 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
7807 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
7808 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7809 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
7810 let table = client.schema.table("tasks").unwrap().clone();
7811 assert!(client
7812 .apply_sqlite_image(
7813 &image,
7814 &table,
7815 true,
7816 &[("project_id".into(), vec!["p1".into()])],
7817 1025,
7818 7,
7819 "digest"
7820 )
7821 .is_err());
7822 assert!(client.conn.is_autocommit());
7823 assert_eq!(
7824 client
7825 .query("SELECT count(*) AS n FROM tasks", &[])
7826 .unwrap()[0]["n"],
7827 0
7828 );
7829 }
7830
7831 #[test]
7832 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
7833 for images in [false, true] {
7834 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7835 "name":name,"primaryKey":"id","columns":[
7836 {"name":"id","type":"string","nullable":false},
7837 {"name":"project_id","type":"string","nullable":false},
7838 {"name":"title","type":"string","nullable":false}],
7839 "scopes":[{"pattern":"project:{project_id}"}],
7840 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
7841 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7842 })))});
7843 let mut actual =
7844 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
7845 let mut reference =
7846 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
7847 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7848 for client in [&mut actual, &mut reference] {
7849 client
7850 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
7851 .unwrap();
7852 client.subs[0].effective = Some(scopes.clone());
7853 for (table, id, title) in [
7854 ("tasks", "a", "original"),
7855 ("tasks", "b", "occupied"),
7856 ("tasks", "c", "deleted"),
7857 ("notes", "n", "untouched"),
7858 ] {
7859 client
7860 .write_base_row(
7861 table,
7862 &vec![
7863 Some(ColumnValue::String(id.into())),
7864 Some(ColumnValue::String("p1".into())),
7865 Some(ColumnValue::String(title.into())),
7866 ],
7867 1,
7868 )
7869 .unwrap();
7870 }
7871 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7872 client
7873 .mutate(vec![
7874 Mutation::Upsert {
7875 table: "tasks".into(),
7876 values: Map::from_iter([
7877 ("id".into(), Value::from("a")),
7878 ("project_id".into(), Value::from("p1")),
7879 ("title".into(), Value::from("available")),
7880 ]),
7881 base_version: None,
7882 },
7883 Mutation::Delete {
7884 table: "tasks".into(),
7885 row_id: "c".into(),
7886 base_version: None,
7887 },
7888 Mutation::Upsert {
7889 table: "tasks".into(),
7890 values: Map::from_iter([
7891 ("id".into(), Value::from("local")),
7892 ("project_id".into(), Value::from("p1")),
7893 ("title".into(), Value::from("local value")),
7894 ]),
7895 base_version: None,
7896 },
7897 Mutation::Upsert {
7898 table: "notes".into(),
7899 values: Map::from_iter([
7900 ("id".into(), Value::from("n")),
7901 ("project_id".into(), Value::from("p1")),
7902 ("title".into(), Value::from("pending note")),
7903 ]),
7904 base_version: None,
7905 },
7906 ])
7907 .unwrap();
7908 }
7909 let pending = actual.pending_commit_ids();
7910 actual.overlay_rebuild_count.set(0);
7911 for (id, title) in [
7912 ("b", "available"),
7913 ("b", "occupied"),
7914 ("a", "server original"),
7915 ("d", "local value"),
7916 ("d", "released"),
7917 ("c", "redelivered"),
7918 ] {
7919 let table = actual.schema.table("tasks").unwrap().clone();
7920 let values = vec![
7921 Some(ColumnValue::String(id.into())),
7922 Some(ColumnValue::String("p1".into())),
7923 Some(ColumnValue::String(title.into())),
7924 ];
7925 if images {
7926 let image = Connection::open_in_memory().unwrap();
7927 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7928 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7929 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7930 image
7931 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
7932 .unwrap();
7933 assert!(matches!(
7934 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
7935 Ok(1)
7936 ));
7937 } else {
7938 let segment = RowsSegment {
7939 table: "tasks".into(),
7940 schema_version: 1,
7941 columns: table.wire_columns.clone(),
7942 blocks: vec![vec![ssp2::segment::SegmentRow {
7943 server_version: 2,
7944 values: values.clone(),
7945 }]],
7946 };
7947 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
7948 }
7949 reference.write_base_row("tasks", &values, 2).unwrap();
7950 reference.rebuild_overlay().expect("rebuild overlay");
7951 for table in ["tasks", "notes"] {
7952 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
7953 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
7954 }
7955 }
7956 assert_eq!(actual.pending_commit_ids(), pending);
7957 }
7958 assert_eq!(
7959 actual.overlay_rebuild_count.get(),
7960 0,
7961 "each chunk must leave unrelated tables alone"
7962 );
7963 let before = actual
7964 .query("SELECT * FROM tasks ORDER BY id", &[])
7965 .unwrap();
7966 let fts_before = actual
7967 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7968 .unwrap();
7969 let revision = actual.local_revision();
7970 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
7971 let table = actual.schema.table("tasks").unwrap().clone();
7972 let segment = RowsSegment {
7973 table: "tasks".into(),
7974 schema_version: 1,
7975 columns: table.wire_columns.clone(),
7976 blocks: vec![vec![ssp2::segment::SegmentRow {
7977 server_version: 3,
7978 values: vec![
7979 Some(ColumnValue::String("b".into())),
7980 Some(ColumnValue::String("p1".into())),
7981 Some(ColumnValue::String("fresh".into())),
7982 ],
7983 }]],
7984 };
7985 assert!(actual.apply_segment(0, &segment, false).is_err());
7986 assert!(actual.conn.is_autocommit());
7987 assert_eq!(
7988 actual
7989 .query("SELECT * FROM tasks ORDER BY id", &[])
7990 .unwrap(),
7991 before
7992 );
7993 assert_eq!(
7994 actual
7995 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7996 .unwrap(),
7997 fts_before
7998 );
7999 assert_eq!(actual.local_revision(), revision);
8000 assert_eq!(actual.pending_commit_ids(), pending);
8001 }
8002 }
8003
8004 #[test]
8005 fn import_reconciliation_preserves_typed_pending_primary_keys() {
8006 for (kind, a, b) in [
8007 ("string", json!("a"), json!("b")),
8008 ("integer", json!(1), json!(2)),
8009 ("boolean", json!(true), json!(false)),
8010 ("json", json!("1"), json!("2")),
8011 ] {
8012 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
8013 for images in [false, true] {
8014 let mut client =
8015 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
8016 let table = client.schema.table("tasks").unwrap().clone();
8017 client
8018 .subscribe("tasks".into(), "tasks".into(), vec![], None)
8019 .unwrap();
8020 for (id, title) in [(&a, "original"), (&b, "occupied")] {
8021 client
8022 .write_base_row(
8023 "tasks",
8024 &vec![
8025 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
8026 Some(ColumnValue::String(title.into())),
8027 ],
8028 1,
8029 )
8030 .unwrap();
8031 }
8032 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
8033 client
8034 .mutate(vec![Mutation::Upsert {
8035 table: "tasks".into(),
8036 values: Map::from_iter([
8037 ("id".into(), a.clone()),
8038 ("title".into(), json!("available")),
8039 ]),
8040 base_version: None,
8041 }])
8042 .unwrap();
8043 client.overlay_rebuild_count.set(0);
8044 for (title, expected) in [("available", "original"), ("occupied", "available")] {
8045 let values = vec![
8046 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
8047 Some(ColumnValue::String(title.into())),
8048 ];
8049 if images {
8050 let image = Connection::open_in_memory().unwrap();
8051 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8052 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8053 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
8054 image
8055 .execute(
8056 "INSERT INTO tasks VALUES (?1,?2,2)",
8057 rusqlite::params![RowParam::Cell(&values[0]), title],
8058 )
8059 .unwrap();
8060 assert!(
8061 matches!(
8062 client.apply_sqlite_image(
8063 &image,
8064 &table,
8065 false,
8066 &[],
8067 1,
8068 7,
8069 "digest"
8070 ),
8071 Ok(1)
8072 ),
8073 "{kind}, images={images}"
8074 );
8075 } else {
8076 let segment = RowsSegment {
8077 table: "tasks".into(),
8078 schema_version: 1,
8079 columns: table.wire_columns.clone(),
8080 blocks: vec![vec![ssp2::segment::SegmentRow {
8081 server_version: 2,
8082 values,
8083 }]],
8084 };
8085 assert!(
8086 matches!(client.apply_segment(0, &segment, false), Ok(1)),
8087 "{kind}"
8088 );
8089 }
8090 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
8091 let actual: String = client
8092 .conn
8093 .query_row(
8094 "SELECT title FROM tasks WHERE id=?1",
8095 [RowParam::Cell(&key)],
8096 |row| row.get(0),
8097 )
8098 .unwrap();
8099 assert_eq!(actual, expected, "{kind}, images={images}");
8100 }
8101 assert_eq!(client.overlay_rebuild_count.get(), 0);
8102 }
8103 }
8104 }
8105
8106 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
8107 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
8108 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
8109 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
8110 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
8111 })))});
8112 let mut client =
8113 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
8114 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
8115 client.rebuild_overlay().expect("rebuild overlay");
8116 client
8117 .mutate(vec![Mutation::Upsert {
8118 table: "tasks".into(),
8119 values: Map::from_iter([
8120 ("id".into(), json!("offline")),
8121 ("project_id".into(), json!("p1")),
8122 ("title".into(), json!("pending edit")),
8123 ]),
8124 base_version: None,
8125 }])
8126 .unwrap();
8127 let image = Connection::open_in_memory().unwrap();
8128 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8129 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
8130 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
8131 image
8132 .execute(
8133 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
8134 [rows],
8135 )
8136 .unwrap();
8137 client.overlay_rebuild_count.set(0);
8138 let table = client.schema.table("tasks").unwrap().clone();
8139 (client, image, table)
8140 }
8141
8142 #[test]
8143 fn sqlite_full_image_import_retains_the_first_code_and_recovers() {
8144 let (mut client, image, table) = unique_import_fixture(1);
8145 image
8146 .execute("UPDATE tasks SET title = ?1", ["full ".repeat(32768)])
8147 .unwrap();
8148 client
8149 .conn
8150 .execute_batch("PRAGMA max_page_count = 1")
8151 .unwrap();
8152 let effective = [("project_id".into(), vec!["p1".into()])];
8153 assert!(client
8154 .apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest")
8155 .is_err());
8156 let failure = client.storage_failure.borrow().clone().unwrap();
8157 assert_eq!(failure.code, Some("client.storage_full"));
8158 assert_eq!(failure.sqlite_code, Some(13));
8159 assert!(!failure.message.contains("rollback"));
8160 assert!(client.conn.is_autocommit());
8161 client
8162 .conn
8163 .execute_batch("PRAGMA max_page_count = 1073741823")
8164 .unwrap();
8165 assert!(matches!(
8166 client.apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest"),
8167 Ok(1)
8168 ));
8169 assert!(client.conn.is_autocommit());
8170 }
8171
8172 #[test]
8173 fn authoring_storage_failures_are_structured_and_scoped() {
8174 let path = std::env::temp_dir().join(format!(
8175 "syncular-authoring-failure-{}.db",
8176 uuid::Uuid::new_v4()
8177 ));
8178 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8179 {"name":"id","type":"string","nullable":false},
8180 {"name":"project_id","type":"string","nullable":false},
8181 {"name":"title","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8182 let mut client = SyncClient::open_path(
8183 "authoring".into(),
8184 &schema,
8185 ClientLimits::default(),
8186 path.to_str().unwrap(),
8187 )
8188 .expect("open client");
8189 client.create_synced_tables().expect("create synced tables");
8190 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
8192
8193 let upsert = |id: &str| {
8194 vec![Mutation::Upsert {
8195 table: "tasks".into(),
8196 values: Map::from_iter([
8197 ("id".into(), json!(id)),
8198 ("project_id".into(), json!("p1")),
8199 ("title".into(), json!("first")),
8200 ]),
8201 base_version: None,
8202 }]
8203 };
8204
8205 let locker = Connection::open(&path).expect("second connection");
8206 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
8207 let busy = client.mutate(upsert("t1")).expect_err("busy write");
8208 assert_eq!(busy.code, "client.storage_busy");
8209 assert!(busy.retryable);
8210 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
8211 assert!(client.pending_commit_ids().is_empty());
8212 assert_eq!(client.local_revision(), 0);
8213
8214 let invalid = client.mutate(Vec::new()).expect_err("empty commit");
8216 assert_eq!(invalid.code, "sync.invalid_request");
8217 assert_eq!(invalid.message, "the authoring request is invalid");
8218 assert!(
8219 invalid.details.as_ref().unwrap()["legacyCause"]
8220 .as_str()
8221 .unwrap()
8222 .contains("at least one operation"),
8223 "{invalid:?}"
8224 );
8225
8226 locker.execute_batch("ROLLBACK").unwrap();
8227 let id = client.mutate(upsert("t1")).expect("retry after busy");
8228 assert_eq!(client.pending_commit_ids(), vec![id]);
8229 assert_eq!(client.local_revision(), 1);
8230
8231 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
8233 let busy_again = client.mutate(upsert("t2")).expect_err("busy write again");
8234 assert_eq!(busy_again.code, "client.storage_busy");
8235 assert!(busy_again.retryable);
8236 locker.execute_batch("ROLLBACK").unwrap();
8237
8238 let full = ClientError::from(QueryReadFailure::from(rusqlite::Error::SqliteFailure(
8240 rusqlite::ffi::Error::new(13),
8241 Some("database or disk is full".to_owned()),
8242 )));
8243 assert_eq!(full.code, "client.storage_full");
8244 assert!(!full.retryable);
8245 assert_eq!(full.details.as_ref().unwrap()["sqliteCode"], 13);
8246
8247 drop(locker);
8248 drop(client);
8249 let _ = std::fs::remove_file(&path);
8250 }
8251
8252 #[test]
8253 fn caller_value_failures_are_invalid_request_and_leave_no_outbox_entry() {
8254 let path = std::env::temp_dir().join(format!(
8255 "syncular-caller-values-{}.db",
8256 uuid::Uuid::new_v4()
8257 ));
8258 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8259 {"name":"id","type":"string","nullable":false},
8260 {"name":"project_id","type":"string","nullable":false},
8261 {"name":"title","type":"string","nullable":false},
8262 {"name":"priority","type":"integer","nullable":false},
8263 {"name":"payload","type":"bytes","nullable":false},
8264 {"name":"extra","type":"bytes","nullable":true}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8265 let mut client = SyncClient::open_path(
8266 "caller-values".into(),
8267 &schema,
8268 ClientLimits::default(),
8269 path.to_str().unwrap(),
8270 )
8271 .expect("open client");
8272 client.create_synced_tables().expect("create synced tables");
8273
8274 let values = |id: &str| {
8275 Map::from_iter([
8276 ("id".to_owned(), json!(id)),
8277 ("project_id".to_owned(), json!("p1")),
8278 ("title".to_owned(), json!("first")),
8279 ("priority".to_owned(), json!(1)),
8280 ("payload".to_owned(), json!({"$bytes": "0a"})),
8281 ])
8282 };
8283 let upsert = |values: Map<String, Value>| {
8284 vec![Mutation::Upsert {
8285 table: "tasks".into(),
8286 values,
8287 base_version: None,
8288 }]
8289 };
8290 let seed = client.mutate(upsert(values("t1"))).expect("seed row");
8291 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
8292
8293 for (label, override_value, cause) in [
8298 ("unknown column", ("nope", json!(1)), "unknown column"),
8299 (
8300 "internal sync column",
8301 ("_sync_version", json!(1)),
8302 "internal sync column",
8303 ),
8304 ("wrong value type", ("title", json!(7)), "expected a string"),
8305 (
8306 "malformed bytes envelope",
8307 ("payload", json!({"$bytes": "zz"})),
8308 "expected exactly",
8309 ),
8310 (
8311 "odd-length bytes envelope",
8312 ("payload", json!({"$bytes": "abc"})),
8313 "expected exactly",
8314 ),
8315 (
8316 "plus-sign bytes envelope",
8317 ("payload", json!({"$bytes": "+a"})),
8318 "expected exactly",
8319 ),
8320 (
8321 "bytes envelope with a second key",
8322 ("payload", json!({"$bytes": "0a", "extra": 1})),
8323 "expected exactly",
8324 ),
8325 (
8326 "no bytes envelope",
8327 ("payload", json!({"nested": true})),
8328 "expected exactly",
8329 ),
8330 ] {
8331 let mut full = values("t2");
8332 full.insert(override_value.0.to_owned(), override_value.1);
8333 let error = client.mutate(upsert(full)).expect_err(label);
8334 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8335 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8336 assert!(!error.retryable, "{label}");
8337 let legacy = error.details.as_ref().unwrap()["legacyCause"]
8338 .as_str()
8339 .unwrap();
8340 assert!(
8341 legacy.starts_with("sync.invalid_request: ") && legacy.contains(cause),
8342 "{label}: {legacy}"
8343 );
8344 assert_eq!(client.pending_commit_ids(), vec![seed.clone()], "{label}");
8345 assert_eq!(client.local_revision(), 1, "{label}");
8346 assert_eq!(
8347 client.query("SELECT * FROM tasks", &[]).unwrap(),
8348 rows,
8349 "{label}"
8350 );
8351 }
8352
8353 let mut missing = values("t3");
8356 missing.remove("payload");
8357 let missing_error = client.mutate(upsert(missing)).expect_err("missing column");
8358 assert_eq!(missing_error.code, "sync.invalid_request");
8359 assert!(missing_error.details.as_ref().unwrap()["legacyCause"]
8360 .as_str()
8361 .unwrap()
8362 .contains("is not nullable"));
8363
8364 let mut bad_row_id = values("t4");
8367 bad_row_id.insert("id".to_owned(), json!({"nested": true}));
8368 let row_id_error = client.mutate(upsert(bad_row_id)).expect_err("bad rowId");
8369 assert_eq!(row_id_error.code, "sync.invalid_request");
8370
8371 for (label, partial, cause) in [
8373 (
8374 "patch wrong value type",
8375 ("priority", json!("high")),
8376 "expected an integer",
8377 ),
8378 ("patch unknown column", ("nope", json!(1)), "unknown column"),
8379 ] {
8380 let error = client
8381 .patch(
8382 "tasks",
8383 "t1",
8384 Map::from_iter([(partial.0.to_owned(), partial.1)]),
8385 None,
8386 )
8387 .expect_err(label);
8388 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8389 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8390 assert!(
8391 error.details.as_ref().unwrap()["legacyCause"]
8392 .as_str()
8393 .unwrap()
8394 .contains(cause),
8395 "{label}"
8396 );
8397 }
8398 assert_eq!(client.pending_commit_ids(), vec![seed.clone()]);
8399 assert_eq!(client.local_revision(), 1);
8400 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
8401
8402 for (label, payload) in [
8405 ("explicit null", json!(null)),
8406 ("empty envelope", json!({"$bytes": ""})),
8407 ("uppercase envelope", json!({"$bytes": "0AFF"})),
8408 ] {
8409 let mut accepted = values("t-ok");
8410 accepted.insert("extra".to_owned(), payload);
8411 let encoded = client.mutate(upsert(accepted)).unwrap_or_else(|error| {
8412 panic!("{label} must be accepted: {error:?}");
8413 });
8414 assert!(!encoded.is_empty(), "{label}");
8415 }
8416
8417 let next = client
8419 .mutate(upsert(values("t5")))
8420 .expect("valid after failures");
8421 assert_ne!(next, seed);
8422
8423 drop(client);
8424 let _ = std::fs::remove_file(&path);
8425 }
8426
8427 #[test]
8428 fn a_legacy_outbox_value_the_codec_refuses_is_dropped_instead_of_wedging() {
8429 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8430 {"name":"id","type":"string","nullable":false},
8431 {"name":"project_id","type":"string","nullable":false},
8432 {"name":"payload","type":"bytes","nullable":true}],
8433 "scopes":[{"pattern":"project:{project_id}"}]}]});
8434 let open = |path: &std::path::Path| {
8435 SyncClient::open_path(
8436 "legacy-values".into(),
8437 &schema,
8438 ClientLimits::default(),
8439 path.to_str().unwrap(),
8440 )
8441 .expect("open client")
8442 };
8443 let row = |id: &str| {
8444 Map::from_iter([
8445 ("id".to_owned(), json!(id)),
8446 ("project_id".to_owned(), json!("p1")),
8447 ("payload".to_owned(), json!({"$bytes": "0a"})),
8448 ])
8449 };
8450 let upsert = |values| {
8451 vec![Mutation::Upsert {
8452 table: "tasks".into(),
8453 values,
8454 base_version: None,
8455 }]
8456 };
8457 for (label, stored_cell, representable) in [
8458 ("plus-sign envelope", json!({"$bytes": "+a"}), false),
8459 ("non-hex envelope", json!({"$bytes": "zz"}), false),
8460 ("odd-length envelope", json!({"$bytes": "abc"}), false),
8461 ("no envelope", json!({"nested": true}), false),
8462 ("wrong declared type", json!("high"), true),
8466 ("stored null for a non-nullable column", json!(null), true),
8467 ] {
8468 let path = std::env::temp_dir().join(format!(
8469 "syncular-legacy-values-{}-{}.db",
8470 label.replace(' ', "-"),
8471 uuid::Uuid::new_v4()
8472 ));
8473 let mut client = open(&path);
8474 client.create_synced_tables().expect("create synced tables");
8475 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8476 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8477 drop(client);
8478
8479 let connection = Connection::open(&path).expect("raw connection");
8481 let (column, cell) = if stored_cell.is_null() {
8482 ("project_id", stored_cell.clone())
8483 } else {
8484 ("payload", stored_cell.clone())
8485 };
8486 let mut planted_values = json!({
8487 "id": "t1", "project_id": "p1", "payload": {"$bytes": "0a"}
8488 });
8489 planted_values[column] = cell;
8490 let planted = json!([{
8491 "op": "upsert",
8492 "table": "tasks",
8493 "rowId": "t1",
8494 "baseVersion": null,
8495 "values": planted_values
8496 }]);
8497 connection
8498 .execute(
8499 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8500 rusqlite::params![planted.to_string(), stale],
8501 )
8502 .expect("plant legacy intent");
8503 drop(connection);
8504
8505 let reopened = open(&path);
8508 assert_eq!(
8509 reopened.pending_commit_ids(),
8510 vec![successor.clone()],
8511 "{label}"
8512 );
8513 let rejection = reopened
8514 .rejections()
8515 .iter()
8516 .find(|rejection| rejection.client_commit_id == stale)
8517 .unwrap_or_else(|| panic!("{label}: rejection"));
8518 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8519 assert_eq!(
8520 rejection.message, "the persisted commit carries values the current codec refuses",
8521 "{label}"
8522 );
8523 assert_eq!(rejection.operation.is_some(), representable, "{label}");
8524 assert_eq!(
8525 rejection
8526 .details
8527 .as_ref()
8528 .and_then(|details| details.reason.as_deref()),
8529 Some(INVALID_STORED_VALUES_REASON),
8530 "{label}"
8531 );
8532 let journal = reopened
8534 .commit_outcome(&stale)
8535 .expect("read outcome")
8536 .expect("journal entry");
8537 assert!(matches!(journal.status, CommitOutcomeStatus::Rejected));
8538 drop(reopened);
8539 let mut reopened = open(&path);
8540 let reread = reopened
8541 .commit_outcome(&stale)
8542 .expect("read outcome")
8543 .expect("journal entry");
8544 assert!(matches!(reread.status, CommitOutcomeStatus::Rejected));
8545
8546 assert_eq!(
8549 reopened
8550 .query("SELECT id FROM tasks ORDER BY id", &[])
8551 .unwrap(),
8552 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8553 "{label}"
8554 );
8555 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8556 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8557 assert!(!reopened
8558 .drop_unencodable_outbox(false, None)
8559 .expect("idempotent prepass"));
8560 let mut transport = CountingRealtimeTransport::default();
8561 let outcome = reopened.sync(&mut transport);
8562 assert!(matches!(outcome, SyncOutcome::Failed { .. }), "{outcome:?}");
8563 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8564 let next = reopened.mutate(upsert(row("t3"))).expect("after the drop");
8565 assert_eq!(reopened.pending_commit_ids(), vec![successor, next]);
8566 assert_eq!(reopened.pending_payloads().len(), 2, "{label}");
8567 drop(reopened);
8568 let _ = std::fs::remove_file(&path);
8569 }
8570 }
8571
8572 #[test]
8573 fn two_legacy_values_on_the_same_row_recover_in_fifo_order() {
8574 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8575 {"name":"id","type":"string","nullable":false},
8576 {"name":"project_id","type":"string","nullable":false},
8577 {"name":"payload","type":"bytes","nullable":true}],
8578 "scopes":[{"pattern":"project:{project_id}"}]}]});
8579 for (label, column, planted) in [
8580 ("malformed envelope", "payload", json!({"$bytes": "zz"})),
8581 (
8582 "stored null for a non-nullable column",
8583 "project_id",
8584 json!(null),
8585 ),
8586 ("wrong declared type", "payload", json!(7)),
8587 ] {
8588 let path = std::env::temp_dir().join(format!(
8589 "syncular-shared-row-{}-{}.db",
8590 label.replace(' ', "-"),
8591 uuid::Uuid::new_v4()
8592 ));
8593 let open = |path: &std::path::Path| {
8594 SyncClient::open_path(
8595 "shared-row".into(),
8596 &schema,
8597 ClientLimits::default(),
8598 path.to_str().unwrap(),
8599 )
8600 .expect("open client")
8601 };
8602 let row = |id: &str| {
8603 Map::from_iter([
8604 ("id".to_owned(), json!(id)),
8605 ("project_id".to_owned(), json!("p1")),
8606 ("payload".to_owned(), json!({"$bytes": "0a"})),
8607 ])
8608 };
8609 let upsert = |values| {
8610 vec![Mutation::Upsert {
8611 table: "tasks".into(),
8612 values,
8613 base_version: None,
8614 }]
8615 };
8616 let mut client = open(&path);
8617 client.create_synced_tables().expect("create synced tables");
8618 let older = client.mutate(upsert(row("t1"))).expect("older commit");
8619 let newer = client.mutate(upsert(row("t1"))).expect("newer commit");
8620 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8621 drop(client);
8622
8623 let connection = Connection::open(&path).expect("raw connection");
8624 for commit_id in [&older, &newer] {
8625 let mut values = row("t1");
8626 values.insert(column.to_owned(), planted.clone());
8627 let planted_ops = json!([{
8628 "op": "upsert",
8629 "table": "tasks",
8630 "rowId": "t1",
8631 "baseVersion": null,
8632 "values": values
8633 }]);
8634 connection
8635 .execute(
8636 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8637 rusqlite::params![planted_ops.to_string(), commit_id],
8638 )
8639 .expect("plant legacy intent");
8640 }
8641 drop(connection);
8642
8643 let mut reopened = open(&path);
8645 assert_eq!(
8646 reopened.pending_commit_ids(),
8647 vec![successor.clone()],
8648 "{label}"
8649 );
8650 assert_eq!(
8651 reopened
8652 .rejections()
8653 .iter()
8654 .map(|rejection| rejection.client_commit_id.clone())
8655 .collect::<Vec<_>>(),
8656 vec![older.clone(), newer.clone()],
8657 "{label}"
8658 );
8659 assert!(
8660 reopened
8661 .rejections()
8662 .iter()
8663 .all(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE
8664 && rejection.details.as_ref().and_then(|d| d.reason.as_deref())
8665 == Some(INVALID_STORED_VALUES_REASON)),
8666 "{label}"
8667 );
8668 for commit_id in [&older, &newer] {
8669 assert!(
8670 matches!(
8671 reopened
8672 .commit_outcome(commit_id)
8673 .expect("outcome")
8674 .map(|o| o.status),
8675 Some(CommitOutcomeStatus::Rejected)
8676 ),
8677 "{label}"
8678 );
8679 }
8680 assert_eq!(
8681 reopened
8682 .query("SELECT id FROM tasks ORDER BY id", &[])
8683 .unwrap(),
8684 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8685 "{label}"
8686 );
8687 let revision = reopened.local_revision();
8688 let batches = reopened.drain_change_batches();
8689 assert!(
8690 batches.iter().any(|batch| batch.rejections_changed
8691 && batch.outcomes_changed
8692 && batch.tables.iter().any(|change| change.table == "tasks")
8693 && batch.revision == revision.to_string()),
8694 "{label}: {batches:?}"
8695 );
8696 drop(reopened);
8697
8698 let mut stable = open(&path);
8701 assert_eq!(stable.local_revision(), revision, "{label}");
8702 assert!(stable.drain_change_batches().is_empty(), "{label}");
8703 assert_eq!(stable.pending_commit_ids(), vec![successor], "{label}");
8704 drop(stable);
8705 let _ = std::fs::remove_file(&path);
8706 }
8707 }
8708
8709 #[test]
8710 fn recovery_omits_a_legacy_operation_the_journal_reader_rejects() {
8711 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8712 {"name":"id","type":"string","nullable":false},
8713 {"name":"project_id","type":"string","nullable":false},
8714 {"name":"payload","type":"bytes","nullable":true}],
8715 "scopes":[{"pattern":"project:{project_id}"}]}]});
8716 for (label, second_op) in [
8717 (
8718 "delete carrying values",
8719 json!({"op": "delete", "table": "tasks", "rowId": "t9", "values": {}}),
8720 ),
8721 (
8722 "baseVersion beyond the safe bound",
8723 json!({"op": "upsert", "table": "tasks", "rowId": "t9",
8724 "baseVersion": MAX_JS_SAFE_INTEGER + 1,
8725 "values": {"id": "t9", "project_id": "p1", "payload": {"$bytes": "0a"}}}),
8726 ),
8727 ] {
8728 let path = std::env::temp_dir().join(format!(
8729 "syncular-journal-shape-{}-{}.db",
8730 label.replace(' ', "-"),
8731 uuid::Uuid::new_v4()
8732 ));
8733 let open = |path: &std::path::Path| {
8734 SyncClient::open_path(
8735 "journal-shape".into(),
8736 &schema,
8737 ClientLimits::default(),
8738 path.to_str().unwrap(),
8739 )
8740 .expect("open client")
8741 };
8742 let values = |id: &str| {
8743 Map::from_iter([
8744 ("id".to_owned(), json!(id)),
8745 ("project_id".to_owned(), json!("p1")),
8746 ("payload".to_owned(), json!({"$bytes": "0a"})),
8747 ])
8748 };
8749 let upsert = |values| {
8750 vec![Mutation::Upsert {
8751 table: "tasks".into(),
8752 values,
8753 base_version: None,
8754 }]
8755 };
8756 let mut client = open(&path);
8757 client.create_synced_tables().expect("create synced tables");
8758 let stale = client.mutate(upsert(values("t1"))).expect("stale commit");
8759 drop(client);
8760
8761 let connection = Connection::open(&path).expect("raw connection");
8762 let planted = json!([
8766 {
8767 "op": "upsert",
8768 "table": "tasks",
8769 "rowId": "t1",
8770 "baseVersion": null,
8771 "values": {"id": "t1", "project_id": null,
8772 "payload": {"$bytes": "0a"}}
8773 },
8774 second_op
8775 ]);
8776 connection
8777 .execute(
8778 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8779 rusqlite::params![planted.to_string(), stale],
8780 )
8781 .expect("plant legacy intent");
8782 drop(connection);
8783
8784 let reopened = open(&path);
8785 let journal = reopened
8786 .commit_outcome(&stale)
8787 .unwrap_or_else(|error| panic!("{label}: journal read: {error}"))
8788 .unwrap_or_else(|| panic!("{label}: journal entry"));
8789 assert!(
8790 matches!(journal.status, CommitOutcomeStatus::Rejected),
8791 "{label}"
8792 );
8793 assert!(journal.operations.is_none(), "{label}");
8794 assert!(
8795 reopened
8796 .commit_outcomes(CommitOutcomeQuery::default())
8797 .is_ok(),
8798 "{label}: outcome list query"
8799 );
8800 assert!(reopened.pending_commit_ids().is_empty(), "{label}");
8801 drop(reopened);
8802 let _ = std::fs::remove_file(&path);
8803 }
8804 }
8805
8806 #[test]
8807 fn legacy_values_recover_at_the_recreate_and_bump_reset_boundaries() {
8808 let schema = |version: i32, extra: bool| {
8809 let mut columns = vec![
8810 json!({"name":"id","type":"string","nullable":false}),
8811 json!({"name":"project_id","type":"string","nullable":false}),
8812 json!({"name":"payload","type":"bytes","nullable":true}),
8813 ];
8814 if extra {
8815 columns.push(json!({"name":"later","type":"string","nullable":true}));
8816 }
8817 json!({"version":version,"tables":[{"name":"tasks","primaryKey":"id","columns":columns,
8818 "scopes":[{"pattern":"project:{project_id}"}]}]})
8819 };
8820 for (label, bump) in [
8821 ("same-version recreation", false),
8822 ("version-bump reset", true),
8823 ("version-only bump reset", true),
8824 ] {
8825 let path = std::env::temp_dir().join(format!(
8826 "syncular-legacy-reset-{}-{}.db",
8827 label.replace(' ', "-"),
8828 uuid::Uuid::new_v4()
8829 ));
8830 let v1 = schema(1, false);
8831 let open = |schema: &Value, path: &std::path::Path| {
8832 SyncClient::open_path(
8833 "legacy-reset".into(),
8834 schema,
8835 ClientLimits::default(),
8836 path.to_str().unwrap(),
8837 )
8838 .expect("open client")
8839 };
8840 let row = |id: &str| {
8841 Map::from_iter([
8842 ("id".to_owned(), json!(id)),
8843 ("project_id".to_owned(), json!("p1")),
8844 ("payload".to_owned(), json!({"$bytes": "0a"})),
8845 ])
8846 };
8847 let upsert = |values| {
8848 vec![Mutation::Upsert {
8849 table: "tasks".into(),
8850 values,
8851 base_version: None,
8852 }]
8853 };
8854 let mut client = open(&v1, &path);
8855 client.create_synced_tables().expect("create synced tables");
8856 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8857 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8858 drop(client);
8859
8860 let connection = Connection::open(&path).expect("raw connection");
8861 let planted = json!([{
8862 "op": "upsert",
8863 "table": "tasks",
8864 "rowId": "t1",
8865 "baseVersion": null,
8866 "values": {"id": "t1", "project_id": 7, "payload": {"$bytes": "0a"}}
8867 }]);
8868 connection
8869 .execute(
8870 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8871 rusqlite::params![planted.to_string(), stale],
8872 )
8873 .expect("plant legacy intent");
8874 drop(connection);
8875
8876 let target = if bump {
8877 schema(2, label == "version-bump reset")
8878 } else {
8879 v1.clone()
8880 };
8881 let reopened = open(&target, &path);
8882 let rejection = reopened
8883 .rejections()
8884 .iter()
8885 .find(|rejection| rejection.client_commit_id == stale)
8886 .unwrap_or_else(|| panic!("{label}: rejection"));
8887 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8888 assert_eq!(
8889 rejection
8890 .details
8891 .as_ref()
8892 .and_then(|details| details.reason.as_deref()),
8893 Some(INVALID_STORED_VALUES_REASON),
8894 "{label}"
8895 );
8896 assert!(rejection.operation.is_some(), "{label}");
8897 assert_eq!(
8898 reopened.pending_commit_ids(),
8899 vec![successor.clone()],
8900 "{label}"
8901 );
8902 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8907 if label == "version-bump reset" {
8908 assert_eq!(
8909 reopened
8910 .query("SELECT id FROM tasks ORDER BY id", &[])
8911 .unwrap(),
8912 Vec::<Map<String, Value>>::new(),
8913 "{label}"
8914 );
8915 let mut next = reopened;
8919 let revision = next.local_revision();
8920 let batches = next.drain_change_batches();
8921 assert!(
8922 batches.iter().any(|batch| batch.rejections_changed
8923 && batch.outcomes_changed
8924 && batch.tables.iter().any(|change| change.table == "tasks")
8925 && batch.revision == revision.to_string()),
8926 "{label}: {batches:?}"
8927 );
8928 next.mutate(upsert(Map::from_iter([
8929 ("id".to_owned(), json!("t3")),
8930 ("project_id".to_owned(), json!("p1")),
8931 ("payload".to_owned(), json!({"$bytes": "0b"})),
8932 ("later".to_owned(), json!("v2")),
8933 ])))
8934 .expect("v2 authoring after the reset");
8935 assert_eq!(next.pending_commit_ids().len(), 2, "{label}");
8936 assert_eq!(next.pending_payloads().len(), 2, "{label}");
8937 drop(next);
8938 } else {
8939 assert_eq!(
8940 reopened
8941 .query("SELECT id FROM tasks ORDER BY id", &[])
8942 .unwrap(),
8943 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8944 "{label}"
8945 );
8946 drop(reopened);
8947 }
8948
8949 if !bump {
8952 let mut recreated = open(&v1, &path);
8953 let fresh = recreated
8954 .mutate(upsert(row("t3")))
8955 .expect("fresh commit while open");
8956 recreated.drain_change_batches();
8957 let planted_fresh = json!([{
8958 "op": "upsert",
8959 "table": "tasks",
8960 "rowId": "t3",
8961 "baseVersion": null,
8962 "values": {"id": "t3", "project_id": "p1",
8963 "payload": {"$bytes": "zz"}}
8964 }]);
8965 let connection = Connection::open(&path).expect("raw connection");
8966 let affected = connection
8967 .execute(
8968 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8969 rusqlite::params![planted_fresh.to_string(), fresh],
8970 )
8971 .expect("plant fresh legacy intent");
8972 assert_eq!(
8973 affected, 1,
8974 "{label}: the in-place plant must hit the queued commit"
8975 );
8976 drop(connection);
8977 let revision_before = recreated.local_revision();
8978 recreated
8979 .recreate_with_schema(&v1)
8980 .unwrap_or_else(|error| panic!("{label}: recreation failed: {error}"));
8981 assert_eq!(recreated.pending_commit_ids(), vec![successor], "{label}");
8982 let fresh_rejection = recreated
8983 .rejections()
8984 .iter()
8985 .find(|rejection| rejection.client_commit_id == fresh)
8986 .expect("the in-place recovery rejection");
8987 assert_eq!(fresh_rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8988 assert_eq!(
8989 fresh_rejection
8990 .details
8991 .as_ref()
8992 .and_then(|details| details.reason.as_deref()),
8993 Some(INVALID_STORED_VALUES_REASON),
8994 "{label}"
8995 );
8996 assert!(
8997 matches!(
8998 recreated
8999 .commit_outcome(&fresh)
9000 .expect("outcome read")
9001 .map(|outcome| outcome.status),
9002 Some(CommitOutcomeStatus::Rejected)
9003 ),
9004 "{label}"
9005 );
9006 assert!(recreated.local_revision() > revision_before, "{label}");
9007 let revision = recreated.local_revision();
9008 let batches = recreated.drain_change_batches();
9009 assert_eq!(batches.len(), 1, "{label}: one recovery batch");
9010 assert!(
9011 batches.iter().any(|batch| batch.rejections_changed
9012 && batch.outcomes_changed
9013 && batch.tables.iter().any(|change| change.table == "tasks")
9014 && batch.revision == revision.to_string()),
9015 "{label}: {batches:?}"
9016 );
9017 assert_eq!(recreated.pending_payloads().len(), 1, "{label}");
9018 assert_eq!(
9019 recreated
9020 .query("SELECT id FROM tasks ORDER BY id", &[])
9021 .unwrap(),
9022 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9023 "{label}: the recovered commit leaves no projection"
9024 );
9025 drop(recreated);
9026 }
9027 let _ = std::fs::remove_file(&path);
9028 }
9029 }
9030
9031 #[test]
9032 fn legacy_values_recover_over_a_synced_authoritative_row() {
9033 let path = std::env::temp_dir().join(format!(
9034 "syncular-authoritative-row-{}.db",
9035 uuid::Uuid::new_v4()
9036 ));
9037 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9038 {"name":"id","type":"string","nullable":false},
9039 {"name":"project_id","type":"string","nullable":false},
9040 {"name":"title","type":"string","nullable":false},
9041 {"name":"payload","type":"bytes","nullable":true}],
9042 "scopes":[{"pattern":"project:{project_id}"}]}]});
9043 let open = |path: &std::path::Path| {
9044 SyncClient::open_path(
9045 "authoritative-row".into(),
9046 &schema,
9047 ClientLimits::default(),
9048 path.to_str().unwrap(),
9049 )
9050 .expect("open client")
9051 };
9052 let row = |title: &str| {
9053 Map::from_iter([
9054 ("id".to_owned(), json!("t1")),
9055 ("project_id".to_owned(), json!("p1")),
9056 ("title".to_owned(), json!(title)),
9057 ("payload".to_owned(), json!({"$bytes": "0a"})),
9058 ])
9059 };
9060 let upsert = |values| {
9061 vec![Mutation::Upsert {
9062 table: "tasks".into(),
9063 values,
9064 base_version: None,
9065 }]
9066 };
9067 let mut client = open(&path);
9068 client.create_synced_tables().expect("create synced tables");
9069 client
9072 .conn
9073 .execute_batch(
9074 "INSERT INTO _syncular_base_tasks VALUES ('t1','p1','authoritative',x'0a',1);
9075 INSERT INTO tasks VALUES ('t1','p1','authoritative',x'0a',1);",
9076 )
9077 .expect("seed authoritative row");
9078 let older = client.mutate(upsert(row("older"))).expect("older");
9079 let newer = client.mutate(upsert(row("newer"))).expect("newer");
9080 let successor = client.mutate(upsert(row("successor"))).expect("successor");
9081 drop(client);
9082
9083 let connection = Connection::open(&path).expect("raw connection");
9084 for commit_id in [&older, &newer] {
9085 let planted = json!([{
9086 "op": "upsert",
9087 "table": "tasks",
9088 "rowId": "t1",
9089 "baseVersion": null,
9090 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
9091 "payload": {"$bytes": "zz"}}
9092 }]);
9093 connection
9094 .execute(
9095 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9096 rusqlite::params![planted.to_string(), commit_id],
9097 )
9098 .expect("plant legacy intent");
9099 }
9100 drop(connection);
9101
9102 let reopened = open(&path);
9103 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
9104 assert_eq!(
9105 reopened
9106 .rejections()
9107 .iter()
9108 .map(|rejection| rejection.client_commit_id.clone())
9109 .collect::<Vec<_>>(),
9110 vec![older.clone(), newer.clone()]
9111 );
9112 assert_eq!(
9115 reopened.query("SELECT id,title,_syncular_version FROM tasks", &[]),
9116 Ok(vec![Map::from_iter([
9117 ("id".to_owned(), json!("t1")),
9118 ("title".to_owned(), json!("successor")),
9119 ("_syncular_version".to_owned(), json!(1)),
9120 ])])
9121 );
9122
9123 drop(reopened);
9126 let connection = Connection::open(&path).expect("raw connection");
9127 connection
9128 .execute(
9129 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9130 rusqlite::params![
9131 json!([{
9132 "op": "upsert",
9133 "table": "tasks",
9134 "rowId": "t1",
9135 "baseVersion": null,
9136 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
9137 "payload": {"$bytes": "zz"}}
9138 }])
9139 .to_string(),
9140 successor
9141 ],
9142 )
9143 .expect("plant successor intent");
9144 drop(connection);
9145 let late = open(&path);
9146 assert!(late.pending_commit_ids().is_empty());
9147 assert_eq!(
9148 late.query("SELECT id,title,_syncular_version FROM tasks", &[]),
9149 Ok(vec![Map::from_iter([
9150 ("id".to_owned(), json!("t1")),
9151 ("title".to_owned(), json!("authoritative")),
9152 ("_syncular_version".to_owned(), json!(1)),
9153 ])])
9154 );
9155 drop(late);
9156 let _ = std::fs::remove_file(&path);
9157 }
9158
9159 #[test]
9160 fn a_failed_value_recovery_commit_restores_every_side_effect() {
9161 let path = std::env::temp_dir().join(format!(
9162 "syncular-value-recovery-commit-{}.db",
9163 uuid::Uuid::new_v4()
9164 ));
9165 let path_str = path.to_str().expect("path");
9166 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9167 {"name":"id","type":"string","nullable":false},
9168 {"name":"project_id","type":"string","nullable":false},
9169 {"name":"payload","type":"bytes","nullable":true}],
9170 "scopes":[{"pattern":"project:{project_id}"}]}]});
9171 let conn = Connection::open(path_str).expect("replica connection");
9174 let mut client = SyncClient::with_connection(
9175 "value-recovery-commit".into(),
9176 &schema,
9177 ClientLimits::default(),
9178 conn,
9179 )
9180 .expect("replica");
9181 assert_eq!(
9182 client
9183 .conn
9184 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
9185 .expect("journal mode"),
9186 "delete"
9187 );
9188 client.create_synced_tables().expect("create synced tables");
9189 let stale = client
9190 .mutate(vec![Mutation::Upsert {
9191 table: "tasks".into(),
9192 values: Map::from_iter([
9193 ("id".to_owned(), json!("t1")),
9194 ("project_id".to_owned(), json!("p1")),
9195 ("payload".to_owned(), json!({"$bytes": "0a"})),
9196 ]),
9197 base_version: None,
9198 }])
9199 .expect("queued write");
9200 client.drain_change_batches();
9201 let affected = client
9202 .conn
9203 .execute(
9204 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9205 rusqlite::params![
9206 json!([{
9207 "op": "upsert",
9208 "table": "tasks",
9209 "rowId": "t1",
9210 "baseVersion": null,
9211 "values": {"id": "t1", "project_id": "p1",
9212 "payload": {"$bytes": "zz"}}
9213 }])
9214 .to_string(),
9215 stale
9216 ],
9217 )
9218 .expect("plant legacy intent");
9219 assert_eq!(affected, 1, "the planted row must exist");
9220 let revision = client.local_revision();
9221 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
9222 let prior_outbox = client.pending_commit_ids();
9223 let prior_rejections = client.rejections().len();
9224 let prior_changes = client.change_queue.len();
9225 let prior_intents = client.sync_intent_queue.len();
9226 let prior_failed = client.failed_commits.len();
9227 let reader = Connection::open(path_str).expect("reader");
9230 reader
9231 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9232 .expect("reader snapshot");
9233 client
9234 .conn
9235 .busy_timeout(std::time::Duration::ZERO)
9236 .expect("busy timeout");
9237 let error = client
9238 .recreate_with_schema(&schema)
9239 .expect_err("a blocked COMMIT must refuse the recovery");
9240 assert!(error.starts_with("client.storage_busy:"), "{error}");
9241 assert!(client.conn.is_autocommit(), "the transaction is released");
9242 assert_eq!(client.local_revision(), revision);
9243 assert_eq!(client.pending_commit_ids(), prior_outbox);
9244 assert_eq!(client.rejections().len(), prior_rejections);
9245 assert_eq!(client.change_queue.len(), prior_changes);
9246 assert_eq!(client.sync_intent_queue.len(), prior_intents);
9247 assert_eq!(client.failed_commits.len(), prior_failed);
9248 assert_eq!(
9249 client.query("SELECT * FROM tasks", &[]).expect("rows"),
9250 rows
9251 );
9252 assert!(
9253 client
9254 .commit_outcome(&stale)
9255 .expect("outcome read")
9256 .is_none(),
9257 "the rolled-back recovery kept no outcome"
9258 );
9259 reader.execute_batch("ROLLBACK").expect("release reader");
9261 client
9262 .recreate_with_schema(&schema)
9263 .expect("retry after the blocked commit");
9264 assert!(client.pending_commit_ids().is_empty());
9265 assert_eq!(client.rejections().len(), prior_rejections + 1);
9266 assert!(client.local_revision() > revision);
9267 let batches = client.drain_change_batches();
9268 assert!(
9269 batches.iter().any(|batch| batch.rejections_changed
9270 && batch.outcomes_changed
9271 && batch.tables.iter().any(|change| change.table == "tasks")),
9272 "{batches:?}"
9273 );
9274 drop(client);
9275 let _ = std::fs::remove_file(&path);
9276 }
9277
9278 #[cfg(feature = "e2ee")]
9279 #[test]
9280 fn a_failed_recovery_restores_retained_failed_commits() {
9281 let path = std::env::temp_dir().join(format!(
9282 "syncular-value-recovery-retention-{}.db",
9283 uuid::Uuid::new_v4()
9284 ));
9285 let path_str = path.to_str().expect("path");
9286 let schema = json!({"version":1,"tables":[{"name":"secrets","primaryKey":"id","columns":[
9287 {"name":"id","type":"string","nullable":false},
9288 {"name":"project_id","type":"string","nullable":false},
9289 {"name":"note","type":"bytes","nullable":true,"encrypted":true,"declaredType":"string"}],
9290 "scopes":[{"pattern":"project:{project_id}"}]}]});
9291 let conn = Connection::open(path_str).expect("replica connection");
9292 let mut client = SyncClient::with_connection(
9293 "value-recovery-retention".into(),
9294 &schema,
9295 ClientLimits::default(),
9296 conn,
9297 )
9298 .expect("replica");
9299 client.create_synced_tables().expect("create synced tables");
9300 client.set_retain_failed_commits(true).expect("retention");
9301 client.set_encryption(crate::values::EncryptionConfig::default());
9303 client
9304 .mutate(vec![Mutation::Upsert {
9305 table: "secrets".into(),
9306 values: Map::from_iter([
9307 ("id".to_owned(), json!("s1")),
9308 ("project_id".to_owned(), json!("p1")),
9309 ("note".to_owned(), json!("secret")),
9310 ]),
9311 base_version: None,
9312 }])
9313 .expect("queued write");
9314 client.drain_change_batches();
9315 let revision = client.local_revision();
9316 let prior_outbox = client.pending_commit_ids();
9317 let prior_rejections = client.rejections().len();
9318 let prior_failed = client.failed_commits.len();
9319 let reader = Connection::open(path_str).expect("reader");
9320 reader
9321 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9322 .expect("reader snapshot");
9323 client
9324 .conn
9325 .busy_timeout(std::time::Duration::ZERO)
9326 .expect("busy timeout");
9327 let error = client
9328 .drop_unencodable_outbox(true, None)
9329 .expect_err("a blocked COMMIT must refuse the recovery");
9330 assert!(error.starts_with("client.storage_busy:"), "{error}");
9331 assert_eq!(client.failed_commits.len(), prior_failed);
9332 assert_eq!(client.pending_commit_ids(), prior_outbox);
9333 assert_eq!(client.rejections().len(), prior_rejections);
9334 assert_eq!(client.local_revision(), revision);
9335 reader.execute_batch("ROLLBACK").expect("release reader");
9337 client
9338 .drop_unencodable_outbox(true, None)
9339 .expect("retry after the blocked commit");
9340 assert_eq!(client.failed_commits.len(), prior_failed + 1);
9341 assert!(client.pending_commit_ids().is_empty());
9342 assert_eq!(client.rejections().len(), prior_rejections + 1);
9343 drop(client);
9344 let _ = std::fs::remove_file(&path);
9345 }
9346
9347 #[test]
9348 fn client_error_maps_legacy_codes_and_static_messages() {
9349 let code_only = ClientError::from("sync.unknown_table".to_owned());
9350 assert_eq!(code_only.code, "sync.unknown_table");
9351 assert_eq!(code_only.message, "the commit targets an unknown table");
9352 assert!(code_only.details.is_none());
9353
9354 let dynamic = ClientError::from(
9355 "sync.invalid_request: table \"tasks\": patch cannot write scope column".to_owned(),
9356 );
9357 assert_eq!(dynamic.code, "sync.invalid_request");
9358 assert_eq!(dynamic.message, "the authoring request is invalid");
9359 assert!(dynamic.details.as_ref().unwrap()["legacyCause"]
9360 .as_str()
9361 .unwrap()
9362 .contains("patch cannot write scope column"));
9363
9364 let unclassified =
9365 ClientError::from("a commit must contain at least one operation".to_owned());
9366 assert_eq!(unclassified.code, "client.failed");
9367 assert_eq!(unclassified.message, "the local authoring operation failed");
9368 assert!(unclassified.details.as_ref().unwrap()["legacyCause"]
9369 .as_str()
9370 .unwrap()
9371 .contains("at least one operation"));
9372
9373 let metadata = ClientError::from(QueryReadFailure {
9375 code: None,
9376 sqlite_code: Some(1),
9377 sqlite_message: Some("generic SQLite prose".to_owned()),
9378 rollback_failure: Some(Box::new(QueryReadFailure::from(
9379 rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(1), None),
9380 ))),
9381 message: "unclassified read".to_owned(),
9382 });
9383 assert_eq!(metadata.code, "client.failed");
9384 assert_eq!(metadata.message, "the local authoring operation failed");
9385 let details = metadata.details.as_ref().unwrap();
9386 assert_eq!(details["sqliteCode"], 1);
9387 assert_eq!(details["sqliteMessage"], "generic SQLite prose");
9388 assert_eq!(details["rollbackFailure"]["sqliteCode"], 1);
9389 assert_eq!(details["legacyCause"], "unclassified read");
9390 }
9391
9392 #[test]
9393 fn schema_marker_read_keeps_storage_classification() {
9394 let busy = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9395 rusqlite::ffi::Error::new(5),
9396 Some("database is locked".to_owned()),
9397 ));
9398 assert!(busy.starts_with("client.storage_busy:"), "{busy}");
9399 let invalid = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9400 rusqlite::ffi::Error::new(1),
9401 Some("unreadable".to_owned()),
9402 ));
9403 assert_eq!(
9404 invalid,
9405 "sync.local_corrupt: persisted local schema marker is unreadable or invalid"
9406 );
9407
9408 let path =
9411 std::env::temp_dir().join(format!("syncular-marker-busy-{}.db", uuid::Uuid::new_v4()));
9412 let setup = Connection::open(&path).expect("setup connection");
9413 setup
9414 .execute_batch(
9415 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value TEXT);\n INSERT INTO _syncular_meta VALUES ('localSchemaVersion','1');",
9416 )
9417 .unwrap();
9418 let locker = Connection::open(&path).expect("locker connection");
9419 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9420 let reader = Connection::open(&path).expect("reader connection");
9421 reader.busy_timeout(std::time::Duration::ZERO).unwrap();
9422 let locked = read_local_schema_version(&reader, 1).expect_err("locked marker read");
9423 assert!(locked.starts_with("client.storage_busy:"), "{locked}");
9424 locker.execute_batch("ROLLBACK").unwrap();
9425 assert_eq!(read_local_schema_version(&reader, 1).unwrap(), Some(1));
9426 let rows = setup
9427 .query_row("SELECT count(*) FROM _syncular_meta", [], |row| {
9428 row.get::<_, i64>(0)
9429 })
9430 .unwrap();
9431 assert_eq!(rows, 1, "the guard performed no writes");
9432 drop(reader);
9433 drop(locker);
9434 drop(setup);
9435 let _ = std::fs::remove_file(&path);
9436 }
9437
9438 #[cfg(feature = "crdt-yjs")]
9439 #[test]
9440 fn crdt_authoring_pre_read_preserves_storage_classification() {
9441 let path =
9442 std::env::temp_dir().join(format!("syncular-crdt-busy-{}.db", uuid::Uuid::new_v4()));
9443 let schema = json!({"version":1,"tables":[{"name":"notes","primaryKey":"id","columns":[
9444 {"name":"id","type":"string","nullable":false},
9445 {"name":"project_id","type":"string","nullable":false},
9446 {"name":"doc","type":"crdt","nullable":true,"crdtType":"yjs-doc"}],
9447 "scopes":[{"pattern":"project:{project_id}"}]}]});
9448 let mut client = SyncClient::open_path(
9449 "crdt-busy".into(),
9450 &schema,
9451 ClientLimits::default(),
9452 path.to_str().unwrap(),
9453 )
9454 .expect("open client");
9455 client.create_synced_tables().expect("create synced tables");
9456 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
9457
9458 let locker = Connection::open(&path).expect("locker connection");
9459 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9460 let busy = client
9461 .crdt_apply_update("notes", "n1", "doc", &[])
9462 .expect_err("busy crdt pre-read");
9463 assert_eq!(busy.code, "client.storage_busy");
9464 assert!(busy.retryable);
9465 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
9466 assert!(client.pending_commit_ids().is_empty());
9467 assert_eq!(client.local_revision(), 0);
9468
9469 locker.execute_batch("ROLLBACK").unwrap();
9470 drop(locker);
9471 drop(client);
9472 let _ = std::fs::remove_file(&path);
9473 }
9474
9475 #[test]
9476 fn large_unique_import_copies_no_unrelated_tables() {
9477 let (mut client, image, table) = unique_import_fixture(10240);
9478 let pending = client.pending_commit_ids();
9479 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
9480 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
9481 assert!(matches!(
9482 client.apply_sqlite_image(
9483 &image,
9484 &table,
9485 true,
9486 &[("project_id".into(), vec!["p1".into()])],
9487 10240,
9488 7,
9489 "digest"
9490 ),
9491 Ok(10240)
9492 ));
9493 assert_eq!(client.overlay_rebuild_count.get(), 0);
9494 assert_eq!(client.pending_commit_ids(), pending);
9495 assert_eq!(
9496 client
9497 .query("SELECT count(*) AS n FROM tasks", &[])
9498 .unwrap()[0]["n"],
9499 10241
9500 );
9501 assert_eq!(
9502 client
9503 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9504 .unwrap()[0]["n"],
9505 10241
9506 );
9507 assert_eq!(
9508 client
9509 .query("SELECT count(*) AS n FROM notes_fts", &[])
9510 .unwrap()[0]["n"],
9511 8192
9512 );
9513 }
9514
9515 #[test]
9516 #[ignore = "manual scaling benchmark; see bench/README.md"]
9517 fn benchmark_unique_image_import() {
9518 let mut samples = Vec::new();
9519 for rows in [10240, 51200, 102400] {
9520 for trial in 0..4 {
9521 let (mut client, image, table) = unique_import_fixture(rows);
9522 let start = std::time::Instant::now();
9523 assert!(
9524 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
9525 );
9526 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
9527 assert_eq!(
9528 client
9529 .query("SELECT count(*) AS n FROM tasks", &[])
9530 .unwrap()[0]["n"],
9531 rows + 1
9532 );
9533 assert_eq!(
9534 client
9535 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9536 .unwrap()[0]["n"],
9537 rows + 1
9538 );
9539 assert_eq!(
9540 client
9541 .query("SELECT count(*) AS n FROM notes_fts", &[])
9542 .unwrap()[0]["n"],
9543 8192
9544 );
9545 assert_eq!(client.pending_commit_ids().len(), 1);
9546 if trial > 0 {
9547 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
9548 }
9549 }
9550 }
9551 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
9552 let output = serde_json::to_string_pretty(&artifact).unwrap();
9553 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
9554 use std::io::Write;
9555 let mut file = std::fs::OpenOptions::new()
9556 .write(true)
9557 .create_new(true)
9558 .open(path)
9559 .unwrap();
9560 file.write_all(output.as_bytes()).unwrap();
9561 } else {
9562 println!("{output}");
9563 }
9564 }
9565
9566 #[test]
9567 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
9568 let schema = json!({
9569 "version": 1,
9570 "tables": [{
9571 "name": "catalogue_codes",
9572 "primaryKey": "id",
9573 "columns": [
9574 { "name": "id", "type": "string", "nullable": false },
9575 { "name": "release_id", "type": "string", "nullable": false },
9576 { "name": "code", "type": "string", "nullable": false },
9577 { "name": "title", "type": "string", "nullable": false }
9578 ],
9579 "scopes": [{ "pattern": "release:{release_id}" }],
9580 "ftsIndexes": [{
9581 "name": "catalogue_codes_fts",
9582 "columns": ["code", "title"],
9583 "tokenize": "unicode61 remove_diacritics 2"
9584 }]
9585 }]
9586 });
9587 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
9588 .expect("FTS5 client");
9589 let insert_trigger: String = client
9590 .conn
9591 .query_row(
9592 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
9593 [],
9594 |row| row.get(0),
9595 )
9596 .expect("insert trigger");
9597 let replace_guard: String = client
9598 .conn
9599 .query_row(
9600 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
9601 [],
9602 |row| row.get(0),
9603 )
9604 .expect("replace guard");
9605 assert!(!insert_trigger.contains("DELETE FROM"));
9606 assert!(replace_guard.contains("BEFORE INSERT"));
9607 assert!(replace_guard.contains("WHEN EXISTS"));
9608 let search = |client: &SyncClient, query: &str| {
9609 client
9610 .query(
9611 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
9612 &[Value::from(query)],
9613 )
9614 .expect("FTS query")
9615 };
9616
9617 client
9618 .mutate(vec![Mutation::Upsert {
9619 table: "catalogue_codes".to_owned(),
9620 values: Map::from_iter([
9621 ("id".to_owned(), Value::from("c1")),
9622 ("release_id".to_owned(), Value::from("r1")),
9623 ("code".to_owned(), Value::from("A01")),
9624 ("title".to_owned(), Value::from("Cholera")),
9625 ]),
9626 base_version: None,
9627 }])
9628 .expect("insert code");
9629 assert_eq!(search(&client, "cholera").len(), 1);
9630 let prior = client
9631 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9632 .unwrap();
9633 client
9634 .conn
9635 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
9636 .unwrap();
9637 client.create_synced_tables().unwrap();
9638 assert_eq!(
9639 client
9640 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9641 .unwrap(),
9642 prior
9643 );
9644 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
9645 let rendered = serde_json::to_string(&plan).unwrap();
9646 assert!(rendered.contains("COVERING INDEX"));
9647 assert!(rendered.contains("INDEX 0:="));
9648 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
9649 assert_eq!(search(&client, "cholera").len(), 1);
9650
9651 client
9652 .mutate(vec![Mutation::Upsert {
9653 table: "catalogue_codes".to_owned(),
9654 values: Map::from_iter([
9655 ("id".to_owned(), Value::from("c1")),
9656 ("release_id".to_owned(), Value::from("r1")),
9657 ("code".to_owned(), Value::from("A01")),
9658 ("title".to_owned(), Value::from("Enteric infection")),
9659 ]),
9660 base_version: None,
9661 }])
9662 .expect("update code");
9663 assert!(search(&client, "cholera").is_empty());
9664 assert_eq!(search(&client, "enteric").len(), 1);
9665
9666 client
9667 .mutate(vec![Mutation::Delete {
9668 table: "catalogue_codes".to_owned(),
9669 row_id: "c1".to_owned(),
9670 base_version: None,
9671 }])
9672 .expect("delete code");
9673 assert!(search(&client, "enteric").is_empty());
9674 }
9675
9676 #[test]
9677 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
9678 let schema = json!({
9679 "version": 1,
9680 "tables": [{
9681 "name": "patient_notes",
9682 "primaryKey": "id",
9683 "columns": [
9684 { "name": "id", "type": "string", "nullable": false },
9685 { "name": "practice_id", "type": "string", "nullable": false },
9686 { "name": "encryption_key_id", "type": "string", "nullable": false },
9687 { "name": "title", "type": "string", "nullable": false }
9688 ],
9689 "scopes": [{ "pattern": "practice:{practice_id}" }],
9690 "ftsIndexes": [{
9691 "name": "patient_notes_fts",
9692 "columns": ["title"],
9693 "tokenize": "unicode61 remove_diacritics 2"
9694 }]
9695 }]
9696 });
9697 let mut client = SyncClient::new(
9698 "local-purge-test".to_owned(),
9699 &schema,
9700 ClientLimits::default(),
9701 )
9702 .expect("local purge client");
9703 client
9704 .conn
9705 .execute(
9706 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
9707 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
9708 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
9709 [],
9710 )
9711 .expect("seed base rows");
9712 client.overlay_dirty.set(true);
9713 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
9714
9715 let note = |id: &str, key_id: &str, title: &str| {
9716 Map::from_iter([
9717 ("id".to_owned(), Value::from(id)),
9718 ("practice_id".to_owned(), Value::from("practice-1")),
9719 ("encryption_key_id".to_owned(), Value::from(key_id)),
9720 ("title".to_owned(), Value::from(title)),
9721 ])
9722 };
9723 let doomed = client
9724 .mutate(vec![
9725 Mutation::Upsert {
9726 table: "patient_notes".to_owned(),
9727 values: note("target", "key-revoked", "Target changed"),
9728 base_version: None,
9729 },
9730 Mutation::Upsert {
9731 table: "patient_notes".to_owned(),
9732 values: note("unrelated", "key-held", "Unrelated changed"),
9733 base_version: None,
9734 },
9735 ])
9736 .expect("doomed commit");
9737 let kept = client
9738 .mutate(vec![Mutation::Upsert {
9739 table: "patient_notes".to_owned(),
9740 values: note("kept", "key-held", "Kept optimistic"),
9741 base_version: None,
9742 }])
9743 .expect("kept commit");
9744 client.drain_change_batches();
9745
9746 let input = LocalDataPurgeInput {
9747 purge_id: "purge-001".to_owned(),
9748 targets: vec![LocalDataPurgeTarget {
9749 table: "patient_notes".to_owned(),
9750 selectors: BTreeMap::from([(
9751 "encryption_key_id".to_owned(),
9752 vec!["key-revoked".to_owned()],
9753 )]),
9754 }],
9755 };
9756 assert_eq!(
9757 client.purge_local_data(&input).expect("apply purge"),
9758 LocalDataPurgeResult {
9759 already_applied: false,
9760 purged_rows: 1,
9761 dropped_commits: 1,
9762 }
9763 );
9764 let rows = client
9765 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
9766 .expect("visible rows");
9767 assert_eq!(rows.len(), 2);
9768 assert_eq!(rows[0]["id"], "kept");
9769 assert_eq!(rows[1]["id"], "unrelated");
9770 assert_eq!(rows[1]["title"], "Unrelated original");
9771 let fts = client
9772 .query(
9773 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
9774 &[],
9775 )
9776 .expect("fts query");
9777 assert!(fts.is_empty());
9778 assert_eq!(client.outbox.len(), 1);
9779 assert_eq!(client.outbox[0].client_commit_id, kept);
9780 let outcome = client
9781 .commit_outcome(&doomed)
9782 .expect("read doomed outcome")
9783 .expect("doomed outcome");
9784 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
9785 match &outcome.results[0] {
9786 CommitOperationOutcome::Error { rejection } => {
9787 assert_eq!(rejection.code, "client.local_data_purged");
9788 assert_eq!(rejection.client_commit_id, doomed);
9789 }
9790 other => panic!("expected local purge rejection, got {other:?}"),
9791 }
9792 assert_eq!(client.drain_change_batches().len(), 1);
9793 assert_eq!(
9794 client.purge_local_data(&input).expect("retry purge"),
9795 LocalDataPurgeResult {
9796 already_applied: true,
9797 purged_rows: 0,
9798 dropped_commits: 0,
9799 }
9800 );
9801 let conflicting = LocalDataPurgeInput {
9802 purge_id: input.purge_id.clone(),
9803 targets: vec![LocalDataPurgeTarget {
9804 table: "patient_notes".to_owned(),
9805 selectors: BTreeMap::from([(
9806 "encryption_key_id".to_owned(),
9807 vec!["key-held".to_owned()],
9808 )]),
9809 }],
9810 };
9811 assert!(client
9812 .purge_local_data(&conflicting)
9813 .expect_err("id collision must fail")
9814 .contains("already used with a different plan"));
9815 }
9816}
9817
9818impl SubState {
9819 fn name(self) -> &'static str {
9820 match self {
9821 SubState::Active => "active",
9822 SubState::Revoked => "revoked",
9823 SubState::Failed => "failed",
9824 }
9825 }
9826
9827 fn parse(value: &str) -> Self {
9828 match value {
9829 "revoked" => Self::Revoked,
9830 "failed" => Self::Failed,
9831 _ => Self::Active,
9832 }
9833 }
9834}
9835
9836#[derive(Debug, Clone)]
9837struct Subscription {
9838 id: String,
9839 table: String,
9840 requested: Vec<(String, Vec<String>)>,
9841 params: Option<String>,
9842 cursor: i64,
9843 bootstrap_state: Option<String>,
9845 state: SubState,
9846 reason_code: Option<String>,
9847 effective: Option<Vec<(String, Vec<String>)>>,
9850 synced_once: bool,
9851}
9852
9853#[cfg(test)]
9854mod subscription_rearm_tests {
9855 use super::*;
9856 use serde_json::json;
9857
9858 fn client() -> SyncClient {
9859 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9860 {"name":"id","type":"string","nullable":false},
9861 {"name":"project_id","type":"string","nullable":false}],
9862 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[],"ftsIndexes":[]}]});
9863 SyncClient::new("rearm".into(), &schema, ClientLimits::default()).expect("client")
9864 }
9865
9866 fn scopes() -> Vec<(String, Vec<String>)> {
9867 vec![("project_id".to_owned(), vec!["p1".to_owned()])]
9868 }
9869
9870 fn store_revoked(client: &mut SyncClient) {
9872 client
9873 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9874 .expect("subscribe");
9875 let sub = &mut client.subs[0];
9876 sub.cursor = 41;
9877 sub.bootstrap_state = Some("resume".to_owned());
9878 sub.effective = Some(scopes());
9879 sub.state = SubState::Revoked;
9880 sub.reason_code = Some("sync.scope_revoked".to_owned());
9881 client.persist_sub(&client.subs[0]).expect("persist");
9882 client.sync_needed = false;
9883 client.sync_intent_queue.clear();
9884 }
9885
9886 #[test]
9887 fn resubscribing_the_same_intent_rearms_a_revoked_registration() {
9888 let mut client = client();
9889 store_revoked(&mut client);
9890
9891 client
9892 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9893 .expect("re-arm");
9894
9895 let view = client.subscription_state("sub").expect("state");
9896 assert_eq!(view.status, "active");
9897 assert_eq!(view.cursor, -1);
9898 assert!(!view.has_resume_token);
9899 assert_eq!(view.effective_scopes, None);
9900 assert_eq!(view.reason_code, None);
9901 assert!(client.sync_needed, "a re-arm asks for a pull");
9902 assert!(matches!(
9903 client.drain_sync_intents().as_slice(),
9904 [SyncIntent::Interactive]
9905 ));
9906 let stored: String = client
9907 .conn
9908 .query_row(
9909 "SELECT state_json FROM _syncular_subscriptions WHERE id = 'sub'",
9910 [],
9911 |row| row.get(0),
9912 )
9913 .expect("persisted row");
9914 let stored: Value = serde_json::from_str(&stored).expect("stored json");
9915 assert_eq!(stored["status"], json!("active"));
9916 assert_eq!(stored["cursor"], json!(-1));
9917 assert_eq!(stored["bootstrapState"], Value::Null);
9918 assert_eq!(stored["effectiveScopes"], Value::Null);
9919 assert_eq!(stored["reasonCode"], Value::Null);
9920 }
9921
9922 #[test]
9923 fn a_failed_registration_is_not_rearmed() {
9924 let mut client = client();
9925 client
9926 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9927 .expect("subscribe");
9928 client.subs[0].state = SubState::Failed;
9929 client.subs[0].reason_code = Some("sync.scope_revoked".to_owned());
9930 client.persist_sub(&client.subs[0]).expect("persist");
9931 client.sync_needed = false;
9932 client.sync_intent_queue.clear();
9933
9934 client
9935 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9936 .expect("subscribe");
9937
9938 let view = client.subscription_state("sub").expect("state");
9939 assert_eq!(view.status, "failed");
9940 assert_eq!(view.reason_code.as_deref(), Some("sync.scope_revoked"));
9941 assert!(!client.sync_needed);
9942 assert!(client.drain_sync_intents().is_empty());
9943 }
9944}
9945
9946#[derive(Debug, Clone)]
9947struct OutboxOp {
9948 upsert: bool,
9949 table: String,
9950 row_id: String,
9951 base_version: Option<i64>,
9952 values: Option<Map<String, Value>>,
9956}
9957
9958impl From<&OutboxOp> for CommitOperation {
9959 fn from(operation: &OutboxOp) -> Self {
9960 Self {
9961 table: operation.table.clone(),
9962 row_id: operation.row_id.clone(),
9963 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
9964 base_version: operation.base_version,
9965 values: operation.values.clone(),
9966 }
9967 }
9968}
9969
9970#[derive(Debug, Clone)]
9971struct OutboxCommit {
9972 client_commit_id: String,
9973 ops: Vec<OutboxOp>,
9974}
9975
9976fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>)> {
9980 commit
9981 .ops
9982 .iter()
9983 .filter(|operation| operation.upsert)
9984 .map(|operation| {
9985 (
9986 operation.table.as_str(),
9987 operation
9988 .values
9989 .as_ref()
9990 .map(|values| values.keys().map(String::as_str).collect())
9991 .unwrap_or_default(),
9992 )
9993 })
9994 .collect()
9995}
9996
9997type FailedCommit = (OutboxCommit, Map<String, Value>);
9998
9999#[derive(Debug, Clone)]
10000struct CompiledLocalDataPurgeTarget {
10001 table: String,
10002 selectors: Vec<(String, Vec<String>)>,
10003}
10004
10005struct StoredCommitOutcomeRow {
10006 sequence: i64,
10007 client_commit_id: String,
10008 status: String,
10009 recorded_at_ms: i64,
10010 results_json: String,
10011 operations_json: Option<String>,
10012 resolution: String,
10013 resolved_at_ms: Option<i64>,
10014 replacement_client_commit_id: Option<String>,
10015}
10016
10017impl StoredCommitOutcomeRow {
10018 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
10019 Ok(Self {
10020 sequence: row.get(0)?,
10021 client_commit_id: row.get(1)?,
10022 status: row.get(2)?,
10023 recorded_at_ms: row.get(3)?,
10024 results_json: row.get(4)?,
10025 operations_json: row.get(5)?,
10026 resolution: row.get(6)?,
10027 resolved_at_ms: row.get(7)?,
10028 replacement_client_commit_id: row.get(8)?,
10029 })
10030 }
10031}
10032
10033enum SectionError {
10036 FailClosed,
10037 Abort(String, String),
10038}
10039
10040#[derive(Debug)]
10041pub(crate) struct RequestMeta {
10042 pushed_ids: Vec<String>,
10043 fresh: Vec<(String, bool)>,
10046 pub(crate) accept: u8,
10047 deferred_commits: usize,
10051}
10052
10053#[derive(Default)]
10054struct ChangeAccumulator {
10055 tables: BTreeMap<String, Option<BTreeSet<String>>>,
10057 windows: BTreeMap<(String, String), BTreeSet<String>>,
10058 status: bool,
10059 conflicts: bool,
10060 rejections: bool,
10061 outcomes: bool,
10062}
10063
10064impl ChangeAccumulator {
10065 fn table(&mut self, table: &str) {
10066 self.tables.insert(table.to_owned(), None);
10067 }
10068
10069 fn scope(&mut self, table: &str, key: String) {
10070 match self.tables.get_mut(table) {
10071 Some(None) => {}
10072 Some(Some(keys)) => {
10073 keys.insert(key);
10074 }
10075 None => {
10076 self.tables
10077 .insert(table.to_owned(), Some(BTreeSet::from([key])));
10078 }
10079 }
10080 }
10081
10082 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
10083 self.windows
10084 .entry((base_key.to_owned(), table.to_owned()))
10085 .or_default()
10086 .insert(unit.to_owned());
10087 }
10088
10089 fn touched(&self) -> bool {
10090 !self.tables.is_empty()
10091 || !self.windows.is_empty()
10092 || self.status
10093 || self.conflicts
10094 || self.rejections
10095 || self.outcomes
10096 }
10097}
10098
10099const PUSH_OPS_PER_REQUEST: usize = 500;
10105const MAX_LOCAL_PURGE_TARGETS: usize = 64;
10106const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
10107const MAX_LOCAL_PURGE_VALUES: usize = 128;
10108const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
10109pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
10110
10111struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
10113
10114impl OverlayDirty {
10115 fn clean() -> Self {
10116 Self(RefCell::new(Some(BTreeSet::new())))
10117 }
10118 fn get(&self) -> bool {
10119 self.0
10120 .borrow()
10121 .as_ref()
10122 .is_none_or(|tables| !tables.is_empty())
10123 }
10124 fn set(&self, dirty: bool) {
10125 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
10126 }
10127 fn table(&self, table: &str) {
10128 if let Some(tables) = self.0.borrow_mut().as_mut() {
10129 tables.insert(table.to_owned());
10130 }
10131 }
10132 fn snapshot(&self) -> Option<BTreeSet<String>> {
10133 self.0.borrow().clone()
10134 }
10135 fn restore(&self, tables: Option<BTreeSet<String>>) {
10136 *self.0.borrow_mut() = tables;
10137 }
10138}
10139
10140pub struct SyncClient {
10141 #[cfg(feature = "bench-internals")]
10142 benchmark_phases: Recorder,
10143 conn: Connection,
10144 schema: ClientSchema,
10145 client_id: String,
10146 limits: ClientLimits,
10147 subs: Vec<Subscription>,
10148 outbox: Vec<OutboxCommit>,
10149 failed_commits: Vec<FailedCommit>,
10150 retain_failed_commits: bool,
10151 conflicts: Vec<ConflictRecord>,
10152 rejections: Vec<RejectionRecord>,
10153 schema_floor: Option<SchemaFloor>,
10154 lease_state: Option<LeaseState>,
10156 stopped: bool,
10158 upgrading: bool,
10160 sync_needed: bool,
10162 realtime_policy: RealtimePolicy,
10165 realtime_state: RealtimeState,
10166 realtime_reason_code: Option<String>,
10167 realtime_retry_delay_ms: Option<u64>,
10168 presence: HashMap<String, HashMap<String, PresencePeer>>,
10170 now_ms: Option<i64>,
10173 encryption: crate::values::EncryptionConfig,
10177 security_preflight: bool,
10180 transport_enabled: bool,
10181 previous_version: Option<PreviousVersionContextConfig>,
10185 insert_sql: RefCell<HashMap<String, String>>,
10191 overlay_dirty: OverlayDirty,
10194 storage_failure: RefCell<Option<QueryReadFailure>>,
10196 #[cfg(test)]
10199 overlay_rebuild_count: Cell<usize>,
10200 #[cfg(test)]
10201 acknowledged_replay_count: Cell<usize>,
10202 #[cfg(test)]
10205 outcome_prune_count: Cell<usize>,
10206 progress: ProgressObserver,
10208 change_queue: VecDeque<ClientChangeBatch>,
10209 sync_intent_queue: VecDeque<SyncIntent>,
10210 retry_delay_ms: u64,
10212 round_retry_delay_ms: Option<u64>,
10214 round_failure: Option<String>,
10219 active_round: Option<uuid::Uuid>,
10220 last_round: Option<DiagnosticLastRound>,
10221 last_change: Option<DiagnosticLastChange>,
10222 query_failures: Vec<DiagnosticQueryFailure>,
10224 coalesced: Option<CoalescedApply>,
10226 coalesced_failure: Option<String>,
10229}
10230
10231struct CoalescedApply {
10235 rows: usize,
10236 subs: Vec<Subscription>,
10237 outbox: Vec<OutboxCommit>,
10238 failed_commits: Vec<FailedCommit>,
10239 conflicts: Vec<ConflictRecord>,
10240 rejections: Vec<RejectionRecord>,
10241 schema_floor: Option<SchemaFloor>,
10242 lease_state: Option<LeaseState>,
10243 stopped: bool,
10244 upgrading: bool,
10245 sync_needed: bool,
10246 overlay_dirty: Option<BTreeSet<String>>,
10247 change_queue: usize,
10248 sync_intent_queue: usize,
10249 last_change: Option<DiagnosticLastChange>,
10250}
10251
10252const COALESCED_APPLY_ROWS: usize = 1024;
10253
10254pub(crate) fn quote_ident(name: &str) -> String {
10255 format!("\"{}\"", name.replace('"', "\"\""))
10256}
10257pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
10261 conn.query_row(
10262 "SELECT value FROM _syncular_meta WHERE key = ?1",
10263 rusqlite::params![key],
10264 |row| row.get::<_, String>(0),
10265 )
10266 .ok()
10267}
10268
10269pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
10270 let _ = conn.execute(
10271 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
10272 rusqlite::params![key, value],
10273 );
10274}
10275
10276pub(crate) fn meta_delete(conn: &Connection, key: &str) {
10277 let _ = conn.execute(
10278 "DELETE FROM _syncular_meta WHERE key = ?1",
10279 rusqlite::params![key],
10280 );
10281}
10282
10283fn row_id_predicate(table: &TableSchema) -> String {
10285 let key = quote_ident(&table.primary_key);
10286 let text_match = format!("CAST({key} AS TEXT) = ?1");
10287 match table.columns[table.pk_index].ty {
10288 ColumnType::String | ColumnType::Json => {
10289 format!("{key} = ?1 AND {text_match}")
10290 }
10291 ColumnType::Integer | ColumnType::Boolean => {
10295 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
10296 }
10297 _ => text_match,
10300 }
10301}
10302
10303fn is_local_operation_code_like(value: &str) -> bool {
10304 let bytes = value.as_bytes();
10305 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
10306 && bytes
10307 .iter()
10308 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
10309}
10310
10311fn base_table(name: &str) -> String {
10312 quote_ident(&format!("_syncular_base_{name}"))
10313}
10314
10315type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
10320
10321fn window_base_key(base: &WindowBase) -> String {
10322 format!(
10323 "{}\0{}\0{}",
10324 base.table,
10325 base.variable,
10326 canonical_scope_json(&base.fixed_scopes)
10327 )
10328}
10329
10330fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
10332 let mut scopes = base.fixed_scopes.clone();
10333 scopes.retain(|(k, _)| k != &base.variable);
10334 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
10335 scopes
10336}
10337
10338fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
10342 let canonical = canonical_scope_json(&unit_scopes(base, unit));
10343 let digest = Sha256::digest(canonical.as_bytes());
10344 let hex = bytes_to_hex(&digest);
10345 format!("w:{}:{}", base.table, &hex[..16])
10346}
10347
10348fn visible_table(name: &str) -> String {
10349 quote_ident(name)
10350}
10351
10352const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
10353
10354fn is_synced_table_name(name: &str) -> bool {
10357 if name.starts_with("sqlite_") {
10358 return false;
10359 }
10360 if name.starts_with("_syncular_base_") {
10361 return true; }
10363 !name.starts_with("_syncular_")
10365}
10366
10367fn blob_id_for(bytes: &[u8]) -> String {
10369 let digest = Sha256::digest(bytes);
10370 format!("sha256:{}", bytes_to_hex(&digest))
10371}
10372
10373enum RowParam<'a> {
10377 Cell(&'a Option<ColumnValue>),
10378 Version(i64),
10379}
10380
10381impl rusqlite::ToSql for RowParam<'_> {
10382 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
10383 Ok(match self {
10384 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
10385 RowParam::Cell(cell) => match cell {
10386 None => ToSqlOutput::Owned(SqlValue::Null),
10387 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
10388 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
10389 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
10390 Some(ColumnValue::Boolean(b)) => {
10391 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
10392 }
10393 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
10394 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
10395 }
10396 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
10398 ToSqlOutput::Borrowed(ValueRef::Blob(b))
10399 }
10400 },
10401 })
10402 }
10403}
10404
10405fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
10407 match param.to_sql().map_err(|error| error.to_string())? {
10408 ToSqlOutput::Owned(value) => Ok(value),
10409 ToSqlOutput::Borrowed(value) => {
10410 SqlValue::try_from(value).map_err(|error| error.to_string())
10411 }
10412 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
10413 }
10414}
10415
10416fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
10424 use ssp2::segment::ColumnType;
10425 let mismatch = || {
10426 Err(format!(
10427 "image column {:?} holds a value of the wrong type",
10428 column.name
10429 ))
10430 };
10431 match value {
10432 ValueRef::Null => {
10433 if !column.nullable {
10434 return Err(format!(
10435 "image column {:?} is NULL but not nullable",
10436 column.name
10437 ));
10438 }
10439 Ok(ToSqlOutput::Owned(SqlValue::Null))
10440 }
10441 ValueRef::Integer(i) => match column.ty {
10442 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
10443 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
10444 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
10445 _ => mismatch(),
10446 },
10447 ValueRef::Real(_) => match column.ty {
10448 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
10449 _ => mismatch(),
10450 },
10451 ValueRef::Text(t) => {
10452 std::str::from_utf8(t)
10453 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
10454 match column.ty {
10455 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
10456 Ok(ToSqlOutput::Borrowed(value))
10457 }
10458 _ => mismatch(),
10459 }
10460 }
10461 ValueRef::Blob(_) => match column.ty {
10462 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
10464 _ => mismatch(),
10465 },
10466 }
10467}
10468
10469pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
10470 use rusqlite::types::ValueRef;
10471 match value {
10472 ValueRef::Null => Value::Null,
10473 ValueRef::Integer(i) => match column.ty {
10474 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
10475 ssp2::segment::ColumnType::Float => {
10476 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
10477 }
10478 _ => Value::from(i),
10479 },
10480 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10481 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10482 ValueRef::Blob(b) => {
10483 let mut map = Map::new();
10484 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10485 Value::Object(map)
10486 }
10487 }
10488}
10489
10490fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
10494 Ok(match value {
10495 Value::Null => SqlValue::Null,
10496 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
10497 Value::Number(n) => {
10498 if let Some(i) = n.as_i64() {
10499 SqlValue::Integer(i)
10500 } else if let Some(f) = n.as_f64() {
10501 SqlValue::Real(f)
10502 } else {
10503 return Err(format!("query param number {n} is out of range"));
10504 }
10505 }
10506 Value::String(s) => SqlValue::Text(s.clone()),
10507 Value::Object(_) => {
10508 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
10509 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
10510 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
10511 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
10512 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
10513 })?)
10514 } else {
10515 return Err(
10516 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
10517 .to_owned(),
10518 );
10519 }
10520 }
10521 Value::Array(_) => return Err("query array params are not supported".to_owned()),
10522 })
10523}
10524
10525fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
10530 use rusqlite::types::ValueRef;
10531 match value {
10532 ValueRef::Null => Value::Null,
10533 ValueRef::Integer(i) => {
10534 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
10538 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
10539 Value::from(i)
10540 } else {
10541 let mut map = Map::new();
10542 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
10543 Value::Object(map)
10544 }
10545 }
10546 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10547 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10548 ValueRef::Blob(b) => {
10549 let mut map = Map::new();
10550 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10551 Value::Object(map)
10552 }
10553 }
10554}
10555
10556fn query_connection(
10558 conn: &Connection,
10559 sql: &str,
10560 params: &[QueryValue],
10561) -> Result<Vec<QueryRow>, QueryReadFailure> {
10562 crate::query_guard::assert_read_only_query(sql)?;
10563 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
10564 let bound: Vec<SqlValue> = params
10565 .iter()
10566 .map(json_param_to_sql)
10567 .collect::<Result<_, _>>()?;
10568 let mut stmt = conn.prepare(&lowered_sql)?;
10569 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
10570 let bound_refs: Vec<&dyn rusqlite::ToSql> =
10571 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
10572 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
10573 let mut out = Vec::new();
10574 while let Some(row) = sql_rows.next()? {
10575 let mut record = Map::new();
10576 for (i, name) in column_names.iter().enumerate() {
10577 let value = row.get_ref(i)?;
10578 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
10579 }
10580 out.push(record);
10581 }
10582 Ok(out)
10583}
10584
10585fn persisted_window_state(
10586 conn: &Connection,
10587 base: &WindowBase,
10588) -> Result<WindowState, QueryReadFailure> {
10589 let mut stmt = conn.prepare(
10590 "SELECT windows.unit, subscriptions.state_json
10591 FROM _syncular_windows AS windows
10592 JOIN _syncular_subscriptions AS subscriptions
10593 ON subscriptions.id = windows.sub_id
10594 WHERE windows.base = ?1
10595 ORDER BY windows.unit ASC",
10596 )?;
10597 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
10598 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
10599 })?;
10600 let mut units = Vec::new();
10601 let mut pending = Vec::new();
10602 for row in rows {
10603 let (unit, raw) = row?;
10604 let state: Value = serde_json::from_str(&raw)
10605 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
10606 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
10607 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
10608 || state
10609 .get("bootstrapState")
10610 .is_some_and(|value| !value.is_null());
10611 if is_pending {
10612 pending.push(unit.clone());
10613 }
10614 units.push(unit);
10615 }
10616 Ok(WindowState { units, pending })
10617}
10618
10619#[derive(Debug, Clone, Copy)]
10621pub struct SnapshotStatement<'a> {
10622 pub sql: &'a str,
10623 pub params: &'a [Value],
10624}
10625
10626#[derive(Debug, Clone, Default)]
10630pub struct SnapshotReadRequest<'a> {
10631 pub statements: Vec<SnapshotStatement<'a>>,
10632 pub coverage: &'a [WindowCoverage],
10633 pub subscriptions: Vec<String>,
10634 pub commit_ids: Vec<String>,
10635}
10636
10637#[cfg(test)]
10638thread_local! {
10639 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
10643 std::cell::RefCell::new(None);
10644}
10645
10646#[cfg(test)]
10647fn run_snapshot_read_hook() {
10648 SNAPSHOT_READ_HOOK.with(|slot| {
10649 if let Some(hook) = slot.borrow_mut().take() {
10650 hook();
10651 }
10652 });
10653}
10654
10655fn snapshot_read_connection(
10660 conn: &Connection,
10661 request: &SnapshotReadRequest<'_>,
10662) -> Result<SnapshotRead, QueryReadFailure> {
10663 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
10664 let corrupt = |message: &str| QueryReadFailure {
10665 code: Some("sync.local_corrupt"),
10666 sqlite_code: None,
10667 sqlite_message: None,
10668 rollback_failure: None,
10669 message: message.to_owned(),
10670 };
10671 let stored_failure = |error: rusqlite::Error, message: &str| match error {
10672 rusqlite::Error::Utf8Error(..)
10673 | rusqlite::Error::InvalidColumnType(..)
10674 | rusqlite::Error::FromSqlConversionFailure(..)
10675 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
10676 _ => QueryReadFailure::from(error),
10677 };
10678 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
10679 let revision: Option<String> = conn
10682 .query_row(
10683 "SELECT value FROM _syncular_meta WHERE key = ?1",
10684 rusqlite::params![LOCAL_REVISION_KEY],
10685 |row| row.get(0),
10686 )
10687 .optional()
10688 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
10689 let revision = match revision {
10690 None => "0".to_owned(),
10691 Some(value)
10692 if value == "0"
10693 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10694 && value.bytes().all(|b| b.is_ascii_digit())
10695 && value.parse::<u64>().is_ok()) =>
10696 {
10697 value
10698 }
10699 Some(_) => return Err(corrupt("persisted local revision is invalid")),
10700 };
10701 let mut queries = Vec::with_capacity(request.statements.len());
10702 for statement in &request.statements {
10703 queries.push(query_connection(conn, statement.sql, statement.params)?);
10704 #[cfg(test)]
10705 run_snapshot_read_hook();
10706 }
10707 let mut pending = Vec::new();
10708 let mut missing = Vec::new();
10709 for requested in request.coverage {
10710 let base_key = window_base_key(&requested.base);
10711 let state = persisted_window_state(conn, &requested.base)?;
10712 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
10713 let reference = WindowUnitRef {
10714 base_key: base_key.clone(),
10715 unit: unit.clone(),
10716 };
10717 if !state.units.iter().any(|held| held == &unit) {
10718 missing.push(reference);
10719 } else if state.pending.iter().any(|held| held == &unit) {
10720 pending.push(reference);
10721 }
10722 }
10723 }
10724 let mut subscriptions = Vec::with_capacity(request.subscriptions.len());
10725 for id in &request.subscriptions {
10726 let row: Option<(String, String)> = conn
10727 .query_row(
10728 "SELECT tbl, state_json FROM _syncular_subscriptions WHERE id = ?1",
10729 rusqlite::params![id],
10730 |row| Ok((row.get(0)?, row.get(1)?)),
10731 )
10732 .optional()
10733 .map_err(|error| {
10734 stored_failure(error, "persisted subscription state is invalid")
10735 })?;
10736 let Some((table, raw)) = row else {
10737 subscriptions.push(SubscriptionCatchup::Unknown { id: id.clone() });
10738 continue;
10739 };
10740 let state: Value = serde_json::from_str(&raw)
10743 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10744 let status = match state.get("status").and_then(Value::as_str) {
10745 Some(value @ ("active" | "revoked" | "failed")) => value,
10746 _ => return Err(corrupt("persisted subscription state is invalid")),
10747 };
10748 let Some(cursor) = state.get("cursor").and_then(Value::as_i64) else {
10749 return Err(corrupt("persisted subscription state is invalid"));
10750 };
10751 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
10752 return Err(corrupt("persisted subscription state is invalid"));
10753 }
10754 let effective_scopes = state
10755 .get("effectiveScopes")
10756 .filter(|value| !value.is_null());
10757 if let Some(scopes) = effective_scopes {
10758 json_to_scope_map(scopes)
10759 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10760 }
10761 let has_resume_token = match state.get("bootstrapState") {
10762 None | Some(Value::Null) => false,
10763 Some(Value::String(_)) => true,
10764 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10765 };
10766 let reason_code = match state.get("reasonCode") {
10769 None | Some(Value::Null) => None,
10770 Some(Value::String(value)) => Some(value.clone()),
10771 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10772 };
10773 let active = status == "active";
10774 subscriptions.push(SubscriptionCatchup::Known {
10775 id: id.clone(),
10776 table,
10777 status: status.to_owned(),
10778 cursor,
10779 has_resume_token,
10780 bootstrap_complete: active && cursor >= 0 && !has_resume_token,
10781 known_pending_pages: active && (has_resume_token || cursor < 0),
10782 effective_scopes: effective_scopes.cloned(),
10783 reason_code,
10784 });
10785 }
10786 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
10787 for id in &request.commit_ids {
10788 let pending: bool = conn.query_row(
10789 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
10790 rusqlite::params![id],
10791 |row| row.get(0),
10792 )?;
10793 if pending {
10794 deliveries.push(CommitDelivery::Pending {
10795 client_commit_id: id.clone(),
10796 });
10797 continue;
10798 }
10799 let row = conn
10800 .query_row(
10801 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
10802 resolution, resolved_at_ms, replacement_client_commit_id
10803 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
10804 rusqlite::params![id],
10805 StoredCommitOutcomeRow::from_row,
10806 )
10807 .optional()
10808 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
10809 deliveries.push(match row {
10810 Some(row) => CommitDelivery::Known {
10811 client_commit_id: id.clone(),
10812 outcome: SyncClient::outcome_from_row(row)
10813 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
10814 },
10815 None => CommitDelivery::Unknown {
10816 client_commit_id: id.clone(),
10817 },
10818 });
10819 }
10820 Ok(SnapshotRead {
10821 revision,
10822 queries,
10823 coverage: CoverageSnapshot {
10824 complete: pending.is_empty() && missing.is_empty(),
10825 pending,
10826 missing,
10827 },
10828 subscriptions,
10829 deliveries,
10830 })
10831 })();
10832 match result {
10833 Ok(read) => {
10834 conn.execute_batch("RELEASE syncular_snapshot_read")?;
10835 Ok(read)
10836 }
10837 Err(mut error) => {
10838 if let Err(rollback) = conn
10839 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
10840 {
10841 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
10842 }
10843 Err(error)
10844 }
10845 }
10846}
10847
10848fn snapshot_connection(
10849 conn: &Connection,
10850 sql: &str,
10851 params: &[Value],
10852 coverage: &[WindowCoverage],
10853) -> Result<QuerySnapshot, QueryReadFailure> {
10854 let read = snapshot_read_connection(
10855 conn,
10856 &SnapshotReadRequest {
10857 statements: vec![SnapshotStatement { sql, params }],
10858 coverage,
10859 subscriptions: Vec::new(),
10860 commit_ids: Vec::new(),
10861 },
10862 )?;
10863 Ok(QuerySnapshot {
10864 revision: read.revision,
10865 rows: read
10866 .queries
10867 .into_iter()
10868 .next()
10869 .expect("single-query snapshot returns one result"),
10870 coverage: read.coverage,
10871 })
10872}
10873
10874pub struct FileQuerySnapshotReader {
10879 path: String,
10880 conn: Option<Connection>,
10881}
10882
10883impl FileQuerySnapshotReader {
10884 #[must_use]
10885 pub fn new(path: impl Into<String>) -> Self {
10886 Self {
10887 path: path.into(),
10888 conn: None,
10889 }
10890 }
10891
10892 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
10893 if self.conn.is_none() {
10894 let conn = Connection::open_with_flags(
10895 &self.path,
10896 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
10897 )
10898 .map_err(QueryReadFailure::from)?;
10899 conn.busy_timeout(std::time::Duration::from_millis(250))
10900 .map_err(QueryReadFailure::from)?;
10901 self.conn = Some(conn);
10902 }
10903 self.conn
10904 .as_ref()
10905 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
10906 }
10907
10908 pub fn snapshot_read(
10911 &mut self,
10912 request: &SnapshotReadRequest<'_>,
10913 ) -> Result<SnapshotRead, QueryReadFailure> {
10914 snapshot_read_connection(self.connection()?, request)
10915 }
10916
10917 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
10920 Ok(self
10921 .snapshot_read(&SnapshotReadRequest::default())?
10922 .revision)
10923 }
10924
10925 pub fn subscription_catchup(
10927 &mut self,
10928 id: &str,
10929 ) -> Result<SubscriptionCatchup, QueryReadFailure> {
10930 let read = self.snapshot_read(&SnapshotReadRequest {
10931 subscriptions: vec![id.to_owned()],
10932 ..Default::default()
10933 })?;
10934 Ok(read
10935 .subscriptions
10936 .into_iter()
10937 .next()
10938 .expect("one requested catch-up"))
10939 }
10940
10941 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
10944 let read = self.snapshot_read(&SnapshotReadRequest {
10945 commit_ids: vec![id.to_owned()],
10946 ..Default::default()
10947 })?;
10948 Ok(read
10949 .deliveries
10950 .into_iter()
10951 .next()
10952 .expect("one requested delivery"))
10953 }
10954
10955 pub fn query_snapshot(
10958 &mut self,
10959 sql: &str,
10960 params: &[Value],
10961 coverage: &[WindowCoverage],
10962 ) -> Result<QuerySnapshot, QueryReadFailure> {
10963 snapshot_connection(self.connection()?, sql, params, coverage)
10964 }
10965}
10966
10967fn validate_authority_reads(
10968 schema: &ClientSchema,
10969 declarations: &[crate::api::AuthorityReadDeclaration],
10970) -> Result<(), String> {
10971 let mut seen = BTreeSet::new();
10972 for read in declarations {
10973 let table = schema
10974 .table(&read.table)
10975 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
10976 let plain = |name: &str| {
10977 table.columns.iter().enumerate().any(|(i, c)| {
10978 c.name == name
10979 && name != "authority_version"
10980 && !name.starts_with("_sync")
10981 && !table.encrypted_columns.iter().any(|e| e.index == i)
10982 && matches!(
10983 c.ty,
10984 ssp2::segment::ColumnType::String
10985 | ssp2::segment::ColumnType::Integer
10986 | ssp2::segment::ColumnType::Float
10987 | ssp2::segment::ColumnType::Boolean
10988 | ssp2::segment::ColumnType::Json
10989 )
10990 })
10991 };
10992 if !seen.insert(&read.table)
10993 || read.columns.is_empty()
10994 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
10995 || !read.columns.contains(&table.primary_key)
10996 || !read.columns.iter().all(|c| plain(c))
10997 || read.scopes.is_empty()
10998 || !read.scopes.iter().all(|(v, values)| {
10999 table
11000 .scope_column(v)
11001 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
11002 && !values.is_empty()
11003 && values.iter().all(|v| !v.is_empty() && v != "*")
11004 })
11005 {
11006 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
11007 }
11008 }
11009 Ok(())
11010}
11011
11012fn schema_marker_read_failure(error: rusqlite::Error) -> String {
11017 let failure = QueryReadFailure::from(error);
11018 if failure.code.is_some() {
11019 failure.to_string()
11020 } else {
11021 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
11022 }
11023}
11024
11025fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
11029 if limits.outcome_retention_max_entries == Some(0) {
11030 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
11031 }
11032 if [
11033 limits.max_push_commits_per_request,
11034 limits.max_push_operations_per_request,
11035 limits.max_push_request_bytes,
11036 ]
11037 .into_iter()
11038 .flatten()
11039 .any(|value| value == 0 || value > u32::MAX as usize)
11040 {
11041 return Err(
11042 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
11043 );
11044 }
11045 Ok(())
11046}
11047
11048fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
11049 let corrupt =
11050 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
11051 let table_type: Option<String> = conn
11052 .query_row(
11053 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
11054 [],
11055 |row| row.get(0),
11056 )
11057 .optional()
11058 .map_err(schema_marker_read_failure)?;
11059 let Some(table_type) = table_type else {
11060 return Ok(None);
11061 };
11062 if table_type != "table" {
11063 return Err(corrupt());
11064 }
11065 let mut statement = conn
11069 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
11070 .map_err(schema_marker_read_failure)?;
11071 let mut rows = statement
11072 .query([LOCAL_SCHEMA_VERSION_KEY])
11073 .map_err(schema_marker_read_failure)?;
11074 let value: Option<String> = rows
11075 .next()
11076 .map_err(schema_marker_read_failure)?
11077 .map(|row| row.get(0))
11078 .transpose()
11079 .map_err(schema_marker_read_failure)?;
11080 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
11081 return Err(corrupt());
11082 }
11083 let Some(value) = value else {
11084 let has_descriptor = conn
11085 .query_row(
11086 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
11087 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
11088 |_| Ok(()),
11089 )
11090 .optional()
11091 .map_err(schema_marker_read_failure)?
11092 .is_some();
11093 if has_descriptor {
11094 return Err(corrupt());
11095 }
11096 return Ok(None);
11097 };
11098 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
11099 || !value.bytes().all(|b| b.is_ascii_digit())
11100 {
11101 return Err(corrupt());
11102 }
11103 let version = value.parse::<i32>().map_err(|_| corrupt())?;
11104 if version > requested {
11105 return Err(
11106 "client.schema_downgrade: persisted local schema is newer than the requested schema"
11107 .to_owned(),
11108 );
11109 }
11110 Ok(Some(version))
11111}
11112
11113impl SyncClient {
11114 pub fn new_with_identity(
11115 client_id: Option<String>,
11116 schema_json: &Value,
11117 limits: ClientLimits,
11118 ) -> Result<Self, String> {
11119 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
11120 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
11121 Self::with_connection(resolved, schema_json, limits, conn)
11122 }
11123
11124 pub fn new(
11125 client_id: String,
11126 schema_json: &Value,
11127 limits: ClientLimits,
11128 ) -> Result<Self, String> {
11129 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
11130 Self::with_connection(client_id, schema_json, limits, conn)
11131 }
11132
11133 pub fn open_path(
11139 client_id: String,
11140 schema_json: &Value,
11141 limits: ClientLimits,
11142 path: &str,
11143 ) -> Result<Self, String> {
11144 validate_client_limits(&limits)?;
11145 parse_schema_json(schema_json)?;
11146 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
11147 Self::with_connection(client_id, schema_json, limits, conn)
11148 }
11149
11150 pub fn open_path_with_identity(
11151 client_id: Option<String>,
11152 schema_json: &Value,
11153 limits: ClientLimits,
11154 path: &str,
11155 ) -> Result<Self, String> {
11156 validate_client_limits(&limits)?;
11157 parse_schema_json(schema_json)?;
11160 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
11161 conn.busy_timeout(std::time::Duration::from_millis(250))
11169 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
11170 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
11171 let journal_mode: String = client
11172 .conn
11173 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
11174 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
11175 if journal_mode != "wal" {
11176 return Err(
11177 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
11178 );
11179 }
11180 Ok(client)
11181 }
11182
11183 #[must_use]
11184 pub fn client_id(&self) -> &str {
11185 &self.client_id
11186 }
11187
11188 pub fn with_connection(
11195 client_id: String,
11196 schema_json: &Value,
11197 limits: ClientLimits,
11198 conn: Connection,
11199 ) -> Result<Self, String> {
11200 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
11201 }
11202
11203 fn with_connection_identity(
11206 client_id: Option<String>,
11207 schema_json: &Value,
11208 limits: ClientLimits,
11209 conn: Connection,
11210 ) -> Result<Self, String> {
11211 validate_client_limits(&limits)?;
11212 let schema = parse_schema_json(schema_json)?;
11213 validate_authority_reads(&schema, &limits.authority_reads)?;
11214 let previous_version = match limits.previous_version_context {
11217 Some(config) => {
11218 config.validate()?;
11219 Some(config)
11220 }
11221 None => None,
11222 };
11223 let mut client = SyncClient {
11224 #[cfg(feature = "bench-internals")]
11225 benchmark_phases: Recorder::default(),
11226 conn,
11227 schema,
11228 client_id: client_id
11229 .clone()
11230 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
11231 limits,
11232 subs: Vec::new(),
11233 outbox: Vec::new(),
11234 failed_commits: Vec::new(),
11235 retain_failed_commits: false,
11236 conflicts: Vec::new(),
11237 rejections: Vec::new(),
11238 schema_floor: None,
11239 lease_state: None,
11240 stopped: false,
11241 upgrading: false,
11242 sync_needed: false,
11243 realtime_policy: RealtimePolicy::Optional,
11244 realtime_state: RealtimeState::Disconnected,
11245 realtime_reason_code: None,
11246 realtime_retry_delay_ms: None,
11247 presence: HashMap::new(),
11248 now_ms: None,
11249 encryption: crate::values::EncryptionConfig::default(),
11250 security_preflight: false,
11251 transport_enabled: true,
11252 previous_version,
11253 insert_sql: RefCell::new(HashMap::new()),
11254 overlay_dirty: OverlayDirty::clean(),
11255 storage_failure: RefCell::new(None),
11256 #[cfg(test)]
11257 overlay_rebuild_count: Cell::new(0),
11258 #[cfg(test)]
11259 acknowledged_replay_count: Cell::new(0),
11260 #[cfg(test)]
11261 outcome_prune_count: Cell::new(0),
11262 progress: ProgressObserver::default(),
11263 change_queue: VecDeque::new(),
11264 sync_intent_queue: VecDeque::new(),
11265 retry_delay_ms: 250,
11266 round_retry_delay_ms: None,
11267 round_failure: None,
11268 active_round: None,
11269 last_round: None,
11270 last_change: None,
11271 query_failures: Vec::new(),
11272 coalesced: None,
11273 coalesced_failure: None,
11274 };
11275 client
11279 .conn
11280 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
11281 client
11287 .conn
11288 .execute_batch("BEGIN IMMEDIATE")
11289 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
11290 let startup = (|| -> Result<(), String> {
11291 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
11294 client.create_bookkeeping_tables()?;
11297 match client.get_meta(CLIENT_ID_KEY) {
11298 Some(existing) => match client_id {
11299 Some(requested) if existing != requested => {
11300 return Err(format!(
11301 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
11302 ))
11303 }
11304 _ => client.client_id = existing,
11305 },
11306 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
11307 }
11308 client.restore_persisted_state()?;
11309 client.failed_commits = client.load_failed_commits()?;
11310 match marker {
11311 None => {
11312 client.create_synced_tables()?;
11313 client.persist_schema_version()?;
11314 client.save_subscription_scope_schema();
11315 }
11316 Some(version) if version == client.schema.version => {
11317 client.create_synced_tables()?;
11318 set_local_schema_descriptor(&client.conn, &client.schema)
11322 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
11323 client.save_subscription_scope_schema();
11324 }
11325 Some(_) => client.run_schema_reset()?,
11326 }
11327 client.reconcile_previous_version_at_boot()?;
11333 client.clear_satisfied_persisted_schema_floor();
11334 client.prune_unknown_subscriptions(false)?;
11335 client.drop_unencodable_outbox(false, None)?;
11344 if marker == Some(client.schema.version)
11345 && (!client.outbox.is_empty()
11346 || !client.failed_commits.is_empty()
11347 || client.has_acknowledged_rows())
11348 {
11349 client.overlay_dirty.set(true);
11353 client.rebuild_overlay()?;
11354 }
11355 client.enqueue_startup_sync_if_needed()?;
11362 Ok(())
11363 })();
11364 match startup {
11365 Ok(()) => match client.conn.execute_batch("COMMIT") {
11366 Ok(()) => Ok(client),
11367 Err(error) => {
11368 let failure = Self::sqlite_failure(&client.storage_failure, error);
11369 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
11370 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
11372 }
11373 Err(failure)
11374 }
11375 },
11376 Err(error) => {
11377 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
11378 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
11380 }
11381 Err(error)
11382 }
11383 }
11384 }
11385
11386 pub fn set_now_ms(&mut self, now_ms: i64) {
11389 self.now_ms = Some(now_ms);
11390 }
11391
11392 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
11396 self.encryption = encryption;
11397 }
11398
11399 #[must_use]
11400 pub fn security_lifecycle(&self) -> &'static str {
11401 if self.security_preflight {
11402 "preflight"
11403 } else {
11404 "active"
11405 }
11406 }
11407
11408 #[must_use]
11409 pub fn security_preflight(&self) -> bool {
11410 self.security_preflight
11411 }
11412
11413 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
11415 if self.transport_enabled != enabled {
11416 self.transport_enabled = enabled;
11417 self.sync_intent_queue.clear();
11418 self.sync_intent_queue
11419 .push_back(if enabled && !self.security_preflight {
11420 SyncIntent::Interactive
11421 } else {
11422 SyncIntent::None
11423 });
11424 }
11425 if !enabled && self.active_round.is_none() {
11427 self.disconnect_realtime(transport);
11428 }
11429 }
11430
11431 pub fn transport_enabled(&self) -> bool {
11432 self.transport_enabled
11433 }
11434
11435 pub fn begin_security_preflight(&mut self) {
11442 self.seal_security_on_teardown();
11443 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
11446 }
11447
11448 pub fn seal_security_on_teardown(&mut self) {
11458 self.cancel_sync_round();
11459 self.security_preflight = true;
11460 self.encryption = crate::values::EncryptionConfig::default();
11461 self.sync_intent_queue.clear();
11462 }
11463
11464 pub fn activate_security(
11466 &mut self,
11467 encryption: crate::values::EncryptionConfig,
11468 ) -> Result<(), String> {
11469 if !self.security_preflight {
11470 return Err(
11471 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
11472 );
11473 }
11474 self.enqueue_startup_sync_if_needed()?;
11475 self.encryption = encryption;
11476 self.security_preflight = false;
11477 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
11478 Ok(())
11479 }
11480
11481 fn clock_now_ms(&self) -> i64 {
11482 self.now_ms.unwrap_or_else(crate::wall_clock_ms)
11483 }
11484
11485 fn create_bookkeeping_tables(&self) -> Result<(), String> {
11486 self.conn
11488 .execute_batch(
11489 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
11490 seq INTEGER PRIMARY KEY AUTOINCREMENT,
11491 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
11492 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
11493 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
11494 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
11495 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
11496 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
11497 PRIMARY KEY(commit_id, idx));
11498 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
11499 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
11500 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
11501 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
11502 seq INTEGER PRIMARY KEY AUTOINCREMENT,
11503 client_commit_id TEXT NOT NULL UNIQUE,
11504 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
11505 recorded_at_ms INTEGER NOT NULL,
11506 results_json TEXT NOT NULL,
11507 operations_json TEXT,
11508 resolution TEXT NOT NULL DEFAULT 'active'
11509 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
11510 resolved_at_ms INTEGER,
11511 replacement_client_commit_id TEXT);
11512 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
11513 ON _syncular_commit_outcomes(resolution, seq);
11514 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
11515 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
11516 CREATE TABLE IF NOT EXISTS _syncular_meta (
11517 key TEXT PRIMARY KEY, value TEXT NOT NULL);
11518 CREATE TABLE IF NOT EXISTS _syncular_windows (
11519 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
11520 PRIMARY KEY (base, unit));
11521 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
11522 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
11523 effective_scopes TEXT NOT NULL);
11524 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
11525 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
11526 PRIMARY KEY(commit_id, blob_id));
11527 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
11528 ON _syncular_blob_commit_refs(blob_id);",
11529 )
11530 .map_err(|e| e.to_string())?;
11531 let _ = self
11534 .conn
11535 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
11536 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
11537 self.set_meta(LOCAL_REVISION_KEY, "0");
11538 }
11539 if self.schema_has_blobs() {
11542 let exists = self
11543 .conn
11544 .query_row(
11545 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
11546 [],
11547 |row| row.get::<_, bool>(0),
11548 )
11549 .map_err(|e| e.to_string())?;
11550 if exists {
11551 let mut statement = self
11552 .conn
11553 .prepare("PRAGMA table_info(_syncular_blobs)")
11554 .map_err(|e| e.to_string())?;
11555 let mut columns = statement
11556 .query_map([], |row| row.get::<_, String>(1))
11557 .map_err(|e| e.to_string())?
11558 .collect::<Result<Vec<_>, _>>()
11559 .map_err(|e| e.to_string())?;
11560 columns.sort();
11561 let mut expected = vec![
11562 "blob_id",
11563 "byte_length",
11564 "bytes",
11565 "created_at_ms",
11566 "media_type",
11567 ];
11568 expected.sort_unstable();
11569 if columns != expected {
11570 return Err(
11571 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
11572 );
11573 }
11574 } else {
11575 self.conn
11576 .execute_batch(
11577 "CREATE TABLE _syncular_blobs(
11578 blob_id TEXT PRIMARY KEY,
11579 bytes BLOB NOT NULL,
11580 byte_length INTEGER NOT NULL,
11581 media_type TEXT,
11582 created_at_ms INTEGER NOT NULL);",
11583 )
11584 .map_err(|e| e.to_string())?;
11585 }
11586 self.conn
11587 .execute_batch(
11588 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
11589 blob_id TEXT PRIMARY KEY,
11590 media_type TEXT,
11591 created_at_ms INTEGER NOT NULL);",
11592 )
11593 .map_err(|e| e.to_string())?;
11594 }
11595 Ok(())
11596 }
11597
11598 fn schema_has_blobs(&self) -> bool {
11600 self.schema
11601 .tables
11602 .iter()
11603 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
11604 }
11605
11606 fn get_meta(&self, key: &str) -> Option<String> {
11609 meta_get(&self.conn, key)
11610 }
11611
11612 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
11613 self.conn
11614 .query_row(
11615 "SELECT value FROM _syncular_meta WHERE key = ?1",
11616 rusqlite::params![key],
11617 |row| row.get::<_, String>(0),
11618 )
11619 .optional()
11620 .map_err(|_| {
11621 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11622 })
11623 }
11624
11625 fn persist_schema_version(&self) -> Result<(), String> {
11628 self.conn
11629 .execute(
11630 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
11631 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
11632 )
11633 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11634 set_local_schema_descriptor(&self.conn, &self.schema)
11635 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11636 }
11637
11638 fn set_meta(&self, key: &str, value: &str) {
11639 meta_set(&self.conn, key, value);
11640 }
11641
11642 fn delete_meta(&self, key: &str) {
11643 meta_delete(&self.conn, key);
11644 }
11645
11646 fn load_outbox(&self) -> Result<Vec<OutboxCommit>, String> {
11648 let mut stmt = self
11649 .conn
11650 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
11651 .map_err(|error| error.to_string())?;
11652 let rows = stmt
11653 .query_map([], |row| {
11654 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11655 })
11656 .map_err(|error| error.to_string())?;
11657 let mut commits = Vec::new();
11658 for row in rows {
11659 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
11660 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
11661 format!("invalid persisted outbox {client_commit_id:?}: {error}")
11662 })?;
11663 let mut ops = Vec::with_capacity(entries.len());
11664 for entry in entries {
11665 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
11666 ops.push(OutboxOp {
11667 upsert: op == "upsert",
11668 table: entry
11669 .get("table")
11670 .and_then(Value::as_str)
11671 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
11672 .to_owned(),
11673 row_id: entry
11674 .get("rowId")
11675 .and_then(Value::as_str)
11676 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
11677 .to_owned(),
11678 base_version: entry.get("baseVersion").and_then(Value::as_i64),
11679 values: entry.get("values").and_then(Value::as_object).cloned(),
11682 });
11683 }
11684 commits.push(OutboxCommit {
11685 client_commit_id,
11686 ops,
11687 });
11688 }
11689 Ok(commits)
11690 }
11691
11692 fn restore_persisted_state(&mut self) -> Result<(), String> {
11693 self.subs = {
11694 let mut stmt = self
11695 .conn
11696 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
11697 .map_err(|error| error.to_string())?;
11698 let rows = stmt
11699 .query_map([], |row| {
11700 Ok((
11701 row.get::<_, String>(0)?,
11702 row.get::<_, String>(1)?,
11703 row.get::<_, String>(2)?,
11704 ))
11705 })
11706 .map_err(|error| error.to_string())?;
11707 let mut subscriptions = Vec::new();
11708 for row in rows {
11709 let (id, table, raw) = row.map_err(|error| error.to_string())?;
11710 let state: Value = serde_json::from_str(&raw)
11711 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
11712 let requested = json_to_scope_map(
11713 state.get("requested").unwrap_or(&Value::Object(Map::new())),
11714 )?;
11715 let effective = state
11716 .get("effectiveScopes")
11717 .filter(|value| !value.is_null())
11718 .map(json_to_scope_map)
11719 .transpose()?;
11720 subscriptions.push(Subscription {
11721 id,
11722 table,
11723 requested,
11724 params: state
11725 .get("params")
11726 .and_then(Value::as_str)
11727 .map(str::to_owned),
11728 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
11729 bootstrap_state: state
11730 .get("bootstrapState")
11731 .and_then(Value::as_str)
11732 .map(str::to_owned),
11733 state: SubState::parse(
11734 state
11735 .get("status")
11736 .and_then(Value::as_str)
11737 .unwrap_or("active"),
11738 ),
11739 reason_code: state
11740 .get("reasonCode")
11741 .and_then(Value::as_str)
11742 .map(str::to_owned),
11743 effective,
11744 synced_once: state
11745 .get("syncedOnce")
11746 .and_then(Value::as_bool)
11747 .unwrap_or(false),
11748 });
11749 }
11750 subscriptions
11751 };
11752
11753 self.outbox = self.load_outbox()?;
11754
11755 self.prune_commit_outcomes()?;
11756 let active = self.commit_outcomes(CommitOutcomeQuery {
11757 active_only: true,
11758 ..CommitOutcomeQuery::default()
11759 })?;
11760 self.conflicts = active
11761 .iter()
11762 .flat_map(|outcome| outcome.results.iter())
11763 .filter_map(|result| match result {
11764 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
11765 _ => None,
11766 })
11767 .collect();
11768 self.rejections = active
11769 .iter()
11770 .flat_map(|outcome| outcome.results.iter())
11771 .filter_map(|result| match result {
11772 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
11773 _ => None,
11774 })
11775 .collect();
11776
11777 self.lease_state = self
11778 .get_meta(LEASE_STATE_KEY)
11779 .map(|raw| serde_json::from_str(&raw))
11780 .transpose()
11781 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
11782 self.schema_floor = self
11783 .get_meta(SCHEMA_FLOOR_KEY)
11784 .map(|raw| serde_json::from_str(&raw))
11785 .transpose()
11786 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
11787 self.stopped = self.schema_floor.is_some();
11788 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
11791 self.security_preflight = true;
11792 }
11793 Ok(())
11794 }
11795
11796 fn save_subscription_scope_schema(&self) {
11797 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
11798 .schema
11799 .tables
11800 .iter()
11801 .map(|table| {
11802 (
11803 table.name.clone(),
11804 table
11805 .scope_variables
11806 .iter()
11807 .map(|scope| {
11808 (
11809 scope.variable.clone(),
11810 [scope.prefix.clone(), scope.column.clone()],
11811 )
11812 })
11813 .collect(),
11814 )
11815 })
11816 .collect();
11817 self.set_meta(
11818 "subscriptionScopeSchema",
11819 &serde_json::to_string(&declarations).expect("scope schema serialization"),
11820 );
11821 }
11822
11823 fn prune_unknown_subscriptions(&mut self, bump: bool) -> Result<(), String> {
11825 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
11826 self.get_meta("subscriptionScopeSchema")
11827 .map(|raw| serde_json::from_str(&raw))
11828 .transpose()
11829 .map_err(|error| format!("invalid persisted scope schema: {error}"))?
11830 } else {
11831 None
11832 };
11833 let compatible = |table_name: &str, requested: &[(String, Vec<String>)]| {
11834 let Some(table) = self.schema.table(table_name) else {
11835 return false;
11836 };
11837 if requested
11838 .iter()
11839 .any(|(variable, _)| table.scope_column(variable).is_none())
11840 {
11841 return false;
11842 }
11843 if !bump {
11844 return true;
11845 }
11846 let Some(old) = previous.as_ref().and_then(|tables| tables.get(table_name)) else {
11847 return false;
11848 };
11849 let variables: BTreeSet<_> = if requested.is_empty() {
11850 old.keys()
11851 .cloned()
11852 .chain(
11853 table
11854 .scope_variables
11855 .iter()
11856 .map(|scope| scope.variable.clone()),
11857 )
11858 .collect()
11859 } else {
11860 requested
11861 .iter()
11862 .map(|(variable, _)| variable.clone())
11863 .collect()
11864 };
11865 variables.iter().all(|variable| {
11866 let current = table
11867 .scope_variables
11868 .iter()
11869 .find(|scope| &scope.variable == variable);
11870 match (old.get(variable), current) {
11871 (Some(old), Some(current)) => {
11872 old == &[current.prefix.clone(), current.column.clone()]
11873 }
11874 _ => false,
11875 }
11876 })
11877 };
11878 let mut stale_ids: BTreeSet<_> = self
11879 .subs
11880 .iter()
11881 .filter(|sub| !compatible(&sub.table, &sub.requested))
11882 .map(|sub| sub.id.clone())
11883 .collect();
11884 for (id, table, effective) in self.load_pending_evictions()? {
11885 if !compatible(&table, &effective) {
11886 stale_ids.insert(id);
11887 }
11888 }
11889 for id in &stale_ids {
11890 for sql in [
11891 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
11892 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
11893 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
11894 ] {
11895 self.conn
11896 .execute(sql, rusqlite::params![id])
11897 .map_err(|error| error.to_string())?;
11898 }
11899 }
11900 self.subs.retain(|sub| !stale_ids.contains(&sub.id));
11901 Ok(())
11902 }
11903
11904 #[must_use]
11905 pub fn local_revision(&self) -> u64 {
11906 self.get_meta(LOCAL_REVISION_KEY)
11907 .and_then(|value| value.parse().ok())
11908 .unwrap_or(0)
11909 }
11910
11911 pub fn snapshot_read(
11917 &mut self,
11918 request: &SnapshotReadRequest<'_>,
11919 owner: Option<QueryOwner<'_>>,
11920 ) -> Result<SnapshotRead, QueryReadFailure> {
11921 let result = snapshot_read_connection(&self.conn, request);
11922 if let Some(owner) = owner {
11923 self.record_query_read(owner, result.as_ref().err());
11924 }
11925 result
11926 }
11927
11928 #[must_use]
11929 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
11930 SyncStatusSnapshot {
11931 current_schema_version: self.schema.version,
11932 outbox: self.outbox.len(),
11933 upgrading: self.upgrading,
11934 lease_state: self.lease_state.clone(),
11935 schema_floor: self.schema_floor.clone(),
11936 sync_needed: self.sync_needed,
11937 previous_version_context: self.previous_version_status(),
11938 }
11939 }
11940
11941 fn previous_version_status(&self) -> PreviousVersionStatus {
11944 let Some(replica_path) = self.previous_version_replica_path() else {
11945 return PreviousVersionStatus {
11946 present: false,
11947 created_at_ms: None,
11948 };
11949 };
11950 match crate::previous_version::read_previous_version_record(&replica_path) {
11951 Ok(Some(record)) => PreviousVersionStatus {
11952 present: true,
11953 created_at_ms: Some(record.created_at_ms),
11954 },
11955 _ => PreviousVersionStatus {
11956 present: std::path::Path::new(
11957 &crate::previous_version::previous_version_container_path(&replica_path),
11958 )
11959 .exists(),
11960 created_at_ms: None,
11961 },
11962 }
11963 }
11964
11965 pub fn previous_version_snapshot(
11971 &self,
11972 spec: &PreviousVersionReadSpec,
11973 ) -> Result<PreviousVersionSnapshot, String> {
11974 let now_ms = self.clock_now_ms();
11975 crate::previous_version::previous_version_snapshot(
11976 &self.conn,
11977 self.previous_version_replica_path().as_deref(),
11978 self.previous_version.as_ref(),
11979 PreviousVersionLifecycle {
11980 lease_inactive: self.previous_version_lease_inactive(now_ms),
11981 scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
11982 coverage_complete: self.previous_version_coverage_complete(),
11983 },
11984 now_ms,
11985 self.schema.version,
11986 spec,
11987 )
11988 }
11989
11990 #[must_use]
11992 pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
11993 crate::previous_version::stored_previous_version_audit(&self.conn)
11994 }
11995
11996 pub fn previous_version_discard(
12000 &self,
12001 ) -> crate::previous_version::PreviousVersionDiscardOutcome {
12002 self.drop_previous_version()
12003 }
12004
12005 fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
12009 let absent = crate::previous_version::PreviousVersionDiscardOutcome {
12010 present: false,
12011 discarded: false,
12012 };
12013 let Some(replica_path) = self.previous_version_replica_path() else {
12014 return absent;
12015 };
12016 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
12017 .unwrap_or(absent)
12018 }
12019
12020 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
12022 let Some(lease) = &self.lease_state else {
12023 return false;
12024 };
12025 if lease.error_code.is_some() {
12026 return true;
12027 }
12028 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
12029 }
12030
12031 fn previous_version_coverage_complete(&self) -> bool {
12035 let active = self
12036 .subs
12037 .iter()
12038 .filter(|sub| sub.state == SubState::Active)
12039 .collect::<Vec<_>>();
12040 if active.is_empty() {
12041 return false;
12042 }
12043 active
12044 .iter()
12045 .all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
12046 }
12047
12048 fn previous_version_lifetime_check(&self) {
12052 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
12053 return;
12054 };
12055 let now_ms = self.clock_now_ms();
12056 if self.previous_version_lease_inactive(now_ms) {
12057 let _ = self.drop_previous_version();
12058 return;
12059 }
12060 let Some(replica_path) = self.previous_version_replica_path() else {
12061 return;
12062 };
12063 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
12064 else {
12065 return;
12066 };
12067 if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
12068 let _ = self.drop_previous_version();
12069 return;
12070 }
12071 if now_ms - record.created_at_ms > config.max_age_ms {
12072 let _ = self.drop_previous_version();
12073 return;
12074 }
12075 if self.previous_version_coverage_complete() {
12076 let _ = self.drop_previous_version();
12077 }
12078 }
12079
12080 pub fn diagnostics_snapshot(
12081 &self,
12082 request: &ClientDiagnosticsRequest,
12083 ) -> Result<ClientDiagnosticsSnapshot, String> {
12084 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
12085 return Err(format!(
12086 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
12087 ));
12088 }
12089 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
12090 for sub in &self.subs {
12091 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
12092 let complete =
12093 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
12094 let state = match sub.state {
12095 SubState::Revoked => "revoked",
12096 SubState::Failed => "failed",
12097 SubState::Active if reset => "reset",
12098 SubState::Active if complete => "complete",
12099 SubState::Active => "bootstrapping",
12100 };
12101 subscriptions.insert(
12102 sub.id.clone(),
12103 DiagnosticSubscription {
12104 id: sub.id.clone(),
12105 table: sub.table.clone(),
12106 state: state.to_owned(),
12107 complete,
12108 cursor: Some(sub.cursor),
12109 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
12110 },
12111 );
12112 }
12113 for expected in &request.expected_subscriptions {
12114 if expected.id.is_empty() || expected.table.is_empty() {
12115 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
12116 }
12117 if subscriptions
12118 .get(&expected.id)
12119 .is_some_and(|registered| registered.table != expected.table)
12120 {
12121 subscriptions.insert(
12122 expected.id.clone(),
12123 DiagnosticSubscription {
12124 id: expected.id.clone(),
12125 table: expected.table.clone(),
12126 state: "failed".to_owned(),
12127 complete: false,
12128 cursor: None,
12129 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
12130 },
12131 );
12132 } else {
12133 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
12134 DiagnosticSubscription {
12135 id: expected.id.clone(),
12136 table: expected.table.clone(),
12137 state: "unregistered".to_owned(),
12138 complete: false,
12139 cursor: None,
12140 reason_code: None,
12141 }
12142 });
12143 }
12144 }
12145 let mut ordered_subscriptions = Vec::new();
12146 let mut included = BTreeSet::new();
12147 for expected in &request.expected_subscriptions {
12148 if included.insert(expected.id.clone()) {
12149 if let Some(subscription) = subscriptions.get(&expected.id) {
12150 ordered_subscriptions.push(subscription.clone());
12151 }
12152 }
12153 }
12154 for (id, subscription) in &subscriptions {
12155 if included.insert(id.clone()) {
12156 ordered_subscriptions.push(subscription.clone());
12157 }
12158 }
12159 let subscriptions_truncated =
12160 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
12161 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
12162 let captured_at_ms = self.clock_now_ms();
12163 let lease = if let Some(error_code) = self
12164 .lease_state
12165 .as_ref()
12166 .and_then(|state| state.error_code.clone())
12167 {
12168 ClientDiagnosticsLease {
12169 state: "stopped".to_owned(),
12170 expires_at_ms: self
12171 .lease_state
12172 .as_ref()
12173 .and_then(|state| state.expires_at_ms),
12174 error_code: Some(Self::diagnostic_code(&error_code)),
12175 }
12176 } else if let Some(expires_at_ms) = self
12177 .lease_state
12178 .as_ref()
12179 .and_then(|state| state.expires_at_ms)
12180 {
12181 ClientDiagnosticsLease {
12182 state: if expires_at_ms <= captured_at_ms {
12183 "expired".to_owned()
12184 } else {
12185 "active".to_owned()
12186 },
12187 expires_at_ms: Some(expires_at_ms),
12188 error_code: None,
12189 }
12190 } else {
12191 ClientDiagnosticsLease {
12192 state: "none".to_owned(),
12193 expires_at_ms: None,
12194 error_code: None,
12195 }
12196 };
12197 let connectivity = match self.last_round.as_ref() {
12198 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
12199 Some(round) if round.status == "succeeded" => "online",
12200 Some(round)
12201 if round.status == "failed"
12202 && round
12203 .error_code
12204 .as_deref()
12205 .is_some_and(Self::retryable_transport_code) =>
12206 {
12207 "offline"
12208 }
12209 _ => "unknown",
12210 };
12211 Ok(ClientDiagnosticsSnapshot {
12212 version: CLIENT_DIAGNOSTICS_VERSION,
12213 captured_at_ms,
12214 host: ClientDiagnosticsHost {
12215 kind: "direct".to_owned(),
12216 role: "single".to_owned(),
12217 connectivity: connectivity.to_owned(),
12218 realtime: self.realtime_state,
12219 realtime_policy: self.realtime_policy,
12220 realtime_reason_code: self.realtime_reason_code.clone(),
12221 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
12222 },
12223 security_lifecycle: self.security_lifecycle().to_owned(),
12224 schema: ClientDiagnosticsSchema {
12225 current_version: self.schema.version,
12226 upgrading: self.upgrading,
12227 required_version: self
12228 .schema_floor
12229 .as_ref()
12230 .and_then(|floor| floor.required_schema_version),
12231 latest_version: self
12232 .schema_floor
12233 .as_ref()
12234 .and_then(|floor| floor.latest_schema_version),
12235 },
12236 replica: ClientDiagnosticsReplica {
12237 local_revision: self.local_revision().to_string(),
12238 sync_needed: self.sync_needed,
12239 pending_outbox: self.outbox.len(),
12240 },
12241 lease,
12242 subscriptions: ordered_subscriptions,
12243 subscriptions_truncated,
12244 last_round: self.last_round.clone(),
12245 last_change: self.last_change.clone(),
12246 storage: self.diagnostics_storage(),
12247 query_failures: self.query_failures.clone(),
12248 })
12249 }
12250
12251 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
12252 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
12253 let page_count: i64 = self
12254 .conn
12255 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
12256 let page_size: i64 = self
12257 .conn
12258 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
12259 let outbox_bytes: i64 = self
12261 .conn
12262 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
12263 .query_row([], |row| row.get(0))?;
12264 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
12265 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
12266 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
12267 let blob_bytes = if self.schema_has_blobs() {
12268 self.conn
12269 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
12270 .query_row([], |row| row.get(0))?
12271 } else {
12272 0
12273 };
12274 let pressure = self
12275 .limits
12276 .blob_cache_max_bytes
12277 .is_some_and(|limit| blob_bytes > limit);
12278 Ok(ClientDiagnosticsStorage {
12279 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
12280 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
12281 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
12282 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
12283 retained_outcome_entries: Some(outcome_entries.max(0)),
12284 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
12285 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
12286 })
12287 };
12288 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
12289 status: "unreadable".to_owned(),
12290 database_bytes_approx: None,
12291 pending_outbox_bytes_approx: None,
12292 retained_outcome_bytes_approx: None,
12293 retained_outcome_entries: None,
12294 blob_cache_bytes_approx: None,
12295 pressure_reason_code: None,
12296 })
12297 }
12298
12299 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
12300 self.change_queue.drain(..).collect()
12301 }
12302
12303 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
12304 let intents = self.sync_intent_queue.drain(..).collect();
12305 if self.transport_enabled {
12306 intents
12307 } else {
12308 vec![SyncIntent::None]
12309 }
12310 }
12311
12312 fn schedule_background_retry(&mut self) -> u64 {
12313 let delay_ms = self.retry_delay_ms;
12314 self.sync_intent_queue
12315 .push_back(SyncIntent::Background { delay_ms });
12316 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
12317 self.round_retry_delay_ms = Some(delay_ms);
12318 delay_ms
12319 }
12320
12321 fn reset_background_retry(&mut self) {
12322 self.retry_delay_ms = 250;
12323 }
12324
12325 fn retryable_transport_code(code: &str) -> bool {
12326 code == "transport.failed"
12327 || code == "transport.timeout"
12328 || code == "transport.unavailable"
12329 || code == "sync.transport_failed"
12330 }
12331
12332 fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
12337 if ssp2::model::valid_round_failure_code(&error.code) {
12338 self.round_failure = Some(error.code.clone());
12339 }
12340 SectionError::Abort(error.code, error.message)
12341 }
12342
12343 fn retryable_failure_code(code: &str) -> bool {
12348 Self::retryable_transport_code(code)
12349 || matches!(
12350 code,
12351 "sync.auth_required"
12352 | "sync.auth_lease_required"
12353 | "sync.auth_lease_revoked"
12354 | "sync.internal_error"
12355 | "sync.idempotency_cache_miss"
12356 | "sync.schema_not_ready"
12357 | "sync.segment_expired"
12358 | "sync.rate_limited"
12359 | "sync.websocket_connection_limit"
12360 )
12361 }
12362
12363 fn diagnostic_code(code: &str) -> String {
12364 let valid = !code.is_empty()
12365 && code.len() <= 96
12366 && code.contains('.')
12367 && code
12368 .bytes()
12369 .next()
12370 .is_some_and(|byte| byte.is_ascii_lowercase())
12371 && code.bytes().all(|byte| {
12372 byte.is_ascii_lowercase()
12373 || byte.is_ascii_digit()
12374 || matches!(byte, b'.' | b'_' | b'-')
12375 });
12376 if valid {
12377 code.to_owned()
12378 } else {
12379 "client.unknown_failure".to_owned()
12380 }
12381 }
12382
12383 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
12384 if self.sync_needed != value {
12385 if self.begin_observation("syncular_status").is_ok() {
12386 self.sync_needed = value;
12387 let batch = ChangeAccumulator {
12388 status: true,
12389 ..ChangeAccumulator::default()
12390 };
12391 if self.finish_observation("syncular_status", batch).is_err() {
12392 self.rollback_observation("syncular_status");
12393 }
12394 } else {
12395 self.sync_needed = value;
12396 }
12397 }
12398 if value && interactive {
12399 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12400 }
12401 }
12402
12403 fn sqlite_failure(
12404 storage_failure: &RefCell<Option<QueryReadFailure>>,
12405 error: rusqlite::Error,
12406 ) -> String {
12407 let message = error.to_string();
12408 let failure = QueryReadFailure::from(error);
12409 let code = failure.code;
12410 if code.is_some() && storage_failure.borrow().is_none() {
12411 *storage_failure.borrow_mut() = Some(failure);
12412 }
12413 match code {
12414 Some(code) => format!("{code}: {message}"),
12415 None => message,
12416 }
12417 }
12418
12419 fn begin_observation(&self, name: &str) -> Result<(), String> {
12420 if let Some(failure) = &self.coalesced_failure {
12421 return Err(failure.clone());
12422 }
12423 if self.coalesced.is_some() && self.conn.is_autocommit() {
12426 return Err("the coalesced apply transaction was rolled back".to_owned());
12427 }
12428 self.conn
12429 .execute_batch(&format!("SAVEPOINT {name}"))
12430 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12431 }
12432
12433 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
12436 if self.coalesced.is_some() || !self.conn.is_autocommit() {
12437 return Ok(());
12438 }
12439 self.conn
12440 .execute_batch("BEGIN")
12441 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12442 self.coalesced = Some(CoalescedApply {
12443 rows: 0,
12444 subs: self.subs.clone(),
12445 outbox: self.outbox.clone(),
12446 failed_commits: self.failed_commits.clone(),
12447 conflicts: self.conflicts.clone(),
12448 rejections: self.rejections.clone(),
12449 schema_floor: self.schema_floor.clone(),
12450 lease_state: self.lease_state.clone(),
12451 stopped: self.stopped,
12452 upgrading: self.upgrading,
12453 sync_needed: self.sync_needed,
12454 overlay_dirty: self.overlay_dirty.snapshot(),
12455 change_queue: self.change_queue.len(),
12456 sync_intent_queue: self.sync_intent_queue.len(),
12457 last_change: self.last_change.clone(),
12458 });
12459 Ok(())
12460 }
12461
12462 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
12465 let Some(open) = &self.coalesced else {
12466 return Ok(());
12467 };
12468 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
12469 self.commit_coalesced_apply()?;
12470 self.begin_coalesced_apply()?;
12471 }
12472 if let Some(open) = &mut self.coalesced {
12473 open.rows += rows;
12474 }
12475 Ok(())
12476 }
12477
12478 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
12483 let Some(open) = self.coalesced.take() else {
12484 return Ok(());
12485 };
12486 let committed = if self.conn.is_autocommit() {
12487 Err("the coalesced apply transaction was rolled back".to_owned())
12488 } else {
12489 self.conn
12490 .execute_batch("COMMIT")
12491 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12492 };
12493 let Err(error) = committed else {
12494 return Ok(());
12495 };
12496 if !self.conn.is_autocommit() {
12497 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12498 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12499 }
12500 }
12501 self.subs = open.subs;
12502 self.outbox = open.outbox;
12503 self.failed_commits = open.failed_commits;
12504 self.conflicts = open.conflicts;
12505 self.rejections = open.rejections;
12506 self.schema_floor = open.schema_floor;
12507 self.lease_state = open.lease_state;
12508 self.stopped = open.stopped;
12509 self.upgrading = open.upgrading;
12510 self.sync_needed = open.sync_needed;
12511 self.overlay_dirty.restore(open.overlay_dirty);
12512 self.change_queue.truncate(open.change_queue);
12513 self.sync_intent_queue.truncate(open.sync_intent_queue);
12514 self.last_change = open.last_change;
12515 self.coalesced_failure = Some(error.clone());
12516 Err(error)
12517 }
12518
12519 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
12521 let committed = self.commit_coalesced_apply();
12522 match self.coalesced_failure.take() {
12523 Some(failure) => Err(failure),
12524 None => committed,
12525 }
12526 }
12527
12528 fn rollback_observation(&self, name: &str) {
12529 if let Err(error) = self
12530 .conn
12531 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
12532 {
12533 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
12534 if failure.rollback_failure.is_none() {
12535 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
12536 }
12537 }
12538 }
12539 }
12540
12541 fn finish_observation(
12542 &mut self,
12543 name: &str,
12544 mut batch: ChangeAccumulator,
12545 ) -> Result<(), String> {
12546 #[cfg(feature = "bench-internals")]
12547 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
12548 if self.failed_commits.iter().any(|(commit, _)| {
12549 commit
12550 .ops
12551 .iter()
12552 .any(|operation| batch.tables.contains_key(&operation.table))
12553 }) {
12554 batch.outcomes = true;
12555 }
12556 if !batch.touched() {
12557 self.conn
12558 .execute_batch(&format!("RELEASE {name}"))
12559 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12560 return Ok(());
12561 }
12562 let revision = self
12563 .local_revision()
12564 .checked_add(1)
12565 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
12566 self.conn
12567 .execute(
12568 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
12569 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
12570 )
12571 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12572 let status = batch.status.then(|| self.status_snapshot());
12573 let event = ClientChangeBatch {
12574 revision: revision.to_string(),
12575 tables: batch
12576 .tables
12577 .into_iter()
12578 .map(|(table, scope_keys)| TableChange {
12579 table,
12580 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
12581 })
12582 .collect(),
12583 windows: batch
12584 .windows
12585 .into_iter()
12586 .map(|((base_key, table), units)| WindowChange {
12587 base_key,
12588 table,
12589 units: units.into_iter().collect(),
12590 })
12591 .collect(),
12592 status,
12593 conflicts_changed: batch.conflicts,
12594 rejections_changed: batch.rejections,
12595 outcomes_changed: batch.outcomes,
12596 };
12597 self.conn
12598 .execute_batch(&format!("RELEASE {name}"))
12599 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12600 let mut diagnostic_tables = event
12601 .tables
12602 .iter()
12603 .map(|entry| entry.table.clone())
12604 .collect::<BTreeSet<_>>()
12605 .into_iter()
12606 .collect::<Vec<_>>();
12607 let mut diagnostic_windows = event
12608 .windows
12609 .iter()
12610 .map(|entry| entry.table.clone())
12611 .collect::<BTreeSet<_>>()
12612 .into_iter()
12613 .collect::<Vec<_>>();
12614 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
12615 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
12616 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
12617 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
12618 self.last_change = Some(DiagnosticLastChange {
12619 revision: event.revision.clone(),
12620 recorded_at_ms: self.clock_now_ms(),
12621 tables: diagnostic_tables,
12622 windows: diagnostic_windows,
12623 domains_truncated,
12624 status_changed: event.status.is_some(),
12625 conflicts_changed: event.conflicts_changed,
12626 rejections_changed: event.rejections_changed,
12627 outcomes_changed: event.outcomes_changed,
12628 });
12629 self.change_queue.push_back(event);
12630 Ok(())
12631 }
12632
12633 fn record_scope_map(
12634 &self,
12635 batch: &mut ChangeAccumulator,
12636 table_name: &str,
12637 scopes: &[(String, Vec<String>)],
12638 ) {
12639 let Some(table) = self.schema.table(table_name) else {
12640 return;
12641 };
12642 for (variable, values) in scopes {
12643 let Some(scope) = table
12644 .scope_variables
12645 .iter()
12646 .find(|scope| &scope.variable == variable)
12647 else {
12648 continue;
12649 };
12650 for value in values {
12651 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12652 }
12653 }
12654 }
12655
12656 fn record_row_scopes(
12658 &self,
12659 batch: &mut ChangeAccumulator,
12660 table_name: &str,
12661 row_id: &str,
12662 base: bool,
12663 ) -> bool {
12664 let Some(table) = self.schema.table(table_name) else {
12665 return false;
12666 };
12667 if table.scope_variables.is_empty() {
12668 return false;
12669 }
12670 let columns = table
12671 .scope_variables
12672 .iter()
12673 .map(|scope| quote_ident(&scope.column))
12674 .collect::<Vec<_>>()
12675 .join(", ");
12676 let full_table = if base {
12677 base_table(table_name)
12678 } else {
12679 visible_table(table_name)
12680 };
12681 let sql = format!(
12682 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
12683 row_id_predicate(table)
12684 );
12685 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
12686 return false;
12687 };
12688 let values = stmt.query_row(rusqlite::params![row_id], |row| {
12689 let mut values = Vec::with_capacity(table.scope_variables.len());
12690 for index in 0..table.scope_variables.len() {
12691 values.push(row.get::<_, Option<String>>(index)?);
12692 }
12693 Ok(values)
12694 });
12695 let Ok(values) = values else {
12696 return false;
12697 };
12698 let mut recorded = false;
12699 for (scope, value) in table.scope_variables.iter().zip(values) {
12700 if let Some(value) = value {
12701 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12702 recorded = true;
12703 }
12704 }
12705 recorded
12706 }
12707
12708 fn record_commit_changes(
12709 &self,
12710 batch: &mut ChangeAccumulator,
12711 tables: &[String],
12712 changes: &[ssp2::model::Change],
12713 ) {
12714 #[cfg(feature = "bench-internals")]
12715 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
12716 for change in changes {
12717 let Some(table_name) = tables.get(change.table_index as usize) else {
12718 continue;
12719 };
12720 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
12721 if let Some(table) = self.schema.table(table_name) {
12722 for (variable, value) in &change.scopes {
12723 if let Some(scope) = table
12724 .scope_variables
12725 .iter()
12726 .find(|scope| &scope.variable == variable)
12727 {
12728 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12729 precise = true;
12730 }
12731 }
12732 }
12733 if !precise {
12734 batch.table(table_name);
12735 }
12736 }
12737 }
12738
12739 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
12740 if effective.is_empty() {
12741 return false;
12742 }
12743 let Some(table) = self.schema.table(table_name) else {
12744 return false;
12745 };
12746 let mut clauses = Vec::new();
12747 let mut params = Vec::new();
12748 for (variable, values) in effective {
12749 let Some(column) = table.scope_column(variable) else {
12750 return false;
12751 };
12752 if values.is_empty() {
12753 return false;
12754 }
12755 let placeholders = values
12756 .iter()
12757 .map(|value| {
12758 params.push(SqlValue::Text(value.clone()));
12759 "?"
12760 })
12761 .collect::<Vec<_>>()
12762 .join(", ");
12763 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
12764 }
12765 let sql = format!(
12766 "SELECT 1 FROM {} WHERE {} LIMIT 1",
12767 base_table(table_name),
12768 clauses.join(" AND ")
12769 );
12770 self.conn
12771 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
12772 .is_ok()
12773 }
12774
12775 pub fn upgrading(&self) -> bool {
12777 self.upgrading
12778 }
12779
12780 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
12788 let new_schema = parse_schema_json(schema_json)?;
12789 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
12790 self.conn
12791 .execute_batch("BEGIN IMMEDIATE")
12792 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12793 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
12794 let prior_subs = self.subs.clone();
12800 let prior_outbox = self.outbox.clone();
12801 let prior_rejections = self.rejections.clone();
12802 let prior_upgrading = self.upgrading;
12803 let prior_stopped = self.stopped;
12804 let prior_schema_floor = self.schema_floor.clone();
12805 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12806 let prior_sync_needed = self.sync_needed;
12807 let prior_active_round = self.active_round;
12808 let prior_changes = self.change_queue.len();
12809 let prior_intents = self.sync_intent_queue.len();
12810 let prior_last_change = self.last_change.clone();
12811 let guarded = (|| -> Result<(), String> {
12812 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
12817 self.create_bookkeeping_tables()?;
12818 match marker {
12819 Some(version) if version < self.schema.version => self.run_schema_reset()?,
12820 None => {
12821 self.create_synced_tables()?;
12822 self.persist_schema_version()?;
12823 }
12824 _ => {}
12825 }
12826 self.outbox = self.load_outbox()?;
12831 self.drop_unencodable_outbox(false, None)?;
12836 if marker == Some(self.schema.version)
12837 && (!self.outbox.is_empty()
12838 || !self.failed_commits.is_empty()
12839 || self.has_acknowledged_rows())
12840 {
12841 self.overlay_dirty.set(true);
12842 self.rebuild_overlay()?;
12843 }
12844 self.prune_unknown_subscriptions(false)?;
12845 self.save_subscription_scope_schema();
12846 self.enqueue_startup_sync_if_needed()?;
12850 Ok(())
12851 })();
12852 let outcome = match guarded {
12853 Ok(()) => self
12854 .conn
12855 .execute_batch("COMMIT")
12856 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
12857 Err(error) => Err(error),
12858 };
12859 if let Err(error) = outcome {
12860 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12861 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12863 }
12864 self.schema = prior_schema;
12865 self.subs = prior_subs;
12866 self.outbox = prior_outbox;
12867 self.rejections = prior_rejections;
12868 self.upgrading = prior_upgrading;
12869 self.stopped = prior_stopped;
12870 self.schema_floor = prior_schema_floor;
12871 self.overlay_dirty.restore(prior_overlay_dirty);
12872 self.sync_needed = prior_sync_needed;
12873 self.active_round = prior_active_round;
12874 self.change_queue.truncate(prior_changes);
12875 self.sync_intent_queue.truncate(prior_intents);
12876 self.last_change = prior_last_change;
12877 self.insert_sql.borrow_mut().clear();
12879 return Err(error);
12880 }
12881 Ok(())
12882 }
12883
12884 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
12885 let startup_work = !self.stopped
12886 && (!self.outbox.is_empty()
12887 || self.has_acknowledged_rows()
12888 || !self.load_pending_evictions()?.is_empty()
12889 || self.subs.iter().any(|sub| sub.state == SubState::Active));
12890 if startup_work {
12891 self.sync_needed = true;
12892 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12893 }
12894 Ok(())
12895 }
12896
12897 fn clear_satisfied_persisted_schema_floor(&mut self) {
12904 let satisfied = self
12905 .schema_floor
12906 .as_ref()
12907 .and_then(|floor| floor.required_schema_version)
12908 .is_some_and(|required| self.schema.version >= required);
12909 if !satisfied {
12910 return;
12911 }
12912 self.schema_floor = None;
12913 self.stopped = false;
12914 self.delete_meta(SCHEMA_FLOOR_KEY);
12915 }
12916
12917 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
12922 let Some(replica_path) = self.previous_version_replica_path() else {
12923 return Ok(());
12924 };
12925 let max_age_ms = self
12926 .previous_version
12927 .filter(|config| config.enabled)
12928 .map(|config| config.max_age_ms);
12929 reconcile_previous_version_at_boot(
12930 &self.conn,
12931 &replica_path,
12932 self.schema.version,
12933 max_age_ms,
12934 self.clock_now_ms(),
12935 )
12936 }
12937
12938 fn previous_version_replica_path(&self) -> Option<String> {
12942 let path = self.conn.path()?;
12943 if path.is_empty() || path.starts_with("file:") {
12944 return None;
12945 }
12946 Some(path.to_owned())
12947 }
12948
12949 fn run_schema_reset(&mut self) -> Result<(), String> {
12955 self.begin_observation("syncular_schema_reset")?;
12956 let prior_subs = self.subs.clone();
12957 let prior_outbox = self.outbox.clone();
12958 let prior_rejections = self.rejections.clone();
12959 let prior_upgrading = self.upgrading;
12960 let prior_stopped = self.stopped;
12961 let prior_schema_floor = self.schema_floor.clone();
12962 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12963 let prior_sync_needed = self.sync_needed;
12964 let prior_active_round = self.active_round;
12965 let mut batch = ChangeAccumulator::default();
12966 let result = self
12967 .run_schema_reset_observed(&mut batch, true)
12968 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
12969 if let Err(error) = result {
12970 self.rollback_observation("syncular_schema_reset");
12971 self.subs = prior_subs;
12972 self.outbox = prior_outbox;
12973 self.rejections = prior_rejections;
12974 self.upgrading = prior_upgrading;
12975 self.stopped = prior_stopped;
12976 self.schema_floor = prior_schema_floor;
12977 self.overlay_dirty.restore(prior_overlay_dirty);
12978 self.sync_needed = prior_sync_needed;
12979 self.active_round = prior_active_round;
12980 return Err(error);
12981 }
12982 Ok(())
12983 }
12984
12985 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
12986 if self.get_meta(LOG_EPOCH_KEY).is_none() {
12987 self.begin_observation("syncular_epoch_acquisition")?;
12988 let previous_sync_needed = self.sync_needed;
12989 if let Err(error) = self.conn.execute(
12990 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12991 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12992 ) {
12993 self.rollback_observation("syncular_epoch_acquisition");
12994 return Err(Self::sqlite_failure(&self.storage_failure, error));
12995 }
12996 self.sync_needed = true;
12997 let batch = ChangeAccumulator {
12998 status: true,
12999 ..ChangeAccumulator::default()
13000 };
13001 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
13002 self.rollback_observation("syncular_epoch_acquisition");
13003 self.sync_needed = previous_sync_needed;
13004 return Err(error);
13005 }
13006 self.sync_intent_queue.push_back(SyncIntent::Interactive);
13007 return Ok(Vec::new());
13008 }
13009 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
13010 self.begin_observation("syncular_log_epoch_reset")?;
13011 let prior_subs = self.subs.clone();
13012 let prior_outbox = self.outbox.clone();
13013 let prior_rejections = self.rejections.clone();
13014 let prior_upgrading = self.upgrading;
13015 let prior_stopped = self.stopped;
13016 let prior_schema_floor = self.schema_floor.clone();
13017 let prior_overlay_dirty = self.overlay_dirty.snapshot();
13018 let prior_sync_needed = self.sync_needed;
13019 let prior_active_round = self.active_round;
13020 let mut batch = ChangeAccumulator::default();
13021 let result = self
13022 .run_schema_reset_observed(&mut batch, false)
13023 .and_then(|()| {
13024 self.conn
13025 .execute(
13026 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
13027 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
13028 )
13029 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13030 self.sync_needed = true;
13031 batch.status = true;
13032 self.finish_observation("syncular_log_epoch_reset", batch)
13033 });
13034 if let Err(error) = result {
13035 self.rollback_observation("syncular_log_epoch_reset");
13036 self.subs = prior_subs;
13037 self.outbox = prior_outbox;
13038 self.rejections = prior_rejections;
13039 self.upgrading = prior_upgrading;
13040 self.stopped = prior_stopped;
13041 self.schema_floor = prior_schema_floor;
13042 self.overlay_dirty.restore(prior_overlay_dirty);
13043 self.sync_needed = prior_sync_needed;
13044 self.active_round = prior_active_round;
13045 return Err(error);
13046 }
13047 self.sync_intent_queue.push_back(SyncIntent::Interactive);
13048 Ok(resets)
13049 }
13050
13051 fn run_schema_reset_observed(
13052 &mut self,
13053 batch: &mut ChangeAccumulator,
13054 capture: bool,
13055 ) -> Result<(), String> {
13056 read_local_schema_version(&self.conn, self.schema.version)?;
13062 self.conn
13063 .execute_batch(
13064 "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
13065 )
13066 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13067 self.cancel_sync_round();
13068 self.upgrading = true;
13069 batch.status = true;
13070 if self.previous_version.is_some_and(|config| config.enabled) {
13078 let previous_version = self
13079 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
13080 .and_then(|value| value.parse::<i32>().ok())
13081 .unwrap_or(self.schema.version);
13082 let pending = self
13083 .outbox
13084 .iter()
13085 .map(|commit| PendingCommitAudit {
13086 commit_id: &commit.client_commit_id,
13087 operations: commit_audit_operations(commit),
13088 })
13089 .collect::<Vec<_>>();
13090 crate::previous_version::write_previous_version_audit(
13091 &self.conn,
13092 &crate::previous_version::build_previous_version_audit(
13093 &self.schema,
13094 &pending,
13095 previous_version,
13096 self.schema.version,
13097 self.clock_now_ms(),
13098 ),
13099 );
13100 }
13101 if let Some(replica_path) = self.previous_version_replica_path() {
13106 sweep_previous_version_container(&self.conn, &replica_path)?;
13107 if capture {
13108 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
13109 let previous_version = self
13110 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
13111 .and_then(|value| value.parse::<i32>().ok())
13112 .unwrap_or(self.schema.version);
13113 capture_previous_version_from_replica(
13114 &self.conn,
13115 &replica_path,
13116 previous_version,
13117 self.schema.version,
13118 &config,
13119 self.clock_now_ms(),
13120 )?;
13121 }
13122 }
13123 }
13124 for table in &self.schema.tables {
13125 batch.table(&table.name);
13126 }
13127 for (base_key, unit, table) in self.load_registered_window_units() {
13128 batch.window(&base_key, &table, &unit);
13129 }
13130 if capture {
13131 self.prune_unknown_subscriptions(true)?;
13132 }
13133 self.insert_sql.borrow_mut().clear();
13135 self.overlay_dirty.set(true);
13136 let virtual_tables: Vec<String> = {
13144 let mut stmt = self
13145 .conn
13146 .prepare(
13147 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
13148 )
13149 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13150 let rows = stmt
13151 .query_map([], |row| row.get::<_, String>(0))
13152 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13153 rows.filter_map(Result::ok)
13154 .filter(|name| is_synced_table_name(name))
13155 .collect()
13156 };
13157 for name in virtual_tables {
13158 self.conn
13159 .execute_batch(&format!(
13160 "DROP TABLE IF EXISTS {}",
13161 quote_ident(&format!("_syncular_fts_{name}"))
13162 ))
13163 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13164 self.conn
13165 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
13166 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13167 }
13168 let existing: Vec<String> = {
13169 let mut stmt = self
13170 .conn
13171 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
13172 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13173 let rows = stmt
13174 .query_map([], |row| row.get::<_, String>(0))
13175 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13176 rows.filter_map(Result::ok)
13177 .filter(|name| is_synced_table_name(name))
13178 .collect()
13179 };
13180 for name in &existing {
13181 if let Some(table) = name.strip_prefix("_syncular_base_") {
13182 batch.table(table);
13183 } else if !name.starts_with("_syncular_") {
13184 batch.table(name);
13185 }
13186 }
13187 for name in existing {
13188 self.conn
13189 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
13190 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13191 }
13192 self.create_synced_tables()?;
13194 for sub in &mut self.subs {
13196 sub.cursor = -1;
13197 sub.bootstrap_state = None;
13198 sub.effective = None;
13199 sub.state = SubState::Active;
13200 sub.reason_code = None;
13201 sub.synced_once = false;
13202 }
13203 let subs = self.subs.clone();
13204 for sub in &subs {
13205 self.persist_sub(sub)?;
13206 }
13207 self.stopped = false;
13209 self.schema_floor = None;
13210 self.delete_meta(SCHEMA_FLOOR_KEY);
13211 self.persist_schema_version()?;
13213 self.save_subscription_scope_schema();
13214 self.drop_unencodable_outbox(false, Some(batch))?;
13225 self.rebuild_overlay()?;
13227 Ok(())
13228 }
13229
13230 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
13237 let incompatible = self
13238 .outbox
13239 .iter()
13240 .filter(|commit| {
13241 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
13242 })
13243 .cloned()
13244 .collect::<Vec<_>>();
13245 if incompatible.is_empty() {
13246 return Ok(false);
13247 }
13248 let prior_outbox = self.outbox.clone();
13254 let prior_rejections = self.rejections.clone();
13255 let prior_overlay_dirty = self.overlay_dirty.snapshot();
13256 let owns_transaction = self.conn.is_autocommit();
13257 self.begin_observation("syncular_outbox_drop")?;
13258 let mut batch = ChangeAccumulator::default();
13259 let dropped = (|| -> Result<(), String> {
13260 let mut rejections = Vec::new();
13261 for commit in &incompatible {
13262 let results = commit
13263 .ops
13264 .iter()
13265 .enumerate()
13266 .map(|(op_index, operation)| {
13267 let rejection = RejectionRecord {
13268 client_commit_id: commit.client_commit_id.clone(),
13269 op_index: op_index as i32,
13270 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
13271 message: "the persisted commit cannot encode under the current schema"
13272 .to_owned(),
13273 retryable: false,
13274 details: None,
13275 operation: Some(CommitOperation::from(operation)),
13276 };
13277 rejections.push(rejection.clone());
13278 CommitOperationOutcome::Error { rejection }
13279 })
13280 .collect::<Vec<_>>();
13281 self.persist_commit_outcome(
13282 &commit.client_commit_id,
13283 CommitOutcomeStatus::Rejected,
13284 &results,
13285 Some(&commit.ops),
13286 )?;
13287 self.delete_outbox_persisted(&commit.client_commit_id)?;
13288 for operation in &commit.ops {
13291 if self.schema.table(&operation.table).is_some() {
13292 batch.table(&operation.table);
13293 }
13294 }
13295 }
13296 self.prune_commit_outcomes()?;
13297 let incompatible_ids = incompatible
13298 .iter()
13299 .map(|commit| commit.client_commit_id.as_str())
13300 .collect::<BTreeSet<_>>();
13301 self.outbox
13302 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
13303 self.rejections.extend(rejections);
13304 self.overlay_dirty.set(true);
13307 self.rebuild_overlay()?;
13308 batch.status = true;
13309 batch.rejections = true;
13310 batch.outcomes = true;
13311 Ok(())
13312 })();
13313 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
13314 if let Err(error) = result {
13315 if owns_transaction {
13319 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
13320 let mut failure = self.storage_failure.borrow_mut();
13321 let failure =
13322 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
13323 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
13324 }
13325 } else {
13326 self.rollback_observation("syncular_outbox_drop");
13327 }
13328 self.outbox = prior_outbox;
13329 self.rejections = prior_rejections;
13330 self.overlay_dirty.restore(prior_overlay_dirty);
13331 return Err(error);
13332 }
13333 Ok(true)
13334 }
13335
13336 fn create_synced_tables(&self) -> Result<(), String> {
13339 for table in &self.schema.tables {
13340 for (full, index_prefix) in [
13344 (base_table(&table.name), "_syncular_base_"),
13345 (visible_table(&table.name), ""),
13346 ] {
13347 let mut cols: Vec<String> =
13348 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
13349 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
13350 let sql = format!(
13351 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
13352 cols.join(", "),
13353 quote_ident(&table.primary_key)
13354 );
13355 self.conn
13356 .execute(&sql, [])
13357 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13358 for index in &table.indexes {
13362 let unique = if index.unique { "UNIQUE " } else { "" };
13363 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
13364 let cols_sql = index
13365 .columns
13366 .iter()
13367 .map(|c| quote_ident(c))
13368 .collect::<Vec<_>>()
13369 .join(", ");
13370 let index_sql = format!(
13371 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
13372 );
13373 self.conn
13374 .execute(&index_sql, [])
13375 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13376 }
13377 }
13378 }
13379 for table in &self.schema.tables {
13382 for index in &table.fts_indexes {
13383 self.create_fts_projection(table, index)?;
13384 }
13385 }
13386 Ok(())
13387 }
13388
13389 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
13390 let count: i64 = self
13391 .conn
13392 .query_row(
13393 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
13394 rusqlite::params![index.name],
13395 |row| row.get(0),
13396 )
13397 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13398 Ok(count > 0)
13399 }
13400
13401 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
13402 for suffix in ["ai", "ad", "au"] {
13403 self.conn
13404 .execute(
13405 &format!(
13406 "DROP TRIGGER IF EXISTS {}",
13407 quote_ident(&format!("{}_{suffix}", index.name))
13408 ),
13409 [],
13410 )
13411 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13412 }
13413 Ok(())
13414 }
13415
13416 fn create_fts_triggers(
13417 &self,
13418 table: &TableSchema,
13419 index: &FtsIndexSchema,
13420 ) -> Result<(), String> {
13421 let fts = quote_ident(&index.name);
13422 let source = visible_table(&table.name);
13423 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
13424 let pk = quote_ident(&table.primary_key);
13425 let projection_columns = std::iter::once(source_id.clone())
13426 .chain(index.columns.iter().map(|column| quote_ident(column)))
13427 .collect::<Vec<_>>()
13428 .join(", ");
13429 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
13430 .chain(
13431 index
13432 .columns
13433 .iter()
13434 .map(|column| format!("new.{}", quote_ident(column))),
13435 )
13436 .collect::<Vec<_>>()
13437 .join(", ");
13438 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13439 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
13440 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
13441 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
13442 let bi = quote_ident(&format!("{}_bi", index.name));
13443 let ai = quote_ident(&format!("{}_ai", index.name));
13444 let ad = quote_ident(&format!("{}_ad", index.name));
13445 let au = quote_ident(&format!("{}_au", index.name));
13446 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
13447 let sql = format!(
13448 "DROP TRIGGER IF EXISTS {bi};
13449 DROP TRIGGER IF EXISTS {ai};
13450 DROP TRIGGER IF EXISTS {ad};
13451 DROP TRIGGER IF EXISTS {au};
13452 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
13453 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
13454 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
13455 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
13456 );
13457 self.conn
13458 .execute_batch(&sql)
13459 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13460 }
13461
13462 fn rebuild_fts_projection(
13463 &self,
13464 table: &TableSchema,
13465 index: &FtsIndexSchema,
13466 ) -> Result<(), String> {
13467 let fts = quote_ident(&index.name);
13468 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
13469 let pk = quote_ident(&table.primary_key);
13470 let indexed_columns = index
13471 .columns
13472 .iter()
13473 .map(|column| quote_ident(column))
13474 .collect::<Vec<_>>();
13475 let projection_columns = std::iter::once(source_id.clone())
13476 .chain(indexed_columns.iter().cloned())
13477 .collect::<Vec<_>>()
13478 .join(", ");
13479 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13480 let sql = format!(
13481 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
13482 columns = indexed_columns.join(", "),
13483 source = visible_table(&table.name),
13484 );
13485 self.conn
13486 .execute_batch(&sql)
13487 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13488 }
13489
13490 fn create_fts_projection(
13491 &self,
13492 table: &TableSchema,
13493 index: &FtsIndexSchema,
13494 ) -> Result<(), String> {
13495 self.begin_observation("syncular_fts_schema")?;
13496 let result = (|| {
13497 let existed = self.fts_projection_exists(index)?;
13498 let tokenizer = index.tokenize.replace('\'', "''");
13499 let columns = index
13500 .columns
13501 .iter()
13502 .map(|column| quote_ident(column))
13503 .collect::<Vec<_>>()
13504 .join(", ");
13505 let sql = format!(
13506 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
13507 fts = quote_ident(&index.name),
13508 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
13509 );
13510 self.conn.execute(&sql, []).map_err(|error| {
13511 format!(
13512 "cannot create local FTS5 projection {:?}: {error}",
13513 index.name
13514 )
13515 })?;
13516 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13517 let mapped: bool = self
13518 .conn
13519 .query_row(
13520 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
13521 [format!("_syncular_fts_{}", index.name)],
13522 |row| row.get(0),
13523 )
13524 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13525 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13526 if !mapped && existed {
13527 self.conn
13528 .execute_batch(&format!(
13529 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
13530 quote_ident(FTS_SOURCE_ID_COLUMN),
13531 quote_ident(&index.name)
13532 ))
13533 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13534 }
13535 self.create_fts_triggers(table, index)?;
13536 if !existed {
13537 self.rebuild_fts_projection(table, index)?;
13538 }
13539 Ok(())
13540 })();
13541 if let Err(error) = result {
13542 self.rollback_observation("syncular_fts_schema");
13543 return Err(error);
13544 }
13545 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
13546 self.rollback_observation("syncular_fts_schema");
13547 return Err(error.to_string());
13548 }
13549 Ok(())
13550 }
13551
13552 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
13555 let state = serde_json::json!({
13556 "requested": scope_map_to_json(&sub.requested),
13557 "params": sub.params,
13558 "cursor": sub.cursor,
13559 "bootstrapState": sub.bootstrap_state,
13560 "status": sub.state.name(),
13561 "reasonCode": sub.reason_code,
13562 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
13563 "syncedOnce": sub.synced_once,
13564 });
13565 self.conn.execute(
13566 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
13567 rusqlite::params![sub.id, sub.table, state.to_string()],
13568 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13569 }
13570
13571 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
13572 let ops: Vec<Value> = commit
13573 .ops
13574 .iter()
13575 .map(|op| {
13576 serde_json::json!({
13577 "op": if op.upsert { "upsert" } else { "delete" },
13578 "table": op.table,
13579 "rowId": op.row_id,
13580 "baseVersion": op.base_version,
13581 "values": op.values.clone().map(Value::Object),
13582 })
13583 })
13584 .collect();
13585 self.conn
13586 .execute(
13587 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
13588 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
13589 )
13590 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13591 for operation in &commit.ops {
13592 let Some(values) = &operation.values else {
13593 continue;
13594 };
13595 let Some(table) = self.schema.table(&operation.table) else {
13596 continue;
13597 };
13598 for column in table
13599 .columns
13600 .iter()
13601 .filter(|column| column.ty == ColumnType::BlobRef)
13602 {
13603 let Some(Value::String(raw)) = values.get(&column.name) else {
13604 continue;
13605 };
13606 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
13607 continue;
13608 };
13609 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
13610 continue;
13611 };
13612 self.conn
13613 .execute(
13614 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
13615 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
13616 rusqlite::params![commit.client_commit_id, blob_id],
13617 )
13618 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13619 }
13620 }
13621 Ok(())
13622 }
13623
13624 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
13625 if let Some(commit) = self
13626 .outbox
13627 .iter()
13628 .find(|commit| commit.client_commit_id == client_commit_id)
13629 {
13630 for operation in &commit.ops {
13631 self.overlay_dirty.table(&operation.table);
13632 }
13633 }
13634
13635 self.conn
13636 .execute(
13637 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
13638 rusqlite::params![client_commit_id],
13639 )
13640 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13641 self.conn
13642 .execute(
13643 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
13644 rusqlite::params![client_commit_id],
13645 )
13646 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13647 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
13648 .map(|_| ())
13649 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13650 }
13651
13652 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
13653 match status {
13654 CommitOutcomeStatus::Applied => "applied",
13655 CommitOutcomeStatus::Cached => "cached",
13656 CommitOutcomeStatus::Conflict => "conflict",
13657 CommitOutcomeStatus::Rejected => "rejected",
13658 }
13659 }
13660
13661 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
13662 match resolution {
13663 CommitOutcomeResolution::Active => "active",
13664 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
13665 CommitOutcomeResolution::Superseded => "superseded",
13666 CommitOutcomeResolution::Dismissed => "dismissed",
13667 }
13668 }
13669
13670 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
13671 match value {
13672 "applied" => Ok(CommitOutcomeStatus::Applied),
13673 "cached" => Ok(CommitOutcomeStatus::Cached),
13674 "conflict" => Ok(CommitOutcomeStatus::Conflict),
13675 "rejected" => Ok(CommitOutcomeStatus::Rejected),
13676 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
13677 }
13678 }
13679
13680 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
13681 match value {
13682 "active" => Ok(CommitOutcomeResolution::Active),
13683 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
13684 "superseded" => Ok(CommitOutcomeResolution::Superseded),
13685 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
13686 _ => Err(format!(
13687 "invalid persisted commit outcome resolution {value:?}"
13688 )),
13689 }
13690 }
13691
13692 fn persist_commit_outcome(
13693 &self,
13694 client_commit_id: &str,
13695 status: CommitOutcomeStatus,
13696 results: &[CommitOperationOutcome],
13697 operations: Option<&[OutboxOp]>,
13698 ) -> Result<(), String> {
13699 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
13700 let operations_json = operations
13701 .map(|items| {
13702 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
13703 })
13704 .transpose()
13705 .map_err(|error| error.to_string())?;
13706 self.conn
13707 .execute(
13708 "INSERT INTO _syncular_commit_outcomes (
13709 client_commit_id, status, recorded_at_ms, results_json,
13710 operations_json, resolution
13711 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
13712 rusqlite::params![
13713 client_commit_id,
13714 Self::outcome_status_name(status),
13715 self.clock_now_ms(),
13716 results_json,
13717 operations_json
13718 ],
13719 )
13720 .map(|_| ())
13721 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13722 }
13723
13724 fn stored_cell_is_representable(cell: &Value) -> bool {
13727 cell.is_null()
13728 || cell.is_string()
13729 || cell.is_boolean()
13730 || cell.is_number()
13731 || is_canonical_bytes_envelope(cell)
13732 }
13733
13734 fn stored_row_is_representable(value: &Value) -> bool {
13735 value
13736 .as_object()
13737 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
13738 }
13739
13740 fn stored_operation_is_representable(value: &Value) -> bool {
13741 value.get("table").is_some_and(Value::is_string)
13742 && value.get("rowId").is_some_and(Value::is_string)
13743 && value.get("baseVersion").is_none_or(|v| {
13744 v.as_i64()
13745 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
13746 })
13747 && match value.get("op").and_then(Value::as_str) {
13748 Some("upsert") => value
13749 .get("values")
13750 .is_some_and(Self::stored_row_is_representable),
13751 Some("delete") => value.get("values").is_none(),
13752 _ => false,
13753 }
13754 }
13755
13756 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
13760 serde_json::to_value(CommitOperation::from(operation))
13761 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
13762 }
13763
13764 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
13765 let StoredCommitOutcomeRow {
13766 sequence,
13767 client_commit_id,
13768 status,
13769 recorded_at_ms,
13770 results_json,
13771 operations_json,
13772 resolution,
13773 resolved_at_ms,
13774 replacement_client_commit_id,
13775 } = row;
13776 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
13777 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
13778 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
13779 {
13780 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
13781 }
13782 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
13783 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
13784 for result in &stored {
13785 let nested = match result.get("status").and_then(Value::as_str) {
13786 Some("conflict") => result.get("conflict"),
13787 Some("error") => result.get("rejection"),
13788 _ => None,
13789 };
13790 if nested
13791 .and_then(|record| record.get("operation"))
13792 .is_some_and(|op| !Self::stored_operation_is_representable(op))
13793 || (result.get("status").and_then(Value::as_str) == Some("conflict")
13794 && nested
13795 .and_then(|record| record.get("serverRow"))
13796 .is_some_and(|row| !Self::stored_row_is_representable(row)))
13797 {
13798 return Err(invalid());
13799 }
13800 }
13801 let results: Vec<CommitOperationOutcome> =
13802 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
13803 let has_conflict = results
13804 .iter()
13805 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
13806 let has_error = results
13807 .iter()
13808 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
13809 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
13810 let redacted = results.is_empty() && operations_json.is_none();
13811 if match status {
13812 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
13813 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
13814 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
13815 } {
13816 return Err(invalid());
13817 }
13818 let operations: Option<Vec<CommitOperation>> = operations_json
13819 .map(|raw| {
13820 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
13821 if !stored.iter().all(Self::stored_operation_is_representable) {
13822 return Err(invalid());
13823 }
13824 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
13825 })
13826 .transpose()?;
13827 for result in &results {
13828 match result {
13829 CommitOperationOutcome::Conflict { conflict }
13830 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
13831 {
13832 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
13833 }
13834 CommitOperationOutcome::Error { rejection } => {
13835 if let Some(details) = &rejection.details {
13836 let encoded = serde_json::to_string(details).map_err(|_| {
13837 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13838 })?;
13839 RejectionDetails::parse(&encoded).map_err(|_| {
13840 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13841 })?;
13842 }
13843 }
13844 _ => {}
13845 }
13846 }
13847 Ok(CommitOutcome {
13848 sequence,
13849 client_commit_id,
13850 status,
13851 recorded_at_ms,
13852 results,
13853 operations,
13854 retained_rows: None,
13855 resolution: Self::parse_outcome_resolution(&resolution)?,
13856 resolved_at_ms,
13857 replacement_client_commit_id,
13858 })
13859 }
13860
13861 fn unique_conflicts(
13863 &self,
13864 table: &TableSchema,
13865 values: &Map<String, Value>,
13866 base: bool,
13867 ) -> Result<Vec<RetainedUniqueConflict>, String> {
13868 let source = if base {
13869 base_table(&table.name)
13870 } else {
13871 visible_table(&table.name)
13872 };
13873 let mut conflicts = Vec::new();
13874 for index in table.indexes.iter().filter(|index| index.unique) {
13875 let mut params = vec![json_param_to_sql(
13876 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
13877 )?];
13878 for name in &index.columns {
13879 let column = table
13880 .columns
13881 .iter()
13882 .find(|column| column.name == *name)
13883 .ok_or("sync.local_corrupt")?;
13884 let cell = json_to_column_value(column, values.get(name))?;
13885 params.push(owned_sql_value(RowParam::Cell(&cell))?);
13886 }
13887 let sql = format!(
13888 "SELECT * FROM {source} WHERE {} != ? AND {}",
13889 quote_ident(&table.primary_key),
13890 index
13891 .columns
13892 .iter()
13893 .map(|column| format!("{} = ?", quote_ident(column)))
13894 .collect::<Vec<_>>()
13895 .join(" AND ")
13896 );
13897 let mut statement = self
13898 .conn
13899 .prepare_cached(&sql)
13900 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13901 let records = statement
13902 .query_map(rusqlite::params_from_iter(params), |record| {
13903 let values = table
13904 .columns
13905 .iter()
13906 .enumerate()
13907 .map(|(index, column)| {
13908 let value = sql_ref_to_json(column, record.get_ref(index)?);
13909 json_to_column_value(column, Some(&value)).map_err(|message| {
13910 rusqlite::Error::ToSqlConversionFailure(message.into())
13911 })
13912 })
13913 .collect::<Result<Row, _>>()?;
13914 Ok((values, record.get::<_, i64>(table.columns.len())?))
13915 })
13916 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13917 for record in records {
13918 let (row, version) =
13919 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13920 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
13921 conflicts.push(RetainedUniqueConflict {
13922 index: index.name.clone(),
13923 columns: index.columns.clone(),
13924 row_id,
13925 server_row: table
13926 .columns
13927 .iter()
13928 .zip(&row)
13929 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
13930 .collect(),
13931 server_version: version,
13932 });
13933 }
13934 }
13935 Ok(conflicts)
13936 }
13937
13938 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
13939 if let Some((commit, initial)) = self
13940 .failed_commits
13941 .iter()
13942 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
13943 {
13944 let mut rows = Vec::new();
13945 for operation in &commit.ops {
13946 let table = self
13947 .schema
13948 .table(&operation.table)
13949 .ok_or_else(|| "sync.unknown_table".to_owned())?;
13950 let key =
13951 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
13952 .map_err(|_| "sync.local_corrupt".to_owned())?;
13953 let base = self.stored_row(table, &operation.row_id, true)?;
13954 rows.push(RetainedCommitRow {
13955 table: operation.table.clone(),
13956 row_id: operation.row_id.clone(),
13957 unique_conflicts: if operation.upsert {
13958 self.unique_conflicts(
13959 table,
13960 initial
13961 .get(&key)
13962 .and_then(Value::as_object)
13963 .or(operation.values.as_ref())
13964 .ok_or("sync.local_corrupt")?,
13965 true,
13966 )?
13967 } else {
13968 Vec::new()
13969 },
13970 local_row: if operation.upsert {
13971 initial
13972 .get(&key)
13973 .and_then(Value::as_object)
13974 .cloned()
13975 .or_else(|| operation.values.clone())
13976 } else {
13977 None
13978 },
13979 server_row: base.as_ref().map(|(values, _)| {
13980 table
13981 .columns
13982 .iter()
13983 .zip(values)
13984 .map(|(column, value)| {
13985 (column.name.clone(), column_value_to_json(value))
13986 })
13987 .collect()
13988 }),
13989 server_version: base.map(|(_, version)| version),
13990 });
13991 }
13992 outcome.retained_rows = Some(rows);
13993 }
13994 Ok(outcome)
13995 }
13996
13997 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
13998 let row = self
13999 .conn
14000 .query_row(
14001 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
14002 resolution, resolved_at_ms, replacement_client_commit_id
14003 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
14004 rusqlite::params![client_commit_id],
14005 |row| {
14006 Ok(StoredCommitOutcomeRow {
14007 sequence: row.get(0)?,
14008 client_commit_id: row.get(1)?,
14009 status: row.get(2)?,
14010 recorded_at_ms: row.get(3)?,
14011 results_json: row.get(4)?,
14012 operations_json: row.get(5)?,
14013 resolution: row.get(6)?,
14014 resolved_at_ms: row.get(7)?,
14015 replacement_client_commit_id: row.get(8)?,
14016 })
14017 },
14018 )
14019 .optional()
14020 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14021 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
14022 .transpose()
14023 }
14024
14025 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
14026 if query.limit == Some(0) {
14027 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
14028 }
14029 let mut sql = String::from(
14030 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
14031 resolution, resolved_at_ms, replacement_client_commit_id
14032 FROM _syncular_commit_outcomes",
14033 );
14034 if query.active_only {
14035 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
14036 }
14037 sql.push_str(" ORDER BY seq DESC");
14038 if let Some(limit) = query.limit {
14039 sql.push_str(&format!(" LIMIT {limit}"));
14040 }
14041 let mut stmt = self
14042 .conn
14043 .prepare(&sql)
14044 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14045 let rows = stmt
14046 .query_map([], |row| {
14047 Ok(StoredCommitOutcomeRow {
14048 sequence: row.get(0)?,
14049 client_commit_id: row.get(1)?,
14050 status: row.get(2)?,
14051 recorded_at_ms: row.get(3)?,
14052 results_json: row.get(4)?,
14053 operations_json: row.get(5)?,
14054 resolution: row.get(6)?,
14055 resolved_at_ms: row.get(7)?,
14056 replacement_client_commit_id: row.get(8)?,
14057 })
14058 })
14059 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14060 let mut outcomes = Vec::new();
14061 for row in rows {
14062 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
14063 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
14064 )?)?);
14065 }
14066 Ok(outcomes)
14067 }
14068
14069 fn prune_commit_outcomes(&self) -> Result<(), String> {
14070 #[cfg(test)]
14071 self.outcome_prune_count
14072 .set(self.outcome_prune_count.get() + 1);
14073 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
14074 let count = self
14075 .conn
14076 .query_row(
14077 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
14078 [],
14079 |row| row.get::<_, i64>(0),
14080 )
14081 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
14082 as usize;
14083 let excess = count.saturating_sub(max_entries);
14084 if excess == 0 {
14085 return Ok(());
14086 }
14087 let mut stmt = self
14088 .conn
14089 .prepare(
14090 "SELECT seq FROM _syncular_commit_outcomes
14091 WHERE status IN ('applied', 'cached') OR resolution != 'active'
14092 ORDER BY seq ASC LIMIT ?1",
14093 )
14094 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14095 let rows = stmt
14096 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
14097 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14098 let sequences = rows
14099 .collect::<Result<Vec<_>, _>>()
14100 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14101 drop(stmt);
14102 for sequence in sequences {
14103 self.conn
14104 .execute(
14105 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
14106 rusqlite::params![sequence],
14107 )
14108 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14109 }
14110 Ok(())
14111 }
14112
14113 pub fn subscribe(
14116 &mut self,
14117 id: String,
14118 table: String,
14119 scopes: Vec<(String, Vec<String>)>,
14120 params: Option<String>,
14121 ) -> Result<(), String> {
14122 if self.schema.table(&table).is_none() {
14123 return Err(format!("unknown table {table:?}"));
14124 }
14125 let registered_at = self
14126 .subs
14127 .iter()
14128 .position(|subscription| subscription.id == id);
14129 if let Some(index) = registered_at {
14130 let existing = &self.subs[index];
14131 let same_intent = existing.table == table
14132 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
14133 && existing.params == params;
14134 if !same_intent {
14135 return Err(
14136 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
14137 .to_owned(),
14138 );
14139 }
14140 if existing.state != SubState::Revoked {
14145 return Ok(());
14146 }
14147 }
14148 let sub = Subscription {
14149 id: id.clone(),
14150 table,
14151 requested: scopes,
14152 params,
14153 cursor: -1,
14154 bootstrap_state: None,
14155 state: SubState::Active,
14156 reason_code: None,
14157 effective: None,
14158 synced_once: false,
14159 };
14160 self.persist_sub(&sub)?;
14161 match registered_at {
14162 Some(index) => self.subs[index] = sub,
14163 None => self.subs.push(sub),
14164 }
14165 if registered_at.is_some() {
14169 self.set_sync_needed(true, true);
14170 }
14171 Ok(())
14172 }
14173
14174 pub fn subscribe_many(&mut self, requests: Vec<SubscriptionRequest>) -> Result<(), String> {
14180 let prior_subs = self.subs.clone();
14181 self.begin_observation("syncular_subscribe")?;
14182 let registered = (|| {
14183 for request in requests {
14184 self.subscribe(request.id, request.table, request.scopes, request.params)?;
14185 }
14186 self.conn
14187 .execute_batch("RELEASE syncular_subscribe")
14188 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14189 })();
14190 if let Err(error) = registered {
14191 self.rollback_observation("syncular_subscribe");
14192 self.subs = prior_subs;
14193 return Err(error);
14194 }
14195 Ok(())
14196 }
14197
14198 pub fn unsubscribe(&mut self, id: &str) {
14199 if self.subs.iter().any(|s| s.id == id) {
14200 self.cancel_sync_round();
14201 }
14202 self.subs.retain(|s| s.id != id);
14203 let _ = self.conn.execute(
14204 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
14205 rusqlite::params![id],
14206 );
14207 }
14208
14209 pub fn set_window(
14217 &mut self,
14218 base: &WindowBase,
14219 units: &[String],
14220 ) -> Result<CommandEffects, String> {
14221 let table = self
14222 .schema
14223 .table(&base.table)
14224 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
14225 if table.scope_column(&base.variable).is_none() {
14226 return Err(format!(
14227 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
14228 base.table, base.variable
14229 ));
14230 }
14231 let base_key = window_base_key(base);
14232 let wanted: std::collections::HashSet<&String> = units.iter().collect();
14233 let live = self.load_window_units(&base_key);
14234 self.begin_observation("syncular_window")?;
14235 let mut batch = ChangeAccumulator::default();
14236 let mut changed = false;
14237
14238 let prior_subs = self.subs.clone();
14239 let widened = (|| {
14240 for unit in units {
14242 if live.iter().any(|(u, _)| u == unit) {
14243 continue;
14244 }
14245 let sub_id = derive_sub_id(base, unit);
14246 self.delete_pending_evict(&sub_id)?;
14247 self.insert_window_unit(&base_key, unit, &sub_id)?;
14248 self.subscribe(
14249 sub_id,
14250 base.table.clone(),
14251 unit_scopes(base, unit),
14252 base.params.clone(),
14253 )?;
14254 batch.window(&base_key, &base.table, unit);
14255 changed = true;
14256 }
14257
14258 self.finish_observation("syncular_window", batch)
14259 })();
14260 if let Err(error) = widened {
14261 self.rollback_observation("syncular_window");
14262 self.subs = prior_subs;
14263 return Err(error);
14264 }
14265 for (unit, sub_id) in live {
14266 if wanted.contains(&unit) {
14267 continue;
14268 }
14269 self.rebuild_overlay_if_dirty()?;
14270 let prior_subs = self.subs.clone();
14271 self.begin_observation("syncular_window")?;
14272 let effective = self
14273 .subs
14274 .iter()
14275 .find(|sub| sub.id == sub_id)
14276 .and_then(|sub| sub.effective.clone())
14277 .unwrap_or_else(|| unit_scopes(base, &unit));
14278 let mut batch = ChangeAccumulator::default();
14279 self.record_scope_map(&mut batch, &base.table, &effective);
14280 batch.window(&base_key, &base.table, &unit);
14281 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
14282 let remaining = match result {
14283 Ok(remaining) => remaining,
14284 Err(error) => {
14285 self.rollback_observation("syncular_window");
14286 self.subs = prior_subs;
14287 return Err(error);
14288 }
14289 };
14290 if let Err(error) = self.finish_observation("syncular_window", batch) {
14291 self.rollback_observation("syncular_window");
14292 self.subs = prior_subs;
14293 return Err(error);
14294 }
14295 if remaining {
14296 self.drain_pending_evictions()?;
14297 }
14298 changed = true;
14299 }
14300 Ok(if changed {
14301 CommandEffects::interactive()
14302 } else {
14303 CommandEffects::none()
14304 })
14305 }
14306
14307 pub fn window_state(&self, base: &WindowBase) -> WindowState {
14312 let mut units = Vec::new();
14313 let mut pending = Vec::new();
14314 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
14315 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
14316 Some(sub) => {
14317 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
14318 }
14319 None => true,
14320 };
14321 if is_pending {
14322 pending.push(unit.clone());
14323 }
14324 units.push(unit);
14325 }
14326 WindowState { units, pending }
14327 }
14328
14329 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
14330 let mut stmt = match self
14331 .conn
14332 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
14333 {
14334 Ok(stmt) => stmt,
14335 Err(_) => return Vec::new(),
14336 };
14337 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
14338 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
14339 });
14340 match rows {
14341 Ok(rows) => rows.filter_map(Result::ok).collect(),
14342 Err(_) => Vec::new(),
14343 }
14344 }
14345
14346 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
14347 let mut stmt = match self.conn.prepare(
14348 "SELECT windows.base, windows.unit, subscriptions.tbl
14349 FROM _syncular_windows AS windows
14350 JOIN _syncular_subscriptions AS subscriptions
14351 ON subscriptions.id = windows.sub_id
14352 ORDER BY windows.base, windows.unit",
14353 ) {
14354 Ok(stmt) => stmt,
14355 Err(_) => return Vec::new(),
14356 };
14357 let rows = stmt.query_map([], |row| {
14358 Ok((
14359 row.get::<_, String>(0)?,
14360 row.get::<_, String>(1)?,
14361 row.get::<_, String>(2)?,
14362 ))
14363 });
14364 match rows {
14365 Ok(rows) => rows.filter_map(Result::ok).collect(),
14366 Err(_) => Vec::new(),
14367 }
14368 }
14369
14370 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
14371 self.conn
14372 .query_row(
14373 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
14374 rusqlite::params![sub_id],
14375 |row| Ok((row.get(0)?, row.get(1)?)),
14376 )
14377 .ok()
14378 }
14379
14380 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
14381 self.conn
14382 .execute(
14383 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
14384 rusqlite::params![base_key, unit, sub_id],
14385 )
14386 .map(|_| ())
14387 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14388 }
14389
14390 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
14391 self.conn
14392 .execute(
14393 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
14394 rusqlite::params![base_key, unit],
14395 )
14396 .map(|_| ())
14397 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14398 }
14399
14400 fn evict_unit(
14407 &mut self,
14408 base_key: &str,
14409 base: &WindowBase,
14410 unit: &str,
14411 sub_id: &str,
14412 ) -> Result<bool, String> {
14413 let effective = self
14414 .subs
14415 .iter()
14416 .find(|s| s.id == sub_id)
14417 .and_then(|s| s.effective.clone())
14418 .unwrap_or_else(|| unit_scopes(base, unit));
14419 let pinned = self.pinned_row_ids(&base.table);
14420 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
14421 self.delete_window_unit(base_key, unit)?;
14422 self.conn
14423 .execute(
14424 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
14425 [sub_id],
14426 )
14427 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14428 self.subs.retain(|sub| sub.id != sub_id);
14429 if deferred {
14430 self.save_pending_evict(sub_id, &base.table, &effective)?;
14431 } else {
14432 self.delete_pending_evict(sub_id)?;
14433 }
14434 Ok(remaining)
14435 }
14436
14437 fn evict_scope_rows(
14441 &self,
14442 table_name: &str,
14443 effective: &[(String, Vec<String>)],
14444 pinned: &std::collections::HashSet<String>,
14445 ) -> Result<(bool, bool), String> {
14446 if effective.is_empty() {
14447 return Ok((false, false));
14448 }
14449 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
14450 let mut clauses = Vec::new();
14451 let mut params: Vec<SqlValue> = Vec::new();
14452 for (variable, values) in effective {
14453 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
14454 if values.is_empty() {
14455 return Ok((false, false));
14456 }
14457 let holes = values
14458 .iter()
14459 .map(|v| {
14460 params.push(SqlValue::Text(v.clone()));
14461 "?"
14462 })
14463 .collect::<Vec<_>>()
14464 .join(", ");
14465 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
14466 }
14467 let scope = clauses.join(" AND ");
14468 let scope_params = params.len();
14469 let pk = quote_ident(&table.primary_key);
14470 if !pinned.is_empty() {
14471 let holes = pinned
14472 .iter()
14473 .map(|id| {
14474 params.push(SqlValue::Text(id.clone()));
14475 "?"
14476 })
14477 .collect::<Vec<_>>()
14478 .join(", ");
14479 clauses.push(format!("{pk} NOT IN ({holes})"));
14480 }
14481 let predicate = clauses.join(" AND ");
14482 let base = base_table(table_name);
14483 let visible = visible_table(table_name);
14484 let mut ids: Vec<SqlValue> = Vec::new();
14485 for source in [&base, &visible] {
14486 let mut stmt = self
14487 .conn
14488 .prepare_cached(&format!(
14489 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
14490 ))
14491 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14492 ids = stmt
14493 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
14494 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
14495 .collect::<Result<_, _>>()
14496 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14497 if !ids.is_empty() {
14498 break;
14499 }
14500 }
14501 if !ids.is_empty() {
14502 for id in &ids {
14503 self.conn
14504 .execute(
14505 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
14506 rusqlite::params![table_name, id],
14507 )
14508 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14509 }
14510 let holes = vec!["?"; ids.len()].join(", ");
14511 for source in [&base, &visible] {
14512 self.conn
14513 .execute(
14514 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
14515 rusqlite::params_from_iter(&ids),
14516 )
14517 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14518 }
14519 }
14520 let mut remaining = false;
14521 let mut deferred = false;
14522 for source in [&base, &visible] {
14523 remaining |= self
14524 .conn
14525 .query_row(
14526 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
14527 rusqlite::params_from_iter(¶ms),
14528 |row| row.get::<_, bool>(0),
14529 )
14530 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14531 deferred |= self
14532 .conn
14533 .query_row(
14534 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
14535 rusqlite::params_from_iter(¶ms[..scope_params]),
14536 |row| row.get::<_, bool>(0),
14537 )
14538 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14539 }
14540 Ok((remaining, deferred))
14541 }
14542
14543 fn drain_pending_evictions(&mut self) -> Result<(), String> {
14545 let pending = self.load_pending_evictions()?;
14546 if pending.is_empty() {
14547 return Ok(());
14548 }
14549 self.rebuild_overlay_if_dirty()?;
14550 for (sub_id, table_name, effective) in pending {
14551 if self.schema.table(&table_name).is_none() {
14552 self.delete_pending_evict(&sub_id)?;
14553 continue;
14554 }
14555 loop {
14556 std::thread::yield_now();
14557 self.begin_observation("syncular_evict")?;
14558 let pinned = self.pinned_row_ids(&table_name);
14559 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
14560 let (remaining, deferred) = match result {
14561 Ok(result) => result,
14562 Err(error) => {
14563 self.rollback_observation("syncular_evict");
14564 return Err(error);
14565 }
14566 };
14567 if !deferred {
14568 if let Err(error) = self.delete_pending_evict(&sub_id) {
14569 self.rollback_observation("syncular_evict");
14570 return Err(error);
14571 }
14572 }
14573 let mut batch = ChangeAccumulator::default();
14574 self.record_scope_map(&mut batch, &table_name, &effective);
14575 if let Err(error) = self.finish_observation("syncular_evict", batch) {
14576 self.rollback_observation("syncular_evict");
14577 return Err(error);
14578 }
14579 if !remaining {
14580 break;
14581 }
14582 }
14583 }
14584 Ok(())
14585 }
14586
14587 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
14590 let mut pinned = std::collections::HashSet::new();
14591 for commit in self
14592 .outbox
14593 .iter()
14594 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
14595 {
14596 for op in &commit.ops {
14597 if op.table == table {
14598 pinned.insert(op.row_id.clone());
14599 }
14600 }
14601 }
14602 pinned
14603 }
14604
14605 fn save_pending_evict(
14606 &self,
14607 sub_id: &str,
14608 table: &str,
14609 effective: &[(String, Vec<String>)],
14610 ) -> Result<(), String> {
14611 self.conn.execute(
14612 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
14613 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
14614 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14615 }
14616
14617 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
14618 self.conn
14619 .execute(
14620 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
14621 rusqlite::params![sub_id],
14622 )
14623 .map(|_| ())
14624 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14625 }
14626
14627 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
14628 let mut stmt = self
14629 .conn
14630 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
14631 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14632 let rows = stmt
14633 .query_map([], |row| {
14634 Ok((
14635 row.get::<_, String>(0)?,
14636 row.get::<_, String>(1)?,
14637 row.get::<_, String>(2)?,
14638 ))
14639 })
14640 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14641 rows.map(|row| {
14642 let (sub_id, table, json) =
14643 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14644 let value: Value = serde_json::from_str(&json)
14645 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14646 let effective = json_to_scope_map(&value)
14647 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14648 Ok((sub_id, table, effective))
14649 })
14650 .collect()
14651 }
14652
14653 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
14655 self.retain_failed_commits = enabled;
14656 for (commit, _) in &self.failed_commits {
14657 for operation in &commit.ops {
14658 self.overlay_dirty.table(&operation.table);
14659 }
14660 }
14661 self.failed_commits = self.load_failed_commits()?;
14662 for (commit, _) in &self.failed_commits {
14663 for operation in &commit.ops {
14664 self.overlay_dirty.table(&operation.table);
14665 }
14666 }
14667 Ok(())
14668 }
14669
14670 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
14671 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
14672 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14673 let rows = statement
14674 .query_map([], |row| {
14675 Ok((
14676 row.get::<_, String>(0)?,
14677 row.get::<_, String>(1)?,
14678 row.get::<_, String>(2)?,
14679 ))
14680 })
14681 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14682 rows.map(|row| {
14683 let (id, operations, initial) =
14684 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14685 let operations: Vec<CommitOperation> =
14686 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
14687 let initial =
14688 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
14689 Ok((
14690 OutboxCommit {
14691 client_commit_id: id,
14692 ops: operations
14693 .into_iter()
14694 .map(|op| OutboxOp {
14695 upsert: op.op == "upsert",
14696 table: op.table,
14697 row_id: op.row_id,
14698 base_version: op.base_version,
14699 values: op.values,
14700 })
14701 .collect(),
14702 },
14703 initial,
14704 ))
14705 })
14706 .collect()
14707 }
14708
14709 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
14710 let mut initial = Map::new();
14711 for operation in operations {
14712 let table = self
14713 .schema
14714 .table(&operation.table)
14715 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14716 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
14717 Some(row) => Some(row),
14718 None => self.visible_row(table, &operation.row_id)?,
14719 };
14720 if let Some((values, _)) = initial_row {
14721 let mut intended: Map<String, Value> = table
14722 .columns
14723 .iter()
14724 .zip(values.iter())
14725 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
14726 .collect();
14727 if let Some(patch) = &operation.values {
14728 intended.extend(patch.clone());
14729 }
14730 initial.insert(
14731 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14732 .map_err(|_| "sync.local_corrupt".to_owned())?,
14733 Value::Object(intended),
14734 );
14735 }
14736 }
14737 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
14738 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
14739 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
14740 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14741 self.failed_commits.push((
14742 OutboxCommit {
14743 client_commit_id: id.to_owned(),
14744 ops: operations.to_vec(),
14745 },
14746 initial,
14747 ));
14748 Ok(())
14749 }
14750
14751 fn drop_failed_commits(
14752 &mut self,
14753 matches: impl Fn(&str, &Map<String, Value>) -> bool,
14754 ) -> Result<Vec<OutboxCommit>, String> {
14755 let mut doomed = self.drop_acknowledged_rows(&matches)?;
14756 for (commit, initial) in &self.failed_commits {
14757 for operation in &commit.ops {
14758 let key =
14759 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14760 .map_err(|_| "sync.local_corrupt".to_owned())?;
14761 if initial
14762 .get(&key)
14763 .and_then(Value::as_object)
14764 .is_some_and(|values| matches(&operation.table, values))
14765 {
14766 doomed.push(commit.clone());
14767 break;
14768 }
14769 }
14770 }
14771 for commit in &doomed {
14772 self.conn
14773 .execute(
14774 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
14775 rusqlite::params![commit.client_commit_id],
14776 )
14777 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14778 }
14779 for commit in &doomed {
14780 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14781 }
14782 self.conflicts.retain(|record| {
14783 !doomed
14784 .iter()
14785 .any(|commit| commit.client_commit_id == record.client_commit_id)
14786 });
14787 self.rejections.retain(|record| {
14788 !doomed
14789 .iter()
14790 .any(|commit| commit.client_commit_id == record.client_commit_id)
14791 });
14792 self.failed_commits.retain(|(commit, _)| {
14793 !doomed
14794 .iter()
14795 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
14796 });
14797 for commit in &doomed {
14798 for operation in &commit.ops {
14799 self.overlay_dirty.table(&operation.table);
14800 }
14801 }
14802 Ok(doomed)
14803 }
14804
14805 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
14809 let prior = self.storage_failure.borrow_mut().take();
14810 let outcome = (|| -> Result<String, String> {
14811 if mutations.is_empty() {
14812 return Err(
14813 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
14814 .to_owned(),
14815 );
14816 }
14817 let mut ops = Vec::with_capacity(mutations.len());
14818 for mutation in mutations {
14819 match mutation {
14820 Mutation::Patch {
14821 table,
14822 values,
14823 base_version,
14824 } => {
14825 let schema_table = self
14826 .schema
14827 .table(&table)
14828 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14829 let values = normalize_values_casing(schema_table, values)?;
14830 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14831 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
14832 }
14833 Mutation::Upsert {
14834 table,
14835 values,
14836 base_version,
14837 } => {
14838 let schema_table = self.schema.table(&table).ok_or_else(|| {
14839 format!("sync.unknown_table: unknown table {table:?}")
14840 })?;
14841 let values = normalize_values_casing(schema_table, values)?;
14845 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14846 let values = full_row_values(schema_table, values)?;
14849 self.validate_author_encode(schema_table, &row_id, &values)?;
14855 ops.push(OutboxOp {
14856 upsert: true,
14857 table,
14858 row_id,
14859 base_version,
14860 values: Some(values),
14861 });
14862 }
14863 Mutation::Delete {
14864 table,
14865 row_id,
14866 base_version,
14867 } => {
14868 if self.schema.table(&table).is_none() {
14869 return Err(format!("sync.unknown_table: unknown table {table:?}"));
14870 }
14871 ops.push(OutboxOp {
14872 upsert: false,
14873 table,
14874 row_id,
14875 base_version,
14876 values: None,
14877 });
14878 }
14879 }
14880 }
14881 self.record_outbox_commit(ops)
14882 })();
14883 self.finish_authoring(prior, outcome)
14884 }
14885
14886 fn finish_authoring(
14892 &self,
14893 prior: Option<QueryReadFailure>,
14894 outcome: Result<String, String>,
14895 ) -> Result<String, ClientError> {
14896 let fresh = self.storage_failure.borrow_mut().take();
14897 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
14898 match outcome {
14899 Ok(id) => Ok(id),
14900 Err(message) => Err(fresh
14901 .map(ClientError::from)
14902 .unwrap_or_else(|| ClientError::from(message))),
14903 }
14904 }
14905
14906 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
14907 let commit = OutboxCommit {
14908 client_commit_id: uuid::Uuid::new_v4().to_string(),
14909 ops,
14910 };
14911 self.begin_observation("syncular_mutation")?;
14912 let mut batch = ChangeAccumulator::default();
14913 for op in &commit.ops {
14914 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
14915 if let Some(values) = &op.values {
14916 if let Some(table) = self.schema.table(&op.table) {
14917 for scope in &table.scope_variables {
14918 if let Some(Value::String(value)) = values.get(&scope.column) {
14919 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
14920 precise = true;
14921 }
14922 }
14923 }
14924 }
14925 if !precise {
14926 batch.table(&op.table);
14927 }
14928 }
14929 if let Err(error) = self.persist_outbox_insert(&commit) {
14930 self.rollback_observation("syncular_mutation");
14931 return Err(error);
14932 }
14933 let was_dirty = self.overlay_dirty.snapshot();
14934 if let Err(error) = self.rebuild_overlay_if_dirty() {
14939 self.rollback_observation("syncular_mutation");
14940 self.overlay_dirty.restore(was_dirty.clone());
14941 return Err(error);
14942 }
14943 if let Some(error) = commit
14944 .ops
14945 .iter()
14946 .find_map(|op| self.apply_outbox_op(op).err())
14947 {
14948 self.rollback_observation("syncular_mutation");
14949 self.overlay_dirty.restore(was_dirty.clone());
14950 return Err(error);
14951 }
14952 let id = commit.client_commit_id.clone();
14953 self.outbox.push(commit);
14954 batch.status = true;
14955 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
14956 self.rollback_observation("syncular_mutation");
14957 self.outbox.pop();
14958 self.overlay_dirty.restore(was_dirty.clone());
14959 return Err(error);
14960 }
14961 Ok(id)
14962 }
14963
14964 pub fn patch(
14970 &mut self,
14971 table: &str,
14972 row_id: &str,
14973 partial: Map<String, Value>,
14974 base_version: Option<i64>,
14975 ) -> Result<String, ClientError> {
14976 let prior = self.storage_failure.borrow_mut().take();
14977 let outcome = self.perform_patch(table, row_id, partial, base_version);
14978 self.finish_authoring(prior, outcome)
14979 }
14980
14981 fn perform_patch(
14984 &mut self,
14985 table: &str,
14986 row_id: &str,
14987 partial: Map<String, Value>,
14988 base_version: Option<i64>,
14989 ) -> Result<String, String> {
14990 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
14991 self.record_outbox_commit(vec![operation])
14992 }
14993
14994 fn prepare_patch(
14995 &self,
14996 table: &str,
14997 row_id: &str,
14998 partial: Map<String, Value>,
14999 base_version: Option<i64>,
15000 ) -> Result<OutboxOp, String> {
15001 let schema_table = self
15002 .schema
15003 .table(table)
15004 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
15005 let partial = normalize_values_casing(schema_table, partial)?;
15006 if let Some(pk) = partial.get(&schema_table.primary_key) {
15007 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
15008 return Err(format!(
15009 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
15010 ));
15011 }
15012 }
15013 let exists: bool = self
15016 .conn
15017 .query_row(
15018 &format!(
15019 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
15020 visible_table(table),
15021 row_id_predicate(schema_table)
15022 ),
15023 rusqlite::params![row_id],
15024 |row| row.get(0),
15025 )
15026 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15027 if !exists {
15028 return Err("sync.row_missing: a sparse patch requires a local row".to_owned());
15029 }
15030 for scope in &schema_table.scope_variables {
15040 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
15041 continue;
15042 }
15043 let Some(column) = schema_table
15044 .columns
15045 .iter()
15046 .find(|column| column.name == scope.column)
15047 else {
15048 continue;
15049 };
15050 let stored = self
15051 .conn
15052 .query_row(
15053 &format!(
15054 "SELECT {} FROM {} WHERE {}",
15055 quote_ident(&scope.column),
15056 visible_table(&schema_table.name),
15057 row_id_predicate(schema_table)
15058 ),
15059 rusqlite::params![row_id],
15060 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
15061 )
15062 .optional()
15063 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15064 let incoming = json_to_column_value(column, partial.get(&scope.column))
15069 .ok()
15070 .map(|value| column_value_to_json(&value));
15071 let equal = stored.as_ref().is_some_and(|stored| {
15075 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
15076 });
15077 if !equal {
15078 return Err(format!(
15079 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
15080 scope.column
15081 ));
15082 }
15083 }
15084 let mut values = partial;
15085 for scope in &schema_table.scope_variables {
15088 values.remove(&scope.column);
15089 }
15090 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
15091 ColumnType::Integer => match row_id.parse::<i64>() {
15093 Ok(v)
15094 if v.to_string() == row_id
15095 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
15096 .contains(&v) =>
15097 {
15098 Value::from(v)
15099 }
15100 _ => {
15101 return Err(format!(
15102 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
15103 ))
15104 }
15105 },
15106 ColumnType::Boolean => match row_id {
15107 "true" => Value::Bool(true),
15108 "false" => Value::Bool(false),
15109 _ => {
15110 return Err(format!(
15111 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
15112 ))
15113 }
15114 },
15115 _ => Value::from(row_id.to_owned()),
15116 };
15117 values.insert(schema_table.primary_key.clone(), pk_value);
15118 self.validate_author_encode(schema_table, row_id, &values)?;
15124 Ok(OutboxOp {
15125 upsert: true,
15126 table: table.to_owned(),
15127 row_id: row_id.to_owned(),
15128 base_version,
15129 values: Some(values),
15130 })
15131 }
15132
15133 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
15138 let selector = self.encryption.key_id_columns.get(&table.name)?;
15139 let index = table
15140 .columns
15141 .iter()
15142 .position(|column| &column.name == selector)?;
15143 let sql = format!(
15144 "SELECT {} FROM {} WHERE {}",
15145 quote_ident(selector),
15146 visible_table(&table.name),
15147 row_id_predicate(table)
15148 );
15149 let key: Option<String> = self
15150 .conn
15151 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
15152 .ok()?;
15153 let key = key.filter(|key| !key.is_empty())?;
15154 let mut fallback: Row = vec![None; table.columns.len()];
15155 fallback[index] = Some(ColumnValue::String(key));
15156 Some(fallback)
15157 }
15158
15159 fn validate_author_encode(
15164 &self,
15165 table: &TableSchema,
15166 row_id: &str,
15167 values: &Map<String, Value>,
15168 ) -> Result<(), String> {
15169 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
15170 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
15171 result => result.map(|_| ()),
15172 }
15173 }
15174
15175 fn drop_unencodable_outbox(
15189 &mut self,
15190 with_encryption: bool,
15191 batch: Option<&mut ChangeAccumulator>,
15192 ) -> Result<bool, String> {
15193 let failures = self
15194 .outbox
15195 .iter()
15196 .filter_map(|commit| {
15197 commit
15198 .ops
15199 .iter()
15200 .find_map(|op| {
15201 let values = op.values.as_ref().filter(|_| op.upsert)?;
15202 let table = self.schema.table(&op.table)?;
15203 let fallback = self.stored_key_fallback(table, &op.row_id);
15204 match encode_sparse_row_json(
15205 table,
15206 &op.row_id,
15207 values,
15208 &self.encryption,
15209 fallback.as_ref(),
15210 ) {
15211 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
15212 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
15213 }
15214 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
15219 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
15220 OUTBOX_INCOMPATIBLE_CODE,
15221 )),
15222 _ => None,
15223 }
15224 })
15225 .map(|(error, code)| (commit.clone(), error, code))
15226 })
15227 .collect::<Vec<_>>();
15228 if failures.is_empty() {
15229 return Ok(false);
15230 }
15231 let prior_outbox = self.outbox.clone();
15237 let prior_rejections = self.rejections.clone();
15238 let prior_failed_commits = self.failed_commits.clone();
15242 let prior_changes = self.change_queue.len();
15243 let prior_intents = self.sync_intent_queue.len();
15244 let prior_last_change = self.last_change.clone();
15245 let prior_overlay_dirty = self.overlay_dirty.snapshot();
15246 let owns_observation = batch.is_none();
15247 let owns_transaction = self.conn.is_autocommit();
15248 let mut domains = ChangeAccumulator::default();
15249 if owns_observation {
15250 self.begin_observation("syncular_value_recovery")?;
15251 }
15252 let dropped = (|| -> Result<(), String> {
15253 for (commit, message, code) in failures {
15254 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
15255 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
15256 }
15257 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
15258 let rejection = RejectionRecord {
15259 client_commit_id: commit.client_commit_id.clone(),
15260 op_index: 0,
15261 code: code.to_owned(),
15262 message,
15263 retryable: false,
15264 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
15265 field_paths: None,
15266 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
15267 required_action: None,
15268 references: None,
15269 }),
15270 operation: representable
15271 .then(|| commit.ops.first())
15272 .flatten()
15273 .map(CommitOperation::from),
15274 };
15275 self.persist_commit_outcome(
15276 &commit.client_commit_id,
15277 CommitOutcomeStatus::Rejected,
15278 &[CommitOperationOutcome::Error {
15279 rejection: rejection.clone(),
15280 }],
15281 representable.then_some(commit.ops.as_slice()),
15282 )?;
15283 self.delete_outbox_persisted(&commit.client_commit_id)?;
15284 self.outbox
15285 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
15286 self.rejections.push(rejection);
15287 for operation in &commit.ops {
15288 domains.table(operation.table.as_str());
15289 }
15290 }
15291 domains.status = true;
15292 domains.rejections = true;
15293 domains.outcomes = true;
15294 self.prune_commit_outcomes()?;
15295 self.overlay_dirty.set(true);
15297 self.rebuild_overlay()?;
15298 Ok(())
15299 })();
15300 let result = match batch {
15301 Some(batch) => {
15302 if dropped.is_ok() {
15303 for table in domains.tables.keys() {
15304 batch.table(table.as_str());
15305 }
15306 batch.status |= domains.status;
15307 batch.rejections |= domains.rejections;
15308 batch.outcomes |= domains.outcomes;
15309 }
15310 dropped
15311 }
15312 None => {
15313 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
15314 }
15315 };
15316 if let Err(error) = result {
15317 if owns_observation {
15318 if owns_transaction {
15319 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
15320 let mut failure = self.storage_failure.borrow_mut();
15321 let failure =
15322 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
15323 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
15324 }
15325 } else {
15326 self.rollback_observation("syncular_value_recovery");
15327 }
15328 }
15329 self.outbox = prior_outbox;
15330 self.rejections = prior_rejections;
15331 self.failed_commits = prior_failed_commits;
15332 self.change_queue.truncate(prior_changes);
15333 self.sync_intent_queue.truncate(prior_intents);
15334 self.last_change = prior_last_change;
15335 self.overlay_dirty.restore(prior_overlay_dirty);
15336 return Err(error);
15337 }
15338 Ok(true)
15339 }
15340
15341 pub fn pending_commit_ids(&self) -> Vec<String> {
15342 self.outbox
15343 .iter()
15344 .map(|c| c.client_commit_id.clone())
15345 .collect()
15346 }
15347
15348 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
15352 let mut payloads = Vec::new();
15353 for commit in &self.outbox {
15354 for op in &commit.ops {
15355 let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
15356 continue;
15357 };
15358 let Some(table) = self.schema.table(&op.table) else {
15359 continue;
15360 };
15361 let fallback = self.stored_key_fallback(table, &op.row_id);
15362 if let Ok(payload) = encode_sparse_row_json(
15363 table,
15364 &op.row_id,
15365 values,
15366 &self.encryption,
15367 fallback.as_ref(),
15368 ) {
15369 payloads.push(payload);
15370 }
15371 }
15372 }
15373 payloads
15374 }
15375
15376 pub fn conflicts(&self) -> &[ConflictRecord] {
15377 &self.conflicts
15378 }
15379
15380 pub fn rejections(&self) -> &[RejectionRecord] {
15381 &self.rejections
15382 }
15383
15384 pub fn resolve_commit_outcome(
15385 &mut self,
15386 input: ResolveCommitOutcomeInput,
15387 ) -> Result<CommitOutcome, String> {
15388 let current = self
15389 .commit_outcome(&input.client_commit_id)?
15390 .ok_or_else(|| {
15391 format!(
15392 "sync.outcome_not_found: no durable outcome exists for {:?}",
15393 input.client_commit_id
15394 )
15395 })?;
15396 if current.resolution != CommitOutcomeResolution::Active {
15397 return Ok(current);
15398 }
15399 if input.resolution == CommitOutcomeResolution::Active {
15400 return Err("sync.invalid_request: resolution must leave active state".to_owned());
15401 }
15402 match input.resolution {
15403 CommitOutcomeResolution::Superseded => {
15404 let replacement = input
15405 .replacement_client_commit_id
15406 .as_deref()
15407 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
15408 .ok_or_else(|| {
15409 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
15410 .to_owned()
15411 })?;
15412 let _ = replacement;
15413 }
15414 _ if input.replacement_client_commit_id.is_some() => {
15415 return Err(
15416 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
15417 .to_owned(),
15418 );
15419 }
15420 _ => {}
15421 }
15422 let allowed = match current.status {
15423 CommitOutcomeStatus::Conflict => matches!(
15424 input.resolution,
15425 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
15426 ),
15427 CommitOutcomeStatus::Rejected => {
15428 matches!(
15429 input.resolution,
15430 CommitOutcomeResolution::Superseded
15431 | CommitOutcomeResolution::ResolvedKeepServer
15432 )
15433 }
15434 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
15435 input.resolution == CommitOutcomeResolution::Dismissed
15436 }
15437 };
15438 if !allowed {
15439 return Err(format!(
15440 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
15441 input.resolution, current.status
15442 ));
15443 }
15444
15445 self.begin_observation("syncular_outcome_resolution")?;
15446 let previous_failed = self.failed_commits.clone();
15447 let previous_conflicts = self.conflicts.clone();
15448 let previous_rejections = self.rejections.clone();
15449 let result = (|| {
15450 self.conn
15451 .execute(
15452 "UPDATE _syncular_commit_outcomes
15453 SET resolution = ?1, resolved_at_ms = ?2,
15454 replacement_client_commit_id = ?3
15455 WHERE client_commit_id = ?4 AND resolution = 'active'",
15456 rusqlite::params![
15457 Self::outcome_resolution_name(input.resolution),
15458 self.clock_now_ms(),
15459 input.replacement_client_commit_id,
15460 input.client_commit_id
15461 ],
15462 )
15463 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15464 let resolved = self
15465 .commit_outcome(¤t.client_commit_id)?
15466 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
15467 self.conn
15468 .execute(
15469 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
15470 rusqlite::params![input.client_commit_id],
15471 )
15472 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15473 for (commit, _) in &self.failed_commits {
15474 if commit.client_commit_id == input.client_commit_id {
15475 for operation in &commit.ops {
15476 self.overlay_dirty.table(&operation.table);
15477 }
15478 }
15479 }
15480 self.failed_commits
15481 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
15482 self.prune_commit_outcomes()?;
15483 Ok(resolved)
15484 })();
15485 let resolved = match result {
15486 Ok(outcome) => outcome,
15487 Err(error) => {
15488 self.rollback_observation("syncular_outcome_resolution");
15489 self.failed_commits = previous_failed;
15490 self.conflicts = previous_conflicts;
15491 self.rejections = previous_rejections;
15492 return Err(error);
15493 }
15494 };
15495 self.conflicts
15496 .retain(|record| record.client_commit_id != current.client_commit_id);
15497 self.rejections
15498 .retain(|record| record.client_commit_id != current.client_commit_id);
15499 let mut batch = ChangeAccumulator {
15500 conflicts: current.status == CommitOutcomeStatus::Conflict,
15501 rejections: current.status == CommitOutcomeStatus::Rejected,
15502 outcomes: true,
15503 ..ChangeAccumulator::default()
15504 };
15505 if let Some(operations) = ¤t.operations {
15506 for operation in operations {
15507 batch.table(&operation.table);
15508 }
15509 }
15510 if let Err(error) = self
15511 .rebuild_overlay_if_dirty()
15512 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
15513 {
15514 self.rollback_observation("syncular_outcome_resolution");
15515 self.failed_commits = previous_failed;
15516 self.conflicts = previous_conflicts;
15517 self.rejections = previous_rejections;
15518 return Err(error);
15519 }
15520 Ok(resolved)
15521 }
15522
15523 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
15524 self.schema_floor.as_ref()
15525 }
15526
15527 pub fn lease_state(&self) -> Option<&LeaseState> {
15529 self.lease_state.as_ref()
15530 }
15531
15532 fn record_lease_error(&mut self, code: &str) {
15535 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
15536 return;
15537 }
15538 let mut next = self.lease_state.clone().unwrap_or_default();
15539 next.error_code = Some(code.to_owned());
15540 self.set_lease_state(Some(next));
15541 }
15542
15543 fn set_lease_state(&mut self, next: Option<LeaseState>) {
15544 if self.lease_state == next {
15545 return;
15546 }
15547 if self.begin_observation("syncular_lease").is_err() {
15548 return;
15549 }
15550 self.lease_state = next;
15551 if let Some(lease) = &self.lease_state {
15552 if let Ok(json) = serde_json::to_string(lease) {
15553 self.set_meta(LEASE_STATE_KEY, &json);
15554 }
15555 } else {
15556 self.delete_meta(LEASE_STATE_KEY);
15557 }
15558 let batch = ChangeAccumulator {
15559 status: true,
15560 ..ChangeAccumulator::default()
15561 };
15562 if self.finish_observation("syncular_lease", batch).is_err() {
15563 self.rollback_observation("syncular_lease");
15564 }
15565 }
15566
15567 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
15568 if self.schema_floor == next {
15569 return;
15570 }
15571 if self.begin_observation("syncular_schema_floor").is_err() {
15572 return;
15573 }
15574 self.schema_floor = next;
15575 self.stopped = self.schema_floor.is_some();
15576 if let Some(floor) = &self.schema_floor {
15577 if let Ok(json) = serde_json::to_string(floor) {
15578 self.set_meta(SCHEMA_FLOOR_KEY, &json);
15579 }
15580 } else {
15581 self.delete_meta(SCHEMA_FLOOR_KEY);
15582 }
15583 let batch = ChangeAccumulator {
15584 status: true,
15585 ..ChangeAccumulator::default()
15586 };
15587 if self
15588 .finish_observation("syncular_schema_floor", batch)
15589 .is_err()
15590 {
15591 self.rollback_observation("syncular_schema_floor");
15592 }
15593 }
15594
15595 fn set_upgrading(&mut self, value: bool) {
15596 if self.upgrading == value {
15597 return;
15598 }
15599 if self.begin_observation("syncular_upgrading").is_err() {
15600 return;
15601 }
15602 self.upgrading = value;
15603 let batch = ChangeAccumulator {
15604 status: true,
15605 ..ChangeAccumulator::default()
15606 };
15607 if self
15608 .finish_observation("syncular_upgrading", batch)
15609 .is_err()
15610 {
15611 self.rollback_observation("syncular_upgrading");
15612 }
15613 }
15614
15615 pub fn sync_needed(&self) -> bool {
15616 self.sync_needed
15617 }
15618
15619 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
15620 let sub = self.subs.iter().find(|s| s.id == id)?;
15621 Some(SubscriptionStateView {
15622 id: sub.id.clone(),
15623 table: sub.table.clone(),
15624 status: sub.state.name().to_owned(),
15625 cursor: sub.cursor,
15626 has_resume_token: sub.bootstrap_state.is_some(),
15627 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
15628 reason_code: sub.reason_code.clone(),
15629 })
15630 }
15631
15632 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
15633 let schema_table = self
15634 .schema
15635 .table(table)
15636 .ok_or_else(|| format!("unknown table {table:?}"))?;
15637 let sql = format!(
15638 "SELECT * FROM {} ORDER BY {} ASC",
15639 visible_table(table),
15640 quote_ident(&schema_table.primary_key)
15641 );
15642 let mut stmt = self
15643 .conn
15644 .prepare(&sql)
15645 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15646 let mut rows = stmt
15647 .query([])
15648 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15649 let mut out = Vec::new();
15650 while let Some(row) = rows
15651 .next()
15652 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15653 {
15654 let mut values = Map::new();
15655 for (i, column) in schema_table.columns.iter().enumerate() {
15656 let value = row
15657 .get_ref(i)
15658 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15659 values.insert(column.name.clone(), sql_ref_to_json(column, value));
15660 }
15661 let version: i64 = row
15662 .get(schema_table.columns.len())
15663 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15664 let row_id = match values.get(&schema_table.primary_key) {
15665 Some(Value::String(s)) => s.clone(),
15666 Some(Value::Number(n)) => n.to_string(),
15667 Some(Value::Bool(b)) => b.to_string(),
15668 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
15669 };
15670 out.push(RowState {
15671 row_id,
15672 version,
15673 values,
15674 });
15675 }
15676 Ok(out)
15677 }
15678
15679 #[cfg(feature = "crdt-yjs")]
15695 fn crdt_column_bytes(
15696 &self,
15697 table: &str,
15698 row_id: &str,
15699 column: &str,
15700 ) -> Result<Option<Vec<u8>>, String> {
15701 let schema_table = self
15702 .schema
15703 .table(table)
15704 .ok_or_else(|| format!("unknown table {table:?}"))?;
15705 let col = schema_table
15706 .columns
15707 .iter()
15708 .find(|c| c.name == column)
15709 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
15710 if col.ty != ColumnType::Crdt {
15711 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
15712 }
15713 let sql = format!(
15714 "SELECT {} FROM {} WHERE {}",
15715 quote_ident(column),
15716 visible_table(table),
15717 row_id_predicate(schema_table)
15718 );
15719 let bytes: Option<Vec<u8>> = self
15720 .conn
15721 .prepare_cached(&sql)
15722 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15723 .query_row(rusqlite::params![row_id], |row| {
15724 row.get::<_, Option<Vec<u8>>>(0)
15725 })
15726 .map_err(|e| match e {
15727 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
15728 other => Self::sqlite_failure(&self.storage_failure, other),
15729 })?;
15730 Ok(bytes)
15731 }
15732
15733 #[cfg(feature = "crdt-yjs")]
15737 pub fn crdt_text(
15738 &self,
15739 table: &str,
15740 row_id: &str,
15741 column: &str,
15742 name: &str,
15743 ) -> Result<String, String> {
15744 let bytes = self
15745 .crdt_column_bytes(table, row_id, column)?
15746 .unwrap_or_default();
15747 crate::crdt::text(&bytes, name)
15748 }
15749
15750 #[cfg(feature = "crdt-yjs")]
15754 pub fn crdt_insert_text(
15755 &mut self,
15756 table: &str,
15757 row_id: &str,
15758 column: &str,
15759 name: &str,
15760 index: u32,
15761 value: &str,
15762 ) -> Result<String, ClientError> {
15763 let prior = self.storage_failure.borrow_mut().take();
15764 let outcome = (|| -> Result<String, String> {
15765 let current = self
15766 .crdt_column_bytes(table, row_id, column)?
15767 .unwrap_or_default();
15768 let update = crate::crdt::insert_text(¤t, name, index, value)?;
15769 self.perform_crdt_push_update(table, row_id, column, &update)
15770 })();
15771 self.finish_authoring(prior, outcome)
15772 }
15773
15774 #[cfg(feature = "crdt-yjs")]
15778 pub fn crdt_delete_text(
15779 &mut self,
15780 table: &str,
15781 row_id: &str,
15782 column: &str,
15783 name: &str,
15784 index: u32,
15785 len: u32,
15786 ) -> Result<String, ClientError> {
15787 let prior = self.storage_failure.borrow_mut().take();
15788 let outcome = (|| -> Result<String, String> {
15789 let current = self
15790 .crdt_column_bytes(table, row_id, column)?
15791 .unwrap_or_default();
15792 let update = crate::crdt::delete_text(¤t, name, index, len)?;
15793 self.perform_crdt_push_update(table, row_id, column, &update)
15794 })();
15795 self.finish_authoring(prior, outcome)
15796 }
15797
15798 #[cfg(feature = "crdt-yjs")]
15803 pub fn crdt_apply_update(
15804 &mut self,
15805 table: &str,
15806 row_id: &str,
15807 column: &str,
15808 update: &[u8],
15809 ) -> Result<String, ClientError> {
15810 let prior = self.storage_failure.borrow_mut().take();
15811 let outcome = (|| -> Result<String, String> {
15812 let current = self
15813 .crdt_column_bytes(table, row_id, column)?
15814 .unwrap_or_default();
15815 let next = crate::crdt::apply_update(¤t, update)?;
15816 self.perform_crdt_push_update(table, row_id, column, &next)
15817 })();
15818 self.finish_authoring(prior, outcome)
15819 }
15820
15821 #[cfg(feature = "crdt-yjs")]
15828 fn perform_crdt_push_update(
15829 &mut self,
15830 table: &str,
15831 row_id: &str,
15832 column: &str,
15833 crdt_bytes: &[u8],
15834 ) -> Result<String, String> {
15835 let mut bytes_obj = Map::new();
15836 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
15837 let mut partial = Map::new();
15838 partial.insert(column.to_owned(), Value::Object(bytes_obj));
15839 self.perform_patch(table, row_id, partial, None)
15840 }
15841
15842 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
15857 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
15858 }
15859
15860 #[cfg(feature = "bench-internals")]
15862 #[doc(hidden)]
15863 pub fn benchmark_connection(&mut self) -> &mut Connection {
15864 &mut self.conn
15865 }
15866
15867 #[cfg(feature = "bench-internals")]
15869 #[doc(hidden)]
15870 pub fn benchmark_phases(
15871 &self,
15872 enabled: Option<bool>,
15873 reset: bool,
15874 ) -> Result<Option<Value>, String> {
15875 self.benchmark_phases.configure(enabled, reset)?;
15876 Ok(self.benchmark_phases.snapshot())
15877 }
15878
15879 pub fn authority_snapshot(&self) -> Result<Value, String> {
15881 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
15882 if self.limits.authority_reads.is_empty() {
15883 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
15884 }
15885 self.conn
15886 .execute_batch("SAVEPOINT authority_snapshot")
15887 .map_err(|e| e.to_string())?;
15888 let result = (|| {
15889 let revision = self
15890 .get_meta(LOCAL_REVISION_KEY)
15891 .ok_or("sync.local_corrupt: authority revision is missing")?;
15892 let parsed_revision = revision
15893 .parse::<u64>()
15894 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
15895 if parsed_revision.to_string() != revision {
15896 return Err("sync.local_corrupt: authority revision is invalid".into());
15897 }
15898 let mut tables = Vec::new();
15899 for read in &self.limits.authority_reads {
15900 let table = self
15901 .schema
15902 .table(&read.table)
15903 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
15904 let scalar = |column: &str, value: &Value| -> Result<String, String> {
15905 let ty = table
15906 .columns
15907 .iter()
15908 .find(|c| c.name == column)
15909 .expect("validated authority column")
15910 .ty;
15911 if ty == ColumnType::Boolean {
15912 if let Some(value) = value.as_bool() {
15913 return Ok(value.to_string());
15914 }
15915 return match value.as_i64() {
15916 Some(0) => Ok("false".into()),
15917 Some(1) => Ok("true".into()),
15918 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
15919 };
15920 }
15921 match value {
15922 Value::String(s) => Ok(s.clone()),
15923 Value::Number(n) => Ok(n.to_string()),
15924 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
15925 .get("$bigint")
15926 .and_then(Value::as_str)
15927 .map(str::to_owned)
15928 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
15929 _ => {
15930 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
15931 }
15932 }
15933 };
15934 let matches = |row: &QueryRow| -> Result<bool, String> {
15935 for (variable, values) in &read.scopes {
15936 let column = table
15937 .scope_column(variable)
15938 .expect("validated authority scope");
15939 let value = row
15940 .get(column)
15941 .ok_or("sync.local_corrupt: authority scope column is missing")?;
15942 if value.is_null() || !values.contains(&scalar(column, value)?) {
15943 return Ok(false);
15944 }
15945 }
15946 Ok(true)
15947 };
15948 let selected = read
15949 .columns
15950 .iter()
15951 .map(|c| quote_ident(c))
15952 .collect::<Vec<_>>()
15953 .join(",");
15954 let mut intent = BTreeSet::new();
15955 let mut relevant = BTreeSet::new();
15956 for row in query_connection(&self.conn,
15957 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
15958 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
15959 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
15960 intent.insert(id.clone());
15961 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
15962 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
15963 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
15964 }
15965 }
15966 let mut rows = BTreeMap::new();
15967 for full in [base_table(&read.table), visible_table(&read.table)] {
15968 for mut values in query_connection(
15969 &self.conn,
15970 &format!(
15971 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
15972 ),
15973 &[],
15974 )
15975 .map_err(|e| e.to_string())?
15976 {
15977 if !matches(&values)? {
15978 continue;
15979 }
15980 let id = scalar(
15981 &table.primary_key,
15982 values
15983 .get(&table.primary_key)
15984 .expect("selected primary key"),
15985 )?;
15986 relevant.insert(id.clone());
15987 if full == base_table(&read.table) {
15988 let version = values
15989 .remove("authority_version")
15990 .expect("selected version");
15991 let version = version
15992 .as_i64()
15993 .ok_or("sync.local_corrupt: authority server version is invalid")?;
15994 if version >= 0 {
15995 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
15996 }
15997 }
15998 }
15999 }
16000 intent.retain(|id| relevant.contains(id));
16001 let rows = rows.into_values().collect::<Vec<_>>();
16002 let mut persisted = Vec::new();
16003 for row in query_connection(
16004 &self.conn,
16005 "SELECT state_json FROM _syncular_subscriptions WHERE tbl=? ORDER BY rowid",
16006 &[Value::String(read.table.clone())],
16007 )
16008 .map_err(|e| e.message)?
16009 {
16010 let state: Value = serde_json::from_str(
16011 row.get("state_json")
16012 .and_then(Value::as_str)
16013 .ok_or("sync.local_corrupt: authority subscription state is missing")?,
16014 )
16015 .map_err(|_| "sync.local_corrupt: authority subscription state is invalid")?;
16016 let effective = state.get("effectiveScopes").and_then(Value::as_object);
16017 let status = state
16018 .get("status")
16019 .and_then(Value::as_str)
16020 .ok_or("sync.local_corrupt: authority subscription status is missing")?;
16021 let cursor = state
16022 .get("cursor")
16023 .and_then(Value::as_i64)
16024 .ok_or("sync.local_corrupt: authority subscription cursor is missing")?;
16025 let complete = effective.is_some_and(|s| s.len() == read.scopes.len())
16026 && status == "active"
16027 && cursor >= 0
16028 && state.get("bootstrapState").is_none_or(Value::is_null)
16029 && state.get("params").is_none_or(Value::is_null);
16030 let exposed = effective.map(|s| {
16031 read.scopes
16032 .keys()
16033 .map(|v| (v.clone(), s.get(v).cloned().unwrap_or(json!([]))))
16034 .collect::<Map<_, _>>()
16035 });
16036 let requested = read
16037 .scopes
16038 .keys()
16039 .map(|v| {
16040 (
16041 v.clone(),
16042 state
16043 .get("requested")
16044 .and_then(|s| s.get(v))
16045 .cloned()
16046 .unwrap_or(json!([])),
16047 )
16048 })
16049 .collect::<Map<_, _>>();
16050 persisted.push(json!({"requestedScopes": requested, "status": status, "cursor": cursor, "effectiveScopes": exposed, "complete": complete}));
16051 }
16052 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
16053 match selectors.split_first() {
16054 None => !candidates.is_empty(),
16055 Some(((variable, values), rest)) => values.iter().all(|value| {
16056 covered(
16057 rest,
16058 &candidates
16059 .iter()
16060 .copied()
16061 .filter(|p| {
16062 p["effectiveScopes"][variable.as_str()]
16063 .as_array()
16064 .is_some_and(|held| {
16065 held.contains(&Value::String(value.clone()))
16066 || held.contains(&Value::String("*".into()))
16067 })
16068 })
16069 .collect::<Vec<_>>(),
16070 )
16071 }),
16072 }
16073 }
16074 let coverage = if covered(
16075 &read.scopes.iter().collect::<Vec<_>>(),
16076 &persisted
16077 .iter()
16078 .filter(|p| p["complete"] == true)
16079 .collect::<Vec<_>>(),
16080 ) {
16081 "complete"
16082 } else if persisted.iter().any(|p| {
16083 p["status"] == "active"
16084 && read.scopes.iter().all(|(v, values)| {
16085 values.iter().any(|value| {
16086 p["requestedScopes"][v].as_array().is_some_and(|held| {
16087 held.contains(&Value::String(value.clone()))
16088 || held.contains(&Value::String("*".into()))
16089 })
16090 })
16091 })
16092 }) {
16093 "pending"
16094 } else {
16095 "missing"
16096 };
16097 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
16098 }
16099 Ok(
16100 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
16101 )
16102 })();
16103 match result {
16104 Ok(snapshot) => {
16105 self.conn
16106 .execute_batch("RELEASE authority_snapshot")
16107 .map_err(|e| e.to_string())?;
16108 Ok(snapshot)
16109 }
16110 Err(error) => {
16111 self.conn
16112 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
16113 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
16114 Err(error)
16115 }
16116 }
16117 }
16118
16119 pub fn query_snapshot(
16123 &mut self,
16124 sql: &str,
16125 params: &[QueryValue],
16126 coverage: &[WindowCoverage],
16127 owner: Option<QueryOwner<'_>>,
16128 ) -> Result<QuerySnapshot, String> {
16129 let result = snapshot_connection(&self.conn, sql, params, coverage);
16130 if let Some(owner) = owner {
16131 self.record_query_read(owner, result.as_ref().err());
16132 }
16133 result.map_err(|failure| failure.to_string())
16134 }
16135
16136 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
16140 let position = self
16141 .query_failures
16142 .iter()
16143 .position(|entry| entry.id == owner.id);
16144 let Some(failure) = failure else {
16145 if let Some(index) = position {
16146 self.query_failures.remove(index);
16147 }
16148 return;
16149 };
16150 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
16151 .into_iter()
16152 .map(str::to_owned)
16153 .collect();
16154 let code = failure.code.unwrap_or("client.query_failed").to_owned();
16155 if position.is_some_and(|index| {
16156 let entry = &self.query_failures[index];
16157 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
16158 }) {
16159 return;
16160 }
16161 if let Some(index) = position {
16162 self.query_failures.remove(index);
16163 }
16164 self.query_failures.push(DiagnosticQueryFailure {
16165 id: owner.id.to_owned(),
16166 tables,
16167 code,
16168 sqlite_code: failure.sqlite_code,
16169 at_ms: self.clock_now_ms(),
16170 });
16171 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
16172 self.query_failures.remove(0);
16173 }
16174 }
16175
16176 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
16179 #[cfg(feature = "bench-internals")]
16180 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
16181 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
16182 let max_commits = self.limits.max_push_commits_per_request;
16183 let max_ops = self
16184 .limits
16185 .max_push_operations_per_request
16186 .unwrap_or(PUSH_OPS_PER_REQUEST);
16187 let byte_cap = self.limits.max_push_request_bytes;
16188 let accept = self.limits.accept.unwrap_or(if url_capable {
16191 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
16192 } else {
16193 DEFAULT_ACCEPT
16194 });
16195 let header = Frame::ReqHeader {
16196 client_id: self.client_id.clone(),
16197 schema_version: self.schema.version,
16198 log_epoch: log_epoch.clone(),
16199 };
16200 let mut lead = vec![header];
16204 lead.extend(
16205 self.round_failure
16206 .iter()
16207 .map(|code| Frame::RoundFailure { code: code.clone() }),
16208 );
16209 let pull = Frame::PullHeader {
16210 limit_commits: self.limits.limit_commits.unwrap_or(0),
16211 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
16212 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
16213 accept,
16214 };
16215 let mut fresh = Vec::new();
16216 let mut sub_frames = Vec::new();
16217 for sub in &self.subs {
16218 if sub.state != SubState::Active {
16219 continue;
16220 }
16221 let mut scopes = sub.requested.clone();
16222 sort_scope_map(&mut scopes);
16223 sub_frames.push(Frame::Subscription {
16224 id: sub.id.clone(),
16225 table: sub.table.clone(),
16226 scopes,
16227 params: sub.params.clone().map(RawJson),
16228 cursor: sub.cursor,
16229 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
16230 });
16231 fresh.push((
16232 sub.id.clone(),
16233 sub.cursor < 0 && sub.bootstrap_state.is_none(),
16234 ));
16235 }
16236 let fixed_bytes = match byte_cap {
16239 Some(_) => {
16240 let mut frames = lead.clone();
16241 frames.push(pull.clone());
16242 frames.extend(sub_frames.iter().cloned());
16243 encode_message(&Message {
16244 wire_version: WIRE_VERSION,
16245 msg_kind: MsgKind::Request,
16246 frames,
16247 })
16248 .len()
16249 }
16250 None => 0,
16251 };
16252 if let Some(cap) = byte_cap {
16253 if fixed_bytes > cap {
16254 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
16255 }
16256 }
16257 let base_bytes = match byte_cap {
16258 Some(_) => {
16259 let mut frames = lead.clone();
16260 frames.push(pull.clone());
16261 encode_message(&Message {
16262 wire_version: WIRE_VERSION,
16263 msg_kind: MsgKind::Request,
16264 frames,
16265 })
16266 .len()
16267 }
16268 None => 0,
16269 };
16270 let total_outbox = if log_epoch.is_some() {
16271 self.outbox.len()
16272 } else {
16273 0
16274 };
16275 let mut push_frames: Vec<Frame> = Vec::new();
16276 let mut pushed_ids = Vec::new();
16277 let mut ops_in_request = 0usize;
16278 let mut bytes_in_request = 0usize;
16279 let mut deferred_commits = 0usize;
16280 #[cfg(feature = "bench-internals")]
16281 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
16282 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
16283 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
16287 deferred_commits = total_outbox - index;
16288 break;
16289 }
16290 if ops_in_request + commit.ops.len() > max_ops {
16291 if push_frames.is_empty() {
16292 return Err(Self::push_capacity_outcome(
16293 "operations",
16294 max_ops,
16295 commit.ops.len(),
16296 Some(commit.client_commit_id.as_str()),
16297 ));
16298 }
16299 deferred_commits = total_outbox - index;
16300 break;
16301 }
16302 let operations = commit
16303 .ops
16304 .iter()
16305 .map(|op| {
16306 let payload = op.values.as_ref().and_then(|values| {
16307 let table = self.schema.table(&op.table)?;
16308 let fallback = self.stored_key_fallback(table, &op.row_id);
16317 encode_sparse_row_json(
16318 table,
16319 &op.row_id,
16320 values,
16321 &self.encryption,
16322 fallback.as_ref(),
16323 )
16324 .ok()
16325 });
16326 ssp2::model::Operation {
16327 table: op.table.clone(),
16328 row_id: op.row_id.clone(),
16329 op: if op.upsert { Op::Upsert } else { Op::Delete },
16330 base_version: op.base_version,
16331 payload,
16332 }
16333 })
16334 .collect();
16335 let frame = Frame::PushCommit {
16336 client_commit_id: commit.client_commit_id.clone(),
16337 operations,
16338 };
16339 let frame_bytes = match byte_cap {
16340 Some(_) => {
16341 let mut frames = lead.clone();
16342 frames.push(frame.clone());
16343 frames.push(pull.clone());
16344 encode_message(&Message {
16345 wire_version: WIRE_VERSION,
16346 msg_kind: MsgKind::Request,
16347 frames,
16348 })
16349 .len()
16350 - base_bytes
16351 }
16352 None => 0,
16353 };
16354 if let Some(cap) = byte_cap {
16358 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
16359 deferred_commits = total_outbox - index;
16360 break;
16361 }
16362 }
16363 push_frames.push(frame);
16364 pushed_ids.push(commit.client_commit_id.clone());
16365 ops_in_request += commit.ops.len();
16366 bytes_in_request += frame_bytes;
16367 }
16368 #[cfg(feature = "bench-internals")]
16369 drop(outbox_phase);
16370 if let Some(cap) = byte_cap {
16371 if fixed_bytes + bytes_in_request > cap {
16372 let blocked = if push_frames.is_empty() {
16375 None
16376 } else {
16377 self.outbox.first().map(|c| c.client_commit_id.as_str())
16378 };
16379 return Err(Self::push_capacity_outcome(
16380 "bytes",
16381 cap,
16382 fixed_bytes + bytes_in_request,
16383 blocked,
16384 ));
16385 }
16386 }
16387 let mut frames = lead;
16388 frames.extend(push_frames);
16389 frames.push(pull);
16390 frames.extend(sub_frames);
16391 let message = Message {
16392 wire_version: WIRE_VERSION,
16393 msg_kind: MsgKind::Request,
16394 frames,
16395 };
16396 Ok((
16397 message,
16398 RequestMeta {
16399 pushed_ids,
16400 fresh,
16401 accept,
16402 deferred_commits,
16403 },
16404 ))
16405 }
16406
16407 fn push_capacity_outcome(
16409 kind: &str,
16410 limit: usize,
16411 size: usize,
16412 client_commit_id: Option<&str>,
16413 ) -> Box<SyncOutcome> {
16414 let mut details = Map::new();
16415 details.insert("kind".to_owned(), Value::from(kind));
16416 details.insert("limit".to_owned(), Value::from(limit as u64));
16417 details.insert("size".to_owned(), Value::from(size as u64));
16418 if let Some(id) = client_commit_id {
16419 details.insert("clientCommitId".to_owned(), Value::from(id));
16420 }
16421 Box::new(SyncOutcome::Failed {
16422 error_code: "client.push_request_too_large".into(),
16423 message: "the push request exceeds a configured capacity limit".into(),
16424 details: Some(Value::Object(details)),
16425 })
16426 }
16427
16428 pub fn progress(&self) -> ProgressObserver {
16432 self.progress.clone()
16433 }
16434
16435 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
16436 let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
16437 Ok(prepared) => prepared,
16438 Err(outcome) => return *outcome,
16439 };
16440 let mut next = prepared;
16445 loop {
16446 match self.apply_sync_round(next.exchange(transport)) {
16447 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
16448 crate::AppliedSyncRound::Complete {
16449 outcome, controls, ..
16450 } => {
16451 if self.transport_enabled {
16452 for text in controls {
16453 let _ = transport.realtime_send(&text);
16454 }
16455 } else {
16456 self.disconnect_realtime(transport);
16457 }
16458 return outcome;
16459 }
16460 }
16461 }
16462 }
16463
16464 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
16465 if let SyncOutcome::Failed {
16466 error_code,
16467 message,
16468 details,
16469 } = &mut outcome
16470 {
16471 if let Some(failure) = self.storage_failure.borrow().as_ref() {
16472 let code = failure
16473 .code
16474 .expect("retained storage failures are classified");
16475 *error_code = code.to_owned();
16476 *message = match code {
16477 "client.storage_busy" => "local SQLite storage is busy",
16478 "client.storage_full" => "local SQLite storage is full",
16479 "client.storage_io" => "local SQLite storage I/O failed",
16480 "client.storage_corrupt" => "local SQLite storage is corrupt",
16481 _ => unreachable!("unrecognized classified storage failure"),
16482 }
16483 .to_owned();
16484 *details = failure.details();
16485 }
16486 }
16487 let retry_delay_ms = self.round_retry_delay_ms;
16488 self.progress.update(|p| match &outcome {
16489 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
16490 if report.failed.is_empty() =>
16491 {
16492 p.state = ProgressState::Complete
16493 }
16494 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
16495 p.state = ProgressState::Failed;
16496 p.error_code = Some("sync.scope_revoked".into());
16497 }
16498 SyncOutcome::Failed { error_code, .. } => {
16499 p.state = ProgressState::Failed;
16500 p.error_code = Some(Self::diagnostic_code(error_code));
16501 p.retry_delay_ms = retry_delay_ms;
16502 }
16503 SyncOutcome::RealtimeUnavailable { .. } => {
16504 p.state = ProgressState::Failed;
16505 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
16506 p.retry_delay_ms = retry_delay_ms;
16507 }
16508 });
16509 let completed_at_ms = self.clock_now_ms();
16510 self.last_round = Some(match &outcome {
16511 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
16512 status: "succeeded".to_owned(),
16513 started_at_ms,
16514 completed_at_ms,
16515 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16516 counters: Some(DiagnosticRoundCounters {
16517 pushed: report.pushed,
16518 applied: report.applied.len(),
16519 rejected: report.rejected.len(),
16520 retryable: report.retryable.len(),
16521 conflicts: report.conflicts,
16522 commits_applied: report.commits_applied,
16523 segment_rows_applied: report.segment_rows_applied,
16524 bootstrapping: report.bootstrapping.len(),
16525 resets: report.resets.len(),
16526 revoked: report.revoked.len(),
16527 failed: report.failed.len(),
16528 deferred_commits: report.deferred_commits,
16529 }),
16530 error_code: None,
16531 },
16532 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
16533 status: "failed".to_owned(),
16534 started_at_ms,
16535 completed_at_ms,
16536 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16537 counters: None,
16538 error_code: Some(Self::diagnostic_code(error_code)),
16539 },
16540 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
16541 status: "failed".to_owned(),
16542 started_at_ms,
16543 completed_at_ms,
16544 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16545 counters: None,
16546 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
16547 },
16548 });
16549 outcome
16550 }
16551
16552 pub fn prepare_sync_round(
16555 &mut self,
16556 url_capable: bool,
16557 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
16558 if !self.transport_enabled {
16559 return Err(Box::new(SyncOutcome::Failed {
16560 error_code: "sync.offline".into(),
16561 message: "transport gate is closed".into(),
16562 details: None,
16563 }));
16564 }
16565 if self.active_round.is_some() {
16566 return Err(Box::new(SyncOutcome::Failed {
16567 error_code: "client.round_in_progress".into(),
16568 message: "a sync round is already in flight".into(),
16569 details: None,
16570 }));
16571 }
16572 self.storage_failure.borrow_mut().take();
16573 self.progress.start();
16574 self.round_retry_delay_ms = None;
16575 let started_at_ms = self.clock_now_ms();
16576 let prepared = (|| {
16577 if self.stopped {
16578 return Err(Box::new(SyncOutcome::Ok(SyncReport {
16579 schema_floor: self.schema_floor.clone(),
16580 ..SyncReport::default()
16581 })));
16582 }
16583 if self.security_preflight {
16584 return Err(Box::new(SyncOutcome::Failed {
16585 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
16586 message: "security preflight must complete before syncing".into(),
16587 details: None,
16588 }));
16589 }
16590 self.set_sync_needed(false, false);
16591 self.drain_pending_evictions()
16592 .and_then(|_| self.drop_incompatible_outbox())
16593 .and_then(|_| self.drop_unencodable_outbox(true, None))
16594 .map_err(|message| {
16595 Box::new(SyncOutcome::Failed {
16596 error_code: "storage.failed".into(),
16597 message,
16598 details: None,
16599 })
16600 })?;
16601 let uploads = if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
16602 self.pending_blob_uploads().map_err(|e| {
16603 Box::new(SyncOutcome::Failed {
16604 error_code: e.code,
16605 message: e.message,
16606 details: e.details,
16607 })
16608 })?
16609 } else {
16610 Vec::new()
16611 };
16612 if self.realtime_state != RealtimeState::Connected
16613 && self.realtime_policy == RealtimePolicy::Required
16614 {
16615 let retry_delay_ms = self.schedule_background_retry();
16616 self.realtime_retry_delay_ms = Some(retry_delay_ms);
16617 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
16618 state: self.realtime_state,
16619 reason_code: self.realtime_reason_code.clone(),
16620 retry_delay_ms,
16621 }));
16622 }
16623 let (message, meta) = self.build_request(url_capable)?;
16624 let id = uuid::Uuid::new_v4();
16625 self.active_round = Some(id);
16626 Ok(crate::PreparedSyncRound {
16627 id,
16628 started_at_ms,
16629 message,
16630 meta,
16631 uploads: uploads.into(),
16632 realtime: self.realtime_state == RealtimeState::Connected,
16633 scopes: self
16634 .subs
16635 .iter()
16636 .map(|s| (s.id.clone(), canonical_scope_json(&s.requested)))
16637 .collect(),
16638 #[cfg(feature = "bench-internals")]
16639 benchmark_phases: self.benchmark_phases.clone(),
16640 progress: self.progress.clone(),
16641 fixed_now: self.now_ms,
16642 round_deadline_at: None,
16643 })
16644 })();
16645 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
16646 }
16647
16648 pub fn cancel_sync_round(&mut self) {
16651 self.active_round = None;
16652 }
16653
16654 pub fn apply_sync_round(
16657 &mut self,
16658 completed: crate::CompletedSyncRound,
16659 ) -> crate::AppliedSyncRound {
16660 let prepared = completed.prepared;
16661 if self.active_round != Some(prepared.id) || self.security_preflight {
16662 return crate::AppliedSyncRound::Complete {
16663 outcome: SyncOutcome::Failed {
16664 error_code: "client.round_cancelled".into(),
16665 message: "sync round belongs to a released client context".into(),
16666 details: None,
16667 },
16668 controls: Vec::new(),
16669 more: false,
16670 bootstrap_advanced: false,
16671 };
16672 }
16673 self.storage_failure.borrow_mut().take();
16674 let (response, mut downloads, transport_failed) = match completed.exchange {
16675 crate::round::ExchangeResult::Upload { id, result } => {
16676 let result = result.and_then(|()| {
16677 self.conn
16678 .execute(
16679 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
16680 rusqlite::params![id],
16681 )
16682 .map(|_| ())
16683 .map_err(|error| {
16684 TransportError::new(
16685 "client.failed",
16686 Self::sqlite_failure(&self.storage_failure, error),
16687 )
16688 })
16689 });
16690 match result {
16691 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
16692 Err(TransportError {
16693 code,
16694 message,
16695 details,
16696 }) => {
16697 self.active_round = None;
16698 if Self::retryable_failure_code(&code) {
16699 self.schedule_background_retry();
16700 }
16701 let outcome = self.finish_sync_round(
16702 prepared.started_at_ms,
16703 SyncOutcome::Failed {
16704 error_code: code,
16705 message,
16706 details,
16707 },
16708 );
16709 return crate::AppliedSyncRound::Complete {
16710 outcome,
16711 controls: Vec::new(),
16712 more: false,
16713 bootstrap_advanced: false,
16714 };
16715 }
16716 }
16717 }
16718 crate::round::ExchangeResult::Reply {
16719 response,
16720 downloads,
16721 transport_failed,
16722 } => (response, downloads, transport_failed),
16723 };
16724 self.active_round = None;
16725 let before = self
16726 .subs
16727 .iter()
16728 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16729 .collect::<Vec<_>>();
16730 let outcome = (|| {
16731 let response = match response {
16732 Ok(response) => {
16733 self.round_failure = None;
16736 response
16737 }
16738 Err(TransportError {
16739 code,
16740 message,
16741 details,
16742 }) => {
16743 if ssp2::model::valid_round_failure_code(&code) {
16748 self.round_failure = Some(code.clone());
16749 }
16750 if transport_failed {
16751 if prepared.realtime {
16752 self.set_realtime_state(RealtimeState::Lost, Some(&code));
16753 }
16754 self.record_lease_error(&code);
16755 }
16756 if Self::retryable_failure_code(&code) {
16757 let delay = self.schedule_background_retry();
16758 if prepared.realtime && transport_failed {
16759 self.realtime_retry_delay_ms = Some(delay);
16760 }
16761 }
16762 return SyncOutcome::Failed {
16763 error_code: code,
16764 message,
16765 details,
16766 };
16767 }
16768 };
16769 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
16770 if let SyncOutcome::Ok(report) = &mut outcome {
16771 report.deferred_commits = prepared.meta.deferred_commits;
16772 }
16773 match &outcome {
16774 SyncOutcome::Ok(_) => self.reset_background_retry(),
16775 SyncOutcome::Failed { error_code, .. }
16776 if Self::retryable_failure_code(error_code) =>
16777 {
16778 self.schedule_background_retry();
16779 }
16780 _ => {}
16781 }
16782 if prepared.meta.deferred_commits > 0 {
16783 self.set_sync_needed(true, true);
16784 }
16785 self.previous_version_lifetime_check();
16786 outcome
16787 })();
16788 let more = self.transport_enabled
16789 && matches!(&outcome, SyncOutcome::Ok(report)
16790 if report.schema_floor.is_none()
16791 && (!report.bootstrapping.is_empty()
16792 || report.commits_applied > 0
16793 || report.segment_rows_applied > 0
16794 || !report.resets.is_empty()
16795 || report.deferred_commits > 0
16796 || self.sync_needed));
16797 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
16798 && before
16799 != self
16800 .subs
16801 .iter()
16802 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16803 .collect::<Vec<_>>();
16804 crate::AppliedSyncRound::Complete {
16805 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
16806 controls: if self.transport_enabled {
16807 downloads.controls
16808 } else {
16809 Vec::new()
16810 },
16811 more,
16812 bootstrap_advanced,
16813 }
16814 }
16815
16816 pub fn sync_until_idle(
16817 &mut self,
16818 transport: &mut dyn Transport,
16819 max_rounds: Option<u32>,
16820 ) -> SyncOutcome {
16821 if max_rounds == Some(0) {
16824 return SyncOutcome::Failed {
16825 details: None,
16826 error_code: "sync.invalid_request".into(),
16827 message: "maxRounds must be a positive integer".into(),
16828 };
16829 }
16830 let rounds = max_rounds.unwrap_or(20);
16831 let mut aggregate = SyncReport::default();
16832 let mut spent = 0;
16833 while spent < rounds {
16834 spent += 1;
16835 let before = self
16836 .subs
16837 .iter()
16838 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16839 .collect::<Vec<_>>();
16840 match self.sync(transport) {
16841 SyncOutcome::Failed {
16842 error_code,
16843 message,
16844 details,
16845 } => {
16846 return SyncOutcome::Failed {
16847 details,
16848 error_code,
16849 message,
16850 };
16851 }
16852 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
16853 SyncOutcome::BudgetExhausted(_) => {
16854 unreachable!("sync() never exhausts a round budget")
16855 }
16856 SyncOutcome::Ok(report) => {
16857 if max_rounds.is_none()
16858 && report.segment_rows_applied > 0
16859 && !report.bootstrapping.is_empty()
16860 && before
16861 != self
16862 .subs
16863 .iter()
16864 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16865 .collect::<Vec<_>>()
16866 {
16867 spent = 0;
16868 }
16869 let stop = !self.transport_enabled() || report.schema_floor.is_some();
16872 aggregate.merge(&report);
16873 if stop {
16874 return SyncOutcome::Ok(aggregate);
16875 }
16876 let more = !report.bootstrapping.is_empty()
16882 || report.commits_applied > 0
16883 || report.segment_rows_applied > 0
16884 || !report.resets.is_empty()
16885 || report.deferred_commits > 0
16886 || self.sync_needed;
16887 if !more {
16888 return SyncOutcome::Ok(aggregate);
16889 }
16890 }
16891 }
16892 }
16893 SyncOutcome::BudgetExhausted(aggregate)
16896 }
16897
16898 fn process_response(
16899 &mut self,
16900 transport: &mut dyn Transport,
16901 response: Message,
16902 meta: &RequestMeta,
16903 ) -> SyncOutcome {
16904 #[cfg(feature = "bench-internals")]
16905 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
16906 let mut report = SyncReport {
16907 pushed: meta.pushed_ids.len() as u32,
16908 ..SyncReport::default()
16909 };
16910 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
16911 HashMap::new();
16912 let pushed_ids = meta
16913 .pushed_ids
16914 .iter()
16915 .map(String::as_str)
16916 .collect::<HashSet<_>>();
16917 let mut last_final_push_result_id: Option<String> = None;
16918 for frame in &response.frames {
16919 match frame {
16920 Frame::PushResultDetails {
16921 client_commit_id,
16922 entries,
16923 } => {
16924 let details = rejection_details_by_commit
16925 .entry(client_commit_id.clone())
16926 .or_default();
16927 for entry in entries {
16928 let parsed = match RejectionDetails::parse(&entry.details.0) {
16929 Ok(value) => value,
16930 Err(message) => {
16931 return SyncOutcome::Failed {
16932 details: None,
16933 error_code: "sync.invalid_request".to_owned(),
16934 message,
16935 };
16936 }
16937 };
16938 details.insert(entry.op_index, parsed);
16939 }
16940 }
16941 Frame::PushResult {
16942 client_commit_id,
16943 status,
16944 results,
16945 ..
16946 } if pushed_ids.contains(client_commit_id.as_str())
16947 && Self::push_result_is_final(*status, results) =>
16948 {
16949 last_final_push_result_id = Some(client_commit_id.clone());
16950 }
16951 _ => {}
16952 }
16953 }
16954 let mut frames = response.frames.into_iter().peekable();
16955 if response.wire_version < 2 {
16956 return SyncOutcome::Failed {
16957 details: None,
16958 error_code: "client.invalid_host_response".to_owned(),
16959 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
16960 };
16961 }
16962 match frames.next() {
16963 Some(Frame::RespHeader {
16964 required_schema_version,
16965 latest_schema_version,
16966 log_epoch,
16967 reset_required,
16968 }) => {
16969 let Some(log_epoch) = log_epoch else {
16970 return SyncOutcome::Failed {
16971 details: None,
16972 error_code: "client.invalid_host_response".to_owned(),
16973 message: "response header omits logEpoch".to_owned(),
16974 };
16975 };
16976 let Some(reset_required) = reset_required else {
16977 return SyncOutcome::Failed {
16978 details: None,
16979 error_code: "client.invalid_host_response".to_owned(),
16980 message: "response header omits resetRequired".to_owned(),
16981 };
16982 };
16983 if let Some(required) = required_schema_version {
16984 let floor = SchemaFloor {
16987 required_schema_version: Some(required),
16988 latest_schema_version,
16989 };
16990 self.set_schema_floor(Some(floor.clone()));
16991 report.schema_floor = Some(floor);
16992 return SyncOutcome::Ok(report);
16993 }
16994 if reset_required {
16995 if frames.next().is_some() {
16996 return SyncOutcome::Failed {
16997 details: None,
16998 error_code: "client.invalid_host_response".to_owned(),
16999 message: "log-epoch reset response contains body frames".to_owned(),
17000 };
17001 }
17002 match self.run_log_epoch_reset(&log_epoch) {
17003 Ok(resets) => {
17004 report.resets = resets;
17005 return SyncOutcome::Ok(report);
17006 }
17007 Err(message) => {
17008 return SyncOutcome::Failed {
17009 details: None,
17010 error_code: "sync.local_corrupt".to_owned(),
17011 message,
17012 };
17013 }
17014 }
17015 }
17016 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
17017 return SyncOutcome::Failed {
17018 details: None,
17019 error_code: "client.invalid_host_response".to_owned(),
17020 message: "server changed logEpoch without requiring a reset".to_owned(),
17021 };
17022 }
17023 }
17024 _ => {
17025 return SyncOutcome::Failed {
17026 details: None,
17027 error_code: "sync.invalid_request".to_owned(),
17028 message: "response does not start with RESP_HEADER".to_owned(),
17029 };
17030 }
17031 }
17032
17033 if let Err(message) = self.begin_coalesced_apply() {
17034 return SyncOutcome::Failed {
17035 details: None,
17036 error_code: "storage.failed".into(),
17037 message,
17038 };
17039 }
17040 let mut failure: Option<(String, String)> = None;
17041 while let Some(frame) = frames.next() {
17042 match frame {
17043 frame @ Frame::PushResult { .. } => {
17044 let successful = matches!(
17045 &frame,
17046 Frame::PushResult {
17047 status: PushStatus::Applied | PushStatus::Cached,
17048 ..
17049 }
17050 );
17051 let mut results = vec![frame];
17052 if successful {
17053 while matches!(
17054 frames.peek(),
17055 Some(Frame::PushResult {
17056 status: PushStatus::Applied | PushStatus::Cached,
17057 ..
17058 })
17059 ) {
17060 if let Some(next) = frames.next() {
17061 results.push(next);
17062 }
17063 }
17064 }
17065 if self
17066 .handle_push_results(
17067 &results,
17068 &pushed_ids,
17069 &rejection_details_by_commit,
17070 &mut report,
17071 last_final_push_result_id.as_deref(),
17072 )
17073 .is_err()
17074 {
17075 failure = Some((
17076 "client.outcome_persistence_failed".to_owned(),
17077 "local commit outcome could not be persisted".to_owned(),
17078 ));
17079 break;
17080 }
17081 }
17082 Frame::PushResultDetails { .. } => {}
17083 Frame::SubStart {
17084 id,
17085 status,
17086 reason_code,
17087 effective_scopes,
17088 bootstrap: _,
17089 } => {
17090 let mut body = Vec::new();
17091 let mut sub_end: Option<(i64, Option<String>)> = None;
17092 for inner in frames.by_ref() {
17093 match inner {
17094 Frame::SubEnd {
17095 next_cursor,
17096 bootstrap_state,
17097 } => {
17098 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
17099 break;
17100 }
17101 Frame::Unknown { .. } => {}
17102 other => body.push(other),
17103 }
17104 }
17105 if let Err(SectionError::Abort(code, message)) = self.process_section(
17106 transport,
17107 &id,
17108 status,
17109 &reason_code,
17110 effective_scopes,
17111 body,
17112 sub_end,
17113 meta,
17114 &mut report,
17115 ) {
17116 failure = Some((code, message));
17117 break;
17118 }
17119 }
17120 Frame::Lease {
17121 lease_id,
17122 expires_at_ms,
17123 } => {
17124 self.set_lease_state(Some(LeaseState {
17127 lease_id: Some(lease_id),
17128 expires_at_ms: Some(expires_at_ms),
17129 error_code: None,
17130 }));
17131 }
17132 Frame::Error { code, message, .. } => {
17133 failure = Some((code, message));
17136 break;
17137 }
17138 Frame::Unknown { .. } => {}
17139 _ => {
17140 failure = Some((
17141 "sync.invalid_request".to_owned(),
17142 "unexpected frame in response".to_owned(),
17143 ));
17144 break;
17145 }
17146 }
17147 }
17148
17149 let applied = match failure {
17150 Some(failure) => Err(failure),
17151 None => self
17156 .rebuild_overlay_if_dirty()
17157 .and_then(|()| self.drain_pending_evictions())
17158 .map_err(|message| ("storage.failed".to_owned(), message)),
17159 };
17160 if let Err(message) = self.finish_coalesced_apply() {
17163 return SyncOutcome::Failed {
17164 details: None,
17165 error_code: "storage.failed".into(),
17166 message,
17167 };
17168 }
17169 if let Err((error_code, message)) = applied {
17170 return SyncOutcome::Failed {
17171 details: None,
17172 error_code,
17173 message,
17174 };
17175 }
17176 self.ack_after_pull(transport);
17177 if self.upgrading && report.bootstrapping.is_empty() {
17180 self.set_upgrading(false);
17181 }
17182 SyncOutcome::Ok(report)
17183 }
17184
17185 fn handle_push_results(
17188 &mut self,
17189 frames: &[Frame],
17190 pushed_ids: &HashSet<&str>,
17191 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
17192 report: &mut SyncReport,
17193 last_final_push_result_id: Option<&str>,
17194 ) -> Result<(), String> {
17195 self.begin_observation("syncular_push_results")?;
17196 let previous_failed = self.failed_commits.clone();
17197 let previous_sync_needed = self.sync_needed;
17198 let conflict_count = self.conflicts.len();
17199 let rejection_count = self.rejections.len();
17200 let applied_count = report.applied.len();
17201 let rejected_count = report.rejected.len();
17202 let retryable_count = report.retryable.len();
17203 let was_dirty = self.overlay_dirty.snapshot();
17204 let mut removed = Vec::new();
17205 let mut batch = ChangeAccumulator::default();
17206 let persisted = (|| {
17207 for frame in frames {
17208 let Frame::PushResult {
17209 client_commit_id,
17210 status,
17211 results,
17212 commit_seq,
17213 } = frame
17214 else {
17215 return Err("unexpected acknowledgement frame".to_owned());
17216 };
17217 if !pushed_ids.contains(client_commit_id.as_str()) {
17218 continue;
17219 }
17220 let Some(index) = self
17221 .outbox
17222 .iter()
17223 .position(|commit| commit.client_commit_id == *client_commit_id)
17224 else {
17225 continue;
17226 };
17227 let status = *status;
17228 let rejection_details = rejection_details_by_commit.get(client_commit_id);
17229 let operations = self.outbox[index].ops.clone();
17230 match status {
17231 PushStatus::Applied | PushStatus::Cached => {
17232 let sequence =
17235 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
17236 let affected: HashSet<_> = operations
17237 .iter()
17238 .map(|op| (op.table.as_str(), op.row_id.as_str()))
17239 .collect();
17240 let prior = self.acknowledged_operations(Some(&affected), None)?;
17241 for (op_index, operation) in operations.iter().enumerate() {
17242 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
17243 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17244 if delivered {
17245 continue;
17246 }
17247 let table = self
17248 .schema
17249 .table(&operation.table)
17250 .ok_or_else(|| "sync.unknown_table".to_owned())?;
17251 let mut intended: Map<String, Value> = self
17252 .stored_row(table, &operation.row_id, true)?
17253 .map(|(row, _)| {
17254 table
17255 .columns
17256 .iter()
17257 .zip(&row)
17258 .map(|(column, value)| {
17259 (column.name.clone(), column_value_to_json(value))
17260 })
17261 .collect()
17262 })
17263 .unwrap_or_default();
17264 let mut scope_image = intended.clone();
17265 for op in prior
17266 .iter()
17267 .chain(
17268 self.failed_commits
17269 .iter()
17270 .flat_map(|(commit, _)| &commit.ops),
17271 )
17272 .chain(
17273 self.outbox
17274 .iter()
17275 .take(index)
17276 .flat_map(|commit| &commit.ops),
17277 )
17278 .chain(operations.iter().take(op_index + 1))
17279 {
17280 if op.table != operation.table || op.row_id != operation.row_id {
17281 continue;
17282 }
17283 if let Some(values) = &op.values {
17284 intended.extend(values.clone());
17285 } else {
17286 scope_image = intended.clone();
17287 intended.clear();
17288 }
17289 }
17290 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
17291 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
17292 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
17293 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
17294 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17295 }
17296 let journal_results = results
17297 .iter()
17298 .map(|result| {
17299 let op_index = match result {
17300 OpResult::Applied { op_index }
17301 | OpResult::Conflict { op_index, .. }
17302 | OpResult::Error { op_index, .. } => *op_index,
17303 };
17304 CommitOperationOutcome::Applied { op_index }
17305 })
17306 .collect::<Vec<_>>();
17307 let outcome_status = if status == PushStatus::Applied {
17308 CommitOutcomeStatus::Applied
17309 } else {
17310 CommitOutcomeStatus::Cached
17311 };
17312 let persisted = self
17313 .persist_commit_outcome(
17314 client_commit_id,
17315 outcome_status,
17316 &journal_results,
17317 None,
17318 )
17319 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
17320 persisted?;
17321 for operation in &operations {
17322 self.overlay_dirty.table(&operation.table);
17323 }
17324 self.sync_needed = true;
17325 batch.status = true;
17326 batch.outcomes = true;
17327 }
17328 PushStatus::Rejected => {
17329 if results.iter().any(|result| {
17330 matches!(
17331 result,
17332 OpResult::Error {
17333 code,
17334 retryable: true,
17335 ..
17336 } if code == "sync.idempotency_cache_miss"
17337 )
17338 }) {
17339 report.retryable.push(client_commit_id.to_owned());
17342 continue;
17343 }
17344
17345 let mut journal_results = Vec::with_capacity(results.len());
17346 let mut conflicts = Vec::new();
17347 let mut rejections = Vec::new();
17348 for result in results {
17349 match result {
17350 OpResult::Applied { op_index } => {
17351 journal_results.push(CommitOperationOutcome::Applied {
17352 op_index: *op_index,
17353 });
17354 }
17355 OpResult::Conflict {
17356 op_index,
17357 code,
17358 message,
17359 server_version,
17360 server_row,
17361 conflict_columns: conflict_columns_bitmap,
17362 } => {
17363 let operation = operations
17364 .get(*op_index as usize)
17365 .map(CommitOperation::from);
17366 let (table, row_id) = operation
17367 .as_ref()
17368 .map(|op| (op.table.clone(), op.row_id.clone()))
17369 .unwrap_or_default();
17370 let server_row_json = self
17371 .schema
17372 .table(&table)
17373 .and_then(|t| {
17374 decode_row_bytes(t, server_row, &self.encryption)
17375 .ok()
17376 .map(|row| (t, row))
17377 })
17378 .map(|(t, row)| {
17379 let mut map = Map::new();
17380 for (i, column) in t.columns.iter().enumerate() {
17381 map.insert(
17382 column.name.clone(),
17383 column_value_to_json(
17384 row.get(i).unwrap_or(&None),
17385 ),
17386 );
17387 }
17388 map
17389 })
17390 .unwrap_or_default();
17391 let conflict_columns = self
17394 .schema
17395 .table(&table)
17396 .map(|t| {
17397 t.columns
17398 .iter()
17399 .enumerate()
17400 .filter(|(index, _)| {
17401 conflict_columns_bitmap
17402 .get(index / 8)
17403 .is_some_and(|byte| {
17404 byte & (1 << (index % 8)) != 0
17405 })
17406 })
17407 .map(|(_, column)| column.name.clone())
17408 .collect::<Vec<_>>()
17409 })
17410 .unwrap_or_default();
17411 let conflict = ConflictRecord {
17412 client_commit_id: client_commit_id.to_owned(),
17413 op_index: *op_index,
17414 table,
17415 row_id,
17416 code: code.clone(),
17417 message: message.clone(),
17418 server_version: *server_version,
17419 server_row: server_row_json,
17420 conflict_columns,
17421 operation,
17422 };
17423 journal_results.push(CommitOperationOutcome::Conflict {
17424 conflict: conflict.clone(),
17425 });
17426 conflicts.push(conflict);
17427 }
17428 OpResult::Error {
17429 op_index,
17430 code,
17431 message,
17432 retryable,
17433 } => {
17434 let rejection = RejectionRecord {
17435 client_commit_id: client_commit_id.to_owned(),
17436 op_index: *op_index,
17437 code: code.clone(),
17438 message: message.clone(),
17439 retryable: *retryable,
17440 details: rejection_details
17441 .and_then(|details| details.get(op_index))
17442 .cloned(),
17443 operation: operations
17444 .get(*op_index as usize)
17445 .map(CommitOperation::from),
17446 };
17447 journal_results.push(CommitOperationOutcome::Error {
17448 rejection: rejection.clone(),
17449 });
17450 rejections.push(rejection);
17451 }
17452 }
17453 }
17454 if self.retain_failed_commits {
17455 self.retain_failed_commit(client_commit_id, &operations)?;
17456 }
17457 let outcome_status = if conflicts.is_empty() {
17458 CommitOutcomeStatus::Rejected
17459 } else {
17460 CommitOutcomeStatus::Conflict
17461 };
17462 let persisted = self
17463 .persist_commit_outcome(
17464 client_commit_id,
17465 outcome_status,
17466 &journal_results,
17467 Some(&operations),
17468 )
17469 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
17470 persisted?;
17471 batch.conflicts = !conflicts.is_empty();
17472 batch.rejections = !rejections.is_empty();
17473 batch.status = true;
17474 batch.outcomes = true;
17475 self.conflicts.extend(conflicts);
17476 self.rejections.extend(rejections);
17477 for operation in &operations {
17478 self.overlay_dirty.table(&operation.table);
17479 }
17480 }
17481 }
17482 removed.push((index, self.outbox.remove(index)));
17483 if batch.status {
17484 for operation in &operations {
17485 if !self.record_row_scopes(
17486 &mut batch,
17487 &operation.table,
17488 &operation.row_id,
17489 false,
17490 ) {
17491 batch.table(&operation.table);
17492 }
17493 }
17494 }
17495 match status {
17496 PushStatus::Applied | PushStatus::Cached => {
17497 report.applied.push(client_commit_id.to_owned())
17498 }
17499 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
17500 }
17501 }
17502 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
17503 self.prune_commit_outcomes()?;
17504 }
17505 self.rebuild_overlay_if_dirty()?;
17506 self.finish_observation("syncular_push_results", batch)
17507 })();
17508 if let Err(error) = persisted {
17509 self.rollback_observation("syncular_push_results");
17510 self.failed_commits = previous_failed;
17511 self.sync_needed = previous_sync_needed;
17512 for (index, commit) in removed.into_iter().rev() {
17513 self.outbox.insert(index, commit);
17514 }
17515 self.conflicts.truncate(conflict_count);
17516 self.rejections.truncate(rejection_count);
17517 self.overlay_dirty.restore(was_dirty.clone());
17518 report.applied.truncate(applied_count);
17519 report.rejected.truncate(rejected_count);
17520 report.retryable.truncate(retryable_count);
17521 return Err(error);
17522 }
17523 if !previous_sync_needed && self.sync_needed {
17524 self.sync_intent_queue.push_back(SyncIntent::Interactive);
17525 }
17526 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
17527 Ok(())
17528 }
17529
17530 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
17531 status != PushStatus::Rejected
17532 || !results.iter().any(|result| {
17533 matches!(
17534 result,
17535 OpResult::Error {
17536 code,
17537 retryable: true,
17538 ..
17539 } if code == "sync.idempotency_cache_miss"
17540 )
17541 })
17542 }
17543
17544 #[allow(clippy::too_many_arguments)]
17547 fn process_section(
17548 &mut self,
17549 transport: &mut dyn Transport,
17550 id: &str,
17551 status: SubStatus,
17552 reason_code: &str,
17553 effective_scopes: Vec<(String, Vec<String>)>,
17554 body: Vec<Frame>,
17555 sub_end: Option<(i64, Option<String>)>,
17556 meta: &RequestMeta,
17557 report: &mut SyncReport,
17558 ) -> Result<(), SectionError> {
17559 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
17560 return Ok(()); };
17562 if status != SubStatus::Active && sub_end.is_none() {
17563 return Err(SectionError::Abort(
17564 "sync.invalid_request".into(),
17565 "subscription section without SUB_END".into(),
17566 ));
17567 }
17568 match status {
17569 SubStatus::Revoked => {
17570 self.begin_observation("syncular_revocation")
17571 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17572 let previous = self.subs[sub_index].clone();
17573 let was_dirty = self.overlay_dirty.snapshot();
17574 let previous_outbox = self.outbox.clone();
17575 let previous_failed = self.failed_commits.clone();
17576 let previous_rejections = self.rejections.clone();
17577 let previous_conflicts = self.conflicts.clone();
17578 let revoked_count = report.revoked.len();
17579 let failed_count = report.failed.len();
17580 let result = (|| {
17581 let mut batch = ChangeAccumulator::default();
17582 let registered = self.window_unit_by_sub_id(id);
17583 let (table, effective) = {
17585 let sub = &self.subs[sub_index];
17586 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17587 };
17588 let scoped_table = self.schema.table(&table).cloned().ok_or_else(|| {
17589 SectionError::Abort(
17590 "sync.unknown_table".to_owned(),
17591 "revoked scope references an unknown table".to_owned(),
17592 )
17593 })?;
17594 let dropped = self
17595 .drop_failed_commits(|name, values| {
17596 name == table
17597 && !effective.is_empty()
17598 && effective.iter().all(|(variable, allowed)| {
17599 scoped_table
17600 .scope_column(variable)
17601 .and_then(|column| values.get(column))
17602 .and_then(Value::as_str)
17603 .is_some_and(|value| {
17604 allowed.iter().any(|allowed| allowed == value)
17605 })
17606 })
17607 })
17608 .map_err(|message| {
17609 SectionError::Abort("storage.failed".to_owned(), message)
17610 })?;
17611 for commit in dropped {
17612 batch.conflicts = true;
17613 batch.rejections = true;
17614 batch.outcomes = true;
17615 for operation in commit.ops {
17616 batch.table(&operation.table);
17617 }
17618 }
17619 let purged = self.purge_scope_rows(&table, &effective);
17620 match purged {
17621 Ok(()) => {
17622 self.record_scope_map(&mut batch, &table, &effective);
17623 let sub = &mut self.subs[sub_index];
17624 sub.state = SubState::Revoked;
17625 sub.reason_code = Some(if reason_code.is_empty() {
17626 "sync.scope_revoked".to_owned()
17627 } else {
17628 reason_code.to_owned()
17629 });
17630 report.revoked.push(id.to_owned());
17631 let doomed_effective = effective;
17632 let sub_table = table;
17633 self.persist_sub(&self.subs[sub_index].clone())
17634 .map_err(|message| {
17635 SectionError::Abort("storage.failed".into(), message)
17636 })?;
17637 let dropped = self
17638 .drop_doomed_outbox(&sub_table, &doomed_effective)
17639 .map_err(|message| {
17640 SectionError::Abort("storage.failed".to_owned(), message)
17641 })?;
17642 if dropped {
17643 batch.status = true;
17644 batch.rejections = true;
17645 batch.outcomes = true;
17646 }
17647 self.delete_unreferenced_cached_blobs().map_err(|message| {
17650 SectionError::Abort("storage.failed".into(), message)
17651 })?;
17652 }
17653 Err(()) => {
17654 let sub = &mut self.subs[sub_index];
17657 sub.state = SubState::Failed;
17658 sub.reason_code = Some("sync.scope_revoked".to_owned());
17659 report.failed.push(id.to_owned());
17660 self.persist_sub(&self.subs[sub_index].clone())
17661 .map_err(|message| {
17662 SectionError::Abort("storage.failed".into(), message)
17663 })?;
17664 }
17665 }
17666 if let Some((base_key, unit)) = registered {
17667 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17668 }
17669 self.rebuild_overlay_if_dirty()
17670 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17671 self.finish_observation("syncular_revocation", batch)
17672 .map_err(|message| {
17673 SectionError::Abort("storage.failed".to_owned(), message)
17674 })?;
17675 Ok(())
17676 })();
17677 if result.is_err() {
17678 self.rollback_observation("syncular_revocation");
17679 self.subs[sub_index] = previous;
17680 self.overlay_dirty.restore(was_dirty.clone());
17681 self.outbox = previous_outbox;
17682 self.failed_commits = previous_failed;
17683 self.rejections = previous_rejections;
17684 self.conflicts = previous_conflicts;
17685 report.revoked.truncate(revoked_count);
17686 report.failed.truncate(failed_count);
17687 }
17688 result
17689 }
17690 SubStatus::Reset => {
17691 self.begin_observation("syncular_reset")
17692 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17693 let previous = self.subs[sub_index].clone();
17694 let was_dirty = self.overlay_dirty.snapshot();
17695 let reset_count = report.resets.len();
17696 let result = (|| {
17697 let mut batch = ChangeAccumulator::default();
17698 let registered = self.window_unit_by_sub_id(id);
17699 let sub = &mut self.subs[sub_index];
17702 sub.cursor = -1;
17703 sub.bootstrap_state = None;
17704 sub.reason_code = Some(if reason_code.is_empty() {
17705 "sync.cursor_expired".to_owned()
17706 } else {
17707 reason_code.to_owned()
17708 });
17709 report.resets.push(id.to_owned());
17710 self.persist_sub(&self.subs[sub_index].clone())
17711 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17712 if let Some((base_key, unit)) = registered {
17713 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17714 }
17715 self.finish_observation("syncular_reset", batch)
17716 .map_err(|message| {
17717 SectionError::Abort("storage.failed".to_owned(), message)
17718 })?;
17719 Ok(())
17720 })();
17721 if result.is_err() {
17722 self.rollback_observation("syncular_reset");
17723 self.subs[sub_index] = previous;
17724 self.overlay_dirty.restore(was_dirty.clone());
17725 report.resets.truncate(reset_count);
17726 }
17727 result
17728 }
17729 SubStatus::Active => {
17730 let fresh = meta
17731 .fresh
17732 .iter()
17733 .find(|(fid, _)| fid == id)
17734 .map(|(_, fresh)| *fresh)
17735 .unwrap_or(false);
17736 let previous = self.subs[sub_index].clone();
17737 self.subs[sub_index].effective = Some(effective_scopes);
17738 let outcome = self
17739 .apply_section_body(transport, sub_index, body, fresh, meta, report)
17740 .and_then(|()| {
17741 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
17742 SectionError::Abort(
17743 "sync.invalid_request".into(),
17744 "subscription section without SUB_END".into(),
17745 )
17746 })?;
17747 self.apply_sub_end(sub_index, cursor, bootstrap_state)
17748 .map_err(|message| {
17749 SectionError::Abort("storage.failed".into(), message)
17750 })
17751 });
17752 match outcome {
17753 Ok(()) => {
17754 if self.subs[sub_index].bootstrap_state.is_some() {
17755 report.bootstrapping.push(id.to_owned());
17756 }
17757 Ok(())
17758 }
17759 Err(SectionError::FailClosed) => {
17760 self.subs[sub_index] = previous.clone();
17761 self.begin_observation("syncular_section_failure")
17762 .map_err(|message| {
17763 SectionError::Abort("storage.failed".into(), message)
17764 })?;
17765 let persisted = (|| {
17766 let mut batch = ChangeAccumulator::default();
17767 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
17768 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17769 }
17770 self.subs[sub_index].state = SubState::Failed;
17771 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
17772 self.persist_sub(&self.subs[sub_index])?;
17773 self.finish_observation("syncular_section_failure", batch)
17774 })();
17775 if let Err(message) = persisted {
17776 self.rollback_observation("syncular_section_failure");
17777 self.subs[sub_index] = previous;
17778 return Err(SectionError::Abort("storage.failed".into(), message));
17779 }
17780 report.failed.push(id.to_owned());
17781 Ok(())
17782 }
17783 Err(error) => {
17784 self.subs[sub_index] = previous;
17787 Err(error)
17788 }
17789 }
17790 }
17791 }
17792 }
17793
17794 fn apply_sub_end(
17795 &mut self,
17796 sub_index: usize,
17797 cursor: i64,
17798 bootstrap_state: Option<String>,
17799 ) -> Result<(), String> {
17800 #[cfg(feature = "bench-internals")]
17801 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
17802 let previous = self.subs[sub_index].clone();
17803 self.begin_observation("syncular_sub_end")?;
17804 let persisted = (|| {
17805 let mut batch = ChangeAccumulator::default();
17806 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
17807 && cursor >= 0
17808 && bootstrap_state.is_none();
17809 if completed {
17810 let table = self
17811 .schema
17812 .table(&previous.table)
17813 .ok_or_else(|| "sync.unknown_table".to_owned())?;
17814 let effective = previous.effective.as_deref().unwrap_or(&[]);
17815 let mut statement = self.conn.prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
17816 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17817 let rows = statement
17818 .query_map(rusqlite::params![previous.table, cursor], |row| {
17819 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
17820 })
17821 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17822 let mut covered = HashSet::new();
17823 for row in rows {
17824 let (id, intent) =
17825 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17826 let values: Map<String, Value> = serde_json::from_str(&intent)
17827 .map_err(|_| "sync.local_corrupt".to_owned())?;
17828 if !effective.is_empty()
17829 && effective.iter().all(|(variable, allowed)| {
17830 table
17831 .scope_column(variable)
17832 .and_then(|column| values.get(column))
17833 .is_some_and(|value| {
17834 allowed.contains(
17835 &value
17836 .as_str()
17837 .map_or_else(|| value.to_string(), str::to_owned),
17838 )
17839 })
17840 })
17841 {
17842 covered.insert(id);
17843 }
17844 }
17845 drop(statement);
17846 for id in covered {
17847 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![previous.table, id, cursor])
17848 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17849 self.overlay_dirty.table(&previous.table);
17850 batch.table(&previous.table);
17851 }
17852 self.rebuild_overlay_if_dirty()?;
17853 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
17854 batch.window(&base_key, &previous.table, &unit);
17855 }
17856 }
17857 let sub = &mut self.subs[sub_index];
17858 sub.cursor = cursor;
17859 sub.bootstrap_state = bootstrap_state;
17860 sub.reason_code = None;
17863 sub.synced_once = true;
17864 self.persist_sub(&self.subs[sub_index])?;
17865 self.finish_observation("syncular_sub_end", batch)
17866 })();
17867 if persisted.is_err() {
17868 self.rollback_observation("syncular_sub_end");
17869 self.subs[sub_index] = previous;
17870 }
17871 persisted
17872 }
17873
17874 fn apply_section_body(
17875 &mut self,
17876 transport: &mut dyn Transport,
17877 sub_index: usize,
17878 body: Vec<Frame>,
17879 fresh: bool,
17880 meta: &RequestMeta,
17881 report: &mut SyncReport,
17882 ) -> Result<(), SectionError> {
17883 let mut saw_segment = false;
17884 for frame in body {
17885 match frame {
17886 Frame::Commit {
17887 tables,
17888 changes,
17889 commit_seq,
17890 ..
17891 } => {
17892 self.apply_commit_frame(&tables, &changes, Some(commit_seq))
17893 .map_err(|(c, m)| SectionError::Abort(c, m))?;
17894 report.commits_applied += 1;
17895 }
17896 Frame::SegmentInline { payload } => {
17897 let segment = decode_rows_segment(&payload)
17898 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
17899 self.progress.update(|p| {
17900 p.phase = ProgressPhase::Import;
17901 p.subscription_id = Some(self.subs[sub_index].id.clone());
17902 p.table = Some(self.subs[sub_index].table.clone());
17903 p.segment_id = None;
17904 p.bytes_received = payload.len() as u64;
17905 p.bytes_total = Some(payload.len() as u64);
17906 p.rows_processed = 0;
17907 p.rows_total =
17908 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
17909 });
17910 let first = !saw_segment;
17911 saw_segment = true;
17912 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17913 report.segment_rows_applied += applied;
17914 }
17915 Frame::SegmentRef {
17916 segment_id,
17917 byte_length,
17918 media_type,
17919 table,
17920 row_count,
17921 as_of_commit_seq,
17922 scope_digest,
17923 row_cursor,
17924 next_row_cursor,
17925 url,
17926 url_expires_at_ms,
17927 ..
17928 } => {
17929 let advertised = match media_type {
17932 MediaType::Rows => {
17933 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
17934 || meta.accept & ACCEPT_INLINE_ROWS != 0
17935 }
17936 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
17937 };
17938 if !advertised {
17939 return Err(SectionError::Abort(
17940 "sync.invalid_request".to_owned(),
17941 format!(
17942 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
17943 media_type.name()
17944 ),
17945 ));
17946 }
17947 self.progress.update(|p| {
17948 p.phase = ProgressPhase::Download;
17949 p.subscription_id = Some(self.subs[sub_index].id.clone());
17950 p.table = Some(table.clone());
17951 p.segment_id = Some(segment_id.clone());
17952 p.bytes_received = 0;
17953 p.bytes_total = u64::try_from(byte_length).ok();
17954 p.rows_processed = 0;
17955 p.rows_total = u64::try_from(row_count).ok();
17956 });
17957 let observer = self.progress.clone();
17958 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
17959 let bytes = if let Some(url) = url {
17960 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
17965 return Err(SectionError::Abort(
17966 "sync.invalid_request".to_owned(),
17967 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
17968 .to_owned(),
17969 ));
17970 }
17971 if url_expires_at_ms.is_some_and(|exp| {
17973 exp <= transport
17974 .segment_fetch_started_at(&url)
17975 .unwrap_or_else(|| self.clock_now_ms())
17976 }) {
17977 return Err(SectionError::Abort(
17978 "sync.segment_expired".to_owned(),
17979 format!(
17980 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
17981 ),
17982 ));
17983 }
17984 transport
17985 .fetch_url(&url, &mut on_progress)
17986 .map_err(|e| self.segment_fetch_failed(e))?
17987 } else {
17988 let requested_scopes_json =
17989 canonical_scope_json(&self.subs[sub_index].requested);
17990 transport
17991 .download_segment(
17992 &SegmentRequest {
17993 segment_id: segment_id.clone(),
17994 table,
17995 requested_scopes_json,
17996 },
17997 &mut on_progress,
17998 )
17999 .map_err(|e| self.segment_fetch_failed(e))?
18000 };
18001 on_progress(bytes.len() as u64);
18003 let digest = Sha256::digest(&bytes);
18004 let expected = segment_id
18005 .strip_prefix("sha256:")
18006 .unwrap_or(segment_id.as_str());
18007 if bytes_to_hex(&digest) != expected {
18008 self.schedule_background_retry();
18010 return Err(SectionError::Abort(
18011 "sync.invalid_request".to_owned(),
18012 "segment bytes do not match the content address (§5.1)".to_owned(),
18013 ));
18014 }
18015 self.progress.update(|p| p.phase = ProgressPhase::Import);
18016 if media_type == MediaType::Sqlite {
18017 if row_cursor.is_some() || next_row_cursor.is_some() {
18020 return Err(SectionError::Abort(
18021 "sync.invalid_request".to_owned(),
18022 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
18023 .to_owned(),
18024 ));
18025 }
18026 let first = !saw_segment;
18027 saw_segment = true;
18028 let applied = self.apply_sqlite_segment(
18029 sub_index,
18030 &bytes,
18031 fresh && first,
18032 row_count,
18033 as_of_commit_seq,
18034 &scope_digest,
18035 )?;
18036 report.segment_rows_applied += applied;
18037 } else {
18038 let segment = decode_rows_segment(&bytes).map_err(|e| {
18039 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
18040 })?;
18041 let first = row_cursor.is_none();
18042 saw_segment = true;
18043 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
18044 report.segment_rows_applied += applied;
18045 }
18046 }
18047 Frame::Unknown { .. } => {}
18048 Frame::Error { code, message, .. } => {
18049 return Err(SectionError::Abort(code, message))
18050 }
18051 _ => {
18052 return Err(SectionError::Abort(
18053 "sync.invalid_request".to_owned(),
18054 "unexpected frame inside a subscription section".to_owned(),
18055 ));
18056 }
18057 }
18058 }
18059 Ok(())
18060 }
18061
18062 fn apply_commit_frame(
18064 &mut self,
18065 tables: &[String],
18066 changes: &[ssp2::model::Change],
18067 commit_seq: Option<i64>,
18068 ) -> Result<(), (String, String)> {
18069 #[cfg(feature = "bench-internals")]
18070 let _phase = self.benchmark_phases.start(Phase::CommitApply);
18071 let was_dirty = self.overlay_dirty.snapshot();
18072 self.coalesce_unit(changes.len())
18073 .and_then(|()| self.begin_observation("syncular_commit"))
18074 .map_err(|message| ("storage.failed".into(), message))?;
18075 let applied = (|| {
18076 let mut batch = ChangeAccumulator::default();
18077 self.record_commit_changes(&mut batch, tables, changes);
18078 if let Some(sequence) = commit_seq {
18079 for change in changes {
18080 let table = tables.get(change.table_index as usize).ok_or_else(|| {
18081 (
18082 "sync.invalid_request".to_owned(),
18083 "change tableIndex out of range".to_owned(),
18084 )
18085 })?;
18086 if self
18087 .outbox
18088 .iter()
18089 .flat_map(|commit| &commit.ops)
18090 .any(|op| op.table == *table && op.row_id == change.row_id)
18091 {
18092 self.conn.execute("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)", rusqlite::params![table, change.row_id, sequence])
18093 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
18094 }
18095 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![table, change.row_id, sequence])
18096 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
18097 }
18098 }
18099 self.apply_commit_changes(tables, changes)?;
18100 self.rebuild_overlay_if_dirty()
18101 .map_err(|message| ("storage.failed".into(), message))?;
18102 self.finish_observation("syncular_commit", batch)
18103 .map_err(|message| ("storage.failed".into(), message))
18104 })();
18105 if applied.is_err() {
18106 self.rollback_observation("syncular_commit");
18107 self.overlay_dirty.restore(was_dirty.clone());
18108 }
18109 applied
18110 }
18111
18112 fn apply_commit_changes(
18113 &mut self,
18114 tables: &[String],
18115 changes: &[ssp2::model::Change],
18116 ) -> Result<(), (String, String)> {
18117 let mirror_visible = !self.overlay_dirty.get()
18122 && self.failed_commits.is_empty()
18123 && ((self.outbox.is_empty() && !self.has_acknowledged_rows())
18124 || changes.iter().all(|change| {
18125 tables
18126 .get(change.table_index as usize)
18127 .and_then(|name| self.schema.table(name))
18128 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
18129 }));
18130 for change in changes {
18131 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
18132 (
18133 "sync.invalid_request".to_owned(),
18134 "change tableIndex out of range".to_owned(),
18135 )
18136 })?;
18137 let table = self.schema.table(table_name).ok_or_else(|| {
18138 (
18139 "sync.schema_mismatch".to_owned(),
18140 format!("change targets unknown table {table_name:?}"),
18141 )
18142 })?;
18143 match change.op {
18144 Op::Upsert => {
18145 let payload = change.row.as_ref().ok_or_else(|| {
18146 (
18147 "sync.invalid_request".to_owned(),
18148 "upsert change without row payload".to_owned(),
18149 )
18150 })?;
18151 let row = {
18153 #[cfg(feature = "bench-internals")]
18154 let _phase = self.benchmark_phases.start(Phase::RowDecode);
18155 decode_row_bytes(table, payload, &self.encryption)
18156 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
18157 };
18158 let version = change.row_version.unwrap_or(0);
18159 let table_name = table.name.clone();
18160 self.write_base_row(&table_name, &row, version)
18161 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
18162 if mirror_visible {
18163 self.write_row(&visible_table(&table_name), &table_name, &row, version)
18164 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
18165 }
18166 }
18167 Op::Delete => {
18168 self.delete_base_row(table_name, &change.row_id)
18169 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
18170 if mirror_visible {
18171 let sql = format!(
18172 "DELETE FROM {} WHERE {}",
18173 visible_table(table_name),
18174 row_id_predicate(table)
18175 );
18176 self.conn
18177 .prepare_cached(&sql)
18178 .and_then(|mut statement| {
18179 statement.execute(rusqlite::params![change.row_id])
18180 })
18181 .map_err(|error| {
18182 ("sync.invalid_request".to_owned(), error.to_string())
18183 })?;
18184 }
18185 }
18186 }
18187 }
18188 if mirror_visible {
18189 {
18190 let changed_rows: HashSet<(&str, &str)> = changes
18191 .iter()
18192 .map(|change| {
18193 (
18194 tables[change.table_index as usize].as_str(),
18195 change.row_id.as_str(),
18196 )
18197 })
18198 .collect();
18199 let acknowledged = self
18200 .acknowledged_operations(Some(&changed_rows), None)
18201 .map_err(|error| ("storage.failed".to_owned(), error))?;
18202 self.apply_outbox_ops(&acknowledged)
18203 .map_err(|message| ("storage.failed".into(), message))?;
18204 self.apply_outbox_ops(
18205 self.outbox
18206 .iter()
18207 .flat_map(|commit| &commit.ops)
18208 .filter(|op| {
18209 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
18210 }),
18211 )
18212 .map_err(|message| ("storage.failed".into(), message))?;
18213 }
18214 self.overlay_dirty.set(false);
18215 }
18216 Ok(())
18217 }
18218
18219 fn apply_segment(
18224 &mut self,
18225 sub_index: usize,
18226 segment: &RowsSegment,
18227 first_fresh_page: bool,
18228 ) -> Result<u32, SectionError> {
18229 let (sub_table, effective) = {
18230 let sub = &self.subs[sub_index];
18231 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
18232 };
18233 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
18234 SectionError::Abort(
18235 "sync.schema_mismatch".to_owned(),
18236 format!("subscription table {sub_table:?} is not in the client schema"),
18237 )
18238 })?;
18239 let matches = segment.table == table.name
18244 && segment.schema_version == self.schema.version
18245 && segment.columns.len() == table.wire_columns.len()
18246 && segment
18247 .columns
18248 .iter()
18249 .zip(table.wire_columns.iter())
18250 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
18251 if !matches {
18252 return Err(SectionError::Abort(
18253 "sync.schema_mismatch".to_owned(),
18254 "segment column table does not match the generated schema (§5.2)".to_owned(),
18255 ));
18256 }
18257 let mut applied = 0u32;
18258 let empty = Vec::new();
18259 let blocks = if segment.blocks.is_empty() {
18260 std::slice::from_ref(&empty)
18261 } else {
18262 &segment.blocks
18263 };
18264 for (index, block) in blocks.iter().enumerate() {
18265 let clear = first_fresh_page && index == 0;
18266 if !clear && block.is_empty() {
18267 continue;
18268 }
18269 let was_dirty = self.overlay_dirty.snapshot();
18270 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
18271 && self
18272 .outbox
18273 .iter()
18274 .flat_map(|commit| &commit.ops)
18275 .any(|op| op.table == table.name);
18276 let mirror_visible = !reconcile_pending
18277 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
18278 self.coalesce_unit(block.len())
18279 .and_then(|()| self.begin_observation("syncular_segment_block"))
18280 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18281 let outcome = (|| {
18282 self.rebuild_overlay_if_dirty()
18283 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18284 let mut changed_keys = Vec::new();
18285 let mut batch = ChangeAccumulator::default();
18286 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
18287 batch.table(&table.name);
18288 }
18289 if clear {
18290 self.purge_scope_rows(&table.name, &effective)
18291 .map_err(|()| SectionError::FailClosed)?;
18292 }
18293 for (row_index, row) in block.iter().enumerate() {
18294 let decrypted;
18295 let values = if table.has_encrypted_columns() {
18296 let mut values = row.values.clone();
18297 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
18298 .map_err(|message| {
18299 SectionError::Abort("client.decrypt_failed".into(), message)
18300 })?;
18301 decrypted = values;
18302 &decrypted
18303 } else {
18304 &row.values
18305 };
18306 self.write_base_row(&table.name, values, row.server_version)
18307 .map_err(|message| {
18308 SectionError::Abort("sync.invalid_request".into(), message)
18309 })?;
18310 if reconcile_pending {
18311 changed_keys.push(
18312 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
18313 |message| SectionError::Abort("storage.failed".into(), message),
18314 )?,
18315 );
18316 }
18317 if mirror_visible {
18318 self.write_row(
18319 &visible_table(&table.name),
18320 &table.name,
18321 values,
18322 row.server_version,
18323 )
18324 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18325 }
18326 let processed = u64::from(applied) + row_index as u64 + 1;
18327 if processed.is_multiple_of(1024) {
18328 self.progress.update(|p| p.rows_processed = processed);
18329 }
18330 }
18331 if reconcile_pending {
18332 self.reconcile_imported_rows(&table, &mut changed_keys)
18333 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18334 }
18335 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
18336 self.apply_outbox_ops(
18337 self.outbox
18338 .iter()
18339 .flat_map(|commit| &commit.ops)
18340 .filter(|op| op.table == table.name),
18341 )
18342 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18343 self.overlay_dirty.set(false);
18344 } else {
18345 self.rebuild_overlay_if_dirty()
18346 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18347 }
18348 self.finish_observation("syncular_segment_block", batch)
18349 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
18350 })();
18351 if let Err(error) = outcome {
18352 self.rollback_observation("syncular_segment_block");
18353 self.overlay_dirty.restore(was_dirty.clone());
18354 return Err(error);
18355 }
18356 applied += block.len() as u32;
18357 self.progress
18358 .update(|p| p.rows_processed = u64::from(applied));
18359 std::thread::yield_now();
18360 }
18361 self.progress
18362 .update(|p| p.rows_processed = u64::from(applied));
18363 Ok(applied)
18364 }
18365
18366 fn apply_sqlite_segment(
18374 &mut self,
18375 sub_index: usize,
18376 bytes: &[u8],
18377 first_fresh_page: bool,
18378 row_count: i64,
18379 as_of_commit_seq: i64,
18380 scope_digest: &str,
18381 ) -> Result<u32, SectionError> {
18382 let invalid = |detail: &str| {
18383 SectionError::Abort(
18384 "sync.invalid_request".to_owned(),
18385 format!("sqlite segment rejected: {detail} (§5.3)"),
18386 )
18387 };
18388 let (sub_table, effective) = {
18389 let sub = &self.subs[sub_index];
18390 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
18391 };
18392 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
18393 SectionError::Abort(
18394 "sync.schema_mismatch".to_owned(),
18395 format!("subscription table {sub_table:?} is not in the client schema"),
18396 )
18397 })?;
18398
18399 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
18400 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
18401 let img = match rusqlite::Connection::open_with_flags(
18402 &path,
18403 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
18404 ) {
18405 Ok(conn) => conn,
18406 Err(_) => {
18407 let _ = std::fs::remove_file(&path);
18408 return Err(invalid("bytes do not open as a SQLite database"));
18409 }
18410 };
18411 let outcome = self.apply_sqlite_image(
18412 &img,
18413 &table,
18414 first_fresh_page,
18415 &effective,
18416 row_count,
18417 as_of_commit_seq,
18418 scope_digest,
18419 );
18420 drop(img);
18421 let _ = std::fs::remove_file(&path);
18422 outcome
18423 }
18424
18425 #[allow(clippy::too_many_arguments)]
18426 fn apply_sqlite_image(
18427 &mut self,
18428 img: &rusqlite::Connection,
18429 table: &crate::schema::TableSchema,
18430 first_fresh_page: bool,
18431 effective: &[(String, Vec<String>)],
18432 row_count: i64,
18433 as_of_commit_seq: i64,
18434 scope_digest: &str,
18435 ) -> Result<u32, SectionError> {
18436 let invalid = |detail: String| {
18437 SectionError::Abort(
18438 "sync.invalid_request".to_owned(),
18439 format!("sqlite segment rejected: {detail} (§5.3)"),
18440 )
18441 };
18442
18443 type MetaRow = (i64, String, i64, i64, String, i64, i64);
18446 let meta: MetaRow = img
18447 .query_row(
18448 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
18449 \"scopeDigest\", \"rowCount\",
18450 (SELECT count(*) FROM _syncular_segment)
18451 FROM _syncular_segment",
18452 [],
18453 |row| {
18454 Ok((
18455 row.get(0)?,
18456 row.get(1)?,
18457 row.get(2)?,
18458 row.get(3)?,
18459 row.get(4)?,
18460 row.get(5)?,
18461 row.get(6)?,
18462 ))
18463 },
18464 )
18465 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
18466 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
18467 if meta_count != 1 {
18468 return Err(invalid(format!(
18469 "_syncular_segment must contain exactly one row, found {meta_count}"
18470 )));
18471 }
18472 if format != 1 {
18473 return Err(invalid(format!("format {format}")));
18474 }
18475 if meta_table != table.name {
18476 return Err(invalid(format!("image table {meta_table:?}")));
18477 }
18478 if schema_version != i64::from(self.schema.version) {
18479 return Err(invalid(format!("schemaVersion {schema_version}")));
18480 }
18481 if pin != as_of_commit_seq {
18482 return Err(invalid(format!("asOfCommitSeq {pin}")));
18483 }
18484 if digest != scope_digest {
18485 return Err(invalid("scopeDigest mismatch".to_owned()));
18486 }
18487 if meta_rows != row_count {
18488 return Err(invalid(format!("rowCount {meta_rows}")));
18489 }
18490
18491 let mut names: Vec<String> = Vec::new();
18493 let mut primary_keys: Vec<String> = Vec::new();
18494 {
18495 let mut stmt = img
18496 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
18497 .map_err(|_| invalid("image data table missing".to_owned()))?;
18498 let mut rows = stmt
18499 .query([])
18500 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
18501 while let Some(row) = rows
18502 .next()
18503 .map_err(|_| invalid("image data table unreadable".to_owned()))?
18504 {
18505 if row
18506 .get::<_, i64>(5)
18507 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
18508 > 0
18509 {
18510 primary_keys.push(row.get(1).map_err(|error| {
18511 invalid(Self::sqlite_failure(&self.storage_failure, error))
18512 })?);
18513 }
18514 names.push(
18515 row.get::<_, String>(1)
18516 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
18517 );
18518 }
18519 }
18520 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
18521 expected.push("_syncular_version");
18522 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
18523 return Err(SectionError::Abort(
18524 "sync.schema_mismatch".to_owned(),
18525 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
18526 ));
18527 }
18528
18529 if primary_keys != vec![table.primary_key.clone()] {
18530 return Err(invalid(
18531 "image primary key does not match generated schema".into(),
18532 ));
18533 }
18534 let actual_count: i64 = img
18535 .query_row(
18536 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
18537 [],
18538 |row| row.get(0),
18539 )
18540 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18541 if actual_count != row_count {
18542 return Err(invalid("image row count does not match descriptor".into()));
18543 }
18544 let insert = self.insert_row_sql(&base_table(&table.name), table);
18545 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
18546 let column_list = names
18547 .iter()
18548 .map(|n| quote_ident(n))
18549 .collect::<Vec<_>>()
18550 .join(", ");
18551 let mut stmt = img
18552 .prepare(&format!(
18553 "SELECT {column_list} FROM {} ORDER BY {}",
18554 quote_ident(&table.name),
18555 quote_ident(&table.primary_key)
18556 ))
18557 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18558 let mut rows = stmt
18559 .query([])
18560 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18561 let mut applied = 0u32;
18562 loop {
18563 let clear = applied == 0 && first_fresh_page;
18564 let was_dirty = self.overlay_dirty.snapshot();
18565 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
18566 && self
18567 .outbox
18568 .iter()
18569 .flat_map(|commit| &commit.ops)
18570 .any(|op| op.table == table.name);
18571 let mirror_visible = !reconcile_pending
18572 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
18573 self.coalesce_unit(1024)
18575 .and_then(|()| self.begin_observation("syncular_image_chunk"))
18576 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18577 let result = (|| {
18578 self.rebuild_overlay_if_dirty()
18579 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18580 let mut changed_keys = Vec::new();
18581 let mut batch = ChangeAccumulator::default();
18582 if clear {
18583 if self.scoped_rows_exist(&table.name, effective) {
18584 batch.table(&table.name);
18585 }
18586 self.purge_scope_rows(&table.name, effective)
18587 .map_err(|()| SectionError::FailClosed)?;
18588 }
18589 let mut processed = 0u32;
18590 {
18591 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
18592 invalid(Self::sqlite_failure(&self.storage_failure, error))
18593 })?;
18594 let mut mirror = if mirror_visible {
18595 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
18596 invalid(Self::sqlite_failure(&self.storage_failure, error))
18597 })?)
18598 } else {
18599 None
18600 };
18601 for _ in 0..1024 {
18602 let Some(row) = rows.next().map_err(|error| {
18603 invalid(Self::sqlite_failure(&self.storage_failure, error))
18604 })?
18605 else {
18606 break;
18607 };
18608 for (i, column) in table.columns.iter().enumerate() {
18609 let cell = row.get_ref(i).map_err(|error| {
18610 invalid(Self::sqlite_failure(&self.storage_failure, error))
18611 })?;
18612 let param = image_cell_param(column, cell).map_err(invalid)?;
18613 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18614 invalid(Self::sqlite_failure(&self.storage_failure, error))
18615 })?;
18616 if let Some(mirror) = &mut mirror {
18617 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18618 invalid(Self::sqlite_failure(&self.storage_failure, error))
18619 })?;
18620 }
18621 }
18622 let version_index = table.columns.len();
18623 let version: i64 = row.get(version_index).map_err(|error| {
18624 invalid(Self::sqlite_failure(&self.storage_failure, error))
18625 })?;
18626 if version < 1 {
18627 return Err(invalid("image row version must be positive".into()));
18628 }
18629 ins.raw_bind_parameter(version_index + 1, version)
18630 .map_err(|error| {
18631 invalid(Self::sqlite_failure(&self.storage_failure, error))
18632 })?;
18633 ins.raw_execute().map_err(|error| {
18634 invalid(Self::sqlite_failure(&self.storage_failure, error))
18635 })?;
18636 if let Some(mirror) = &mut mirror {
18637 mirror
18638 .raw_bind_parameter(version_index + 1, version)
18639 .map_err(|error| {
18640 invalid(Self::sqlite_failure(&self.storage_failure, error))
18641 })?;
18642 mirror.raw_execute().map_err(|error| {
18643 invalid(Self::sqlite_failure(&self.storage_failure, error))
18644 })?;
18645 }
18646 if reconcile_pending {
18647 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
18648 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
18649 )?);
18650 }
18651 processed += 1;
18652 }
18653 }
18654 if processed > 0 {
18655 batch.table(&table.name);
18656 self.overlay_dirty.table(&table.name);
18657 }
18658 if reconcile_pending {
18659 self.reconcile_imported_rows(table, &mut changed_keys)
18660 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18661 }
18662 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
18663 self.apply_outbox_ops(
18664 self.outbox
18665 .iter()
18666 .flat_map(|commit| &commit.ops)
18667 .filter(|op| op.table == table.name),
18668 )
18669 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18670 self.overlay_dirty.set(false);
18671 } else {
18672 self.rebuild_overlay_if_dirty()
18673 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18674 }
18675 self.finish_observation("syncular_image_chunk", batch)
18676 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18677 Ok(processed)
18678 })();
18679 let processed = match result {
18680 Ok(processed) => processed,
18681 Err(error) => {
18682 self.rollback_observation("syncular_image_chunk");
18683 self.overlay_dirty.restore(was_dirty.clone());
18684 return Err(error);
18685 }
18686 };
18687 applied += processed;
18688 self.progress
18689 .update(|p| p.rows_processed = u64::from(applied));
18690 std::thread::yield_now();
18691 if i64::from(applied) == actual_count {
18692 break;
18693 }
18694 }
18695 Ok(applied)
18696 }
18697
18698 fn reconcile_imported_rows(
18702 &self,
18703 table: &TableSchema,
18704 keys: &mut Vec<SqlValue>,
18705 ) -> Result<(), String> {
18706 let visible = visible_table(&table.name);
18707 let base = base_table(&table.name);
18708 let pk = quote_ident(&table.primary_key);
18709 let predicate = row_id_predicate(table);
18710 let mut lookup = self.conn.prepare_cached(&format!(
18711 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
18712 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18713 let mut deleted = HashSet::new();
18714 for op in self
18715 .outbox
18716 .iter()
18717 .flat_map(|commit| &commit.ops)
18718 .filter(|op| op.table == table.name)
18719 {
18720 if op.upsert {
18721 if let Some(values) = &op.values {
18722 let key = json_to_column_value(
18723 &table.columns[table.pk_index],
18724 values.get(&table.primary_key),
18725 )?;
18726 keys.push(owned_sql_value(RowParam::Cell(&key))?);
18727 }
18728 } else if deleted.insert(&op.row_id) {
18729 keys.extend(
18730 lookup
18731 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
18732 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
18733 .collect::<Result<Vec<_>, _>>()
18734 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18735 );
18736 }
18737 }
18738 for chunk in keys.chunks(400) {
18741 let holes = vec!["?"; chunk.len()].join(",");
18742 self.conn
18743 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
18744 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18745 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18746 }
18747 let updates = table
18748 .columns
18749 .iter()
18750 .map(|column| column.name.as_str())
18751 .chain(std::iter::once("_syncular_version"))
18752 .filter(|name| *name != table.primary_key)
18753 .map(|name| {
18754 let name = quote_ident(name);
18755 format!("{name}=excluded.{name}")
18756 })
18757 .collect::<Vec<_>>()
18758 .join(",");
18759 for chunk in keys.chunks(400) {
18760 let holes = vec!["?"; chunk.len()].join(",");
18761 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
18762 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18763 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18764 }
18765 Ok(())
18766 }
18767
18768 fn compile_local_data_purge(
18771 &self,
18772 input: &LocalDataPurgeInput,
18773 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
18774 let invalid = |message: String| format!("sync.invalid_request: {message}");
18775 if input.purge_id.is_empty()
18776 || input.purge_id.len() > 128
18777 || !is_local_operation_code_like(&input.purge_id)
18778 {
18779 return Err(invalid(
18780 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
18781 ));
18782 }
18783 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
18784 return Err(invalid(format!(
18785 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
18786 )));
18787 }
18788
18789 let mut deduplicated: BTreeMap<
18790 String,
18791 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
18792 > = BTreeMap::new();
18793 for target in &input.targets {
18794 let table = self.schema.table(&target.table).ok_or_else(|| {
18795 invalid(format!(
18796 "local purge names unknown table {:?}",
18797 target.table
18798 ))
18799 })?;
18800 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
18801 return Err(invalid(format!(
18802 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
18803 target.table
18804 )));
18805 }
18806 let mut selectors = Vec::with_capacity(target.selectors.len());
18807 let mut canonical_selectors = BTreeMap::new();
18808 for (column_name, raw_values) in &target.selectors {
18809 let Some((column_index, column)) = table
18810 .columns
18811 .iter()
18812 .enumerate()
18813 .find(|(_, column)| column.name == *column_name)
18814 else {
18815 return Err(invalid(format!(
18816 "local purge target {:?} names unknown column {:?}",
18817 target.table, column_name
18818 )));
18819 };
18820 let encrypted = table
18821 .encrypted_columns
18822 .iter()
18823 .any(|candidate| candidate.index == column_index);
18824 if column.ty != ColumnType::String || encrypted {
18825 return Err(invalid(format!(
18826 "local purge selector {:?}.{:?} must be a plaintext string column",
18827 target.table, column_name
18828 )));
18829 }
18830 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
18831 return Err(invalid(format!(
18832 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
18833 target.table, column_name
18834 )));
18835 }
18836 let mut values = raw_values.clone();
18837 values.sort();
18838 values.dedup();
18839 if values.iter().any(|value| {
18840 value.is_empty()
18841 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
18842 || !is_local_operation_code_like(value)
18843 }) {
18844 return Err(invalid(format!(
18845 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
18846 )));
18847 }
18848 selectors.push((column_name.clone(), values.clone()));
18849 canonical_selectors.insert(column_name.clone(), values);
18850 }
18851 let canonical = LocalDataPurgeTarget {
18852 table: target.table.clone(),
18853 selectors: canonical_selectors,
18854 };
18855 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18856 deduplicated.insert(
18857 key,
18858 (
18859 CompiledLocalDataPurgeTarget {
18860 table: target.table.clone(),
18861 selectors,
18862 },
18863 canonical,
18864 ),
18865 );
18866 }
18867 let targets = deduplicated
18868 .values()
18869 .map(|(compiled, _)| compiled.clone())
18870 .collect::<Vec<_>>();
18871 let canonical = deduplicated
18872 .values()
18873 .map(|(_, target)| target.clone())
18874 .collect::<Vec<_>>();
18875 let canonical_plan =
18876 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18877 Ok((targets, canonical_plan))
18878 }
18879
18880 fn local_purge_base_row_ids(
18881 &self,
18882 targets: &[CompiledLocalDataPurgeTarget],
18883 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
18884 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
18885 for target in targets {
18886 let table = self
18887 .schema
18888 .table(&target.table)
18889 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
18890 let mut clauses = Vec::with_capacity(target.selectors.len());
18891 let mut params = Vec::new();
18892 for (column, values) in &target.selectors {
18893 clauses.push(format!(
18894 "{} IN ({})",
18895 quote_ident(column),
18896 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
18897 ));
18898 params.extend(values.iter().cloned().map(SqlValue::Text));
18899 }
18900 let sql = format!(
18901 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
18902 quote_ident(&table.primary_key),
18903 base_table(&target.table),
18904 clauses.join(" AND ")
18905 );
18906 let mut statement = self
18907 .conn
18908 .prepare(&sql)
18909 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18910 let rows = statement
18911 .query_map(rusqlite::params_from_iter(params), |row| {
18912 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
18913 render_row_id_json(Some(&value))
18914 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
18915 })
18916 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18917 let ids = by_table.entry(target.table.clone()).or_default();
18918 for row in rows {
18919 ids.insert(
18920 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18921 );
18922 }
18923 }
18924 Ok(by_table)
18925 }
18926
18927 fn local_purge_values_match(
18928 target: &CompiledLocalDataPurgeTarget,
18929 values: &Map<String, Value>,
18930 ) -> bool {
18931 target.selectors.iter().all(|(column, allowed)| {
18932 values
18933 .get(column)
18934 .and_then(Value::as_str)
18935 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
18936 })
18937 }
18938
18939 pub fn purge_local_data(
18943 &mut self,
18944 input: &LocalDataPurgeInput,
18945 ) -> Result<LocalDataPurgeResult, String> {
18946 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
18947 let meta_key = format!("localPurge:{}", input.purge_id);
18948 let applied_plan = self
18949 .conn
18950 .query_row(
18951 "SELECT value FROM _syncular_meta WHERE key = ?1",
18952 rusqlite::params![meta_key],
18953 |row| row.get::<_, String>(0),
18954 )
18955 .optional()
18956 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18957 if let Some(applied_plan) = applied_plan {
18958 if applied_plan != canonical_plan {
18959 return Err(format!(
18960 "sync.invalid_request: local purge id {:?} was already used with a different plan",
18961 input.purge_id
18962 ));
18963 }
18964 return Ok(LocalDataPurgeResult {
18965 already_applied: true,
18966 purged_rows: 0,
18967 dropped_commits: 0,
18968 });
18969 }
18970
18971 let _ = self.drop_previous_version();
18975
18976 let prior_outbox = self.outbox.clone();
18977 let prior_failed = self.failed_commits.clone();
18978 let prior_rejections = self.rejections.clone();
18979 let prior_conflicts = self.conflicts.clone();
18980 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18981 self.begin_observation("syncular_local_purge")?;
18982 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
18983 let row_ids = self.local_purge_base_row_ids(&targets)?;
18984 let retained_dropped = self.drop_failed_commits(|table, values| {
18985 targets.iter().any(|target| {
18986 target.table == table && Self::local_purge_values_match(target, values)
18987 })
18988 })?;
18989 let doomed = self
18990 .outbox
18991 .iter()
18992 .filter(|commit| {
18993 commit.ops.iter().any(|operation| {
18994 let matching_targets = targets
18995 .iter()
18996 .filter(|target| target.table == operation.table)
18997 .collect::<Vec<_>>();
18998 if matching_targets.is_empty() {
18999 return false;
19000 }
19001 if row_ids
19002 .get(&operation.table)
19003 .is_some_and(|ids| ids.contains(&operation.row_id))
19004 {
19005 return true;
19006 }
19007 operation.values.as_ref().is_some_and(|values| {
19008 matching_targets
19009 .iter()
19010 .any(|target| Self::local_purge_values_match(target, values))
19011 })
19012 })
19013 })
19014 .cloned()
19015 .collect::<Vec<_>>();
19016 let doomed_ids = doomed
19017 .iter()
19018 .map(|commit| commit.client_commit_id.clone())
19019 .collect::<BTreeSet<_>>();
19020 let mut batch = ChangeAccumulator::default();
19021 for commit in &retained_dropped {
19022 batch.conflicts = true;
19023 batch.rejections = true;
19024 batch.outcomes = true;
19025 for operation in &commit.ops {
19026 batch.table(&operation.table);
19027 }
19028 }
19029 let mut rejections = Vec::new();
19030 for commit in &doomed {
19031 for operation in &commit.ops {
19032 batch.table(&operation.table);
19033 }
19034 let results = commit
19035 .ops
19036 .iter()
19037 .enumerate()
19038 .map(|(op_index, operation)| {
19039 let rejection = RejectionRecord {
19040 client_commit_id: commit.client_commit_id.clone(),
19041 op_index: op_index as i32,
19042 code: "client.local_data_purged".to_owned(),
19043 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
19044 retryable: false,
19045 details: None,
19046 operation: Some(CommitOperation::from(operation)),
19047 };
19048 rejections.push(rejection.clone());
19049 CommitOperationOutcome::Error { rejection }
19050 })
19051 .collect::<Vec<_>>();
19052 self.persist_commit_outcome(
19053 &commit.client_commit_id,
19054 CommitOutcomeStatus::Rejected,
19055 &results,
19056 Some(&commit.ops),
19057 )?;
19058 self.delete_outbox_persisted(&commit.client_commit_id)?;
19059 }
19060 if !doomed.is_empty() {
19061 self.outbox
19062 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
19063 self.rejections.extend(rejections);
19064 self.prune_commit_outcomes()?;
19065 batch.status = true;
19066 batch.rejections = true;
19067 batch.outcomes = true;
19068 }
19069
19070 let mut purged_rows = 0usize;
19071 for (table, ids) in &row_ids {
19072 if ids.is_empty() {
19073 continue;
19074 }
19075 batch.table(table);
19076 for row_id in ids {
19077 self.delete_base_row(table, row_id)?;
19078 purged_rows += 1;
19079 }
19080 }
19081 self.rebuild_overlay_if_dirty()?;
19082 self.delete_unreferenced_cached_blobs()?;
19083 self.conn
19084 .execute(
19085 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
19086 rusqlite::params![meta_key, canonical_plan],
19087 )
19088 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19089 Ok((
19090 batch,
19091 LocalDataPurgeResult {
19092 already_applied: false,
19093 purged_rows,
19094 dropped_commits: doomed.len(),
19095 },
19096 ))
19097 })();
19098 let (batch, result) = match applied {
19099 Ok(value) => value,
19100 Err(error) => {
19101 self.rollback_observation("syncular_local_purge");
19102 self.outbox = prior_outbox;
19103 self.failed_commits = prior_failed;
19104 self.rejections = prior_rejections;
19105 self.conflicts = prior_conflicts;
19106 self.overlay_dirty.restore(prior_overlay_dirty.clone());
19107 return Err(error);
19108 }
19109 };
19110 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
19111 self.rollback_observation("syncular_local_purge");
19112 self.outbox = prior_outbox;
19113 self.failed_commits = prior_failed;
19114 self.rejections = prior_rejections;
19115 self.conflicts = prior_conflicts;
19116 self.overlay_dirty.restore(prior_overlay_dirty.clone());
19117 return Err(error);
19118 }
19119 self.cancel_sync_round();
19120 Ok(result)
19121 }
19122
19123 pub fn rebootstrap_local_data(
19129 &mut self,
19130 input: &LocalDataRebootstrapInput,
19131 ) -> Result<LocalDataRebootstrapResult, String> {
19132 if input.rebootstrap_id.is_empty()
19133 || input.rebootstrap_id.len() > 128
19134 || !is_local_operation_code_like(&input.rebootstrap_id)
19135 {
19136 return Err(
19137 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
19138 .to_owned(),
19139 );
19140 }
19141 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
19142 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
19143 let (retained_commits, reset_subscriptions) =
19144 decode_local_rebootstrap_receipt(&persisted)?;
19145 return Ok(LocalDataRebootstrapResult {
19146 already_applied: true,
19147 retained_commits,
19148 reset_subscriptions,
19149 });
19150 }
19151 if self.stopped || self.schema_floor.is_some() {
19152 return Err(
19153 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
19154 .to_owned(),
19155 );
19156 }
19157
19158 let retained_commits = self.outbox.len();
19159 let reset_subscriptions = self.subs.len();
19160 let prior_subs = self.subs.clone();
19161 let prior_upgrading = self.upgrading;
19162 let prior_stopped = self.stopped;
19163 let prior_schema_floor = self.schema_floor.clone();
19164 let prior_overlay_dirty = self.overlay_dirty.snapshot();
19165 let prior_sync_needed = self.sync_needed;
19166 let prior_sync_intents = self.sync_intent_queue.clone();
19167 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
19168
19169 let prior_active_round = self.active_round;
19170 self.begin_observation("syncular_local_rebootstrap")?;
19171 let mut batch = ChangeAccumulator::default();
19172 let applied = (|| -> Result<(), String> {
19173 self.run_schema_reset_observed(&mut batch, false)?;
19174 self.conn
19175 .execute(
19176 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
19177 rusqlite::params![meta_key, receipt],
19178 )
19179 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19180 self.sync_needed = true;
19181 self.sync_intent_queue.push_back(SyncIntent::Interactive);
19182 batch.status = true;
19183 Ok(())
19184 })();
19185 if let Err(error) = applied {
19186 self.rollback_observation("syncular_local_rebootstrap");
19187 self.active_round = prior_active_round;
19188 self.subs = prior_subs;
19189 self.upgrading = prior_upgrading;
19190 self.stopped = prior_stopped;
19191 self.schema_floor = prior_schema_floor;
19192 self.overlay_dirty.restore(prior_overlay_dirty.clone());
19193 self.sync_needed = prior_sync_needed;
19194 self.sync_intent_queue = prior_sync_intents;
19195 return Err(error);
19196 }
19197 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
19198 self.rollback_observation("syncular_local_rebootstrap");
19199 self.active_round = prior_active_round;
19200 self.subs = prior_subs;
19201 self.upgrading = prior_upgrading;
19202 self.stopped = prior_stopped;
19203 self.schema_floor = prior_schema_floor;
19204 self.overlay_dirty.restore(prior_overlay_dirty.clone());
19205 self.sync_needed = prior_sync_needed;
19206 self.sync_intent_queue = prior_sync_intents;
19207 return Err(error);
19208 }
19209
19210 Ok(LocalDataRebootstrapResult {
19211 already_applied: false,
19212 retained_commits,
19213 reset_subscriptions,
19214 })
19215 }
19216
19217 fn purge_scope_rows(
19222 &mut self,
19223 table_name: &str,
19224 effective: &[(String, Vec<String>)],
19225 ) -> Result<(), ()> {
19226 if effective.is_empty() {
19227 return Ok(());
19228 }
19229 let table = self.schema.table(table_name).ok_or(())?.clone();
19230 let mut clauses = Vec::new();
19231 let mut params: Vec<SqlValue> = Vec::new();
19232 for (variable, values) in effective {
19233 let column = table.scope_column(variable).ok_or(())?;
19234 let placeholders: Vec<String> = values
19235 .iter()
19236 .map(|v| {
19237 params.push(SqlValue::Text(v.clone()));
19238 "?".to_owned()
19239 })
19240 .collect();
19241 clauses.push(format!(
19242 "{} IN ({})",
19243 quote_ident(column),
19244 placeholders.join(", ")
19245 ));
19246 }
19247 let sql = format!(
19248 "DELETE FROM {} WHERE {}",
19249 base_table(table_name),
19250 clauses.join(" AND ")
19251 );
19252 self.overlay_dirty.table(table_name);
19253 self.conn
19254 .execute(&sql, rusqlite::params_from_iter(¶ms))
19255 .map_err(|_| ())?;
19256 self.conn
19257 .execute(
19258 &format!(
19259 "DELETE FROM {} WHERE {}",
19260 visible_table(table_name),
19261 clauses.join(" AND ")
19262 ),
19263 rusqlite::params_from_iter(¶ms),
19264 )
19265 .map_err(|_| ())?;
19266 Ok(())
19267 }
19268
19269 fn drop_doomed_outbox(
19272 &mut self,
19273 table_name: &str,
19274 effective: &[(String, Vec<String>)],
19275 ) -> Result<bool, String> {
19276 if effective.is_empty() {
19277 return Ok(false);
19278 }
19279 let Some(table) = self.schema.table(table_name).cloned() else {
19280 return Ok(false);
19281 };
19282 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
19283 for (variable, values) in effective {
19284 match table.scope_column(variable) {
19285 Some(column) => mappings.push((column, values)),
19286 None => return Ok(false), }
19288 }
19289 let doomed: Vec<OutboxCommit> = self
19290 .outbox
19291 .iter()
19292 .filter(|commit| {
19293 commit.ops.iter().any(|op| {
19294 op.upsert
19295 && op.table == table_name
19296 && op.values.as_ref().is_some_and(|values| {
19297 mappings
19298 .iter()
19299 .all(|(column, allowed)| match values.get(*column) {
19300 Some(Value::String(s)) => allowed.contains(s),
19301 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
19302 Some(Value::Bool(b)) => allowed.contains(&b.to_string()),
19303 _ => false,
19304 })
19305 })
19306 })
19307 })
19308 .cloned()
19309 .collect();
19310 if doomed.is_empty() {
19311 return Ok(false);
19312 }
19313 let mut rejections = Vec::new();
19314 for commit in &doomed {
19315 let results = commit
19316 .ops
19317 .iter()
19318 .enumerate()
19319 .map(|(op_index, operation)| {
19320 let rejection = RejectionRecord {
19321 client_commit_id: commit.client_commit_id.clone(),
19322 op_index: op_index as i32,
19323 code: "sync.scope_revoked".to_owned(),
19324 message: "the commit was dropped because its effective scope was revoked"
19325 .to_owned(),
19326 retryable: false,
19327 details: None,
19328 operation: Some(CommitOperation::from(operation)),
19329 };
19330 rejections.push(rejection.clone());
19331 CommitOperationOutcome::Error { rejection }
19332 })
19333 .collect::<Vec<_>>();
19334 self.persist_commit_outcome(
19335 &commit.client_commit_id,
19336 CommitOutcomeStatus::Rejected,
19337 &results,
19338 Some(&commit.ops),
19339 )?;
19340 self.delete_outbox_persisted(&commit.client_commit_id)?;
19341 }
19342 self.prune_commit_outcomes()?;
19343 let doomed_ids = doomed
19344 .iter()
19345 .map(|commit| commit.client_commit_id.as_str())
19346 .collect::<BTreeSet<_>>();
19347 self.outbox
19348 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
19349 self.rejections.extend(rejections);
19350 Ok(true)
19351 }
19352
19353 pub fn upload_blob(
19360 &mut self,
19361 bytes: &[u8],
19362 media_type: Option<String>,
19363 name: Option<String>,
19364 ) -> Result<Value, String> {
19365 let blob_id = blob_id_for(bytes);
19366 let now = self.clock_now_ms();
19367 let tx = self
19368 .conn
19369 .transaction()
19370 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19371 tx.execute(
19372 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
19373 ON CONFLICT(blob_id) DO NOTHING",
19374 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
19375 )
19376 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19377 tx.execute(
19378 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
19379 rusqlite::params![blob_id, media_type, now],
19380 )
19381 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19382 tx.commit()
19383 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19384 self.enforce_blob_cache_cap();
19385 let mut obj = Map::new();
19386 obj.insert("blobId".to_owned(), Value::from(blob_id));
19387 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
19388 if let Some(mt) = media_type {
19389 obj.insert("mediaType".to_owned(), Value::from(mt));
19390 }
19391 if let Some(n) = name {
19392 obj.insert("name".to_owned(), Value::from(n));
19393 }
19394 Ok(Value::Object(obj))
19395 }
19396
19397 pub fn fetch_blob_bytes(
19401 &mut self,
19402 transport: &mut dyn Transport,
19403 blob_id_or_ref: &str,
19404 ) -> Result<FetchedBlob, (String, String)> {
19405 let simple = |m: String| ("client.failed".to_owned(), m);
19406 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
19407 blob_id_or_ref.to_owned()
19408 } else {
19409 let value: Value = serde_json::from_str(blob_id_or_ref)
19410 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
19411 value
19412 .get("blobId")
19413 .and_then(Value::as_str)
19414 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
19415 .to_owned()
19416 };
19417 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
19418 return Ok(cached);
19419 }
19420 if !self.transport_enabled {
19421 return Err(("sync.offline".into(), "transport gate is closed".into()));
19422 }
19423 #[cfg(feature = "bench-internals")]
19429 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
19430 let bytes = match transport
19431 .blob_download(&blob_id)
19432 .map_err(|e| (e.code, e.message))?
19433 {
19434 BlobDownload::Bytes(bytes) => bytes,
19435 BlobDownload::Url {
19436 url,
19437 url_expires_at_ms,
19438 } => {
19439 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
19441 return Err((
19442 "sync.segment_expired".to_owned(),
19443 format!(
19444 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
19445 ),
19446 ));
19447 }
19448 transport
19449 .fetch_blob_url(&url)
19450 .map_err(|e| (e.code, e.message))?
19451 }
19452 };
19453 #[cfg(feature = "bench-internals")]
19454 drop(download_phase);
19455 #[cfg(feature = "bench-internals")]
19456 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
19457 if blob_id_for(&bytes) != blob_id {
19459 return Err(simple(format!(
19460 "blob content address mismatch for {blob_id}"
19461 )));
19462 }
19463 #[cfg(feature = "bench-internals")]
19464 drop(validate_phase);
19465 #[cfg(feature = "bench-internals")]
19466 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
19467 let now = self.clock_now_ms();
19468 self.conn
19469 .execute(
19470 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
19471 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
19472 )
19473 .map_err(|e| simple(e.to_string()))?;
19474 self.conn
19475 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
19476 .map_err(|e| simple(e.to_string()))?;
19477 self.enforce_blob_cache_cap();
19478 #[cfg(feature = "bench-internals")]
19479 drop(insert_phase);
19480 self.get_cached_blob_bytes(&blob_id)
19481 .map_err(simple)?
19482 .ok_or_else(|| simple("blob cache write failed".to_owned()))
19483 }
19484
19485 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
19486 #[cfg(feature = "bench-internals")]
19487 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
19488 let mut stmt = self
19489 .conn
19490 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
19491 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19492 let mut rows = stmt
19493 .query(rusqlite::params![blob_id])
19494 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19495 if let Some(row) = rows
19496 .next()
19497 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
19498 {
19499 let bytes: Vec<u8> = row
19500 .get(0)
19501 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19502 let byte_length: i64 = row
19503 .get(1)
19504 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19505 let media_type: Option<String> = row
19506 .get(2)
19507 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19508 return Ok(Some(FetchedBlob {
19509 blob_id: blob_id.to_owned(),
19510 byte_length,
19511 bytes,
19512 media_type,
19513 }));
19514 }
19515 Ok(None)
19516 }
19517
19518 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
19520 let pending: Vec<(String, Option<String>)> = {
19521 let mut stmt = self
19522 .conn
19523 .prepare(
19524 "SELECT blob_id, media_type FROM (
19525 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
19526 UNION ALL
19527 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
19528 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
19529 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
19530 ) ORDER BY created_at_ms, blob_id",
19531 )
19532 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19533 let rows = stmt
19534 .query_map([], |row| {
19535 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
19536 })
19537 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19538 rows.collect::<Result<_, _>>()
19539 .map_err(|error| match error {
19540 rusqlite::Error::InvalidColumnType(..)
19541 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
19542 "sync.local_corrupt",
19543 "Pending blob upload metadata is invalid",
19544 ),
19545 _ => TransportError::new("client.failed", error.to_string()),
19546 })?
19547 };
19548 let mut uploads = Vec::new();
19549 for (blob_id, media_type) in pending {
19550 let bytes = (|| {
19551 let stored: Option<(SqlValue, SqlValue)> = self
19552 .conn
19553 .query_row(
19554 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
19555 rusqlite::params![blob_id],
19556 |row| Ok((row.get(0)?, row.get(1)?)),
19557 )
19558 .optional()
19559 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19560 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
19561 return Err(TransportError::new(
19562 "sync.local_corrupt",
19563 "Pending blob upload body is missing or corrupt",
19564 ));
19565 };
19566 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
19567 return Err(TransportError::new(
19568 "sync.local_corrupt",
19569 "Pending blob upload body is missing or corrupt",
19570 ));
19571 }
19572 Ok(bytes)
19573 })();
19574 let failed = bytes.is_err();
19575 uploads.push(crate::round::PendingUpload {
19576 id: blob_id,
19577 bytes,
19578 media_type,
19579 });
19580 if failed {
19581 break;
19582 }
19583 }
19584 Ok(uploads)
19585 }
19586
19587 #[cfg(test)]
19588 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
19589 for upload in self.pending_blob_uploads()? {
19590 self.upload_one(
19591 transport,
19592 &upload.id,
19593 &upload.bytes?,
19594 upload.media_type.as_deref(),
19595 )?;
19596 self.conn
19597 .execute(
19598 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
19599 rusqlite::params![upload.id],
19600 )
19601 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19602 }
19603 Ok(())
19604 }
19605
19606 #[cfg(test)]
19613 fn upload_one(
19614 &self,
19615 transport: &mut dyn Transport,
19616 blob_id: &str,
19617 bytes: &[u8],
19618 media_type: Option<&str>,
19619 ) -> Result<(), TransportError> {
19620 crate::round::upload_one(transport, blob_id, bytes, media_type, self.now_ms)
19621 }
19622
19623 fn visible_blob_ids_sql(&self) -> String {
19624 let mut selects = Vec::new();
19625 for table in &self.schema.tables {
19626 for column in table
19627 .columns
19628 .iter()
19629 .filter(|column| column.ty == ColumnType::BlobRef)
19630 {
19631 let identifier = quote_ident(&column.name);
19632 selects.push(format!(
19633 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
19634 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
19635 AND json_type({identifier}, '$.blobId') = 'text'",
19636 quote_ident(&table.name),
19637 ));
19638 }
19639 }
19640 if selects.is_empty() {
19641 "SELECT NULL AS blob_id WHERE 0".to_owned()
19642 } else {
19643 selects.join(" UNION ")
19644 }
19645 }
19646
19647 fn enforce_blob_cache_cap(&self) {
19650 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
19651 return;
19652 };
19653 let mut total: i64 = self
19654 .conn
19655 .query_row(
19656 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
19657 [],
19658 |row| row.get(0),
19659 )
19660 .unwrap_or(0);
19661 if total <= max_bytes {
19662 return;
19663 }
19664 let query = format!(
19665 "SELECT blob_id, byte_length FROM _syncular_blobs
19666 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19667 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19668 AND blob_id NOT IN ({})
19669 ORDER BY created_at_ms ASC, blob_id ASC",
19670 self.visible_blob_ids_sql(),
19671 );
19672 let candidates: Vec<(String, i64)> = {
19673 let Ok(mut stmt) = self.conn.prepare(&query) else {
19674 return;
19675 };
19676 let Ok(rows) = stmt.query_map([], |row| {
19677 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
19678 }) else {
19679 return;
19680 };
19681 rows.filter_map(Result::ok).collect()
19682 };
19683 for (blob_id, byte_length) in candidates {
19684 if total <= max_bytes {
19685 break;
19686 }
19687 let _ = self.conn.execute(
19688 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
19689 rusqlite::params![blob_id],
19690 );
19691 total -= byte_length;
19692 }
19693 }
19694
19695 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
19696 #[cfg(feature = "bench-internals")]
19697 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
19698 if !self.schema_has_blobs() {
19699 return Ok(());
19700 }
19701 self.rebuild_overlay_if_dirty()?;
19702 self.conn
19703 .execute(
19704 &format!(
19705 "DELETE FROM _syncular_blobs
19706 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19707 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19708 AND blob_id NOT IN ({})",
19709 self.visible_blob_ids_sql(),
19710 ),
19711 [],
19712 )
19713 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19714 Ok(())
19715 }
19716
19717 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
19722 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
19723 return sql.clone();
19724 }
19725 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
19726 columns.push(quote_ident("_syncular_version"));
19727 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
19728 let primary_key = quote_ident(&table.primary_key);
19729 let updates = columns
19730 .iter()
19731 .filter(|column| **column != primary_key)
19732 .map(|column| format!("{column}=excluded.{column}"))
19733 .collect::<Vec<_>>()
19734 .join(", ");
19735 let sql = format!(
19736 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
19737 columns.join(", "),
19738 placeholders.join(", ")
19739 );
19740 self.insert_sql
19741 .borrow_mut()
19742 .insert(full_table.to_owned(), sql.clone());
19743 sql
19744 }
19745
19746 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
19747 self.overlay_dirty.table(table_name);
19748 self.write_row(&base_table(table_name), table_name, row, version)
19749 }
19750
19751 fn write_row(
19752 &self,
19753 full_table: &str,
19754 table_name: &str,
19755 row: &Row,
19756 version: i64,
19757 ) -> Result<(), String> {
19758 #[cfg(feature = "bench-internals")]
19759 let _phase = self.benchmark_phases.start(Phase::RowWrite);
19760 let table = self
19761 .schema
19762 .table(table_name)
19763 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19764 let sql = self.insert_row_sql(full_table, table);
19765 let mut stmt = self
19766 .conn
19767 .prepare_cached(&sql)
19768 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19769 let params = row
19770 .iter()
19771 .map(RowParam::Cell)
19772 .chain(std::iter::once(RowParam::Version(version)));
19773 stmt.execute(rusqlite::params_from_iter(params))
19774 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19775 Ok(())
19776 }
19777
19778 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
19779 let table = self
19780 .schema
19781 .table(table_name)
19782 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19783 self.overlay_dirty.table(table_name);
19784 let sql = format!(
19785 "DELETE FROM {} WHERE {}",
19786 base_table(table_name),
19787 row_id_predicate(table)
19788 );
19789 let mut stmt = self
19790 .conn
19791 .prepare_cached(&sql)
19792 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19793 stmt.execute(rusqlite::params![row_id])
19794 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19795 Ok(())
19796 }
19797
19798 fn has_acknowledged_rows(&self) -> bool {
19799 self.conn
19800 .query_row(
19801 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
19802 [],
19803 |row| row.get(0),
19804 )
19805 .expect("read acknowledged intent")
19806 }
19807
19808 fn acknowledged_operations(
19809 &self,
19810 rows: Option<&HashSet<(&str, &str)>>,
19811 tables: Option<&BTreeSet<&str>>,
19812 ) -> Result<Vec<OutboxOp>, String> {
19813 let keys = rows.map(|rows| {
19814 serde_json::to_string(
19815 &rows
19816 .iter()
19817 .map(|(table, id)| serde_json::to_string(&(table, id)).expect("row key"))
19818 .collect::<Vec<_>>(),
19819 )
19820 .expect("row keys")
19821 });
19822 let mut statement = self
19823 .conn
19824 .prepare("SELECT op_json,intent_json FROM _syncular_acked_rows WHERE (?1 IS NULL OR json_array(tbl,id) IN(SELECT value FROM json_each(?1))) AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx")
19825 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19826 let rows = statement
19827 .query_map(
19828 rusqlite::params![
19829 keys,
19830 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
19831 ],
19832 |row| {
19833 #[cfg(test)]
19834 self.acknowledged_replay_count
19835 .set(self.acknowledged_replay_count.get() + 1);
19836 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
19837 },
19838 )
19839 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19840 rows.map(|row| {
19841 let (op, intent) =
19842 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19843 let op: CommitOperation =
19844 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
19845 let table = self
19846 .schema
19847 .table(&op.table)
19848 .ok_or_else(|| "sync.unknown_table".to_owned())?;
19849 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
19850 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
19851 } else {
19852 op.values
19853 };
19854 Ok(OutboxOp {
19855 table: op.table,
19856 row_id: op.row_id,
19857 upsert: op.op == "upsert",
19858 base_version: op.base_version,
19859 values,
19860 })
19861 })
19862 .collect()
19863 }
19864
19865 fn drop_acknowledged_rows(
19866 &self,
19867 matches: impl Fn(&str, &Map<String, Value>) -> bool,
19868 ) -> Result<Vec<OutboxCommit>, String> {
19869 let mut statement = self
19870 .conn
19871 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
19872 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19873 let rows = statement
19874 .query_map([], |row| {
19875 Ok((
19876 row.get::<_, String>(0)?,
19877 row.get::<_, String>(1)?,
19878 row.get::<_, String>(2)?,
19879 row.get::<_, String>(3)?,
19880 ))
19881 })
19882 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19883 let mut doomed = HashSet::new();
19884 for row in rows {
19885 let (id, table, intent, row_id) =
19886 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19887 let values =
19888 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
19889 let base_matches = self
19890 .schema
19891 .table(&table)
19892 .map(|schema| self.stored_row(schema, &row_id, true))
19893 .transpose()?
19894 .flatten()
19895 .is_some_and(|(base, _)| {
19896 let schema = self.schema.table(&table).expect("acknowledged table");
19897 let values = schema
19898 .columns
19899 .iter()
19900 .zip(base.iter())
19901 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
19902 .collect();
19903 matches(&table, &values)
19904 });
19905 if matches(&table, &values) || base_matches {
19906 doomed.insert(id);
19907 }
19908 }
19909 let mut dropped = Vec::new();
19910 for id in doomed {
19911 let mut statement = self
19912 .conn
19913 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
19914 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19915 let operations = statement
19916 .query_map([&id], |row| row.get::<_, String>(0))
19917 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19918 let mut ops = Vec::new();
19919 for operation in operations {
19920 let json = operation
19921 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19922 let op: CommitOperation =
19923 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
19924 self.overlay_dirty.table(&op.table);
19925 ops.push(OutboxOp {
19926 table: op.table,
19927 row_id: op.row_id,
19928 upsert: op.op == "upsert",
19929 base_version: op.base_version,
19930 values: op.values,
19931 });
19932 }
19933 dropped.push(OutboxCommit {
19934 client_commit_id: id.clone(),
19935 ops,
19936 });
19937 self.conn
19938 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
19939 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19940 }
19941 Ok(dropped)
19942 }
19943
19944 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
19947 if self.overlay_dirty.get() {
19948 self.rebuild_overlay()?;
19949 }
19950 Ok(())
19951 }
19952
19953 fn rebuild_overlay(&mut self) -> Result<(), String> {
19957 #[cfg(feature = "bench-internals")]
19958 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
19959 #[cfg(test)]
19960 self.overlay_rebuild_count
19961 .set(self.overlay_rebuild_count.get() + 1);
19962 let dirty = self.overlay_dirty.snapshot();
19963 let tables: Vec<_> = self
19964 .schema
19965 .tables
19966 .iter()
19967 .filter(|table| {
19968 dirty
19969 .as_ref()
19970 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
19971 })
19972 .cloned()
19973 .collect();
19974 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
19975 self.begin_observation("syncular_overlay")?;
19976 let rebuilt = (|| {
19977 for table in &tables {
19978 for index in &table.fts_indexes {
19979 self.drop_fts_triggers(index)?;
19980 }
19981 let visible = visible_table(&table.name);
19982 let base = base_table(&table.name);
19983 self.conn
19984 .execute_batch(&format!(
19985 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
19986 ))
19987 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19988 }
19989 let acknowledged = self.acknowledged_operations(None, Some(&names))?;
19990 self.apply_outbox_ops(
19991 acknowledged
19992 .iter()
19993 .filter(|op| names.contains(op.table.as_str())),
19994 )?;
19995 for (commit, initial) in &self.failed_commits {
19996 for operation in &commit.ops {
19997 if !names.contains(operation.table.as_str()) || !operation.upsert {
19998 continue;
19999 }
20000 if let Some(table) = self.schema.table(&operation.table) {
20001 if self.visible_row(table, &operation.row_id)?.is_none()
20002 && operation.values.as_ref().is_some_and(|values| {
20003 table
20004 .columns
20005 .iter()
20006 .all(|column| values.contains_key(&column.name))
20007 })
20008 {
20009 let key = serde_json::to_string(&(
20010 operation.table.clone(),
20011 operation.row_id.clone(),
20012 ))
20013 .expect("row key serialization");
20014 if let Some(Value::Object(values)) = initial.get(&key) {
20015 self.apply_outbox_ops(&[OutboxOp {
20016 values: Some(values.clone()),
20017 ..operation.clone()
20018 }])?;
20019 }
20020 }
20021 }
20022 }
20023 self.apply_outbox_ops(
20024 commit
20025 .ops
20026 .iter()
20027 .filter(|op| names.contains(op.table.as_str())),
20028 )?;
20029 }
20030 for commit in &self.outbox {
20031 self.apply_outbox_ops(
20032 commit
20033 .ops
20034 .iter()
20035 .filter(|op| names.contains(op.table.as_str())),
20036 )?;
20037 }
20038 for table in &tables {
20039 for index in &table.fts_indexes {
20040 self.rebuild_fts_projection(table, index)?;
20041 self.create_fts_triggers(table, index)?;
20042 }
20043 }
20044 self.conn
20045 .execute_batch("RELEASE syncular_overlay")
20046 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20047 Ok(())
20048 })();
20049 if rebuilt.is_err() {
20050 self.conn
20051 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
20052 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20053 } else {
20054 self.overlay_dirty.set(false);
20055 }
20056 rebuilt
20057 }
20058
20059 fn apply_outbox_ops<'a>(
20064 &self,
20065 ops: impl IntoIterator<Item = &'a OutboxOp>,
20066 ) -> Result<(), String> {
20067 #[cfg(feature = "bench-internals")]
20068 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
20069 for op in ops {
20070 #[cfg(feature = "bench-internals")]
20071 if let Some(phase) = &mut phase {
20072 phase.unit();
20073 }
20074 if op.upsert {
20075 if let (Some(table), Some(values)) = (
20076 self.schema
20077 .table(&op.table)
20078 .filter(|table| table.indexes.iter().any(|index| index.unique)),
20079 op.values.as_ref(),
20080 ) {
20081 let mut intended: Map<String, Value> = self
20082 .visible_row(table, &op.row_id)?
20083 .map(|(row, _)| {
20084 table
20085 .columns
20086 .iter()
20087 .zip(&row)
20088 .map(|(column, value)| {
20089 (column.name.clone(), column_value_to_json(value))
20090 })
20091 .collect()
20092 })
20093 .unwrap_or_default();
20094 intended.extend(values.clone());
20095 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
20096 continue;
20097 }
20098 }
20099 }
20100 self.apply_outbox_op(op)?;
20101 }
20102 Ok(())
20103 }
20104
20105 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
20106 let Some(table) = self.schema.table(&op.table) else {
20107 return Ok(());
20108 };
20109 let visible = visible_table(&table.name);
20110 if !op.upsert {
20111 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
20112 self.conn
20113 .prepare_cached(&sql)
20114 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
20115 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20116 return Ok(());
20117 }
20118 let Some(values) = op.values.as_ref() else {
20119 return Ok(());
20120 };
20121 let full = table
20126 .columns
20127 .iter()
20128 .all(|column| values.contains_key(&column.name));
20129 let local = self.visible_row(table, &op.row_id)?;
20130 if local.is_none() && !full {
20131 return Ok(());
20132 }
20133 let mut row: Row = Vec::with_capacity(table.columns.len());
20134 for (index, column) in table.columns.iter().enumerate() {
20135 let incoming = match values.get(&column.name) {
20136 Some(raw) => json_to_column_value(
20137 if raw.get("$bytes").is_some()
20138 && table
20139 .encrypted_columns
20140 .iter()
20141 .any(|encrypted| encrypted.index == index)
20142 {
20143 &table.wire_columns[index]
20144 } else {
20145 column
20146 },
20147 Some(raw),
20148 ),
20149 None => Ok(local
20150 .as_ref()
20151 .and_then(|(values, _)| values.get(index).cloned())
20152 .flatten()),
20153 };
20154 row.push(incoming.map_err(|_| {
20155 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
20156 })?);
20157 }
20158 let version = local.as_ref().map_or(-1, |(_, version)| *version);
20159 self.write_row(&visible, &table.name, &row, version)
20160 .map_err(|error| {
20161 if error.starts_with("UNIQUE constraint failed:") {
20162 "sync.constraint_violation: local write violates a unique constraint".to_owned()
20163 } else {
20164 error
20165 }
20166 })
20167 }
20168
20169 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
20172 self.stored_row(table, row_id, false)
20173 }
20174
20175 fn stored_row(
20176 &self,
20177 table: &TableSchema,
20178 row_id: &str,
20179 base: bool,
20180 ) -> Result<Option<(Row, i64)>, String> {
20181 let sql = format!(
20182 "SELECT * FROM {} WHERE {}",
20183 if base {
20184 base_table(&table.name)
20185 } else {
20186 visible_table(&table.name)
20187 },
20188 row_id_predicate(table)
20189 );
20190 let mut stmt = self
20191 .conn
20192 .prepare_cached(&sql)
20193 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20194 let mut rows = stmt
20195 .query(rusqlite::params![row_id])
20196 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20197 let Some(row) = rows
20198 .next()
20199 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
20200 else {
20201 return Ok(None);
20202 };
20203 let mut values = Vec::with_capacity(table.columns.len());
20204 for (index, column) in table.columns.iter().enumerate() {
20205 let raw = row
20206 .get_ref(index)
20207 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20208 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
20209 if table
20210 .encrypted_columns
20211 .iter()
20212 .any(|encrypted| encrypted.index == index)
20213 {
20214 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
20215 continue;
20216 }
20217 }
20218 let json = sql_ref_to_json(column, raw);
20219 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
20220 "sync.local_corrupt: persisted local row value is invalid".to_owned()
20221 })?);
20222 }
20223 let version: i64 = row
20224 .get(table.columns.len())
20225 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20226 Ok(Some((values, version)))
20227 }
20228
20229 #[must_use]
20233 pub fn realtime_policy(&self) -> RealtimePolicy {
20234 self.realtime_policy
20235 }
20236
20237 #[must_use]
20239 pub fn realtime_state(&self) -> RealtimeState {
20240 self.realtime_state
20241 }
20242
20243 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
20246 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
20247 return Err(
20248 "sync.invalid_request: disconnect realtime before changing the realtime policy"
20249 .to_owned(),
20250 );
20251 }
20252 self.realtime_policy = policy;
20253 match policy {
20254 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
20255 RealtimePolicy::Required | RealtimePolicy::Optional
20256 if self.realtime_state == RealtimeState::Disabled =>
20257 {
20258 self.set_realtime_state(RealtimeState::Disconnected, None);
20259 }
20260 RealtimePolicy::Required | RealtimePolicy::Optional => {}
20261 }
20262 Ok(())
20263 }
20264
20265 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
20268 self.realtime_state = state;
20269 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
20270 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
20271 self.realtime_reason_code = None;
20272 self.realtime_retry_delay_ms = None;
20273 }
20274 }
20275
20276 #[must_use]
20277 fn realtime_connected(&self) -> bool {
20278 self.realtime_state == RealtimeState::Connected
20279 }
20280
20281 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
20282 if !self.transport_enabled {
20283 return Err("sync.offline: transport gate is closed".into());
20284 }
20285 if self.realtime_policy == RealtimePolicy::Off {
20286 return Err(
20287 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
20288 );
20289 }
20290 if self.realtime_connected() {
20291 return Ok(());
20292 }
20293 self.realtime_state = RealtimeState::Connecting;
20294 match transport.realtime_connect_for_client(&self.client_id) {
20295 Ok(()) => {
20296 self.set_realtime_state(RealtimeState::Connected, None);
20297 Ok(())
20298 }
20299 Err(error) => {
20300 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
20301 Err(format!("{}: {}", error.code, error.message))
20302 }
20303 }
20304 }
20305
20306 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
20307 self.cancel_sync_round();
20308 if self.realtime_state == RealtimeState::Disconnected
20311 || self.realtime_state == RealtimeState::Disabled
20312 {
20313 return;
20314 }
20315 let _ = transport.realtime_close();
20316 self.set_realtime_state(
20317 if self.realtime_policy == RealtimePolicy::Off {
20318 RealtimeState::Disabled
20319 } else {
20320 RealtimeState::Disconnected
20321 },
20322 None,
20323 );
20324 self.presence.clear(); }
20326
20327 pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
20333 if self.realtime_state != RealtimeState::Connected {
20334 return;
20335 }
20336 self.cancel_sync_round();
20337 let _ = transport.realtime_close();
20338 self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
20339 self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
20340 self.presence.clear();
20341 }
20342
20343 pub fn set_presence(
20349 &mut self,
20350 transport: &mut dyn Transport,
20351 scope_key: &str,
20352 doc: Option<&Value>,
20353 ) -> Result<(), String> {
20354 if !self.transport_enabled {
20355 return Err("sync.offline: transport gate is closed".into());
20356 }
20357 if !self.realtime_connected() {
20358 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
20359 }
20360 let text = encode_presence_publish(scope_key, doc);
20361 transport
20362 .realtime_send(&text)
20363 .map_err(|e| format!("{}: {}", e.code, e.message))
20364 }
20365
20366 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
20368 self.presence
20369 .get(scope_key)
20370 .map(|peers| peers.values().cloned().collect())
20371 .unwrap_or_default()
20372 }
20373
20374 fn apply_presence(
20376 &mut self,
20377 scope_key: String,
20378 kind: Option<PresenceKind>,
20379 actor_id: Option<String>,
20380 client_id: Option<String>,
20381 doc: Option<Value>,
20382 error: Option<String>,
20383 ) {
20384 if error.is_some() {
20387 return;
20388 }
20389 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
20390 return;
20391 };
20392 let peer_key = format!("{actor_id} {client_id}");
20393 match kind {
20394 PresenceKind::Leave => {
20395 if let Some(peers) = self.presence.get_mut(&scope_key) {
20396 peers.remove(&peer_key);
20397 if peers.is_empty() {
20398 self.presence.remove(&scope_key);
20399 }
20400 }
20401 }
20402 _ => {
20403 let doc = match doc {
20404 Some(Value::Object(_)) => doc.unwrap(),
20405 _ => return,
20406 };
20407 self.presence.entry(scope_key).or_default().insert(
20408 peer_key,
20409 PresencePeer {
20410 actor_id,
20411 client_id,
20412 doc,
20413 },
20414 );
20415 }
20416 }
20417 }
20418
20419 pub fn on_realtime_text(&mut self, text: &str) {
20421 if self.stopped || !self.transport_enabled || self.security_preflight {
20422 return;
20423 }
20424 match parse_control(text) {
20425 Ok(ControlMessage::Hello { requires_sync, .. }) => {
20426 if requires_sync {
20427 self.set_sync_needed(true, true);
20429 }
20430 }
20431 Ok(ControlMessage::Presence {
20432 scope_key,
20433 kind,
20434 actor_id,
20435 client_id,
20436 doc,
20437 error,
20438 ..
20439 }) => {
20440 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
20441 }
20442 Ok(ControlMessage::Wake { .. }) => {
20443 self.set_sync_needed(true, true);
20445 }
20446 _ => {}
20447 }
20448 }
20449
20450 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
20454 let mut transport = crate::round::DownloadResults::default();
20455 self.on_realtime_binary(&mut transport, bytes);
20456 transport.controls
20457 }
20458
20459 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
20460 if self.stopped || !self.transport_enabled || self.security_preflight {
20461 return;
20462 }
20463 #[cfg(feature = "bench-internals")]
20464 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
20465 let message = match decode_message(bytes) {
20466 Ok(m) if m.msg_kind == MsgKind::Response => m,
20467 _ => {
20468 self.set_sync_needed(true, true);
20469 return;
20470 }
20471 };
20472 #[cfg(feature = "bench-internals")]
20473 drop(decode_phase);
20474 let mut frames = message.frames.into_iter();
20475 let mut applied_cursor: Option<i64> = None;
20476 let mut any_covered = false;
20477 let mut dropped = self.begin_coalesced_apply().is_err();
20478 while let Some(frame) = frames.next() {
20479 let Frame::SubStart {
20480 id,
20481 status,
20482 effective_scopes,
20483 ..
20484 } = frame
20485 else {
20486 continue;
20487 };
20488 let mut body = Vec::new();
20489 let mut next_cursor: Option<i64> = None;
20490 for inner in frames.by_ref() {
20491 match inner {
20492 Frame::SubEnd {
20493 next_cursor: nc, ..
20494 } => {
20495 next_cursor = Some(nc);
20496 break;
20497 }
20498 Frame::Unknown { .. } => {}
20499 other => body.push(other),
20500 }
20501 }
20502 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
20503 dropped = true;
20504 continue;
20505 };
20506 let sub = &self.subs[sub_index];
20507 if status != SubStatus::Active
20510 || sub.state != SubState::Active
20511 || sub.bootstrap_state.is_some()
20512 || !sub.synced_once
20513 {
20514 dropped = true;
20515 continue;
20516 }
20517 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
20518 any_covered = true;
20520 continue;
20521 }
20522 let previous_effective = self.subs[sub_index].effective.clone();
20523 let previous_cursor = self.subs[sub_index].cursor;
20524 self.subs[sub_index].effective = Some(effective_scopes);
20525 let mut failed = false;
20526 for inner in body {
20527 match inner {
20528 Frame::Commit {
20529 tables,
20530 changes,
20531 commit_seq,
20532 ..
20533 } => {
20534 if self
20535 .apply_commit_frame(&tables, &changes, Some(commit_seq))
20536 .is_err()
20537 {
20538 failed = true;
20539 break;
20540 }
20541 }
20542 _ => {
20543 failed = true;
20544 break;
20545 }
20546 }
20547 }
20548 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
20549 self.apply_sub_end(sub_index, cursor, None)
20550 .ok()
20551 .map(|()| cursor)
20552 });
20553 let Some(next_cursor) = committed else {
20554 self.subs[sub_index].effective = previous_effective;
20555 self.subs[sub_index].cursor = previous_cursor;
20556 dropped = true;
20557 continue;
20558 };
20559 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
20560 }
20561 if self.finish_coalesced_apply().is_err() {
20564 applied_cursor = None;
20565 dropped = true;
20566 }
20567 if let Some(cursor) = applied_cursor {
20568 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
20570 let _ = transport.realtime_send(&ack);
20571 } else if !any_covered || dropped {
20572 self.set_sync_needed(true, true);
20574 }
20575 }
20576
20577 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
20581 if !self.realtime_connected() {
20582 return;
20583 }
20584 let floor = self
20585 .subs
20586 .iter()
20587 .filter(|s| {
20588 s.state == SubState::Active
20589 && s.bootstrap_state.is_none()
20590 && s.synced_once
20591 && s.cursor >= 0
20592 })
20593 .map(|s| s.cursor)
20594 .min();
20595 if let Some(cursor) = floor {
20596 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
20597 let _ = transport.realtime_send(&ack);
20598 }
20599 }
20600}
20601
20602#[cfg(test)]
20603mod previous_version_wiring_tests {
20604 use super::*;
20605 use serde_json::{json, Value};
20606
20607 fn previous_version_schema(version: i32) -> Value {
20608 json!({
20609 "version": version,
20610 "tables": [{
20611 "name": "tasks",
20612 "primaryKey": "id",
20613 "columns": [
20614 { "name": "id", "type": "string", "nullable": false },
20615 { "name": "note", "type": "string", "nullable": true }
20616 ],
20617 "scopes": []
20618 }]
20619 })
20620 }
20621
20622 fn previous_version_limits() -> crate::api::ClientLimits {
20623 crate::api::ClientLimits {
20624 previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
20625 enabled: true,
20626 ..Default::default()
20627 }),
20628 ..Default::default()
20629 }
20630 }
20631
20632 fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
20634 previous_version_limits()
20635 .previous_version_context
20636 .expect("configured")
20637 }
20638
20639 fn recapture_previous_version(client: &SyncClient) {
20642 let replica_path = client
20643 .previous_version_replica_path()
20644 .expect("file-backed replica");
20645 let outcome = crate::previous_version::capture_previous_version_from_replica(
20646 &client.conn,
20647 &replica_path,
20648 2,
20649 2,
20650 &previous_version_config(),
20651 1,
20652 )
20653 .expect("capture");
20654 assert!(matches!(
20655 outcome,
20656 crate::previous_version::CaptureOutcome::Captured(_)
20657 ));
20658 }
20659
20660 fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
20661 Subscription {
20662 id: "wiring-sub".to_owned(),
20663 table: "tasks".to_owned(),
20664 requested: Vec::new(),
20665 params: None,
20666 cursor,
20667 bootstrap_state: bootstrap_state.map(str::to_owned),
20668 state,
20669 reason_code: None,
20670 effective: None,
20671 synced_once: false,
20672 }
20673 }
20674
20675 #[test]
20676 fn schema_downgrade_refuses_all_opens_and_recreate_without_mutation() {
20677 let path =
20678 std::env::temp_dir().join(format!("syncular-downgrade-{}.db", uuid::Uuid::new_v4()));
20679 let path_str = path.to_str().expect("temp path");
20680 let schema = previous_version_schema(3);
20681 let mut older = previous_version_schema(2);
20682 older["tables"][0]["columns"] =
20683 json!([{ "name": "id", "type": "string", "nullable": false }]);
20684 let mut client = SyncClient::open_path(
20685 "downgrade".into(),
20686 &previous_version_schema(1),
20687 previous_version_limits(),
20688 path_str,
20689 )
20690 .expect("fresh");
20691 client
20692 .mutate(vec![Mutation::Upsert {
20693 table: "tasks".into(),
20694 values: Map::from_iter([
20695 ("id".into(), json!("accepted")),
20696 ("note".into(), json!("old")),
20697 ]),
20698 base_version: None,
20699 }])
20700 .expect("v1 write");
20701 client
20702 .recreate_with_schema(&schema)
20703 .expect("upgrade captures context");
20704 let commit = client
20705 .mutate(vec![Mutation::Upsert {
20706 table: "tasks".into(),
20707 values: Map::from_iter([
20708 ("id".into(), json!("queued")),
20709 ("note".into(), json!("v3-only")),
20710 ]),
20711 base_version: None,
20712 }])
20713 .expect("queued v3 write");
20714 let changes = client.conn.total_changes();
20715 let outbox = client.pending_commit_ids();
20716 assert!(outbox.contains(&commit));
20717 let descriptor = client.get_meta("localSchemaDescriptor");
20718 let container = format!(
20719 "{path_str}{}",
20720 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
20721 );
20722 let context = std::fs::read(&container).expect("captured context");
20723 assert!(client
20724 .recreate_with_schema(&older)
20725 .expect_err("refuse recreate")
20726 .starts_with("client.schema_downgrade:"));
20727 assert_eq!(client.schema.version, 3);
20728 assert_eq!(client.conn.total_changes(), changes);
20729 assert_eq!(client.pending_commit_ids(), outbox);
20730 assert_eq!(client.get_meta("localSchemaDescriptor"), descriptor);
20731 assert_eq!(
20732 client
20733 .conn
20734 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20735 row.get::<_, String>(0)
20736 })
20737 .expect("queued row"),
20738 "v3-only"
20739 );
20740 assert_eq!(std::fs::read(&container).expect("context remains"), context);
20741 drop(client);
20742 let before = std::fs::read(&path).expect("replica bytes");
20743 for entry in 0..3 {
20746 let error = match entry {
20747 0 => SyncClient::open_path(
20748 "downgrade".into(),
20749 &older,
20750 previous_version_limits(),
20751 path_str,
20752 ),
20753 1 => SyncClient::open_path_with_identity(
20754 None,
20755 &older,
20756 previous_version_limits(),
20757 path_str,
20758 ),
20759 _ => SyncClient::with_connection(
20760 "downgrade".into(),
20761 &older,
20762 previous_version_limits(),
20763 Connection::open(&path).expect("connection"),
20764 ),
20765 }
20766 .err()
20767 .expect("refuse open");
20768 assert!(error.starts_with("client.schema_downgrade:"));
20769 let conn = Connection::open(&path).expect("inspect");
20770 assert_eq!(
20771 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20772 .expect("journal mode"),
20773 "delete"
20774 );
20775 drop(conn);
20776 assert_eq!(std::fs::read(&path).expect("replica unchanged"), before);
20777 assert_eq!(
20778 std::fs::read(&container).expect("context unchanged"),
20779 context
20780 );
20781 }
20782 let reopened = SyncClient::open_path(
20783 "downgrade".into(),
20784 &schema,
20785 previous_version_limits(),
20786 path_str,
20787 )
20788 .expect("compatible reopen");
20789 assert_eq!(reopened.pending_commit_ids(), outbox);
20790 assert_eq!(
20791 reopened
20792 .conn
20793 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20794 row.get::<_, String>(0)
20795 })
20796 .expect("queued row remains"),
20797 "v3-only"
20798 );
20799 reopened.delete_meta(LOCAL_SCHEMA_VERSION_KEY);
20800 reopened.delete_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20801 drop(reopened);
20802 let legacy = SyncClient::open_path(
20803 "downgrade".into(),
20804 &schema,
20805 previous_version_limits(),
20806 path_str,
20807 )
20808 .expect("legacy no-marker reopen");
20809 assert_eq!(legacy.pending_commit_ids(), outbox);
20810 assert!(!legacy.upgrading());
20811 assert_eq!(
20812 legacy
20813 .conn
20814 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20815 row.get::<_, String>(0)
20816 })
20817 .expect("legacy row remains"),
20818 "v3-only"
20819 );
20820 drop(legacy);
20821 std::fs::remove_file(path).expect("remove replica");
20822 let _ = std::fs::remove_file(container);
20823 }
20824
20825 #[test]
20826 fn schema_marker_errors_refuse_open_and_recreate_without_mutation() {
20827 for marker in [
20828 "''",
20829 "'bad'",
20830 "'0'",
20831 "'-1'",
20832 "'2.5'",
20833 "'03'",
20834 "'3\n'",
20835 "' 3'",
20836 "'+3'",
20837 "'3e0'",
20838 "'2147483648'",
20839 "NULL",
20840 "X'33'",
20841 ] {
20842 let conn = Connection::open_in_memory().expect("connection");
20843 conn.execute_batch(&format!("CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value); INSERT INTO _syncular_meta VALUES ('localSchemaVersion', {marker});")).expect("invalid marker fixture");
20844 let error = SyncClient::with_connection(
20845 "marker".into(),
20846 &previous_version_schema(3),
20847 ClientLimits::default(),
20848 conn,
20849 )
20850 .err()
20851 .expect("refuse marker");
20852 assert!(
20853 error.starts_with("sync.local_corrupt:"),
20854 "{marker}: {error}"
20855 );
20856 }
20857 for fault in [
20858 "UPDATE _syncular_meta SET value = 'bad' WHERE key = 'localSchemaVersion'",
20859 "ALTER TABLE _syncular_meta RENAME COLUMN value TO unreadable_value",
20860 ] {
20861 let mut client = SyncClient::new(
20862 "marker".into(),
20863 &previous_version_schema(3),
20864 ClientLimits::default(),
20865 )
20866 .expect("fresh");
20867 let commit = client
20868 .mutate(vec![Mutation::Upsert {
20869 table: "tasks".into(),
20870 values: Map::from_iter([("id".into(), json!("queued"))]),
20871 base_version: None,
20872 }])
20873 .expect("queued write");
20874 client.conn.execute_batch(fault).expect("marker fault");
20875 let changes = client.conn.total_changes();
20876 let error = client
20877 .recreate_with_schema(&previous_version_schema(4))
20878 .expect_err("refuse corrupt marker");
20879 assert!(error.starts_with("sync.local_corrupt:"));
20880 assert_eq!(client.conn.total_changes(), changes);
20881 assert_eq!(client.schema.version, 3);
20882 assert_eq!(client.pending_commit_ids(), vec![commit]);
20883 }
20884 for setup in [
20885 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY)",
20886 "CREATE VIEW _syncular_meta AS SELECT 'localSchemaVersion' AS key, '3' AS value",
20887 ] {
20888 let path =
20889 std::env::temp_dir().join(format!("syncular-marker-{}.db", uuid::Uuid::new_v4()));
20890 let conn = Connection::open(&path).expect("connection");
20891 conn.execute_batch(setup)
20892 .expect("unreadable metadata fixture");
20893 drop(conn);
20894 let before = std::fs::read(&path).expect("before");
20895 let error = SyncClient::open_path_with_identity(
20896 None,
20897 &previous_version_schema(3),
20898 ClientLimits::default(),
20899 path.to_str().expect("path"),
20900 )
20901 .err()
20902 .expect("refuse unreadable marker");
20903 assert!(error.starts_with("sync.local_corrupt:"));
20904 assert_eq!(std::fs::read(&path).expect("after"), before);
20905 std::fs::remove_file(path).expect("remove fixture");
20906 }
20907 }
20908
20909 std::thread_local! {
20910 static CONCURRENT_UPGRADE: RefCell<Option<Connection>> =
20913 const { RefCell::new(None) };
20914 }
20915
20916 fn commit_concurrent_upgrade(attempts: i32) -> bool {
20917 CONCURRENT_UPGRADE.with(|slot| {
20918 if let Some(banner) = slot.borrow_mut().take() {
20919 banner
20920 .execute_batch(
20921 "UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion'; COMMIT",
20922 )
20923 .expect("commit the concurrent upgrade");
20924 }
20925 });
20926 attempts < 10
20927 }
20928
20929 fn journal_mode(conn: &Connection) -> String {
20930 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20931 .expect("journal mode")
20932 }
20933
20934 fn temp_replica(label: &str) -> std::path::PathBuf {
20935 std::env::temp_dir().join(format!("syncular-{label}-{}.db", uuid::Uuid::new_v4()))
20936 }
20937
20938 #[test]
20939 fn file_identity_constructor_requires_the_returned_wal_mode() {
20940 for path in [":memory:", ""] {
20941 let error = SyncClient::open_path_with_identity(
20942 None,
20943 &previous_version_schema(1),
20944 ClientLimits::default(),
20945 path,
20946 )
20947 .err()
20948 .expect("temporary SQLite databases cannot enter WAL mode");
20949 assert_eq!(
20950 error,
20951 "sync.invalid_request: file-backed client requires WAL journal mode"
20952 );
20953 }
20954 let path = temp_replica("confirmed-wal");
20955 let client = SyncClient::open_path_with_identity(
20956 None,
20957 &previous_version_schema(1),
20958 ClientLimits::default(),
20959 path.to_str().unwrap(),
20960 )
20961 .unwrap();
20962 assert_eq!(journal_mode(&client.conn), "wal");
20963 drop(client);
20964 std::fs::remove_file(path).unwrap();
20965 }
20966
20967 #[test]
20968 fn non_positive_generated_schema_versions_are_refused_before_storage() {
20969 for version in [json!(0), json!(-1), json!(i32::MIN), json!(2147483648i64)] {
20970 let mut schema = previous_version_schema(1);
20971 schema["version"] = version.clone();
20972 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20973 .err()
20974 .expect("refuse an out-of-range generated version");
20975 assert_eq!(
20976 error,
20977 "sync.invalid_request: generated schema version must be an integer in 1..=2147483647"
20978 );
20979 let path = temp_replica("bad-version");
20981 let path_str = path.to_str().expect("path");
20982 for identity in [false, true] {
20983 let error = if identity {
20984 SyncClient::open_path_with_identity(
20985 None,
20986 &schema,
20987 ClientLimits::default(),
20988 path_str,
20989 )
20990 } else {
20991 SyncClient::open_path(
20992 "bad-version".into(),
20993 &schema,
20994 ClientLimits::default(),
20995 path_str,
20996 )
20997 }
20998 .err()
20999 .expect("refuse before storage");
21000 assert!(
21001 error.starts_with("sync.invalid_request:"),
21002 "{version}: {error}"
21003 );
21004 assert!(
21005 !path.exists(),
21006 "{version}: a refused constructor creates no replica file"
21007 );
21008 }
21009 }
21010 let mut schema = previous_version_schema(1);
21012 schema
21013 .as_object_mut()
21014 .expect("schema object")
21015 .remove("version");
21016 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
21017 .err()
21018 .expect("refuse a missing version");
21019 assert!(error.starts_with("sync.invalid_request:"), "{error}");
21020 }
21021
21022 #[test]
21023 fn a_duplicated_schema_marker_is_refused_without_a_reset() {
21024 let path = temp_replica("duplicate-marker");
21025 let path_str = path.to_str().expect("path");
21026 {
21027 let conn = Connection::open(path_str).expect("fixture");
21030 conn.execute_batch(
21031 "CREATE TABLE _syncular_meta(key TEXT, value);
21032 INSERT INTO _syncular_meta VALUES
21033 ('localSchemaVersion', '1'),
21034 ('localSchemaVersion', '2'),
21035 ('clientId', 'duplicate');",
21036 )
21037 .expect("duplicate marker fixture");
21038 }
21039 let error = SyncClient::open_path_with_identity(
21040 None,
21041 &previous_version_schema(3),
21042 ClientLimits::default(),
21043 path_str,
21044 )
21045 .err()
21046 .expect("refuse a duplicated marker");
21047 assert!(error.starts_with("sync.local_corrupt:"), "{error}");
21048 let probe = Connection::open(path_str).expect("probe");
21049 assert_eq!(
21050 probe
21051 .query_row(
21052 "SELECT COUNT(*) FROM _syncular_meta WHERE key = 'localSchemaVersion'",
21053 [],
21054 |row| row.get::<_, i64>(0),
21055 )
21056 .expect("marker rows"),
21057 2,
21058 "the duplicated rows are left for the operator, not collapsed"
21059 );
21060 assert_eq!(
21061 probe
21062 .query_row(
21063 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_syncular_%' AND name NOT LIKE 'sqlite_%'",
21064 [],
21065 |row| row.get::<_, i64>(0),
21066 )
21067 .expect("synced tables"),
21068 0,
21069 "a duplicated marker never drives the reset that recreates v3 tables"
21070 );
21071 assert_eq!(
21072 journal_mode(&probe),
21073 "delete",
21074 "the refusal leaves the journal mode alone"
21075 );
21076 drop(probe);
21077 std::fs::remove_file(path).expect("remove replica");
21078 }
21079
21080 #[test]
21081 fn a_refused_open_does_not_persist_wal_or_mutate_the_replica() {
21082 let path = temp_replica("refused-wal");
21083 let path_str = path.to_str().expect("path");
21084 drop(
21087 SyncClient::open_path(
21088 "owner".into(),
21089 &previous_version_schema(2),
21090 ClientLimits::default(),
21091 path_str,
21092 )
21093 .expect("v2 replica"),
21094 );
21095 let before = std::fs::read(&path).expect("before");
21096 let error = SyncClient::open_path_with_identity(
21097 Some("intruder".into()),
21098 &previous_version_schema(2),
21099 ClientLimits::default(),
21100 path_str,
21101 )
21102 .err()
21103 .expect("refuse the identity mismatch");
21104 assert!(error.starts_with("client.identity_mismatch:"), "{error}");
21105 let probe = Connection::open(path_str).expect("probe");
21106 assert_eq!(
21107 journal_mode(&probe),
21108 "delete",
21109 "a refused open must not persist WAL"
21110 );
21111 assert_eq!(std::fs::read(&path).expect("after"), before);
21112 assert_eq!(
21113 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
21114 Some("2")
21115 );
21116 drop(probe);
21117 std::fs::remove_file(path).expect("remove replica");
21118 }
21119
21120 #[test]
21121 fn a_concurrent_marker_upgrade_refuses_a_stale_open() {
21122 let path = temp_replica("marker-race");
21123 let path_str = path.to_str().expect("path");
21124 drop(
21125 SyncClient::open_path_with_identity(
21126 Some("stale".into()),
21127 &previous_version_schema(1),
21128 ClientLimits::default(),
21129 path_str,
21130 )
21131 .expect("v1 replica"),
21132 );
21133 let banner = Connection::open(path_str).expect("banner");
21137 banner
21138 .execute_batch(
21139 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
21140 )
21141 .expect("pending upgrade");
21142 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
21143 let conn = Connection::open(path_str).expect("stale connection");
21144 conn.busy_handler(Some(commit_concurrent_upgrade))
21145 .expect("busy handler");
21146 let error = SyncClient::with_connection(
21147 "stale".into(),
21148 &previous_version_schema(1),
21149 ClientLimits::default(),
21150 conn,
21151 )
21152 .err()
21153 .expect("a concurrent upgrade must refuse the stale open");
21154 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
21155 let probe = Connection::open(path_str).expect("probe");
21156 assert_eq!(
21157 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
21158 Some("2"),
21159 "the concurrent upgrade survives"
21160 );
21161 drop(probe);
21162 std::fs::remove_file(path).expect("remove replica");
21163 }
21164
21165 #[test]
21166 fn a_concurrent_marker_upgrade_refuses_a_stale_recreate() {
21167 let path = temp_replica("recreate-race");
21168 let path_str = path.to_str().expect("path");
21169 let mut client = SyncClient::open_path_with_identity(
21170 Some("stale".into()),
21171 &previous_version_schema(1),
21172 ClientLimits::default(),
21173 path_str,
21174 )
21175 .expect("v1 replica");
21176 client
21177 .mutate(vec![Mutation::Upsert {
21178 table: "tasks".into(),
21179 values: Map::from_iter([("id".into(), json!("queued"))]),
21180 base_version: None,
21181 }])
21182 .expect("queued write");
21183 let commit = client.pending_commit_ids();
21184 let banner = Connection::open(path_str).expect("banner");
21185 banner
21186 .execute_batch(
21187 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
21188 )
21189 .expect("pending upgrade");
21190 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
21191 client
21192 .conn
21193 .busy_handler(Some(commit_concurrent_upgrade))
21194 .expect("busy handler");
21195 let error = client
21196 .recreate_with_schema(&previous_version_schema(1))
21197 .expect_err("a concurrent upgrade must refuse the stale recreate");
21198 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
21199 assert_eq!(client.schema.version, 1, "the compiled schema is restored");
21200 assert_eq!(
21201 client.pending_commit_ids(),
21202 commit,
21203 "the outbox is preserved"
21204 );
21205 let probe = Connection::open(path_str).expect("probe");
21206 assert_eq!(
21207 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
21208 Some("2")
21209 );
21210 drop(probe);
21211 drop(client);
21212 std::fs::remove_file(path).expect("remove replica");
21213 }
21214
21215 #[test]
21216 fn failed_schema_record_writes_roll_back_recreation() {
21217 for key in [
21218 LOCAL_SCHEMA_VERSION_KEY,
21219 crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY,
21220 ] {
21221 let mut client = SyncClient::new(
21222 "schema-record".into(),
21223 &previous_version_schema(1),
21224 ClientLimits::default(),
21225 )
21226 .expect("v1");
21227 let marker = client.get_meta(LOCAL_SCHEMA_VERSION_KEY);
21228 let descriptor = client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
21229 client.conn.execute_batch(&format!("CREATE TRIGGER refuse_schema_record BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{key}' BEGIN SELECT RAISE(ABORT, 'refuse schema record'); END")).expect("fault");
21230 client
21231 .recreate_with_schema(&previous_version_schema(2))
21232 .expect_err("both record writes must propagate failure");
21233 assert_eq!(client.schema.version, 1);
21234 assert_eq!(client.get_meta(LOCAL_SCHEMA_VERSION_KEY), marker);
21235 assert_eq!(
21236 client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY),
21237 descriptor
21238 );
21239 client
21240 .conn
21241 .execute_batch("DROP TRIGGER refuse_schema_record")
21242 .expect("remove fault");
21243 client
21244 .recreate_with_schema(&previous_version_schema(2))
21245 .expect("retry");
21246 assert_eq!(client.schema.version, 2);
21247 }
21248 }
21249
21250 #[test]
21251 fn a_failed_outer_commit_restores_every_reset_side_effect() {
21252 let path = temp_replica("outer-commit");
21253 let path_str = path.to_str().expect("path");
21254 let conn = Connection::open(path_str).expect("replica connection");
21257 let mut client = SyncClient::with_connection(
21258 "outer-commit".into(),
21259 &previous_version_schema(1),
21260 previous_version_limits(),
21261 conn,
21262 )
21263 .expect("v1 replica");
21264 assert_eq!(journal_mode(&client.conn), "delete");
21265 let commit = client
21266 .mutate(vec![Mutation::Upsert {
21267 table: "tasks".into(),
21268 values: Map::from_iter([
21269 ("id".into(), json!("queued")),
21270 ("note".into(), json!("dropped-by-v2")),
21271 ]),
21272 base_version: None,
21273 }])
21274 .expect("queued write");
21275 client.drain_change_batches();
21276 let mut dropped = previous_version_schema(2);
21278 dropped["tables"][0]["columns"] =
21279 json!([{ "name": "id", "type": "string", "nullable": false }]);
21280 let marker = meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY);
21283 let revision = client.local_revision();
21284 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
21285 let columns: i64 = client
21286 .conn
21287 .query_row(
21288 "SELECT count(*) FROM pragma_table_info('tasks')",
21289 [],
21290 |row| row.get(0),
21291 )
21292 .expect("column count");
21293 let rebuilds = client.overlay_rebuild_count.get();
21294 let prior_subs = client.subs.len();
21295 let prior_changes = client.change_queue.len();
21296 let prior_intents = client.sync_intent_queue.len();
21297 let prior_last_change = client.last_change.clone();
21298 let prior_overlay_dirty = client.overlay_dirty.get();
21299 let prior_sync_needed = client.sync_needed;
21300 let prior_upgrading = client.upgrading;
21301 let prior_stopped = client.stopped;
21302 let prior_schema_floor = client.schema_floor.clone();
21303 let prior_active_round = client.active_round;
21304 let reader = Connection::open(path_str).expect("reader");
21307 reader
21308 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
21309 .expect("reader snapshot");
21310 client
21311 .conn
21312 .busy_timeout(std::time::Duration::ZERO)
21313 .expect("busy timeout");
21314 let error = client
21315 .recreate_with_schema(&dropped)
21316 .expect_err("a blocked COMMIT must refuse the recreation");
21317 assert!(error.starts_with("client.storage_busy:"), "{error}");
21318 assert!(
21319 client.overlay_rebuild_count.get() > rebuilds,
21320 "the reset ran to its overlay rebuild inside the transaction"
21321 );
21322 assert!(client.conn.is_autocommit(), "the transaction is released");
21323 assert_eq!(meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY), marker);
21325 assert_eq!(client.local_revision(), revision);
21326 assert_eq!(
21327 client.query("SELECT * FROM tasks", &[]).expect("rows"),
21328 rows
21329 );
21330 assert_eq!(
21331 client
21332 .conn
21333 .query_row(
21334 "SELECT count(*) FROM pragma_table_info('tasks')",
21335 [],
21336 |row| { row.get::<_, i64>(0) }
21337 )
21338 .expect("column count"),
21339 columns,
21340 "the previous table layout survives the rollback"
21341 );
21342 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
21343 assert_eq!(client.schema.version, 1, "compiled schema");
21346 assert_eq!(client.subs.len(), prior_subs);
21347 assert_eq!(client.change_queue.len(), prior_changes);
21348 assert_eq!(client.sync_intent_queue.len(), prior_intents);
21349 assert_eq!(client.last_change, prior_last_change);
21350 assert_eq!(client.overlay_dirty.get(), prior_overlay_dirty);
21351 assert_eq!(client.sync_needed, prior_sync_needed);
21352 assert_eq!(client.upgrading, prior_upgrading);
21353 assert_eq!(client.stopped, prior_stopped);
21354 assert_eq!(client.schema_floor, prior_schema_floor);
21355 assert_eq!(client.active_round, prior_active_round);
21356 assert!(
21357 client.rejections.is_empty(),
21358 "the dropped-column rejection is rolled back too"
21359 );
21360 let container = crate::previous_version::previous_version_container_path(path_str);
21363 assert!(std::path::Path::new(&container).exists());
21364 drop(reader);
21367 let snapshot = client
21368 .previous_version_snapshot(&PreviousVersionReadSpec {
21369 table: "tasks".into(),
21370 ..Default::default()
21371 })
21372 .expect("refuse stale capture");
21373 assert!(!snapshot.available);
21374 assert_eq!(snapshot.current_version, 1);
21375 assert!(!std::path::Path::new(&container).exists());
21376 client
21381 .conn
21382 .busy_timeout(std::time::Duration::from_secs(1))
21383 .expect("busy timeout");
21384 client
21385 .recreate_with_schema(&dropped)
21386 .expect("the recreation succeeds once the reader releases");
21387 assert_eq!(client.schema.version, 2);
21388 assert_eq!(
21389 client.pending_commit_ids(),
21390 vec![commit.clone()],
21391 "the reset kept the incompatible commit pending"
21392 );
21393 assert!(
21394 client.rejections.is_empty(),
21395 "no rejection exists before the send-time prepass"
21396 );
21397 client
21399 .prepare_sync_round(false)
21400 .expect("the send prepass runs during round preparation");
21401 assert!(client.pending_commit_ids().is_empty());
21402 assert!(
21403 client
21404 .rejections
21405 .iter()
21406 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE),
21407 "the dropped-column commit surfaced as a rejection at send time"
21408 );
21409 client.previous_version_discard();
21410 drop(client);
21411 std::fs::remove_file(path).expect("remove replica");
21412 }
21413
21414 #[test]
21415 fn the_send_time_incompatible_drop_is_atomic_and_publishes_once() {
21416 let v1 = previous_version_schema(1);
21417 let mut v2 = previous_version_schema(2);
21418 v2["tables"][0]["columns"] = json!([{ "name": "id", "type": "string", "nullable": false }]);
21420 let mut client = SyncClient::new("drop".to_string(), &v1, ClientLimits::default()).unwrap();
21421 let dropped = client
21422 .mutate(vec![Mutation::Upsert {
21423 table: "tasks".into(),
21424 values: Map::from_iter([
21425 ("id".into(), json!("dropped")),
21426 ("note".into(), json!("gone")),
21427 ]),
21428 base_version: None,
21429 }])
21430 .unwrap();
21431 let survivor = client
21435 .mutate(vec![Mutation::Delete {
21436 table: "tasks".into(),
21437 row_id: "gone".into(),
21438 base_version: None,
21439 }])
21440 .unwrap();
21441 client
21442 .recreate_with_schema(&v2)
21443 .expect("the reset preserves the outbox");
21444 assert_eq!(
21445 client.pending_commit_ids(),
21446 vec![dropped.clone(), survivor.clone()],
21447 "the reset preserves both commits"
21448 );
21449 assert!(client.rejections.is_empty(), "the reset classifies nothing");
21450 client.drain_change_batches();
21451 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21452 assert_eq!(rows.len(), 1, "only the dropped upsert has a visible row");
21453
21454 for fault in [
21460 "outcome-insert",
21461 "outbox-delete",
21462 "overlay-rebuild",
21463 "publish-revision",
21464 ] {
21465 client.set_sync_needed(false, false);
21466 client.drain_change_batches();
21467 let revision = client.local_revision();
21468 let trigger = match fault {
21469 "outcome-insert" => {
21470 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21471 }
21472 "outbox-delete" => {
21473 "CREATE TRIGGER fail_drop BEFORE DELETE ON _syncular_outbox BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21474 }
21475 "publish-revision" => {
21480 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21481 }
21482 _ => {
21483 "CREATE TRIGGER fail_drop BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21484 }
21485 };
21486 client.conn.execute_batch(trigger).unwrap();
21487 let outcome = client
21488 .prepare_sync_round(false)
21489 .err()
21490 .unwrap_or_else(|| panic!("{fault}: a fault in the drop must fail the round"));
21491 assert!(
21492 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "storage.failed"),
21493 "{fault}: {outcome:?}"
21494 );
21495 client.conn.execute_batch("DROP TRIGGER fail_drop").unwrap();
21496 assert!(client.conn.is_autocommit(), "{fault}: no leaked savepoint");
21497 assert_eq!(
21498 client.pending_commit_ids(),
21499 vec![dropped.clone(), survivor.clone()],
21500 "{fault}: the outbox is restored"
21501 );
21502 assert!(
21503 client.rejections.is_empty(),
21504 "{fault}: no in-memory rejection survives"
21505 );
21506 assert!(
21507 client.commit_outcome(&dropped).unwrap().is_none(),
21508 "{fault}: the durable outcome rolled back"
21509 );
21510 assert_eq!(client.local_revision(), revision, "{fault}: revision");
21511 assert_eq!(
21512 client.query("SELECT * FROM tasks", &[]).unwrap(),
21513 rows,
21514 "{fault}: the visible projection is unchanged"
21515 );
21516 assert!(
21517 client.drain_change_batches().is_empty(),
21518 "{fault}: nothing is published"
21519 );
21520 }
21521
21522 client
21525 .conn
21526 .execute_batch(
21527 "BEGIN; INSERT INTO _syncular_meta(key,value) VALUES ('outerProbe','kept');
21528 CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes
21529 BEGIN SELECT RAISE(ABORT,'nested drop fault'); END;",
21530 )
21531 .unwrap();
21532 assert!(client.drop_incompatible_outbox().is_err());
21533 assert!(!client.conn.is_autocommit());
21534 assert_eq!(client.get_meta("outerProbe").as_deref(), Some("kept"));
21535 assert_eq!(
21536 client.pending_commit_ids(),
21537 vec![dropped.clone(), survivor.clone()]
21538 );
21539 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21540 client
21541 .conn
21542 .execute_batch("DROP TRIGGER fail_drop; COMMIT")
21543 .unwrap();
21544
21545 client.set_meta(LOG_EPOCH_KEY, "epoch");
21550 client.set_sync_needed(false, false);
21551 client.drain_change_batches();
21552 let revision = client.local_revision();
21553 let prepared = client.prepare_sync_round(false).expect("the drop succeeds");
21554 assert_eq!(client.pending_commit_ids(), vec![survivor.clone()]);
21555 assert_eq!(
21556 prepared.meta.pushed_ids,
21557 vec![survivor.clone()],
21558 "the incompatible commit is classified before any operation is sent"
21559 );
21560 let rejection = client
21561 .rejections
21562 .iter()
21563 .find(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE)
21564 .expect("a sync.outbox_incompatible rejection");
21565 assert_eq!(rejection.client_commit_id, dropped);
21566 assert!(
21567 !rejection.retryable,
21568 "a schema-incompatible commit is final"
21569 );
21570 assert_eq!(
21571 client
21572 .commit_outcome(&dropped)
21573 .unwrap()
21574 .expect("a durable rejected outcome")
21575 .status,
21576 CommitOutcomeStatus::Rejected
21577 );
21578 assert_eq!(client.local_revision(), revision + 1, "one revision");
21579 let batches = client.drain_change_batches();
21580 assert_eq!(batches.len(), 1, "one change batch");
21581 assert!(
21582 batches[0].rejections_changed
21583 && batches[0].outcomes_changed
21584 && batches[0].status.is_some(),
21585 "{:?}",
21586 batches[0]
21587 );
21588 assert!(
21589 batches[0].tables.iter().any(|entry| entry.table == "tasks"),
21590 "{:?}",
21591 batches[0]
21592 );
21593 let visible = client.query("SELECT * FROM tasks", &[]).unwrap();
21594 assert!(
21595 visible.is_empty(),
21596 "the dropped projection is undone: {visible:?}"
21597 );
21598
21599 let path = temp_replica("outbox-drop-commit");
21603 let path_str = path.to_str().expect("path");
21604 let conn = Connection::open(path_str).expect("replica connection");
21605 let mut client =
21606 SyncClient::with_connection("drop-commit".into(), &v1, ClientLimits::default(), conn)
21607 .expect("v1 replica");
21608 let dropped = client
21609 .mutate(vec![Mutation::Upsert {
21610 table: "tasks".into(),
21611 values: Map::from_iter([
21612 ("id".into(), json!("dropped")),
21613 ("note".into(), json!("gone")),
21614 ]),
21615 base_version: None,
21616 }])
21617 .unwrap();
21618 client
21619 .recreate_with_schema(&v2)
21620 .expect("the reset preserves the outbox");
21621 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21622 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21623 assert_eq!(rows.len(), 1, "the optimistic row is visible");
21624 client.set_sync_needed(false, false);
21625 client.drain_change_batches();
21626 let revision = client.local_revision();
21627 let reader = Connection::open(path_str).expect("reader");
21628 reader
21629 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
21630 .expect("reader snapshot");
21631 client
21632 .conn
21633 .busy_timeout(std::time::Duration::ZERO)
21634 .expect("busy timeout");
21635 let outcome = client
21636 .prepare_sync_round(false)
21637 .err()
21638 .expect("a blocked COMMIT must fail the round");
21639 assert!(
21640 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "client.storage_busy"),
21641 "{outcome:?}"
21642 );
21643 assert!(client.conn.is_autocommit(), "the savepoint is released");
21644 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21645 assert!(client.rejections.is_empty());
21646 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21647 assert_eq!(client.local_revision(), revision);
21648 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
21649 assert!(client.drain_change_batches().is_empty());
21650 drop(reader);
21652 client
21653 .conn
21654 .busy_timeout(std::time::Duration::from_secs(1))
21655 .expect("busy timeout");
21656 client
21657 .prepare_sync_round(false)
21658 .expect("the drop succeeds after the reader releases");
21659 assert!(client.pending_commit_ids().is_empty());
21660 assert!(client
21661 .rejections
21662 .iter()
21663 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE));
21664 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
21665 drop(client);
21666 std::fs::remove_file(path).expect("remove replica");
21667 }
21668
21669 #[test]
21670 fn previous_version_read_reasons_come_from_live_client_state() {
21671 use crate::previous_version::{
21672 PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
21673 };
21674 let path = std::env::temp_dir()
21675 .join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
21676 .to_string_lossy()
21677 .into_owned();
21678 let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
21679 {
21680 let mut setup = SyncClient::open_path(
21681 "wiring".to_owned(),
21682 &previous_version_schema(1),
21683 previous_version_limits(),
21684 &path,
21685 )
21686 .expect("v1 install");
21687 setup
21688 .mutate(vec![Mutation::Upsert {
21689 table: "tasks".to_owned(),
21690 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21691 base_version: None,
21692 }])
21693 .expect("seed a row");
21694 }
21695 let mut client = SyncClient::open_path(
21696 "wiring".to_owned(),
21697 &previous_version_schema(2),
21698 previous_version_limits(),
21699 &path,
21700 )
21701 .expect("v2 bump captures");
21702 assert!(std::path::Path::new(&container).exists());
21703 let spec = PreviousVersionReadSpec {
21704 table: "tasks".to_owned(),
21705 ..Default::default()
21706 };
21707
21708 recapture_previous_version(&client);
21710 client.lease_state = Some(LeaseState {
21711 lease_id: None,
21712 expires_at_ms: None,
21713 error_code: Some("lease.expired".to_owned()),
21714 });
21715 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21716 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21717 assert!(!snapshot.available);
21718 assert!(!std::path::Path::new(&container).exists());
21719
21720 recapture_previous_version(&client);
21722 client.set_now_ms(1_000);
21723 client.lease_state = Some(LeaseState {
21724 lease_id: Some("lease".to_owned()),
21725 expires_at_ms: Some(1_000),
21726 error_code: None,
21727 });
21728 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21729 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21730 assert!(!std::path::Path::new(&container).exists());
21731 client.lease_state = None;
21732
21733 recapture_previous_version(&client);
21735 client.subs.push(subscription(SubState::Revoked, 0, None));
21736 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21737 assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
21738 assert!(!std::path::Path::new(&container).exists());
21739 client.subs.clear();
21740
21741 recapture_previous_version(&client);
21744 client
21745 .subs
21746 .push(subscription(SubState::Active, 0, Some("resume")));
21747 assert!(!client.previous_version_coverage_complete());
21748 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21749 assert!(snapshot.available);
21750 assert_eq!(snapshot.previous_version, Some(2));
21751 assert_eq!(snapshot.rows.len(), 1);
21752
21753 recapture_previous_version(&client);
21754 client.subs[0].bootstrap_state = None;
21755 assert!(client.previous_version_coverage_complete());
21756 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21757 assert_eq!(
21758 snapshot.reason,
21759 Some(PreviousVersionReason::CoverageComplete)
21760 );
21761 assert!(!std::path::Path::new(&container).exists());
21762 client.subs.clear();
21763
21764 assert!(!client.previous_version_coverage_complete());
21766
21767 drop(client);
21768 let _ = std::fs::remove_file(&path);
21769 }
21770
21771 #[test]
21778 fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
21779 let path = std::env::temp_dir()
21780 .join(format!(
21781 "syncular-prev-lifetime-{}.db",
21782 uuid::Uuid::new_v4()
21783 ))
21784 .to_string_lossy()
21785 .into_owned();
21786 let container = format!(
21787 "{path}{}",
21788 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
21789 );
21790 let exists = || std::path::Path::new(&container).exists();
21791 {
21792 let mut setup = SyncClient::open_path(
21793 "lifetime".to_owned(),
21794 &previous_version_schema(1),
21795 previous_version_limits(),
21796 &path,
21797 )
21798 .expect("v1 install");
21799 setup
21800 .mutate(vec![Mutation::Upsert {
21801 table: "tasks".to_owned(),
21802 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21803 base_version: None,
21804 }])
21805 .expect("seed a row");
21806 }
21807 let mut client = SyncClient::open_path(
21808 "lifetime".to_owned(),
21809 &previous_version_schema(2),
21810 previous_version_limits(),
21811 &path,
21812 )
21813 .expect("v2 bump captures");
21814 client.set_now_ms(1_000);
21817 assert!(exists());
21818
21819 client.lease_state = Some(LeaseState {
21821 lease_id: None,
21822 expires_at_ms: None,
21823 error_code: Some("lease.expired".to_owned()),
21824 });
21825 client.previous_version_lifetime_check();
21826 assert!(!exists());
21827 client.lease_state = None;
21828
21829 recapture_previous_version(&client);
21832 client.subs.push(subscription(SubState::Active, 0, None));
21833 client.previous_version_lifetime_check();
21834 assert!(!exists());
21835 client.subs.clear();
21836
21837 recapture_previous_version(&client);
21839 client.subs.push(subscription(SubState::Revoked, 0, None));
21840 client.previous_version_lifetime_check();
21841 assert!(!exists());
21842 client.subs.clear();
21843
21844 recapture_previous_version(&client);
21847 client
21848 .subs
21849 .push(subscription(SubState::Active, 0, Some("resume")));
21850 client.previous_version_lifetime_check();
21851 assert!(exists());
21852 client.subs.clear();
21853
21854 drop(client);
21855 let _ = std::fs::remove_file(&path);
21856 let _ = std::fs::remove_file(&container);
21857 let _ = std::fs::remove_file(format!("{container}-wal"));
21858 let _ = std::fs::remove_file(format!("{container}-shm"));
21859 }
21860}