1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12#[cfg(test)]
13use std::cell::Cell;
14use std::cell::RefCell;
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
16
17use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
18use rusqlite::{Connection, OpenFlags, OptionalExtension};
19use serde::{Deserialize, Serialize};
20use serde_json::{json, Map, Value};
21use sha2::{Digest, Sha256};
22use ssp2::decode::WIRE_VERSION;
23use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
24use ssp2::primitives::RawJson;
25use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
26use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
27
28use crate::api::{
29 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
30 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
31 ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects, CommitDelivery,
32 CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
33 CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
34 DiagnosticLastChange, DiagnosticLastRound, DiagnosticQueryFailure, DiagnosticRoundCounters,
35 DiagnosticSubscription, FetchedBlob, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
36 LocalDataPurgeTarget, LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation,
37 PresencePeer, PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot,
38 QueryValue, RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
39 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SubscriptionCatchup,
40 SubscriptionRequest, SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport,
41 SyncStatusSnapshot, TableChange, WindowBase, WindowChange, WindowCoverage, WindowState,
42 WindowUnitRef, CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
43 MAX_DIAGNOSTIC_QUERY_FAILURES,
44};
45use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
46#[cfg(feature = "bench-internals")]
47use crate::bench::{Phase, Recorder};
48use crate::previous_version::{
49 capture_previous_version_from_replica, first_incompatibility,
50 reconcile_previous_version_at_boot, set_local_schema_descriptor,
51 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
52 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
53};
54use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
55use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
56use crate::values::{
57 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
58 encode_sparse_row_json, full_row_values, is_canonical_bytes_envelope, json_to_column_value,
59 json_to_scope_map, normalize_values_casing, render_row_id_json, scope_map_to_json,
60 sort_scope_map,
61};
62
63const DEFAULT_ACCEPT: u8 = 0b0111;
67const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
68const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
69const ACCEPT_SQLITE: u8 = 1 << 2;
70const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
71const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
72
73pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
75const LOCAL_REVISION_KEY: &str = "localRevision";
76const CLIENT_ID_KEY: &str = "clientId";
77const LEASE_STATE_KEY: &str = "leaseState";
78const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
79const LOG_EPOCH_KEY: &str = "logEpoch";
80const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
85const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
86const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
87const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
90const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
93const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
94const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
96 "the persisted commit carries values the current codec refuses";
97const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
99
100#[derive(Debug, Serialize, Deserialize)]
101#[serde(rename_all = "camelCase", deny_unknown_fields)]
102struct PersistedLocalDataRebootstrapReceipt {
103 version: u8,
104 retained_commits: u64,
105 reset_subscriptions: u64,
106}
107
108fn invalid_local_rebootstrap_receipt() -> String {
109 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
110}
111
112fn encode_local_rebootstrap_receipt(
113 retained_commits: usize,
114 reset_subscriptions: usize,
115) -> Result<String, String> {
116 let retained_commits =
117 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
118 let reset_subscriptions =
119 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
120 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
121 return Err(invalid_local_rebootstrap_receipt());
122 }
123 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
124 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
125 retained_commits,
126 reset_subscriptions,
127 })
128 .map_err(|_| invalid_local_rebootstrap_receipt())
129}
130
131fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
132 if value == "v1" {
134 return Ok((0, 0));
135 }
136 let receipt: PersistedLocalDataRebootstrapReceipt =
137 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
138 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
139 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
140 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
141 {
142 return Err(invalid_local_rebootstrap_receipt());
143 }
144 Ok((
145 usize::try_from(receipt.retained_commits)
146 .map_err(|_| invalid_local_rebootstrap_receipt())?,
147 usize::try_from(receipt.reset_subscriptions)
148 .map_err(|_| invalid_local_rebootstrap_receipt())?,
149 ))
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153enum SubState {
154 Active,
155 Revoked,
156 Failed,
157}
158
159#[cfg(test)]
160mod observation_tests {
161 use super::*;
162 use crate::native_transport::HostTransport;
163 use crate::values::encode_row_json;
164 use serde_json::json;
165
166 #[test]
167 fn authority_policy_refuses_encrypted_and_undeclared_columns_before_open() {
168 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
169 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "secret", "type": "bytes", "encrypted": true, "declaredType": "string", "nullable": true}
170 ]}]});
171 for name in [
172 "secret",
173 "missing",
174 "*",
175 "id FROM authority; DELETE FROM authority",
176 "_syncular_version",
177 ] {
178 let limits = ClientLimits {
179 authority_reads: vec![crate::api::AuthorityReadDeclaration {
180 table: "authority".into(),
181 columns: vec!["id".into(), "actor_id".into(), name.into()],
182 scopes: [("actor_id".into(), vec!["a".into()])].into(),
183 }],
184 ..Default::default()
185 };
186 let error = SyncClient::new("authority".into(), &schema, limits)
187 .err()
188 .expect("forbidden policy");
189 assert!(
190 error.starts_with("client.authority_read_forbidden:"),
191 "{error}"
192 );
193 }
194 }
195
196 #[test]
197 fn authority_snapshot_observes_one_file_transaction_during_concurrent_writes() {
198 let path = std::env::temp_dir().join(format!(
199 "syncular-authority-snapshot-{}.db",
200 uuid::Uuid::new_v4()
201 ));
202 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
203 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "revision", "type": "integer", "nullable": false}
204 ]}]});
205 let limits = ClientLimits {
206 authority_reads: vec![crate::api::AuthorityReadDeclaration {
207 table: "authority".into(),
208 columns: vec!["id".into(), "actor_id".into(), "revision".into()],
209 scopes: [("actor_id".into(), vec!["a".into()])].into(),
210 }],
211 ..Default::default()
212 };
213 let mut client =
214 SyncClient::open_path("authority".into(), &schema, limits, path.to_str().unwrap())
215 .unwrap();
216 client.begin_security_preflight();
217 let update = |conn: &Connection, revision: i64| {
218 conn.execute_batch("BEGIN IMMEDIATE").unwrap();
219 conn.execute(
220 "INSERT OR REPLACE INTO _syncular_base_authority VALUES('one','a',?1,1)",
221 [revision],
222 )
223 .unwrap();
224 conn.execute(
225 "UPDATE _syncular_meta SET value=?1 WHERE key='localRevision'",
226 [revision.to_string()],
227 )
228 .unwrap();
229 conn.execute("INSERT OR REPLACE INTO _syncular_subscriptions VALUES('s','authority',?1)", [json!({"requested": {"actor_id": ["a"]}, "effectiveScopes": {"actor_id": ["a"]}, "status": "active", "cursor": revision}).to_string()]).unwrap();
230 conn.execute_batch("COMMIT").unwrap();
231 };
232 update(&client.conn, 1);
233 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
234 let writer_barrier = barrier.clone();
235 let writer_path = path.clone();
236 let writer = std::thread::spawn(move || {
237 let conn = Connection::open(writer_path).unwrap();
238 conn.busy_timeout(std::time::Duration::from_secs(5))
239 .unwrap();
240 writer_barrier.wait();
241 for revision in 2..=100 {
242 update(&conn, revision);
243 }
244 });
245 barrier.wait();
246 for _ in 0..100 {
247 let snapshot = client.authority_snapshot().unwrap();
248 let revision = snapshot["revision"]
249 .as_str()
250 .unwrap()
251 .parse::<i64>()
252 .unwrap();
253 assert_eq!(
254 snapshot["tables"][0]["rows"][0]["values"]["revision"],
255 revision
256 );
257 assert_eq!(snapshot["tables"][0]["persisted"][0]["cursor"], revision);
258 assert_eq!(snapshot["complete"], true);
259 }
260 writer.join().unwrap();
261 assert_eq!(client.authority_snapshot().unwrap()["revision"], "100");
262 drop(client);
263 let _ = std::fs::remove_file(path);
264 }
265
266 fn client() -> SyncClient {
267 SyncClient::new(
268 "retry-test".to_owned(),
269 &json!({
270 "version": 1,
271 "tables": [{
272 "name": "tasks",
273 "primaryKey": "id",
274 "columns": [
275 { "name": "id", "type": "string", "nullable": false },
276 { "name": "project_id", "type": "string", "nullable": false }
277 ],
278 "scopes": [{ "pattern": "project:{project_id}" }]
279 }]
280 }),
281 ClientLimits::default(),
282 )
283 .expect("test client")
284 }
285
286 #[test]
287 fn subscribe_many_registers_all_or_nothing() {
288 let request = |id: &str, project: &str| SubscriptionRequest {
289 id: id.to_owned(),
290 table: "tasks".to_owned(),
291 scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
292 params: None,
293 };
294 let mut client = client();
295 client
296 .subscribe_many(vec![request("a", "p1"), request("b", "p2")])
297 .unwrap();
298 let persisted = |client: &SyncClient| -> i64 {
299 client
300 .conn
301 .query_row("SELECT count(*) FROM _syncular_subscriptions", [], |row| {
302 row.get(0)
303 })
304 .unwrap()
305 };
306 assert_eq!(persisted(&client), 2);
307 let error = client
309 .subscribe_many(vec![request("c", "p3"), request("b", "p9")])
310 .unwrap_err();
311 assert!(error.starts_with("client.subscription_intent_mismatch:"));
312 assert!(client.subscription_state("c").is_none());
313 assert_eq!(persisted(&client), 2);
314 assert!(client.conn.is_autocommit());
315 }
316
317 #[cfg(feature = "e2ee")]
318 #[test]
319 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
320 let schema = json!({
321 "version": 1,
322 "tables": [{
323 "name": "secrets",
324 "primaryKey": "id",
325 "columns": [
326 { "name": "id", "type": "string", "nullable": false },
327 { "name": "encryption_key_id", "type": "string", "nullable": true },
328 { "name": "note", "type": "bytes", "nullable": true,
329 "encrypted": true, "declaredType": "string" }
330 ],
331 "scopes": []
332 }]
333 });
334 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
335 .expect("test client");
336 let mut config = crate::values::EncryptionConfig::default();
337 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
338 config
339 .key_id_columns
340 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
341 client.set_encryption(config);
342 client
345 .write_base_row(
346 "secrets",
347 &vec![Some(ColumnValue::String("ghost".into())), None, None],
348 1,
349 )
350 .unwrap();
351 client.rebuild_overlay().expect("rebuild overlay");
352 let commit_id = client
353 .patch(
354 "secrets",
355 "ghost",
356 Map::from_iter([("note".to_owned(), json!("x"))]),
357 None,
358 )
359 .expect("patch records without an author-time encrypt failure");
360 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
361 assert!(client.drop_unencodable_outbox(true, None).unwrap());
363 assert!(client.pending_commit_ids().is_empty());
364 let rejections = client.rejections();
365 assert_eq!(rejections.len(), 1);
366 assert_eq!(rejections[0].code, "client.encrypt_failed");
367 assert_eq!(rejections[0].client_commit_id, commit_id);
368 assert_eq!(
369 client
370 .commit_outcome(&commit_id)
371 .unwrap()
372 .map(|outcome| outcome.status),
373 Some(CommitOutcomeStatus::Rejected)
374 );
375 }
376
377 #[test]
378 fn patch_scope_read_preserves_sqlite_failure() {
379 let mut client = client();
380 client
381 .conn
382 .execute_batch(
383 "DROP TABLE tasks;
384 CREATE VIEW tasks AS SELECT 't1' AS id, abs(-9223372036854775808) AS project_id;",
385 )
386 .unwrap();
387 let error = client
388 .patch(
389 "tasks",
390 "t1",
391 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
392 None,
393 )
394 .expect_err("the scope read must fail");
395 assert_ne!(error.code, "sync.invalid_request");
396 assert_eq!(error.code, "client.failed");
397 assert!(error.details.as_ref().unwrap()["legacyCause"]
398 .as_str()
399 .unwrap()
400 .contains("integer overflow"));
401 assert!(client.pending_commit_ids().is_empty());
402 assert!(client.conn.is_autocommit());
403 }
404
405 #[test]
406 fn patch_with_a_scope_column_matches_the_stored_local_row() {
407 let mut client = client();
408 client.create_synced_tables().unwrap();
409 client
410 .conn
411 .execute(
412 "INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
413 [],
414 )
415 .unwrap();
416
417 client
420 .patch(
421 "tasks",
422 "t1",
423 Map::from_iter([
424 ("project_id".to_owned(), json!("p1")),
425 ("id".to_owned(), json!("t1")),
426 ]),
427 None,
428 )
429 .expect("an equal scope column is a no-op");
430 let values = client.outbox[0].ops[0]
431 .values
432 .clone()
433 .expect("the patch records values");
434 assert_eq!(values.get("id"), Some(&json!("t1")));
435 assert!(!values.contains_key("project_id"));
436
437 let differing = client.patch(
439 "tasks",
440 "t1",
441 Map::from_iter([("project_id".to_owned(), json!("p2"))]),
442 None,
443 );
444 let differing = differing.expect_err("a differing scope column is rejected");
445 assert_eq!(differing.code, "sync.invalid_request");
446 assert_eq!(differing.message, "the authoring request is invalid");
447 assert!(
448 differing
449 .details
450 .as_ref()
451 .and_then(|details| details["legacyCause"].as_str())
452 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
453 "{differing:?}"
454 );
455
456 let absent = client.patch(
459 "tasks",
460 "ghost",
461 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
462 None,
463 );
464 let absent = absent.expect_err("an absent local row is rejected");
465 assert_eq!(absent.code, "sync.row_missing");
466 assert!(absent.message.contains("requires a local row"));
467 assert_eq!(client.pending_commit_ids().len(), 1);
468 }
469
470 #[test]
471 fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
472 let schema = json!({
473 "version": 1,
474 "tables": [
475 {
476 "name": "tenants",
477 "primaryKey": "tenant_id",
478 "columns": [
479 { "name": "tenant_id", "type": "string", "nullable": false },
480 { "name": "body", "type": "string", "nullable": false }
481 ],
482 "scopes": [{ "pattern": "tenant:{tenant_id}" }]
483 },
484 {
485 "name": "buckets",
486 "primaryKey": "id",
487 "columns": [
488 { "name": "id", "type": "string", "nullable": false },
489 { "name": "bucket", "type": "float", "nullable": false },
490 { "name": "body", "type": "string", "nullable": false }
491 ],
492 "scopes": [{ "pattern": "bucket:{bucket}" }]
493 }
494 ]
495 });
496 let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
497 .expect("test client");
498 client.create_synced_tables().unwrap();
499
500 client
502 .conn
503 .execute(
504 "INSERT INTO tenants (tenant_id, body, _syncular_version) VALUES ('t1', 'seed', 1)",
505 [],
506 )
507 .unwrap();
508 client
509 .patch(
510 "tenants",
511 "t1",
512 Map::from_iter([
513 ("tenant_id".to_owned(), json!("t1")),
514 ("body".to_owned(), json!("created")),
515 ]),
516 None,
517 )
518 .expect("the primary-key scope column is proven equal by construction");
519
520 client
522 .conn
523 .execute(
524 "INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
525 [],
526 )
527 .unwrap();
528 client
529 .patch(
530 "buckets",
531 "b1",
532 Map::from_iter([
533 ("bucket".to_owned(), json!(2)),
534 ("body".to_owned(), json!("coerced")),
535 ]),
536 None,
537 )
538 .expect("the stored REAL value coerces from a supplied integer");
539 let values = client.outbox[1].ops[0]
540 .values
541 .clone()
542 .expect("the patch records values");
543 assert!(!values.contains_key("bucket"));
544
545 let differing = client.patch(
547 "buckets",
548 "b1",
549 Map::from_iter([("bucket".to_owned(), json!(3))]),
550 None,
551 );
552 let differing = differing.expect_err("a differing scope value is rejected");
553 assert_eq!(differing.code, "sync.invalid_request");
554 assert_eq!(differing.message, "the authoring request is invalid");
555 assert!(
556 differing
557 .details
558 .as_ref()
559 .and_then(|details| details["legacyCause"].as_str())
560 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
561 "{differing:?}"
562 );
563 }
564
565 #[cfg(feature = "e2ee")]
566 #[test]
567 fn stored_key_fallback_resolves_an_integer_primary_key() {
568 let schema = json!({
569 "version": 1,
570 "tables": [{
571 "name": "patients",
572 "primaryKey": "id",
573 "columns": [
574 { "name": "id", "type": "integer", "nullable": false },
575 { "name": "encryption_key_id", "type": "string", "nullable": true },
576 { "name": "note", "type": "bytes", "nullable": true,
577 "encrypted": true, "declaredType": "string" }
578 ],
579 "scopes": []
580 }]
581 });
582 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
583 .expect("test client");
584 client.create_synced_tables().unwrap();
585 client
586 .conn
587 .execute(
588 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
589 VALUES (5, 'k1', NULL, 1)",
590 [],
591 )
592 .unwrap();
593 let mut config = crate::values::EncryptionConfig::default();
594 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
595 config
596 .key_id_columns
597 .insert("patients".to_owned(), "encryption_key_id".to_owned());
598 client.set_encryption(config);
599 let table = client.schema.table("patients").unwrap();
600 let fallback = client
601 .stored_key_fallback(table, "5")
602 .expect("the stored integer key resolves the fallback");
603 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
604 }
605
606 #[test]
607 fn patch_types_an_integer_primary_key_from_its_canonical_row_id() {
608 let schema = json!({
609 "version": 1,
610 "tables": [{
611 "name": "counters",
612 "primaryKey": "id",
613 "columns": [
614 { "name": "id", "type": "integer", "nullable": false },
615 { "name": "title", "type": "string", "nullable": false }
616 ],
617 "scopes": []
618 }]
619 });
620 let mut client = SyncClient::new("int-patch".into(), &schema, ClientLimits::default())
621 .expect("test client");
622 client
623 .write_base_row(
624 "counters",
625 &vec![
626 Some(ColumnValue::Integer(42)),
627 Some(ColumnValue::String("before".into())),
628 ],
629 1,
630 )
631 .unwrap();
632 client.rebuild_overlay().expect("rebuild overlay");
633 client
634 .patch(
635 "counters",
636 "42",
637 Map::from_iter([
638 ("id".to_owned(), json!(42)),
639 ("title".to_owned(), json!("after")),
640 ]),
641 None,
642 )
643 .expect("an integer key that renders as the rowId is not a change");
644 let rows = client.query("SELECT id, title FROM counters", &[]).unwrap();
645 assert_eq!(Value::from(rows), json!([{"id": 42, "title": "after"}]));
646 for row_id in ["042", "+42", "42.0"] {
647 let error = client
648 .patch(
649 "counters",
650 row_id,
651 Map::from_iter([("title".to_owned(), json!("x"))]),
652 None,
653 )
654 .expect_err("a non-canonical integer rowId names no row");
655 assert_eq!(error.code, "sync.row_missing", "{row_id}");
656 }
657 }
658
659 #[test]
660 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
661 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
662 {"name":"id","type":"string","nullable":false},
663 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
664 for state in ["absent", "cached", "pinned"] {
665 let path = std::env::temp_dir().join(format!(
666 "syncular-blob-stage-{}.sqlite",
667 uuid::Uuid::new_v4()
668 ));
669 let mut client = SyncClient::open_path(
670 "stage-test".into(),
671 &schema,
672 Default::default(),
673 path.to_str().unwrap(),
674 )
675 .unwrap();
676 let bytes = b"atomic stage";
677 if state != "absent" {
678 client
679 .upload_blob(bytes, Some("text/plain".into()), None)
680 .unwrap();
681 if state == "cached" {
682 client
683 .conn
684 .execute("DELETE FROM _syncular_blob_uploads", [])
685 .unwrap();
686 }
687 }
688 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
689 client
690 .conn
691 .execute_batch(
692 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
693 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
694 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
695 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
696 )
697 .unwrap();
698 let error = client
699 .upload_blob(bytes, Some("application/octet-stream".into()), None)
700 .unwrap_err();
701 assert!(error.contains("injected stage failure"), "{state}: {error}");
702 assert!(client.conn.is_autocommit());
703 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
704 drop(client);
705 let mut reopened = SyncClient::open_path(
706 "stage-test".into(),
707 &schema,
708 Default::default(),
709 path.to_str().unwrap(),
710 )
711 .unwrap();
712 let durable = reopened
713 .query("SELECT * FROM _syncular_blobs", &[])
714 .unwrap();
715 reopened
716 .conn
717 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
718 .unwrap();
719 let retried = reopened.upload_blob(bytes, None, None).unwrap();
720 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
721 assert_eq!(
722 reopened
723 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
724 .unwrap(),
725 vec![Map::from_iter([(
726 "blob_id".into(),
727 Value::from(blob_id_for(bytes)),
728 )])]
729 );
730 drop(reopened);
731 std::fs::remove_file(&path).unwrap();
732 assert_eq!(durable, bodies, "{state}: reopened body state");
733 assert_eq!(immediate, bodies, "{state}: immediate body state");
734 }
735 }
736
737 #[test]
738 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
739 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
740 {"name":"id","type":"string","nullable":false},
741 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
742 for fault in [
743 "missing",
744 "body-type",
745 "length",
746 "hash",
747 "pin-metadata",
748 "read",
749 "pin-delete",
750 ] {
751 let path = std::env::temp_dir().join(format!(
752 "syncular-blob-upload-{}.sqlite",
753 uuid::Uuid::new_v4()
754 ));
755 let mut client = SyncClient::open_path(
756 "upload-test".into(),
757 &schema,
758 Default::default(),
759 path.to_str().unwrap(),
760 )
761 .unwrap();
762 let bytes = b"durable upload";
763 let ref_value = client
764 .upload_blob(bytes, Some("text/plain".into()), None)
765 .unwrap();
766 let commit = client
767 .mutate(vec![Mutation::Upsert {
768 table: "attachments".into(),
769 values: Map::from_iter([
770 ("id".into(), Value::from("pending")),
771 (
772 "file".into(),
773 Value::from(format!(
774 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
775 blob_id_for(bytes),
776 bytes.len()
777 )),
778 ),
779 ]),
780 base_version: None,
781 }])
782 .unwrap();
783 client
784 .conn
785 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
786 .unwrap();
787 client.conn.execute_batch(match fault {
788 "missing" => "DELETE FROM _syncular_blobs",
789 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
790 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
791 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
792 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
793 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
794 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
795 }).unwrap();
796 let state = if fault == "read" {
797 Vec::new()
798 } else {
799 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
800 };
801 let mut transport = CountingRealtimeTransport::default();
802 let error = client.flush_blob_uploads(&mut transport).unwrap_err();
803 assert_eq!(
804 error.code,
805 if matches!(fault, "read" | "pin-delete") {
806 "client.failed"
807 } else {
808 "sync.local_corrupt"
809 },
810 "{fault}"
811 );
812 assert_eq!(
813 transport.blob_uploads.len(),
814 usize::from(fault == "pin-delete"),
815 "{fault}: corrupt bytes must never be uploaded"
816 );
817 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
818 if fault != "read" {
819 assert_eq!(
820 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
821 state
822 );
823 } else {
824 client
825 .conn
826 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
827 .unwrap();
828 }
829 drop(client);
830 let mut reopened = SyncClient::open_path(
831 "upload-test".into(),
832 &schema,
833 Default::default(),
834 path.to_str().unwrap(),
835 )
836 .unwrap();
837 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
838 if fault != "read" {
839 assert_eq!(
840 reopened
841 .query("SELECT * FROM _syncular_blobs", &[])
842 .unwrap(),
843 state
844 );
845 }
846 if fault == "pin-delete" {
847 reopened
848 .conn
849 .execute_batch("DROP TRIGGER fail_pin_delete")
850 .unwrap();
851 }
852 if fault == "pin-metadata" {
853 reopened
854 .conn
855 .execute(
856 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
857 [],
858 )
859 .unwrap();
860 }
861 reopened
862 .conn
863 .execute_batch(
864 "DELETE FROM _syncular_blobs;
865 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
866 )
867 .unwrap();
868 transport.blob_uploads.clear();
869 reopened.flush_blob_uploads(&mut transport).unwrap();
870 assert_eq!(
871 transport.blob_uploads,
872 [(blob_id_for(bytes), bytes.to_vec())]
873 );
874 assert_eq!(
875 reopened
876 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
877 .unwrap(),
878 Vec::<Map<String, Value>>::new()
879 );
880 assert_eq!(reopened.pending_commit_ids(), [commit]);
881 assert_eq!(
882 reopened
883 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
884 .unwrap()
885 .unwrap()
886 .bytes,
887 bytes
888 );
889 drop(reopened);
890 std::fs::remove_file(path).unwrap();
891 }
892 }
893
894 fn coalesced_fixture(sections: usize, commits: usize, rows: usize) -> (SyncClient, Message) {
898 let mut client = client();
899 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
900 let mut response = Message {
901 wire_version: WIRE_VERSION,
902 msg_kind: MsgKind::Response,
903 frames: vec![Frame::RespHeader {
904 required_schema_version: None,
905 latest_schema_version: None,
906 log_epoch: Some("epoch-1".into()),
907 reset_required: Some(false),
908 }],
909 };
910 let mut seq = 0;
911 for section in 0..sections {
912 let project = format!("p{section}");
913 let scopes = vec![("project_id".to_owned(), vec![project.clone()])];
914 client
915 .subscribe(format!("s{section}"), "tasks".into(), scopes.clone(), None)
916 .unwrap();
917 let sub = client.subs.last_mut().unwrap();
918 sub.cursor = 0;
919 sub.synced_once = true;
920 sub.effective = Some(scopes.clone());
921 let sub = sub.clone();
922 client.persist_sub(&sub).unwrap();
923 response.frames.push(Frame::SubStart {
924 id: format!("s{section}"),
925 status: SubStatus::Active,
926 reason_code: "".into(),
927 effective_scopes: scopes,
928 bootstrap: false,
929 });
930 for _ in 0..commits {
931 seq += 1;
932 let changes = (0..rows)
933 .map(|row| {
934 let id = format!("{project}-{seq}-{row}");
935 let values = Map::from_iter([
936 ("id".into(), Value::from(id.clone())),
937 ("project_id".into(), Value::from(project.clone())),
938 ]);
939 ssp2::model::Change {
940 table_index: 0,
941 row_id: id.clone(),
942 op: Op::Upsert,
943 row_version: Some(seq),
944 scopes: vec![("project_id".into(), project.clone())],
945 row: Some(
946 encode_row_json(
947 client.schema.table("tasks").unwrap(),
948 &id,
949 &values,
950 &client.encryption,
951 )
952 .unwrap(),
953 ),
954 }
955 })
956 .collect();
957 response.frames.push(Frame::Commit {
958 commit_seq: seq,
959 actor_id: "writer".into(),
960 created_at_ms: 1,
961 tables: vec!["tasks".into()],
962 changes,
963 });
964 }
965 response.frames.push(Frame::SubEnd {
966 next_cursor: seq,
967 bootstrap_state: None,
968 });
969 }
970 client.drain_change_batches();
971 (client, response)
972 }
973
974 fn count_commits(client: &SyncClient) -> std::sync::Arc<std::sync::atomic::AtomicUsize> {
975 let commits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
976 let counter = commits.clone();
977 client
978 .conn
979 .commit_hook(Some(move || {
980 counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
981 false
982 }))
983 .expect("commit hook");
984 commits
985 }
986
987 #[test]
988 fn a_response_of_many_small_sections_commits_once() {
989 for realtime in [false, true] {
990 let (mut client, response) = coalesced_fixture(70, 1, 1);
991 client.realtime_state = RealtimeState::Connected;
992 client.realtime_reason_code = None;
993 let mut transport = CountingRealtimeTransport::default();
994 let (_, meta) = client.build_request(false).unwrap();
995 let commits = count_commits(&client);
996 if realtime {
997 client.on_realtime_binary(&mut transport, &encode_message(&response));
998 } else {
999 assert!(matches!(
1000 client.process_response(&mut transport, response, &meta),
1001 SyncOutcome::Ok(_)
1002 ));
1003 }
1004 assert_eq!(
1005 commits.load(std::sync::atomic::Ordering::SeqCst),
1006 1,
1007 "{realtime}"
1008 );
1009 assert!(client.conn.is_autocommit());
1010 assert!(client.subs.iter().all(|sub| sub.cursor > 0));
1011 let revisions = client
1013 .drain_change_batches()
1014 .into_iter()
1015 .filter(|batch| !batch.tables.is_empty())
1016 .map(|batch| batch.revision)
1017 .collect::<Vec<_>>();
1018 assert_eq!(
1019 revisions,
1020 (1..=70).map(|r: u64| r.to_string()).collect::<Vec<_>>()
1021 );
1022 assert_eq!(client.query("SELECT id FROM tasks", &[]).unwrap().len(), 70);
1023 assert_eq!(
1025 transport
1026 .messages
1027 .iter()
1028 .filter(|m| m.contains("\"ack\""))
1029 .count(),
1030 1,
1031 "{realtime}"
1032 );
1033 }
1034 }
1035
1036 #[test]
1037 fn a_response_past_the_row_bound_commits_per_bound() {
1038 let (mut client, response) = coalesced_fixture(1, 3, 600);
1040 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1041 let (_, meta) = client.build_request(false).unwrap();
1042 let commits = count_commits(&client);
1043 assert!(matches!(
1044 client.process_response(&mut transport, response, &meta),
1045 SyncOutcome::Ok(_)
1046 ));
1047 assert_eq!(commits.load(std::sync::atomic::Ordering::SeqCst), 3);
1048 assert_eq!(
1049 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1050 1800
1051 );
1052 assert_eq!(client.subs[0].cursor, 3);
1053 }
1054
1055 #[test]
1056 fn a_failed_coalesced_commit_restores_memory_and_publishes_nothing() {
1057 let (mut client, response) = coalesced_fixture(2, 2, 600);
1061 client
1062 .conn
1063 .execute_batch(
1064 "PRAGMA foreign_keys = ON; CREATE TABLE fail_parent(id INTEGER PRIMARY KEY); CREATE TABLE fail_child(id INTEGER REFERENCES fail_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_commit AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'p0-1-0' BEGIN INSERT INTO fail_child VALUES(1); END",
1065 )
1066 .unwrap();
1067 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1068 let (_, meta) = client.build_request(false).unwrap();
1069 let subs = client.subs.clone();
1070 let revision = client.local_revision();
1071 let outcome = client.process_response(&mut transport, response.clone(), &meta);
1072 assert!(
1073 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == "storage.failed"),
1074 "{outcome:?}"
1075 );
1076 assert!(client.conn.is_autocommit());
1077 assert!(client.coalesced.is_none() && client.coalesced_failure.is_none());
1078 assert_eq!(client.local_revision(), revision);
1079 assert!(client
1080 .drain_change_batches()
1081 .iter()
1082 .all(|batch| batch.tables.is_empty()));
1083 assert!(client
1084 .query("SELECT id FROM tasks", &[])
1085 .unwrap()
1086 .is_empty());
1087 assert_eq!(
1088 client.subs.iter().map(|s| s.cursor).collect::<Vec<_>>(),
1089 subs.iter().map(|s| s.cursor).collect::<Vec<_>>()
1090 );
1091 client
1093 .conn
1094 .execute_batch("DROP TRIGGER fail_commit")
1095 .unwrap();
1096 let (_, meta) = client.build_request(false).unwrap();
1097 assert!(matches!(
1098 client.process_response(&mut transport, response, &meta),
1099 SyncOutcome::Ok(_)
1100 ));
1101 assert_eq!(
1102 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1103 2400
1104 );
1105 }
1106
1107 #[test]
1108 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
1109 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1110 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
1111 "scopes":[{"pattern":"project:{project_id}"}]}]});
1112 for realtime in [false, true] {
1113 for fault in [
1114 "none",
1115 "row",
1116 "revision",
1117 "commit",
1118 "cursor",
1119 "error",
1120 "missing-end",
1121 ] {
1122 if realtime && fault == "missing-end" {
1123 continue;
1124 }
1125 let path = std::env::temp_dir()
1126 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
1127 let mut client = SyncClient::open_path(
1128 "frame-test".into(),
1129 &schema,
1130 Default::default(),
1131 path.to_str().unwrap(),
1132 )
1133 .unwrap();
1134 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1135 client
1136 .subscribe(
1137 "tasks".into(),
1138 "tasks".into(),
1139 vec![("project_id".into(), vec!["p1".into()])],
1140 None,
1141 )
1142 .unwrap();
1143 client.subs[0].cursor = 0;
1144 client.subs[0].synced_once = true;
1145 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1146 client.persist_sub(&client.subs[0]).unwrap();
1147 client.realtime_state = RealtimeState::Connected;
1148 client.realtime_reason_code = None;
1149 client.drain_change_batches();
1150 let mut transport = CountingRealtimeTransport::default();
1151 let (_, meta) = client.build_request(false).unwrap();
1152 let mut response = Message {
1153 wire_version: WIRE_VERSION,
1154 msg_kind: MsgKind::Response,
1155 frames: vec![
1156 Frame::RespHeader {
1157 required_schema_version: None,
1158 latest_schema_version: None,
1159 log_epoch: Some("epoch-1".into()),
1160 reset_required: Some(false),
1161 },
1162 Frame::SubStart {
1163 id: "tasks".into(),
1164 status: SubStatus::Active,
1165 reason_code: "".into(),
1166 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1167 bootstrap: false,
1168 },
1169 ],
1170 };
1171 for (index, id) in ["first", "second"].iter().enumerate() {
1172 let values = Map::from_iter([
1173 ("id".into(), Value::from(*id)),
1174 ("project_id".into(), Value::from("p1")),
1175 ]);
1176 response.frames.push(Frame::Commit {
1177 commit_seq: index as i64 + 1,
1178 actor_id: "writer".into(),
1179 created_at_ms: 1,
1180 tables: vec!["tasks".into()],
1181 changes: vec![ssp2::model::Change {
1182 table_index: 0,
1183 row_id: (*id).into(),
1184 op: Op::Upsert,
1185 row_version: Some(index as i64 + 1),
1186 scopes: vec![("project_id".into(), "p1".into())],
1187 row: Some(
1188 encode_row_json(
1189 client.schema.table("tasks").unwrap(),
1190 id,
1191 &values,
1192 &client.encryption,
1193 )
1194 .unwrap(),
1195 ),
1196 }],
1197 });
1198 }
1199 response.frames.push(Frame::SubEnd {
1200 next_cursor: 2,
1201 bootstrap_state: None,
1202 });
1203 let valid = response.clone();
1204 match fault {
1205 "row" => {
1206 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
1207 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
1208 },
1209 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
1210 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
1211 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
1212 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
1213 "missing-end" => {response.frames.truncate(3);},
1214 _ => {}
1215 }
1216 if realtime {
1217 client.on_realtime_binary(&mut transport, &encode_message(&response));
1218 } else {
1219 assert_eq!(
1220 matches!(
1221 client.process_response(&mut transport, response, &meta),
1222 SyncOutcome::Ok(_)
1223 ),
1224 fault == "none",
1225 "{fault}"
1226 );
1227 }
1228 assert!(
1229 client.conn.is_autocommit(),
1230 "no open transaction after {realtime}/{fault}"
1231 );
1232 let count = match fault {
1236 "none" | "cursor" => 2,
1237 "commit" => 0,
1238 _ => 1,
1239 };
1240 let expected_revision = count + u64::from(realtime && fault != "none");
1241 assert_eq!(
1242 client.local_revision(),
1243 expected_revision,
1244 "{realtime}/{fault}"
1245 );
1246 let batches = client.drain_change_batches();
1247 assert_eq!(
1248 batches.len(),
1249 expected_revision as usize,
1250 "{realtime}/{fault}"
1251 );
1252 let row_batches: Vec<_> = batches
1253 .iter()
1254 .filter(|batch| !batch.tables.is_empty())
1255 .collect();
1256 assert_eq!(row_batches.len(), count as usize);
1257 assert_eq!(
1258 row_batches
1259 .iter()
1260 .map(|b| b.revision.clone())
1261 .collect::<Vec<_>>(),
1262 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
1263 );
1264 for table in ["tasks", "_syncular_base_tasks"] {
1265 assert_eq!(
1266 client
1267 .conn
1268 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
1269 .get::<_, u64>(0))
1270 .unwrap(),
1271 count,
1272 "{realtime}/{fault}/{table}"
1273 );
1274 }
1275 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1276 assert_eq!(
1277 transport
1278 .messages
1279 .iter()
1280 .filter(|m| m.contains("\"ack\""))
1281 .count(),
1282 usize::from(fault == "none")
1283 );
1284 if matches!(fault, "revision" | "commit" | "cursor") {
1285 client
1286 .conn
1287 .execute_batch("DROP TRIGGER fail_frame")
1288 .unwrap();
1289 }
1290 drop(client);
1291 let mut reopened = SyncClient::open_path(
1292 "frame-test".into(),
1293 &schema,
1294 Default::default(),
1295 path.to_str().unwrap(),
1296 )
1297 .unwrap();
1298 assert_eq!(reopened.local_revision(), expected_revision);
1299 assert_eq!(
1300 reopened
1301 .query("SELECT id FROM tasks ORDER BY id", &[])
1302 .unwrap()
1303 .len(),
1304 count as usize
1305 );
1306 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1307 let (_, meta) = reopened.build_request(false).unwrap();
1308 assert!(matches!(
1309 reopened.process_response(&mut transport, valid, &meta),
1310 SyncOutcome::Ok(_)
1311 ));
1312 assert_eq!(
1313 reopened
1314 .query("SELECT id FROM tasks ORDER BY id", &[])
1315 .unwrap()
1316 .len(),
1317 2
1318 );
1319 assert_eq!(reopened.subs[0].cursor, 2);
1320 drop(reopened);
1321 std::fs::remove_file(path).unwrap();
1322 }
1323 }
1324 }
1325
1326 #[test]
1327 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
1328 for status in [SubStatus::Reset, SubStatus::Revoked] {
1329 for fault in ["subscription", "revision"] {
1330 let mut client = client();
1331 client
1332 .subscribe(
1333 "tasks".into(),
1334 "tasks".into(),
1335 vec![("project_id".into(), vec!["p1".into()])],
1336 None,
1337 )
1338 .unwrap();
1339 client.subs[0].cursor = 41;
1340 client.subs[0].synced_once = true;
1341 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1342 client.persist_sub(&client.subs[0]).unwrap();
1343 client
1344 .mutate(vec![Mutation::Upsert {
1345 table: "tasks".into(),
1346 values: Map::from_iter([
1347 ("id".into(), Value::from("pending")),
1348 ("project_id".into(), Value::from("p1")),
1349 ]),
1350 base_version: None,
1351 }])
1352 .unwrap();
1353 let ids = client.pending_commit_ids();
1354 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
1355 let revision = client.local_revision();
1356 client.drain_change_batches();
1357 let (_, meta) = client.build_request(false).unwrap();
1358 client.conn.execute_batch(if fault == "subscription" {
1359 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
1360 } else {
1361 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
1362 }).unwrap();
1363 let mut transport = CountingRealtimeTransport::default();
1364 let mut report = SyncReport::default();
1365 if status == SubStatus::Reset {
1367 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
1368 }
1369 let result = client.process_section(
1370 &mut transport,
1371 "tasks",
1372 status,
1373 "sync.scope_revoked",
1374 vec![],
1375 vec![],
1376 Some((42, None)),
1377 &meta,
1378 &mut report,
1379 );
1380 assert!(result.is_err(), "{status:?}/{fault}");
1381 assert!(client.conn.is_autocommit());
1382 assert_eq!(client.subs[0].cursor, 41);
1383 assert_eq!(client.subs[0].state, SubState::Active);
1384 assert_eq!(client.pending_commit_ids(), ids);
1385 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
1386 assert!(client.rejections.is_empty());
1387 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
1388 assert_eq!(client.local_revision(), revision);
1389 assert!(client.drain_change_batches().is_empty());
1390 assert!(report.revoked.is_empty() && report.resets.is_empty());
1391 }
1392 }
1393 }
1394
1395 #[test]
1396 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
1397 let mut client = client();
1398 client
1399 .subscribe(
1400 "tasks".into(),
1401 "tasks".into(),
1402 vec![("project_id".into(), vec!["p1".into()])],
1403 None,
1404 )
1405 .unwrap();
1406 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1407 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
1408 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
1409 let row = |id: &str| ssp2::segment::SegmentRow {
1410 server_version: 1,
1411 values: vec![
1412 Some(ColumnValue::String(id.into())),
1413 Some(ColumnValue::String("p1".into())),
1414 ],
1415 };
1416 let segment = RowsSegment {
1417 table: "tasks".into(),
1418 schema_version: 1,
1419 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
1420 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
1421 };
1422 assert!(client.apply_segment(0, &segment, true).is_err());
1423 assert!(client.conn.is_autocommit());
1424 assert_eq!(
1425 client
1426 .query("SELECT id FROM tasks ORDER BY id", &[])
1427 .unwrap(),
1428 vec![Map::from_iter([("id".into(), Value::from("first"))])]
1429 );
1430 assert_eq!(client.local_revision(), 1);
1431 assert_eq!(client.drain_change_batches().len(), 1);
1432 assert_eq!(client.subs[0].cursor, -1);
1433 client
1434 .conn
1435 .execute_batch("DROP TRIGGER fail_block")
1436 .unwrap();
1437 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
1438 assert_eq!(client.local_revision(), 3);
1439 assert_eq!(client.drain_change_batches().len(), 2);
1440 }
1441
1442 struct SegmentBatchTransport {
1446 response: Message,
1447 bodies: BTreeMap<String, Vec<u8>>,
1448 batches: Vec<Vec<String>>,
1449 single_fetches: usize,
1450 }
1451
1452 impl Transport for SegmentBatchTransport {
1453 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1454 Ok(encode_message(&self.response))
1455 }
1456
1457 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1458 self.sync(request)
1459 }
1460
1461 fn download_segment(
1462 &mut self,
1463 _request: &SegmentRequest,
1464 _on_progress: &mut dyn FnMut(u64),
1465 ) -> Result<Vec<u8>, TransportError> {
1466 self.single_fetches += 1;
1467 Err(TransportError::new(
1468 "sync.transport_failed",
1469 "per-descriptor fetch",
1470 ))
1471 }
1472
1473 fn download_segments(
1474 &mut self,
1475 fetches: &[crate::SegmentFetch],
1476 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1477 ) -> Vec<Result<Vec<u8>, TransportError>> {
1478 self.batches.push(
1479 fetches
1480 .iter()
1481 .map(|fetch| match fetch {
1482 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
1483 crate::SegmentFetch::Url(url) => url.clone(),
1484 })
1485 .collect(),
1486 );
1487 fetches
1488 .iter()
1489 .enumerate()
1490 .map(|(index, fetch)| {
1491 let crate::SegmentFetch::Direct(request) = fetch else {
1492 panic!("the descriptors are direct-endpoint requests")
1493 };
1494 let bytes = self.bodies[&request.segment_id].clone();
1495 on_progress(index, bytes.len() as u64);
1496 Ok(bytes)
1497 })
1498 .collect()
1499 }
1500
1501 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1502 Ok(())
1503 }
1504
1505 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1506 Ok(())
1507 }
1508
1509 fn realtime_close(&mut self) -> Result<(), TransportError> {
1510 Ok(())
1511 }
1512 }
1513
1514 #[test]
1515 fn a_response_fetches_all_its_segment_pages_in_one_ordered_batch() {
1516 let mut client = client();
1517 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1518 client
1519 .subscribe(
1520 "tasks".into(),
1521 "tasks".into(),
1522 vec![("project_id".into(), vec!["p1".into()])],
1523 None,
1524 )
1525 .unwrap();
1526 client.subs[0].cursor = 0;
1527 client.subs[0].synced_once = true;
1528 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1529 client.persist_sub(&client.subs[0]).unwrap();
1530 client.drain_change_batches();
1531 let columns = client.schema.table("tasks").unwrap().wire_columns.clone();
1532
1533 let pages = ["one", "two", "three", "four"].map(|id| {
1537 let bytes = ssp2::segment::encode_rows_segment(&RowsSegment {
1538 table: "tasks".into(),
1539 schema_version: 1,
1540 columns: columns.clone(),
1541 blocks: vec![vec![ssp2::segment::SegmentRow {
1542 server_version: 1,
1543 values: vec![
1544 Some(ColumnValue::String(id.into())),
1545 Some(ColumnValue::String("p1".into())),
1546 ],
1547 }]],
1548 });
1549 (
1550 format!("sha256:{}", bytes_to_hex(&Sha256::digest(&bytes))),
1551 bytes,
1552 )
1553 });
1554
1555 let mut frames = vec![
1556 Frame::RespHeader {
1557 required_schema_version: None,
1558 latest_schema_version: None,
1559 log_epoch: Some("epoch-1".into()),
1560 reset_required: Some(false),
1561 },
1562 Frame::SubStart {
1563 id: "tasks".into(),
1564 status: SubStatus::Active,
1565 reason_code: String::new(),
1566 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1567 bootstrap: false,
1568 },
1569 ];
1570 for (segment_id, bytes) in &pages {
1571 frames.push(Frame::SegmentRef {
1572 segment_id: segment_id.clone(),
1573 media_type: MediaType::Rows,
1574 table: "tasks".into(),
1575 byte_length: bytes.len() as i64,
1576 row_count: 1,
1577 as_of_commit_seq: 1,
1578 scope_digest: "digest".into(),
1579 row_cursor: None,
1580 next_row_cursor: None,
1581 url: None,
1582 url_expires_at_ms: None,
1583 });
1584 }
1585 frames.push(Frame::SubEnd {
1586 next_cursor: 1,
1587 bootstrap_state: None,
1588 });
1589 let mut transport = SegmentBatchTransport {
1590 response: Message {
1591 wire_version: WIRE_VERSION,
1592 msg_kind: MsgKind::Response,
1593 frames,
1594 },
1595 bodies: pages
1596 .iter()
1597 .map(|(segment_id, bytes)| (segment_id.clone(), bytes.clone()))
1598 .collect(),
1599 batches: Vec::new(),
1600 single_fetches: 0,
1601 };
1602
1603 let outcome = client.sync(&mut transport);
1604 assert!(matches!(outcome, SyncOutcome::Ok(_)), "{outcome:?}");
1605 assert_eq!(
1606 transport.batches,
1607 vec![pages
1608 .iter()
1609 .map(|(segment_id, _)| segment_id.clone())
1610 .collect::<Vec<_>>()],
1611 "one batch per response, in wire order"
1612 );
1613 assert_eq!(
1614 transport.single_fetches, 0,
1615 "the batch seam replaces the per-descriptor calls"
1616 );
1617 assert_eq!(
1618 client
1619 .query("SELECT id FROM tasks ORDER BY id", &[])
1620 .unwrap(),
1621 ["four", "one", "three", "two"]
1622 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
1623 .to_vec()
1624 );
1625 assert_eq!(client.subs[0].cursor, 1);
1626 }
1627
1628 #[test]
1629 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
1630 assert_eq!(
1631 client().diagnostics_storage().blob_cache_bytes_approx,
1632 Some(0)
1633 );
1634 let client = SyncClient::new(
1635 "diagnostic-storage".to_owned(),
1636 &json!({
1637 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
1638 "columns": [{"name": "id", "type": "string", "nullable": false},
1639 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
1640 }),
1641 ClientLimits {
1642 blob_cache_max_bytes: Some(5),
1643 ..Default::default()
1644 },
1645 )
1646 .unwrap();
1647 let empty = client.diagnostics_storage();
1648 assert_eq!(empty.status, "healthy");
1649 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
1650 assert_eq!(empty.retained_outcome_entries, Some(0));
1651 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
1652 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
1653 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
1654 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
1655 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
1656 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
1657 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
1658 let written = client.diagnostics_storage();
1659 assert_eq!(written.status, "pressure");
1660 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
1661 assert_eq!(written.retained_outcome_entries, Some(2));
1662 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
1663 assert_eq!(written.blob_cache_bytes_approx, Some(8));
1664 assert_eq!(
1665 written.pressure_reason_code.as_deref(),
1666 Some("client.blob_cache_over_limit")
1667 );
1668 client
1669 .conn
1670 .execute_batch(
1671 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
1672 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
1673 )
1674 .unwrap();
1675 let staged = client.diagnostics_storage();
1676 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
1677 assert_eq!(staged.retained_outcome_entries, Some(0));
1678 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
1679 assert_eq!(staged.status, "healthy");
1680 client.conn.execute_batch("ROLLBACK").unwrap();
1681 assert_eq!(client.diagnostics_storage(), written);
1682 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
1683 let unreadable = client.diagnostics_storage();
1684 assert_eq!(unreadable.status, "unreadable");
1685 assert!(unreadable.database_bytes_approx.is_none());
1686 assert!(unreadable.pending_outbox_bytes_approx.is_none());
1687 assert!(unreadable.retained_outcome_entries.is_none());
1688 assert!(unreadable.retained_outcome_bytes_approx.is_none());
1689 assert!(unreadable.blob_cache_bytes_approx.is_none());
1690 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
1691 let restored = client.diagnostics_storage();
1692 assert_eq!(restored.status, written.status);
1693 assert_eq!(
1694 restored.retained_outcome_bytes_approx,
1695 written.retained_outcome_bytes_approx
1696 );
1697 client.conn.set_prepared_statement_cache_capacity(0);
1698 assert_eq!(client.diagnostics_storage(), restored);
1699 }
1700
1701 struct ServerErrorTransport {
1703 code: &'static str,
1704 retryable: bool,
1705 }
1706
1707 impl Transport for ServerErrorTransport {
1708 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1709 Ok(encode_message(&Message {
1710 wire_version: WIRE_VERSION,
1711 msg_kind: MsgKind::Response,
1712 frames: vec![
1713 Frame::RespHeader {
1714 required_schema_version: None,
1715 latest_schema_version: None,
1716 log_epoch: Some("epoch-1".to_owned()),
1717 reset_required: Some(false),
1718 },
1719 Frame::Error {
1720 code: self.code.to_owned(),
1721 message: "server error".to_owned(),
1722 category: "internal".to_owned(),
1723 retryable: self.retryable,
1724 recommended_action: "retryLater".to_owned(),
1725 details: None,
1726 },
1727 ],
1728 }))
1729 }
1730
1731 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1732 self.sync(request)
1733 }
1734
1735 fn download_segment(
1736 &mut self,
1737 _request: &SegmentRequest,
1738 _on_progress: &mut dyn FnMut(u64),
1739 ) -> Result<Vec<u8>, TransportError> {
1740 Err(TransportError::new("sync.transport_failed", "offline"))
1741 }
1742
1743 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1744 Ok(())
1745 }
1746
1747 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1748 Ok(())
1749 }
1750
1751 fn realtime_close(&mut self) -> Result<(), TransportError> {
1752 Ok(())
1753 }
1754 }
1755
1756 #[test]
1757 fn a_retryable_in_band_failure_keeps_the_socket_connected_and_still_asks_for_a_retry() {
1758 let mut client = client();
1759 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1760 let mut transport = ServerErrorTransport {
1761 code: "sync.schema_not_ready",
1762 retryable: true,
1763 };
1764 client.connect_realtime(&mut transport).expect("connect");
1765 assert!(matches!(
1766 client.sync(&mut transport),
1767 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.schema_not_ready"
1768 ));
1769 assert_eq!(
1770 client.realtime_state(),
1771 RealtimeState::Connected,
1772 "the socket delivered the answer, so it is not lost"
1773 );
1774 assert!(
1775 matches!(
1776 client.drain_sync_intents().as_slice(),
1777 [SyncIntent::Background { delay_ms: 250 }]
1778 ),
1779 "the retry deadline is a sync round and no reconnect satisfies it"
1780 );
1781 assert_eq!(
1782 client.progress().snapshot().unwrap().retry_delay_ms,
1783 Some(250)
1784 );
1785 }
1786
1787 #[test]
1788 fn retryable_server_error_schedules_a_background_retry() {
1789 let mut client = client();
1790 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1791 let mut transport = ServerErrorTransport {
1792 code: "sync.internal_error",
1793 retryable: true,
1794 };
1795 assert!(matches!(
1796 client.sync(&mut transport),
1797 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
1798 ));
1799 assert!(matches!(
1800 client.drain_sync_intents().as_slice(),
1801 [SyncIntent::Background { delay_ms: 250 }]
1802 ));
1803 assert_eq!(
1804 client.progress().snapshot().unwrap().retry_delay_ms,
1805 Some(250)
1806 );
1807 let mut transport = ServerErrorTransport {
1808 code: "sync.invalid_request",
1809 retryable: false,
1810 };
1811 assert!(matches!(
1812 client.sync(&mut transport),
1813 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
1814 ));
1815 assert!(client.drain_sync_intents().is_empty());
1816 let progress = client.progress().snapshot().unwrap();
1817 assert_eq!(progress.state, ProgressState::Failed);
1818 assert_eq!(progress.retry_delay_ms, None);
1819 }
1820
1821 #[test]
1822 fn sync_until_idle_reports_budget_exhaustion_or_failure() {
1823 let exhausted = SyncOutcome::BudgetExhausted(SyncReport::default()).to_json();
1826 assert_eq!(exhausted["ok"], json!(true));
1827 assert_eq!(exhausted["budgetExhausted"], json!(true));
1828 assert!(exhausted["report"].is_object());
1829
1830 let mut client = client();
1831 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1832 let mut idle_transport = HostTransport::new_from_config(&json!({})).unwrap();
1834 let zero = client.sync_until_idle(&mut idle_transport, Some(0));
1835 assert!(
1836 matches!(zero, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
1837 "{zero:?}"
1838 );
1839 let mut erroring = ServerErrorTransport {
1841 code: "sync.internal_error",
1842 retryable: false,
1843 };
1844 let failed = client.sync_until_idle(&mut erroring, Some(3));
1845 assert!(
1846 matches!(failed, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1847 "{failed:?}"
1848 );
1849 }
1850
1851 struct ScriptedSyncTransport {
1854 responses: std::collections::VecDeque<Message>,
1855 calls: usize,
1856 }
1857
1858 impl Transport for ScriptedSyncTransport {
1859 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1860 self.calls += 1;
1861 let response = self.responses.pop_front().expect("scripted response");
1862 Ok(encode_message(&response))
1863 }
1864
1865 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1866 self.sync(request)
1867 }
1868
1869 fn download_segment(
1870 &mut self,
1871 _request: &SegmentRequest,
1872 _on_progress: &mut dyn FnMut(u64),
1873 ) -> Result<Vec<u8>, TransportError> {
1874 Err(TransportError::new("sync.transport_failed", "offline"))
1875 }
1876
1877 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1878 Ok(())
1879 }
1880
1881 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1882 Ok(())
1883 }
1884
1885 fn realtime_close(&mut self) -> Result<(), TransportError> {
1886 Ok(())
1887 }
1888 }
1889
1890 #[test]
1891 fn sync_until_idle_success_then_failure_keeps_earlier_work() {
1892 let path = std::env::temp_dir().join(format!(
1893 "syncular-budget-script-{}.sqlite",
1894 uuid::Uuid::new_v4()
1895 ));
1896 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1897 {"name":"id","type":"string","nullable":false},
1898 {"name":"project_id","type":"string","nullable":false}],
1899 "scopes":[{"pattern":"project:{project_id}"}]}]});
1900 let mut client = SyncClient::open_path(
1901 "budget-script".into(),
1902 &schema,
1903 ClientLimits::default(),
1904 path.to_str().unwrap(),
1905 )
1906 .expect("open client");
1907 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1908 client
1909 .subscribe(
1910 "tasks".into(),
1911 "tasks".into(),
1912 vec![("project_id".into(), vec!["p1".into()])],
1913 None,
1914 )
1915 .unwrap();
1916 client.subs[0].cursor = 0;
1917 client.subs[0].synced_once = true;
1918 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1919 client.persist_sub(&client.subs[0]).unwrap();
1920 client.drain_change_batches();
1921
1922 let row = encode_row_json(
1923 client.schema.table("tasks").unwrap(),
1924 "kept",
1925 &Map::from_iter([
1926 ("id".into(), json!("kept")),
1927 ("project_id".into(), json!("p1")),
1928 ]),
1929 &client.encryption,
1930 )
1931 .unwrap();
1932 let success = Message {
1933 wire_version: WIRE_VERSION,
1934 msg_kind: MsgKind::Response,
1935 frames: vec![
1936 Frame::RespHeader {
1937 required_schema_version: None,
1938 latest_schema_version: None,
1939 log_epoch: Some("epoch-1".into()),
1940 reset_required: Some(false),
1941 },
1942 Frame::SubStart {
1943 id: "tasks".into(),
1944 status: SubStatus::Active,
1945 reason_code: String::new(),
1946 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1947 bootstrap: false,
1948 },
1949 Frame::Commit {
1950 commit_seq: 1,
1951 created_at_ms: 1,
1952 actor_id: "writer".into(),
1953 tables: vec!["tasks".into()],
1954 changes: vec![ssp2::model::Change {
1955 table_index: 0,
1956 row_id: "kept".into(),
1957 op: Op::Upsert,
1958 row_version: Some(1),
1959 scopes: vec![("project_id".into(), "p1".into())],
1960 row: Some(row),
1961 }],
1962 },
1963 Frame::SubEnd {
1964 next_cursor: 1,
1965 bootstrap_state: None,
1966 },
1967 ],
1968 };
1969 let failure = Message {
1970 wire_version: WIRE_VERSION,
1971 msg_kind: MsgKind::Response,
1972 frames: vec![
1973 Frame::RespHeader {
1974 required_schema_version: None,
1975 latest_schema_version: None,
1976 log_epoch: Some("epoch-1".into()),
1977 reset_required: Some(false),
1978 },
1979 Frame::Error {
1980 code: "sync.internal_error".into(),
1981 message: "second round failed".into(),
1982 category: "internal".into(),
1983 retryable: false,
1984 recommended_action: "retryLater".into(),
1985 details: None,
1986 },
1987 ],
1988 };
1989 let mut transport = ScriptedSyncTransport {
1990 responses: std::collections::VecDeque::from([success, failure]),
1991 calls: 0,
1992 };
1993 let outcome = client.sync_until_idle(&mut transport, Some(2));
1994 assert!(
1995 matches!(outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1996 "{outcome:?}"
1997 );
1998 assert_eq!(
1999 transport.calls, 2,
2000 "one successful round then the failing round"
2001 );
2002 assert_eq!(
2003 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
2004 json!("kept"),
2005 "the earlier round's durable work survives the later failure"
2006 );
2007 drop(client);
2008 let _ = std::fs::remove_file(&path);
2009 }
2010
2011 #[test]
2012 fn sync_until_idle_stops_on_a_schema_floor() {
2013 let mut client = client();
2014 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2015 let floor = Message {
2016 wire_version: WIRE_VERSION,
2017 msg_kind: MsgKind::Response,
2018 frames: vec![Frame::RespHeader {
2019 required_schema_version: Some(2),
2020 latest_schema_version: Some(2),
2021 log_epoch: Some("epoch-1".into()),
2022 reset_required: Some(false),
2023 }],
2024 };
2025 let mut transport = ScriptedSyncTransport {
2026 responses: std::collections::VecDeque::from([floor.clone(), floor]),
2027 calls: 0,
2028 };
2029 let outcome = client.sync_until_idle(&mut transport, Some(3));
2030 match outcome {
2031 SyncOutcome::Ok(report) => assert_eq!(
2032 report
2033 .schema_floor
2034 .as_ref()
2035 .and_then(|floor| floor.required_schema_version),
2036 Some(2),
2037 "the stop retains the latest schema floor"
2038 ),
2039 other => panic!("expected a schema-floor stop, got {other:?}"),
2040 }
2041 assert_eq!(
2042 transport.calls, 1,
2043 "a schema floor stops the budget loop after one round"
2044 );
2045 }
2046
2047 #[test]
2048 fn sync_report_merge_schema_floor_is_latest_state() {
2049 let mut aggregate = SyncReport {
2050 schema_floor: Some(SchemaFloor {
2051 required_schema_version: Some(2),
2052 latest_schema_version: Some(2),
2053 }),
2054 ..SyncReport::default()
2055 };
2056 aggregate.merge(&SyncReport::default());
2057 assert!(
2058 aggregate.schema_floor.is_none(),
2059 "a later round without a floor clears it"
2060 );
2061 }
2062
2063 #[test]
2064 fn push_capacity_limits_are_hard_and_keep_intent() {
2065 let mut c = client();
2067 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2068 c.limits.max_push_operations_per_request = Some(1);
2069 let id = c
2070 .mutate(vec![
2071 Mutation::Upsert {
2072 table: "tasks".into(),
2073 values: Map::from_iter([
2074 ("id".into(), json!("a")),
2075 ("project_id".into(), json!("p1")),
2076 ]),
2077 base_version: None,
2078 },
2079 Mutation::Upsert {
2080 table: "tasks".into(),
2081 values: Map::from_iter([
2082 ("id".into(), json!("b")),
2083 ("project_id".into(), json!("p1")),
2084 ]),
2085 base_version: None,
2086 },
2087 ])
2088 .unwrap();
2089 let error = c.build_request(false).unwrap_err();
2090 let SyncOutcome::Failed {
2091 error_code,
2092 details,
2093 ..
2094 } = *error
2095 else {
2096 panic!("expected a capacity failure");
2097 };
2098 assert_eq!(error_code, "client.push_request_too_large");
2099 let details = details.expect("capacity details");
2100 assert_eq!(details["kind"], json!("operations"));
2101 assert_eq!(details["limit"], json!(1));
2102 assert_eq!(details["size"], json!(2));
2103 assert_eq!(details["clientCommitId"], json!(id));
2104 assert_eq!(c.pending_commit_ids(), vec![id]);
2105
2106 let mut c = client();
2108 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2109 c.limits.max_push_commits_per_request = Some(1);
2110 let first = c
2111 .mutate(vec![Mutation::Upsert {
2112 table: "tasks".into(),
2113 values: Map::from_iter([
2114 ("id".into(), json!("one")),
2115 ("project_id".into(), json!("p1")),
2116 ]),
2117 base_version: None,
2118 }])
2119 .unwrap();
2120 let second = c
2121 .mutate(vec![Mutation::Upsert {
2122 table: "tasks".into(),
2123 values: Map::from_iter([
2124 ("id".into(), json!("two")),
2125 ("project_id".into(), json!("p1")),
2126 ]),
2127 base_version: None,
2128 }])
2129 .unwrap();
2130 let (_, meta) = c.build_request(false).unwrap();
2131 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2132 assert_eq!(meta.deferred_commits, 1);
2133 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2134 }
2135
2136 #[test]
2137 fn push_request_byte_cap_defers_the_prefix_boundary_and_errors_the_head() {
2138 let mut c = client();
2141 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2142 let first = c
2143 .mutate(vec![Mutation::Upsert {
2144 table: "tasks".into(),
2145 values: Map::from_iter([
2146 ("id".into(), json!("one")),
2147 ("project_id".into(), json!("p1")),
2148 ]),
2149 base_version: None,
2150 }])
2151 .unwrap();
2152 let second = c
2153 .mutate(vec![Mutation::Upsert {
2154 table: "tasks".into(),
2155 values: Map::from_iter([
2156 ("id".into(), json!("two")),
2157 ("project_id".into(), json!("p1")),
2158 ]),
2159 base_version: None,
2160 }])
2161 .unwrap();
2162 let (message, meta) = c.build_request(false).unwrap();
2163 assert_eq!(meta.pushed_ids.len(), 2);
2164 let full = encode_message(&message).len();
2165 c.limits.max_push_request_bytes = Some(full - 1);
2166 let (_, meta) = c.build_request(false).unwrap();
2167 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2168 assert_eq!(meta.deferred_commits, 1);
2169 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2170
2171 let mut c = client();
2174 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2175 let huge = c
2176 .mutate(vec![Mutation::Upsert {
2177 table: "tasks".into(),
2178 values: Map::from_iter([
2179 ("id".into(), json!("x".repeat(20_000))),
2180 ("project_id".into(), json!("p1")),
2181 ]),
2182 base_version: None,
2183 }])
2184 .unwrap();
2185 let (message, meta) = c.build_request(false).unwrap();
2186 assert_eq!(meta.pushed_ids, vec![huge.clone()]);
2187 let with_huge = encode_message(&message).len();
2188 c.limits.max_push_request_bytes = Some(with_huge - 1);
2189 let error = c.build_request(false).unwrap_err();
2190 let SyncOutcome::Failed {
2191 error_code,
2192 details,
2193 ..
2194 } = *error
2195 else {
2196 panic!("expected a capacity failure");
2197 };
2198 assert_eq!(error_code, "client.push_request_too_large");
2199 let details = details.expect("capacity details");
2200 assert_eq!(details["kind"], json!("bytes"));
2201 assert_eq!(details["limit"], json!((with_huge - 1) as u64));
2202 assert_eq!(details["size"], json!(with_huge as u64));
2203 assert_eq!(details["clientCommitId"], json!(huge));
2204 assert_eq!(c.pending_commit_ids(), vec![huge]);
2205
2206 let mut c = client();
2208 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2209 c.limits.max_push_request_bytes = Some(1);
2210 let error = c.build_request(false).unwrap_err();
2211 let SyncOutcome::Failed { details, .. } = *error else {
2212 panic!("expected a capacity failure");
2213 };
2214 let details = details.expect("capacity details");
2215 assert_eq!(details["kind"], json!("bytes"));
2216 assert!(details.get("clientCommitId").is_none(), "{details}");
2217 }
2218
2219 #[test]
2220 fn push_limits_validate_at_construction() {
2221 for limits in [
2222 ClientLimits {
2223 max_push_commits_per_request: Some(0),
2224 ..Default::default()
2225 },
2226 ClientLimits {
2227 max_push_operations_per_request: Some(0),
2228 ..Default::default()
2229 },
2230 ClientLimits {
2231 max_push_request_bytes: Some(0),
2232 ..Default::default()
2233 },
2234 ClientLimits {
2235 max_push_request_bytes: Some(u32::MAX as usize + 1),
2236 ..Default::default()
2237 },
2238 ] {
2239 let schema = json!({"version":1,"tables":[]});
2240 let error = SyncClient::with_connection(
2241 "bad-limits".into(),
2242 &schema,
2243 limits,
2244 Connection::open_in_memory().unwrap(),
2245 )
2246 .err();
2247 assert_eq!(
2248 error.as_deref(),
2249 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2250 );
2251 }
2252 }
2253
2254 #[test]
2255 fn open_path_rejects_invalid_push_limits_before_storage() {
2256 for identity in [false, true] {
2257 let path = std::env::temp_dir().join(format!(
2258 "syncular-bad-limits-{}.sqlite",
2259 uuid::Uuid::new_v4()
2260 ));
2261 let schema = json!({"version":1,"tables":[]});
2262 let limits = ClientLimits {
2263 max_push_request_bytes: Some(0),
2264 ..Default::default()
2265 };
2266 let error = if identity {
2267 SyncClient::open_path_with_identity(None, &schema, limits, path.to_str().unwrap())
2268 } else {
2269 SyncClient::open_path("bad-limits".into(), &schema, limits, path.to_str().unwrap())
2270 }
2271 .err();
2272 assert_eq!(
2273 error.as_deref(),
2274 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2275 );
2276 assert!(
2277 !path.exists(),
2278 "an invalid limit is rejected before the database file is created"
2279 );
2280 }
2281 }
2282
2283 #[test]
2284 fn failed_progress_retry_delay_doubles_to_the_cap() {
2285 let mut client = client();
2286 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2287 let mut transport = ServerErrorTransport {
2288 code: "sync.internal_error",
2289 retryable: true,
2290 };
2291 let delays: Vec<Option<u64>> = (0..9)
2292 .map(|_| {
2293 client.sync(&mut transport);
2294 client.progress().snapshot().unwrap().retry_delay_ms
2295 })
2296 .collect();
2297 assert_eq!(
2298 delays,
2299 [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
2300 );
2301 }
2302
2303 #[test]
2304 fn background_retry_deadlines_back_off_and_reset() {
2305 let mut client = client();
2306 client.schedule_background_retry();
2307 assert!(matches!(
2308 client.drain_sync_intents().as_slice(),
2309 [SyncIntent::Background { delay_ms: 250 }]
2310 ));
2311 client.schedule_background_retry();
2312 assert!(matches!(
2313 client.drain_sync_intents().as_slice(),
2314 [SyncIntent::Background { delay_ms: 500 }]
2315 ));
2316 client.reset_background_retry();
2317 client.schedule_background_retry();
2318 assert!(matches!(
2319 client.drain_sync_intents().as_slice(),
2320 [SyncIntent::Background { delay_ms: 250 }]
2321 ));
2322 }
2323
2324 #[test]
2325 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
2326 let mut client = client();
2327 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2328 client
2329 .subscribe(
2330 "epoch-tasks".to_owned(),
2331 "tasks".to_owned(),
2332 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2333 None,
2334 )
2335 .expect("subscribe");
2336 client.drain_sync_intents();
2337
2338 client
2339 .run_log_epoch_reset("epoch-2")
2340 .expect("reset partition log epoch");
2341
2342 assert!(matches!(
2343 client.drain_sync_intents().as_slice(),
2344 [SyncIntent::Interactive]
2345 ));
2346 }
2347
2348 #[derive(Default)]
2349 struct CountingRealtimeTransport {
2350 connects: usize,
2351 closes: usize,
2352 messages: Vec<String>,
2353 blob_uploads: Vec<(String, Vec<u8>)>,
2354 blob_download: Option<Vec<u8>>,
2355 sync_calls: usize,
2356 realtime_sync_calls: usize,
2357 refuse_connect: bool,
2358 }
2359
2360 impl Transport for CountingRealtimeTransport {
2361 fn blob_upload(
2362 &mut self,
2363 blob_id: &str,
2364 bytes: &[u8],
2365 _media_type: Option<&str>,
2366 ) -> Result<(), TransportError> {
2367 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
2368 Ok(())
2369 }
2370
2371 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
2372 self.blob_download
2373 .clone()
2374 .map(BlobDownload::Bytes)
2375 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
2376 }
2377
2378 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2379 self.sync_calls += 1;
2380 Err(TransportError::new("sync.transport_failed", "offline"))
2381 }
2382
2383 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2384 self.realtime_sync_calls += 1;
2385 Err(TransportError::new("sync.transport_failed", "offline"))
2386 }
2387
2388 fn download_segment(
2389 &mut self,
2390 _request: &SegmentRequest,
2391 _on_progress: &mut dyn FnMut(u64),
2392 ) -> Result<Vec<u8>, TransportError> {
2393 Err(TransportError::new("sync.transport_failed", "offline"))
2394 }
2395
2396 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2397 self.connects += 1;
2398 if self.refuse_connect {
2399 return Err(TransportError::new("transport.failed", "refused"));
2400 }
2401 Ok(())
2402 }
2403
2404 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2405 self.messages.push(text.to_owned());
2406 Ok(())
2407 }
2408
2409 fn realtime_close(&mut self) -> Result<(), TransportError> {
2410 self.closes += 1;
2411 Ok(())
2412 }
2413 }
2414
2415 #[test]
2416 fn blob_results_own_downloaded_bytes() {
2417 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
2418 {"name":"id","type":"string","nullable":false},
2419 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
2420 let bytes = b"owned typed result".to_vec();
2421 let blob_id = blob_id_for(&bytes);
2422 let mut transport = CountingRealtimeTransport {
2423 blob_download: Some(bytes.clone()),
2424 ..Default::default()
2425 };
2426 let result = {
2427 let mut client =
2428 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
2429 .unwrap();
2430 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
2431 assert_eq!(result.blob_id, blob_id);
2432 assert_eq!(result.byte_length, bytes.len() as i64);
2433 assert_eq!(result.bytes, bytes);
2434 assert_eq!(result.media_type, None);
2435 assert_eq!(
2436 client
2437 .conn
2438 .query_row(
2439 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
2440 rusqlite::params![blob_id],
2441 |row| row.get::<_, i64>(0),
2442 )
2443 .unwrap(),
2444 0
2445 );
2446 result
2447 };
2448 assert_eq!(result.bytes, bytes);
2449
2450 let mut client =
2451 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
2452 assert_eq!(
2453 client
2454 .fetch_blob_bytes(&mut transport, "not json")
2455 .unwrap_err(),
2456 (
2457 "client.failed".to_owned(),
2458 "blob ref is not JSON".to_owned()
2459 )
2460 );
2461 }
2462
2463 #[test]
2464 fn visible_blob_references_survive_cache_trimming() {
2465 let schema = json!({"version":1,"tables":[
2466 {"name":"attachments","primaryKey":"id","columns":[
2467 {"name":"id","type":"string","nullable":false},
2468 {"name":"body","type":"blob_ref","nullable":true},
2469 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
2470 {"name":"avatars","primaryKey":"id","columns":[
2471 {"name":"id","type":"string","nullable":false},
2472 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
2473 ]});
2474 let bytes = b"referenced above cap".to_vec();
2475 let blob_id = blob_id_for(&bytes);
2476 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
2477 let mut client = SyncClient::new(
2478 "visible-blob-refs".to_owned(),
2479 &schema,
2480 ClientLimits {
2481 blob_cache_max_bytes: Some(1),
2482 ..Default::default()
2483 },
2484 )
2485 .unwrap();
2486 client.conn.execute(
2487 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
2488 rusqlite::params![reference, reference, reference, reference],
2489 ).unwrap();
2490 client
2491 .conn
2492 .execute(
2493 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
2494 rusqlite::params![reference],
2495 )
2496 .unwrap();
2497 client.conn.execute(
2498 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
2499 [],
2500 ).unwrap();
2501 let mut transport = CountingRealtimeTransport {
2502 blob_download: Some(bytes.clone()),
2503 ..Default::default()
2504 };
2505
2506 assert_eq!(
2507 client
2508 .fetch_blob_bytes(&mut transport, &blob_id)
2509 .unwrap()
2510 .bytes,
2511 bytes
2512 );
2513 let rows = client
2514 .conn
2515 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
2516 .unwrap()
2517 .query_map([], |row| row.get::<_, String>(0))
2518 .unwrap()
2519 .collect::<Result<Vec<_>, _>>()
2520 .unwrap();
2521 assert_eq!(rows, vec![blob_id]);
2522 }
2523
2524 #[test]
2525 fn closed_transport_never_invokes_network_and_reopen_is_host_owned() {
2526 let mut transport = CountingRealtimeTransport::default();
2527 let path = std::env::temp_dir().join(format!(
2528 "syncular-transport-gate-{}.db",
2529 uuid::Uuid::new_v4()
2530 ));
2531 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
2532 {"name":"id","type":"string","nullable":false},
2533 {"name":"project_id","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
2534 let mut client = SyncClient::open_path(
2535 "gate".into(),
2536 &schema,
2537 ClientLimits::default(),
2538 path.to_str().unwrap(),
2539 )
2540 .unwrap();
2541 client.connect_realtime(&mut transport).unwrap();
2542 client.set_transport_enabled(&mut transport, false);
2543 assert_eq!(transport.closes, 1);
2544 client.begin_security_preflight();
2545 client.activate_security(Default::default()).unwrap();
2546 client
2547 .subscribe(
2548 "tasks".into(),
2549 "tasks".into(),
2550 vec![("project_id".into(), vec!["p1".into()])],
2551 None,
2552 )
2553 .unwrap();
2554 for id in ["one", "two"] {
2555 client
2556 .mutate(vec![Mutation::Upsert {
2557 table: "tasks".into(),
2558 values: Map::from_iter([
2559 ("id".into(), json!(id)),
2560 ("project_id".into(), json!("p1")),
2561 ]),
2562 base_version: None,
2563 }])
2564 .unwrap();
2565 }
2566 assert_eq!(
2567 client
2568 .query("SELECT id FROM tasks ORDER BY id", &[])
2569 .unwrap()
2570 .len(),
2571 2
2572 );
2573 assert!(
2574 matches!(client.sync(&mut transport), SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.offline")
2575 );
2576 assert!(client
2577 .connect_realtime(&mut transport)
2578 .unwrap_err()
2579 .starts_with("sync.offline:"));
2580 assert!(client
2581 .set_presence(&mut transport, "project:p1", None)
2582 .unwrap_err()
2583 .starts_with("sync.offline:"));
2584 assert_eq!(transport.sync_calls, 0);
2585 assert_eq!(transport.realtime_sync_calls, 0);
2586 assert_eq!(transport.connects, 1);
2587 assert!(transport.messages.is_empty());
2588 assert!(matches!(
2589 client.drain_sync_intents().as_slice(),
2590 [SyncIntent::None]
2591 ));
2592 let pending = client.pending_commit_ids();
2593 drop(client);
2594 let mut reopened = SyncClient::open_path(
2595 "gate".into(),
2596 &schema,
2597 ClientLimits::default(),
2598 path.to_str().unwrap(),
2599 )
2600 .unwrap();
2601 assert!(reopened.transport_enabled());
2602 reopened.set_transport_enabled(&mut transport, false);
2603 assert_eq!(reopened.pending_commit_ids(), pending);
2604 assert_eq!(
2605 reopened.query("SELECT id FROM tasks", &[]).unwrap().len(),
2606 2
2607 );
2608 reopened.set_transport_enabled(&mut transport, true);
2609 assert!(matches!(
2610 reopened.drain_sync_intents().as_slice(),
2611 [SyncIntent::Interactive]
2612 ));
2613 reopened.set_transport_enabled(&mut transport, true);
2614 assert!(reopened.drain_sync_intents().is_empty());
2615 drop(reopened);
2616 std::fs::remove_file(path).unwrap();
2617 }
2618
2619 #[test]
2620 fn realtime_connection_ownership_is_idempotent() {
2621 let mut client = client();
2622 let mut transport = CountingRealtimeTransport {
2623 connects: 0,
2624 closes: 0,
2625 ..Default::default()
2626 };
2627 client
2628 .connect_realtime(&mut transport)
2629 .expect("first connect");
2630 client
2631 .connect_realtime(&mut transport)
2632 .expect("idempotent connect");
2633 assert_eq!(transport.connects, 1);
2634 client.disconnect_realtime(&mut transport);
2635 client.disconnect_realtime(&mut transport);
2636 assert_eq!(transport.closes, 1);
2637 client
2638 .connect_realtime(&mut transport)
2639 .expect("deliberate reconnect");
2640 assert_eq!(transport.connects, 2);
2641 }
2642
2643 #[test]
2644 fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
2645 let mut client = client();
2646 client
2647 .set_realtime_policy(RealtimePolicy::Required)
2648 .expect("set policy");
2649 let mut transport = CountingRealtimeTransport::default();
2650 match client.sync(&mut transport) {
2651 SyncOutcome::RealtimeUnavailable {
2652 state,
2653 reason_code,
2654 retry_delay_ms,
2655 } => {
2656 assert_eq!(state, RealtimeState::Disconnected);
2657 assert_eq!(reason_code, None);
2658 assert_eq!(retry_delay_ms, 250);
2659 }
2660 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2661 }
2662 assert_eq!(transport.sync_calls, 0);
2663 assert!(matches!(
2664 client.drain_sync_intents().as_slice(),
2665 [SyncIntent::Background { delay_ms: 250 }]
2666 ));
2667 let snapshot = client
2668 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2669 .expect("diagnostics");
2670 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2671 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
2672 assert_eq!(snapshot.host.realtime_reason_code, None);
2673 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2674 assert_eq!(
2675 snapshot
2676 .last_round
2677 .as_ref()
2678 .and_then(|round| round.error_code.clone()),
2679 Some(REALTIME_UNAVAILABLE_CODE.to_owned())
2680 );
2681 }
2682
2683 #[test]
2684 fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
2685 let mut client = client();
2686 client
2687 .set_realtime_policy(RealtimePolicy::Required)
2688 .expect("set policy");
2689 let mut transport = CountingRealtimeTransport {
2690 refuse_connect: true,
2691 ..Default::default()
2692 };
2693 assert!(client.connect_realtime(&mut transport).is_err());
2694 assert_eq!(client.realtime_state(), RealtimeState::Refused);
2695 let snapshot = client
2696 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2697 .expect("diagnostics");
2698 assert_eq!(
2699 snapshot.host.realtime_reason_code.as_deref(),
2700 Some("transport.failed")
2701 );
2702 match client.sync(&mut transport) {
2703 SyncOutcome::RealtimeUnavailable {
2704 state, reason_code, ..
2705 } => {
2706 assert_eq!(state, RealtimeState::Refused);
2707 assert_eq!(reason_code.as_deref(), Some("transport.failed"));
2708 }
2709 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2710 }
2711 assert_eq!(transport.sync_calls, 0);
2712 assert_eq!(transport.realtime_sync_calls, 0);
2713 }
2714
2715 #[test]
2716 fn an_unsolicited_socket_end_is_lost_with_the_transport_code_and_a_retry_intent() {
2717 let mut client = client();
2718 let mut transport = CountingRealtimeTransport::default();
2719 let ended = TransportError::new("transport.failed", "realtime connection closed");
2720 client.realtime_lost(&mut transport, &ended);
2721 assert_eq!(
2722 client.realtime_state(),
2723 RealtimeState::Disconnected,
2724 "a socket that was never connected has nothing to lose"
2725 );
2726 assert!(client.drain_sync_intents().is_empty());
2727
2728 client.connect_realtime(&mut transport).expect("connect");
2729 client.realtime_lost(&mut transport, &ended);
2730 assert_eq!(client.realtime_state(), RealtimeState::Lost);
2731 assert_eq!(transport.closes, 1);
2732 assert!(matches!(
2733 client.drain_sync_intents().as_slice(),
2734 [SyncIntent::Background { delay_ms: 250 }]
2735 ));
2736 let snapshot = client
2737 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2738 .expect("diagnostics");
2739 assert_eq!(snapshot.host.realtime, RealtimeState::Lost);
2740 assert_eq!(
2741 snapshot.host.realtime_reason_code.as_deref(),
2742 Some("transport.failed")
2743 );
2744 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2745
2746 client.realtime_lost(&mut transport, &ended);
2747 assert!(
2748 client.drain_sync_intents().is_empty(),
2749 "a second report for the same loss schedules nothing"
2750 );
2751 client.connect_realtime(&mut transport).expect("reconnect");
2752 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2753 }
2754
2755 #[test]
2756 fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
2757 let mut client = client();
2758 client
2759 .set_realtime_policy(RealtimePolicy::Required)
2760 .expect("set policy");
2761 let mut transport = CountingRealtimeTransport::default();
2762 client.connect_realtime(&mut transport).expect("connect");
2763 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2764 assert!(
2765 client.set_realtime_policy(RealtimePolicy::Off).is_err(),
2766 "a policy change cannot silently re-route a connected socket"
2767 );
2768 assert!(matches!(
2769 client.sync(&mut transport),
2770 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
2771 ));
2772 assert_eq!(client.realtime_state(), RealtimeState::Lost);
2773 assert_eq!(transport.realtime_sync_calls, 1);
2774 assert_eq!(transport.sync_calls, 0);
2775 let snapshot = client
2776 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2777 .expect("diagnostics");
2778 assert_eq!(
2779 snapshot.host.realtime_reason_code.as_deref(),
2780 Some("sync.transport_failed")
2781 );
2782 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2783 match client.sync(&mut transport) {
2784 SyncOutcome::RealtimeUnavailable {
2785 state, reason_code, ..
2786 } => {
2787 assert_eq!(state, RealtimeState::Lost);
2788 assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
2789 }
2790 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2791 }
2792 assert_eq!(transport.sync_calls, 0);
2793 client.disconnect_realtime(&mut transport);
2794 assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
2795 assert_eq!(transport.closes, 1);
2796 client.connect_realtime(&mut transport).expect("reconnect");
2797 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2798 assert_eq!(transport.connects, 2);
2799 }
2800
2801 #[test]
2802 fn off_policy_uses_http_and_refuses_an_explicit_connect() {
2803 let mut client = client();
2804 client
2805 .set_realtime_policy(RealtimePolicy::Off)
2806 .expect("set policy");
2807 let mut transport = CountingRealtimeTransport::default();
2808 assert!(client.connect_realtime(&mut transport).is_err());
2809 assert_eq!(client.realtime_state(), RealtimeState::Disabled);
2810 assert!(matches!(
2811 client.sync(&mut transport),
2812 SyncOutcome::Failed { .. }
2813 ));
2814 assert_eq!(transport.sync_calls, 1);
2815 assert_eq!(transport.realtime_sync_calls, 0);
2816 let snapshot = client
2817 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2818 .expect("diagnostics");
2819 assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
2820 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
2821 }
2822
2823 #[test]
2824 fn optional_policy_keeps_the_http_round() {
2825 let mut client = client();
2826 let mut transport = CountingRealtimeTransport::default();
2827 assert!(matches!(
2828 client.sync(&mut transport),
2829 SyncOutcome::Failed { .. }
2830 ));
2831 assert_eq!(transport.sync_calls, 1);
2832 assert_eq!(transport.realtime_sync_calls, 0);
2833 let snapshot = client
2834 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2835 .expect("diagnostics");
2836 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2837 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
2838 }
2839
2840 #[test]
2841 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
2842 let mut client = client();
2843 client
2844 .subscribe(
2845 "repair-tasks".to_owned(),
2846 "tasks".to_owned(),
2847 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2848 None,
2849 )
2850 .expect("subscribe");
2851 {
2852 let sub = client
2853 .subs
2854 .iter_mut()
2855 .find(|sub| sub.id == "repair-tasks")
2856 .expect("subscription");
2857 sub.cursor = 42;
2858 sub.synced_once = true;
2859 let persisted = sub.clone();
2860 client.persist_sub(&persisted).unwrap();
2861 }
2862 for table in ["_syncular_base_tasks", "tasks"] {
2863 client
2864 .conn
2865 .execute(
2866 &format!(
2867 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
2868 ),
2869 rusqlite::params!["server-row", "p1"],
2870 )
2871 .expect("seed server row");
2872 }
2873 let pending = client
2874 .mutate(vec![Mutation::Upsert {
2875 table: "tasks".to_owned(),
2876 values: Map::from_iter([
2877 ("id".to_owned(), Value::from("offline-row")),
2878 ("project_id".to_owned(), Value::from("p1")),
2879 ]),
2880 base_version: None,
2881 }])
2882 .expect("queue offline work");
2883 client.drain_change_batches();
2884 client.drain_sync_intents();
2885
2886 assert_eq!(
2887 client
2888 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2889 rebootstrap_id: "support-case-001".to_owned(),
2890 })
2891 .expect("rebootstrap"),
2892 LocalDataRebootstrapResult {
2893 already_applied: false,
2894 retained_commits: 1,
2895 reset_subscriptions: 1,
2896 }
2897 );
2898 let visible_ids = client
2899 .conn
2900 .prepare("SELECT id FROM tasks ORDER BY id")
2901 .expect("prepare visible ids")
2902 .query_map([], |row| row.get::<_, String>(0))
2903 .expect("query visible ids")
2904 .collect::<Result<Vec<_>, _>>()
2905 .expect("collect visible ids");
2906 assert_eq!(visible_ids, vec!["offline-row"]);
2907 assert_eq!(client.pending_commit_ids(), vec![pending]);
2908 assert_eq!(
2909 client
2910 .subscription_state("repair-tasks")
2911 .expect("subscription")
2912 .cursor,
2913 -1
2914 );
2915 assert!(client.upgrading());
2916 assert!(client.sync_needed());
2917 assert!(matches!(
2918 client.drain_sync_intents().as_slice(),
2919 [SyncIntent::Interactive]
2920 ));
2921 assert_eq!(client.drain_change_batches().len(), 1);
2922
2923 assert_eq!(
2924 client
2925 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2926 rebootstrap_id: "support-case-001".to_owned(),
2927 })
2928 .expect("idempotent retry"),
2929 LocalDataRebootstrapResult {
2930 already_applied: true,
2931 retained_commits: 1,
2932 reset_subscriptions: 1,
2933 }
2934 );
2935 assert!(client.drain_change_batches().is_empty());
2936 }
2937
2938 #[test]
2939 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
2940 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
2941 assert_eq!(
2942 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
2943 (3, 4)
2944 );
2945 assert_eq!(
2946 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
2947 (0, 0)
2948 );
2949 for malformed in [
2950 "",
2951 "{}",
2952 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
2953 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
2954 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
2955 ] {
2956 assert_eq!(
2957 decode_local_rebootstrap_receipt(malformed)
2958 .expect_err("malformed receipt must fail"),
2959 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
2960 );
2961 }
2962 }
2963
2964 #[test]
2965 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
2966 let path = std::env::temp_dir().join(format!(
2967 "syncular-rebootstrap-receipt-{}.db",
2968 uuid::Uuid::new_v4()
2969 ));
2970 let schema = json!({
2971 "version": 1,
2972 "tables": [{
2973 "name": "tasks",
2974 "primaryKey": "id",
2975 "columns": [
2976 { "name": "id", "type": "string", "nullable": false },
2977 { "name": "project_id", "type": "string", "nullable": false }
2978 ],
2979 "scopes": [{ "pattern": "project:{project_id}" }]
2980 }]
2981 });
2982 let path_string = path.to_str().expect("UTF-8 temp path");
2983
2984 {
2985 let mut first = SyncClient::open_path(
2986 "repair-restart-client".to_owned(),
2987 &schema,
2988 ClientLimits::default(),
2989 path_string,
2990 )
2991 .expect("first open");
2992 first
2993 .subscribe(
2994 "repair-tasks".to_owned(),
2995 "tasks".to_owned(),
2996 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2997 None,
2998 )
2999 .expect("subscribe");
3000 first
3001 .mutate(vec![Mutation::Upsert {
3002 table: "tasks".to_owned(),
3003 values: Map::from_iter([
3004 ("id".to_owned(), Value::from("offline-row")),
3005 ("project_id".to_owned(), Value::from("p1")),
3006 ]),
3007 base_version: None,
3008 }])
3009 .expect("queue offline work");
3010 assert_eq!(
3011 first
3012 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3013 rebootstrap_id: "restart-receipt".to_owned(),
3014 })
3015 .expect("first rebootstrap"),
3016 LocalDataRebootstrapResult {
3017 already_applied: false,
3018 retained_commits: 1,
3019 reset_subscriptions: 1,
3020 }
3021 );
3022 }
3023
3024 let mut reopened = SyncClient::open_path(
3025 "repair-restart-client".to_owned(),
3026 &schema,
3027 ClientLimits::default(),
3028 path_string,
3029 )
3030 .expect("reopen");
3031 reopened.drain_change_batches();
3032 reopened.drain_sync_intents();
3033 assert_eq!(
3034 reopened
3035 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3036 rebootstrap_id: "restart-receipt".to_owned(),
3037 })
3038 .expect("receipt replay"),
3039 LocalDataRebootstrapResult {
3040 already_applied: true,
3041 retained_commits: 1,
3042 reset_subscriptions: 1,
3043 }
3044 );
3045 assert!(reopened.drain_change_batches().is_empty());
3046 assert!(reopened.drain_sync_intents().is_empty());
3047 drop(reopened);
3048 std::fs::remove_file(path).expect("remove temp database");
3049 }
3050
3051 #[test]
3052 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
3053 let mut malformed = client();
3054 malformed
3055 .subscribe(
3056 "repair-tasks".to_owned(),
3057 "tasks".to_owned(),
3058 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3059 None,
3060 )
3061 .expect("subscribe");
3062 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
3063 malformed.drain_change_batches();
3064 malformed.drain_sync_intents();
3065 let malformed_error = malformed
3066 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3067 rebootstrap_id: "malformed".to_owned(),
3068 })
3069 .expect_err("malformed receipt must fail closed");
3070 assert_eq!(
3071 malformed_error,
3072 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3073 );
3074 assert!(!malformed.upgrading());
3075 assert_eq!(
3076 malformed
3077 .subscription_state("repair-tasks")
3078 .expect("unchanged subscription")
3079 .cursor,
3080 -1
3081 );
3082 assert!(malformed.drain_change_batches().is_empty());
3083 assert!(malformed.drain_sync_intents().is_empty());
3084
3085 let mut unreadable = client();
3086 unreadable
3087 .conn
3088 .execute(
3089 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
3090 rusqlite::params!["server-row", "p1"],
3091 )
3092 .expect("seed visible row");
3093 unreadable
3094 .conn
3095 .execute("DROP TABLE _syncular_meta", [])
3096 .expect("break marker storage");
3097 let unreadable_error = unreadable
3098 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3099 rebootstrap_id: "unreadable".to_owned(),
3100 })
3101 .expect_err("unreadable marker storage must fail closed");
3102 assert_eq!(
3103 unreadable_error,
3104 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
3105 );
3106 let visible_rows = unreadable
3107 .conn
3108 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
3109 .expect("visible projection remains");
3110 assert_eq!(visible_rows, 1);
3111 assert!(!unreadable.upgrading());
3112 assert!(unreadable.drain_change_batches().is_empty());
3113 assert!(unreadable.drain_sync_intents().is_empty());
3114 }
3115
3116 #[test]
3117 fn local_rebootstrap_cannot_bypass_schema_floor() {
3118 let mut client = client();
3119 client.set_schema_floor(Some(SchemaFloor {
3120 required_schema_version: Some(2),
3121 latest_schema_version: Some(2),
3122 }));
3123 let error = client
3124 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3125 rebootstrap_id: "blocked-floor".to_owned(),
3126 })
3127 .expect_err("schema floor must block repair");
3128 assert!(error.contains("cannot bypass an active schema-floor stop"));
3129 }
3130
3131 #[test]
3132 fn diego_applied_sparse_moves_stay_visible_until_pull_image_arrives() {
3133 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3134 "columns":[{"name":"id","type":"string","nullable":false},
3135 {"name":"project_id","type":"string","nullable":false},
3136 {"name":"starts_at_ms","type":"integer","nullable":false},
3137 {"name":"version","type":"integer","nullable":false}],
3138 "scopes":[{"pattern":"project:{project_id}"}]}]});
3139 let mut client =
3140 SyncClient::new("diego-ack-gap".into(), &schema, ClientLimits::default()).unwrap();
3141 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3142 client
3143 .write_base_row(
3144 "tasks",
3145 &vec![
3146 Some(ColumnValue::String("t1".into())),
3147 Some(ColumnValue::String("p1".into())),
3148 Some(ColumnValue::Integer(100)),
3149 Some(ColumnValue::Integer(1)),
3150 ],
3151 1,
3152 )
3153 .unwrap();
3154 client.rebuild_overlay().expect("rebuild overlay");
3155 let first = client
3156 .patch(
3157 "tasks",
3158 "t1",
3159 Map::from_iter([
3160 ("starts_at_ms".into(), json!(200)),
3161 ("version".into(), json!(2)),
3162 ]),
3163 None,
3164 )
3165 .unwrap();
3166 let (_, first_request) = client.build_request(false).unwrap();
3167 let second = client
3169 .patch(
3170 "tasks",
3171 "t1",
3172 Map::from_iter([
3173 ("starts_at_ms".into(), json!(300)),
3174 ("version".into(), json!(3)),
3175 ]),
3176 None,
3177 )
3178 .unwrap();
3179 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3180 for (id, seq, meta) in [
3181 (first, 2, first_request),
3182 (second, 3, client.build_request(false).unwrap().1),
3183 ] {
3184 let response = Message {
3185 wire_version: WIRE_VERSION,
3186 msg_kind: MsgKind::Response,
3187 frames: vec![
3188 Frame::RespHeader {
3189 required_schema_version: None,
3190 latest_schema_version: None,
3191 log_epoch: Some("epoch-1".into()),
3192 reset_required: Some(false),
3193 },
3194 Frame::PushResult {
3195 client_commit_id: id,
3196 status: PushStatus::Applied,
3197 commit_seq: Some(seq),
3198 results: vec![OpResult::Applied { op_index: 0 }],
3199 },
3200 ],
3201 };
3202 assert!(matches!(
3203 client.process_response(&mut transport, response, &meta),
3204 SyncOutcome::Ok(_)
3205 ));
3206 let row = &client
3207 .query("SELECT starts_at_ms, version FROM tasks WHERE id='t1'", &[])
3208 .unwrap()[0];
3209 assert_eq!(
3210 row["starts_at_ms"],
3211 json!(300),
3212 "an applied ACK without a pull image must not expose the old base"
3213 );
3214 assert_eq!(
3215 row["version"],
3216 json!(3),
3217 "the queued domain version must remain monotonic"
3218 );
3219 }
3220 }
3221
3222 #[test]
3223 fn first_epoch_acquisition_preserves_tables_subscriptions_and_local_intent() {
3224 for stored in [None, Some("epoch-old")] {
3225 let mut client = client();
3226 client
3227 .subscribe(
3228 "tasks".into(),
3229 "tasks".into(),
3230 vec![("project_id".into(), vec!["p1".into()])],
3231 None,
3232 )
3233 .unwrap();
3234 client
3235 .conn
3236 .execute_batch(
3237 "CREATE TRIGGER epoch_table_identity AFTER INSERT ON tasks BEGIN SELECT 1; END",
3238 )
3239 .unwrap();
3240 let id = client
3241 .mutate(vec![Mutation::Upsert {
3242 table: "tasks".into(),
3243 values: Map::from_iter([
3244 ("id".into(), json!("offline")),
3245 ("project_id".into(), json!("p1")),
3246 ]),
3247 base_version: None,
3248 }])
3249 .unwrap();
3250 if let Some(epoch) = stored {
3251 client.set_meta(LOG_EPOCH_KEY, epoch);
3252 }
3253 let before = serde_json::to_value(client.subscription_state("tasks")).unwrap();
3254 let (_, meta) = client.build_request(false).unwrap();
3255 client.drain_change_batches();
3256 client.drain_sync_intents();
3257 let response = Message {
3258 wire_version: WIRE_VERSION,
3259 msg_kind: MsgKind::Response,
3260 frames: vec![Frame::RespHeader {
3261 required_schema_version: None,
3262 latest_schema_version: None,
3263 log_epoch: Some("epoch-new".into()),
3264 reset_required: Some(true),
3265 }],
3266 };
3267 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3268 let outcome = client.process_response(&mut transport, response, &meta);
3269 let SyncOutcome::Ok(report) = outcome else {
3270 panic!("epoch round failed: {outcome:?}");
3271 };
3272 assert_eq!(
3273 report.resets,
3274 if stored.is_none() {
3275 Vec::<String>::new()
3276 } else {
3277 vec!["tasks".into()]
3278 }
3279 );
3280 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3281 assert_eq!(client.pending_commit_ids(), vec![id]);
3282 assert_eq!(
3283 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3284 json!("offline")
3285 );
3286 let trigger: i64 = client
3287 .conn
3288 .query_row(
3289 "SELECT count(*) FROM sqlite_master WHERE name='epoch_table_identity'",
3290 [],
3291 |row| row.get(0),
3292 )
3293 .unwrap();
3294 if stored.is_none() {
3295 assert!(!client.upgrading);
3296 assert!(client
3297 .drain_change_batches()
3298 .iter()
3299 .all(|batch| batch.status.as_ref().is_none_or(|status| !status.upgrading)));
3300 assert_eq!(
3301 serde_json::to_value(client.subscription_state("tasks")).unwrap(),
3302 before
3303 );
3304 assert_eq!(trigger, 1, "acquisition must not drop the synced table");
3305 } else {
3306 assert!(client.upgrading);
3307 assert_eq!(trigger, 0, "an epoch change must drop the old table");
3308 }
3309 assert!(matches!(
3310 client.drain_sync_intents().as_slice(),
3311 [SyncIntent::Interactive]
3312 ));
3313 }
3314 }
3315
3316 #[test]
3317 fn epoch_acquisition_storage_failures_preserve_ready_state_and_intent() {
3318 for key in ["logEpoch", "localRevision"] {
3319 let mut client = client();
3320 let id = client
3321 .mutate(vec![Mutation::Upsert {
3322 table: "tasks".into(),
3323 values: Map::from_iter([
3324 ("id".into(), json!("offline")),
3325 ("project_id".into(), json!("p1")),
3326 ]),
3327 base_version: None,
3328 }])
3329 .unwrap();
3330 let revision = client.local_revision();
3331 let needed = client.sync_needed;
3332 client.drain_change_batches();
3333 client.drain_sync_intents();
3334 client.conn.execute_batch(&format!("CREATE TRIGGER fail_acquisition BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected acquisition failure'); END")).unwrap();
3335 assert!(client.run_log_epoch_reset("epoch-new").is_err());
3336 assert!(client.get_meta(LOG_EPOCH_KEY).is_none());
3337 assert!(!client.upgrading);
3338 assert_eq!(client.sync_needed, needed);
3339 assert_eq!(client.local_revision(), revision);
3340 assert_eq!(client.pending_commit_ids(), [id]);
3341 assert!(client.drain_change_batches().is_empty());
3342 assert!(client.drain_sync_intents().is_empty());
3343 }
3344 }
3345
3346 #[test]
3347 fn failed_log_epoch_resets_restore_memory_and_durable_state() {
3348 for key in [
3349 "localSchemaVersion",
3350 "localSchemaDescriptor",
3351 "logEpoch",
3352 "localRevision",
3353 ] {
3354 let mut client = client();
3355 client.set_meta(LOG_EPOCH_KEY, "epoch-old");
3356 let commit = client
3357 .mutate(vec![Mutation::Upsert {
3358 table: "tasks".into(),
3359 values: Map::from_iter([
3360 ("id".into(), json!("offline")),
3361 ("project_id".into(), json!("p1")),
3362 ]),
3363 base_version: None,
3364 }])
3365 .unwrap();
3366 client.active_round = Some(uuid::Uuid::new_v4());
3367 client.overlay_dirty.set(false);
3368 let round = client.active_round;
3369 let needed = client.sync_needed;
3370 let revision = client.local_revision();
3371 let rows = serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap();
3372 client.drain_change_batches();
3373 client.drain_sync_intents();
3374 client.conn.execute_batch(&format!("CREATE TRIGGER fail_reset BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected reset failure'); END")).unwrap();
3375 assert!(client.run_log_epoch_reset("epoch-new").is_err(), "{key}");
3376 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-old"));
3377 assert!(!client.upgrading);
3378 assert_eq!(client.active_round, round);
3379 assert!(!client.overlay_dirty.get());
3380 assert_eq!(client.sync_needed, needed);
3381 assert_eq!(client.local_revision(), revision);
3382 assert_eq!(
3383 client.pending_commit_ids().as_slice(),
3384 std::slice::from_ref(&commit)
3385 );
3386 assert_eq!(
3387 serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap(),
3388 rows
3389 );
3390 assert!(client.drain_change_batches().is_empty());
3391 assert!(client.drain_sync_intents().is_empty());
3392 assert!(client.conn.is_autocommit());
3393 client
3394 .conn
3395 .execute_batch("DROP TRIGGER fail_reset")
3396 .unwrap();
3397 client.run_log_epoch_reset("epoch-new").unwrap();
3398 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3399 assert_eq!(client.pending_commit_ids(), [commit]);
3400 }
3401 }
3402
3403 #[test]
3404 fn acknowledged_restore_and_replay_ignore_unrelated_subscription_count() {
3405 for subscriptions in [0, 2, 6, 12] {
3406 let mut client = SyncClient::new("ack-scope".into(), &json!({ "version": 1, "tables": [
3407 { "name": "tasks", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] },
3408 { "name": "docs", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] }
3409 ] }), ClientLimits::default()).unwrap();
3410 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3411 client.conn.execute_batch("CREATE TABLE ack_writes(kind TEXT); CREATE TRIGGER count_ack_restore AFTER DELETE ON tasks BEGIN INSERT INTO ack_writes VALUES('restore'); END; CREATE TRIGGER count_ack_replay AFTER INSERT ON tasks BEGIN INSERT INTO ack_writes VALUES('replay'); END;").unwrap();
3412 for index in 0..32 {
3413 client
3414 .mutate(vec![Mutation::Upsert {
3415 table: "tasks".into(),
3416 values: Map::from_iter([
3417 ("id".into(), json!(format!("task-{index}"))),
3418 ("project_id".into(), json!("p1")),
3419 ]),
3420 base_version: None,
3421 }])
3422 .unwrap();
3423 }
3424 let (_, meta) = client.build_request(false).unwrap();
3425 let mut frames = vec![Frame::RespHeader {
3426 required_schema_version: None,
3427 latest_schema_version: None,
3428 log_epoch: Some("epoch-1".into()),
3429 reset_required: Some(false),
3430 }];
3431 frames.extend(meta.pushed_ids.iter().enumerate().map(|(index, id)| {
3432 Frame::PushResult {
3433 client_commit_id: id.clone(),
3434 status: PushStatus::Applied,
3435 commit_seq: Some(index as i64 + 1),
3436 results: vec![OpResult::Applied { op_index: 0 }],
3437 }
3438 }));
3439 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3440 client.acknowledged_replay_count.set(0);
3441 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3442 assert!(matches!(
3443 client.process_response(
3444 &mut transport,
3445 Message {
3446 wire_version: WIRE_VERSION,
3447 msg_kind: MsgKind::Response,
3448 frames
3449 },
3450 &meta
3451 ),
3452 SyncOutcome::Ok(_)
3453 ));
3454 assert_eq!(client.acknowledged_replay_count.get(), 32);
3455 assert_eq!(
3456 client
3457 .query(
3458 "SELECT count(*) AS n FROM ack_writes WHERE kind='restore'",
3459 &[]
3460 )
3461 .unwrap()[0]["n"],
3462 json!(32)
3463 );
3464 assert_eq!(
3465 client
3466 .query(
3467 "SELECT count(*) AS n FROM ack_writes WHERE kind='replay'",
3468 &[]
3469 )
3470 .unwrap()[0]["n"],
3471 json!(32)
3472 );
3473 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3474 client.acknowledged_replay_count.set(0);
3475 for index in 0..subscriptions {
3476 client
3477 .subscribe(
3478 format!("docs-{index}"),
3479 "docs".into(),
3480 vec![("project_id".into(), vec![format!("scope-{index}")])],
3481 None,
3482 )
3483 .unwrap();
3484 let sub = client.subs.len() - 1;
3485 client.subs[sub].effective =
3486 Some(vec![("project_id".into(), vec![format!("scope-{index}")])]);
3487 let segment = RowsSegment {
3488 table: "docs".into(),
3489 schema_version: 1,
3490 columns: client.schema.table("docs").unwrap().wire_columns.clone(),
3491 blocks: if index % 2 == 0 {
3492 vec![vec![ssp2::segment::SegmentRow {
3493 values: vec![
3494 Some(ColumnValue::String(format!("doc-{index}"))),
3495 Some(ColumnValue::String(format!("scope-{index}"))),
3496 ],
3497 server_version: 1,
3498 }]]
3499 } else {
3500 vec![]
3501 },
3502 };
3503 assert!(client.apply_segment(sub, &segment, true).is_ok());
3504 }
3505 assert_eq!(
3506 client
3507 .query("SELECT count(*) AS n FROM ack_writes", &[])
3508 .unwrap()[0]["n"],
3509 json!(0),
3510 "unrelated bootstrap must not restore/replay tasks"
3511 );
3512 assert_eq!(
3513 client.acknowledged_replay_count.get(),
3514 0,
3515 "unrelated bootstrap must not decode task ACKs"
3516 );
3517 assert_eq!(
3518 client
3519 .query("SELECT count(*) AS n FROM tasks", &[])
3520 .unwrap()[0]["n"],
3521 json!(32)
3522 );
3523 assert_eq!(
3524 client
3525 .query("SELECT count(*) AS n FROM _syncular_acked_rows", &[])
3526 .unwrap()[0]["n"],
3527 json!(32)
3528 );
3529 }
3530 }
3531
3532 #[test]
3533 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
3534 let mut client = client();
3535 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3536 const COMMIT_COUNT: usize = 32;
3537
3538 for index in 0..COMMIT_COUNT {
3539 client
3540 .mutate(vec![Mutation::Upsert {
3541 table: "tasks".to_owned(),
3542 values: Map::from_iter([
3543 ("id".to_owned(), Value::from(format!("task-{index}"))),
3544 ("project_id".to_owned(), Value::from("project-1")),
3545 ]),
3546 base_version: None,
3547 }])
3548 .expect("queue commit");
3549 }
3550
3551 let (_, request_meta) = client.build_request(false).unwrap();
3552 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
3553 let mut frames = vec![Frame::RespHeader {
3554 required_schema_version: None,
3555 latest_schema_version: None,
3556 log_epoch: Some("epoch-1".to_owned()),
3557 reset_required: Some(false),
3558 }];
3559 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
3560 |(index, client_commit_id)| Frame::PushResult {
3561 client_commit_id: client_commit_id.clone(),
3562 status: PushStatus::Applied,
3563 commit_seq: Some(index as i64 + 1),
3564 results: vec![OpResult::Applied { op_index: 0 }],
3565 },
3566 ));
3567 let response = Message {
3568 wire_version: WIRE_VERSION,
3569 msg_kind: MsgKind::Response,
3570 frames,
3571 };
3572 let mut transport =
3573 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
3574
3575 client.overlay_rebuild_count.set(0);
3576 client.outcome_prune_count.set(0);
3577 let outcome = client.process_response(&mut transport, response, &request_meta);
3578 assert!(matches!(outcome, SyncOutcome::Ok(_)));
3579 assert!(client.pending_commit_ids().is_empty());
3580 assert_eq!(
3581 client.overlay_rebuild_count.get(),
3582 1,
3583 "one response must reconcile its acknowledged commits with one overlay rebuild"
3584 );
3585 assert_eq!(
3586 client.outcome_prune_count.get(),
3587 1,
3588 "one response must enforce outcome retention once"
3589 );
3590 }
3591
3592 #[test]
3593 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
3594 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
3595 "columns": [
3596 {"name":"id", "type":"string", "nullable":false},
3597 {"name":"project_id", "type":"string", "nullable":false},
3598 {"name":"title", "type":"string", "nullable":false}
3599 ], "scopes": [{"pattern":"project:{project_id}"}],
3600 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
3601 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
3602 }] });
3603 let mut incremental =
3604 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3605 let mut reference =
3606 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3607 let tables = vec!["tasks".to_owned()];
3608 for index in 0..32 {
3609 let id = format!("task-{}", index % 8);
3610 let values = Map::from_iter([
3611 ("id".to_owned(), Value::from(id.clone())),
3612 (
3613 "project_id".to_owned(),
3614 Value::from(format!("p{}", index % 2)),
3615 ),
3616 ("title".to_owned(), Value::from(format!("needle{index}"))),
3617 ]);
3618 let payload = encode_row_json(
3619 incremental.schema.table("tasks").unwrap(),
3620 &id,
3621 &values,
3622 &incremental.encryption,
3623 )
3624 .unwrap();
3625 let change = ssp2::model::Change {
3626 table_index: 0,
3627 row_id: id,
3628 op: if index % 5 == 0 {
3629 Op::Delete
3630 } else {
3631 Op::Upsert
3632 },
3633 row_version: Some(index + 1),
3634 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
3635 row: Some(payload),
3636 };
3637 reference.overlay_dirty.set(true);
3638 for instance in [&mut incremental, &mut reference] {
3639 instance.begin_observation("mirror_test").unwrap();
3640 let mut batch = ChangeAccumulator::default();
3641 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
3642 instance
3643 .apply_commit_changes(&tables, std::slice::from_ref(&change))
3644 .unwrap();
3645 instance
3646 .rebuild_overlay_if_dirty()
3647 .expect("rebuild overlay");
3648 instance.finish_observation("mirror_test", batch).unwrap();
3649 }
3650 for sql in [
3651 "SELECT * FROM tasks ORDER BY id",
3652 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
3653 ] {
3654 assert_eq!(
3655 incremental.query(sql, &[]).unwrap(),
3656 reference.query(sql, &[]).unwrap()
3657 );
3658 }
3659 assert_eq!(incremental.local_revision(), reference.local_revision());
3660 assert_eq!(
3661 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3662 serde_json::to_value(reference.drain_change_batches()).unwrap()
3663 );
3664 }
3665 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
3666 assert_eq!(reference.overlay_rebuild_count.get(), 32);
3667 }
3668
3669 #[test]
3670 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
3671 let cases = [
3672 (
3673 "string",
3674 vec![
3675 json!("1"),
3676 json!("01"),
3677 json!(""),
3678 json!("nul\u{0}id"),
3679 json!("é"),
3680 ],
3681 ),
3682 (
3683 "integer",
3684 vec![
3685 json!(i64::MIN),
3686 json!(-1),
3687 json!(0),
3688 json!(1),
3689 json!(i64::MAX),
3690 ],
3691 ),
3692 ("boolean", vec![json!(false), json!(true)]),
3693 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
3694 ];
3695 for (kind, ids) in cases {
3696 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3697 "columns":[{"name":"id","type":kind,"nullable":false},
3698 {"name":"project_id","type":"string","nullable":false}],
3699 "scopes":[{"pattern":"project:{project_id}"}]}]});
3700 let mut client =
3701 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
3702 for (index, id) in ids.iter().enumerate() {
3703 let table = client.schema.table("tasks").unwrap();
3704 client
3705 .write_base_row(
3706 "tasks",
3707 &vec![
3708 if kind == "integer" {
3709 Some(ColumnValue::Integer(id.as_i64().unwrap()))
3712 } else {
3713 json_to_column_value(&table.columns[0], Some(id)).unwrap()
3714 },
3715 Some(ColumnValue::String(format!("p{index}"))),
3716 ],
3717 1,
3718 )
3719 .unwrap();
3720 }
3721 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3722 for base in [true, false] {
3723 let full = if base {
3724 base_table("tasks")
3725 } else {
3726 visible_table("tasks")
3727 };
3728 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
3729 for row_id in [
3730 "1",
3731 "01",
3732 "1.0",
3733 "1.0000000000000002",
3734 "+1",
3735 "1e0",
3736 " 1",
3737 "-1",
3738 "0",
3739 "9223372036854775807",
3740 "-9223372036854775808",
3741 "true",
3742 "false",
3743 "",
3744 "nul\u{0}id",
3745 "é",
3746 "null",
3747 "{\"a\":1}",
3748 ] {
3749 let old_sql =
3750 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
3751 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
3752 let old = client
3753 .conn
3754 .prepare(&old_sql)
3755 .unwrap()
3756 .query_map([row_id], |row| row.get::<_, String>(0))
3757 .unwrap()
3758 .collect::<Result<Vec<_>, _>>()
3759 .unwrap();
3760 let new = client
3761 .conn
3762 .prepare_cached(&new_sql)
3763 .unwrap()
3764 .query_map([row_id], |row| row.get::<_, String>(0))
3765 .unwrap()
3766 .collect::<Result<Vec<_>, _>>()
3767 .unwrap();
3768 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
3769 let mut batch = ChangeAccumulator::default();
3770 assert_eq!(
3771 client.record_row_scopes(&mut batch, "tasks", row_id, base),
3772 !old.is_empty()
3773 );
3774 let expected = old.first().map(|scope| {
3775 BTreeMap::from([(
3776 "tasks".to_owned(),
3777 Some(BTreeSet::from([format!("project:{scope}")])),
3778 )])
3779 });
3780 assert_eq!(batch.tables, expected.unwrap_or_default());
3781 client
3782 .conn
3783 .execute_batch("SAVEPOINT lookup_delete")
3784 .unwrap();
3785 if base {
3786 client.delete_base_row("tasks", row_id).unwrap();
3787 } else {
3788 client
3789 .apply_outbox_ops(&[OutboxOp {
3790 upsert: false,
3791 table: "tasks".into(),
3792 row_id: row_id.into(),
3793 base_version: None,
3794 values: None,
3795 }])
3796 .expect("replay delete");
3797 }
3798 let remaining: i64 = client
3799 .conn
3800 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
3801 row.get(0)
3802 })
3803 .unwrap();
3804 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
3805 client
3806 .conn
3807 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
3808 .unwrap();
3809 client.overlay_dirty.set(false);
3810 }
3811 for (predicate, expected) in [
3812 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
3813 (predicate, "SEARCH"),
3814 ] {
3815 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
3816 let details = client
3817 .conn
3818 .prepare(&sql)
3819 .unwrap()
3820 .query_map(["1"], |row| row.get::<_, String>(3))
3821 .unwrap()
3822 .collect::<Result<Vec<_>, _>>()
3823 .unwrap();
3824 assert!(
3825 details.iter().any(|detail| detail.contains(expected)),
3826 "{kind}: {details:?}"
3827 );
3828 }
3829 }
3830 }
3831 }
3832
3833 #[test]
3834 fn primary_keys_reject_every_ineligible_column_type() {
3835 for ty in ["float", "bytes", "crdt", "blob_ref"] {
3839 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3840 "columns":[{"name":"id","type":ty,"nullable":false},
3841 {"name":"project_id","type":"string","nullable":false}],
3842 "scopes":[{"pattern":"project:{project_id}"}]}]});
3843 let error =
3844 match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
3845 Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
3846 Err(error) => error,
3847 };
3848 assert!(
3849 error.contains("has an ineligible column type (§2.4)"),
3850 "{ty}: {error}"
3851 );
3852 }
3853 for ty in ["string", "integer", "boolean", "json"] {
3854 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3855 "columns":[{"name":"id","type":ty,"nullable":false},
3856 {"name":"project_id","type":"string","nullable":false}],
3857 "scopes":[{"pattern":"project:{project_id}"}]}]});
3858 SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
3859 .unwrap_or_else(|error| panic!("{ty}: {error}"));
3860 }
3861 }
3862
3863 #[test]
3864 fn pending_row_reconciliation_matches_full_fifo_replay() {
3865 for unique in [false, true] {
3866 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
3867 "name": name, "primaryKey": "id", "columns": [
3868 {"name":"id", "type":"string", "nullable":false},
3869 {"name":"project_id", "type":"string", "nullable":false},
3870 {"name":"title", "type":"string", "nullable":false}],
3871 "scopes": [{"pattern":"project:{project_id}"}],
3872 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
3873 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
3874 })))});
3875 let mut incremental =
3876 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
3877 let mut reference =
3878 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
3879 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
3880 for instance in [&mut incremental, &mut reference] {
3881 for table in &tables {
3882 for index in 0..12 {
3883 instance
3884 .write_base_row(
3885 table,
3886 &vec![
3887 Some(ColumnValue::String(format!("row{index}"))),
3888 Some(ColumnValue::String("p0".into())),
3889 Some(ColumnValue::String(format!("base{index}"))),
3890 ],
3891 1,
3892 )
3893 .unwrap();
3894 }
3895 }
3896 instance
3897 .rebuild_overlay_if_dirty()
3898 .expect("rebuild overlay");
3899 }
3900 for index in 0..80 {
3901 let mutations = (0..3)
3902 .map(|offset| {
3903 let table = tables[(index + offset) % 2].clone();
3904 let id = format!("row{}", (index + offset) % 16);
3905 if (index + offset) % 7 == 0 {
3906 Mutation::Delete {
3907 table,
3908 row_id: id,
3909 base_version: None,
3910 }
3911 } else {
3912 Mutation::Upsert {
3913 table,
3914 values: Map::from_iter([
3915 ("id".into(), Value::from(id)),
3916 ("project_id".into(), Value::from(format!("p{}", index % 3))),
3917 (
3918 "title".into(),
3919 Value::from(format!("needle{}", (index + offset) % 16)),
3920 ),
3921 ]),
3922 base_version: None,
3923 }
3924 }
3925 })
3926 .collect::<Vec<_>>();
3927 for instance in [&mut incremental, &mut reference] {
3928 instance.mutate(mutations.clone()).unwrap();
3929 }
3930 let changes = (0..3)
3933 .map(|offset| {
3934 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
3935 let id = format!("row{}", (index + offset) % 16);
3936 let values = Map::from_iter([
3937 ("id".into(), Value::from(id.clone())),
3938 (
3939 "project_id".into(),
3940 Value::from(format!("p{}", (index + 1) % 3)),
3941 ),
3942 (
3943 "title".into(),
3944 Value::from(format!("remote{index}_{offset}")),
3945 ),
3946 ]);
3947 ssp2::model::Change {
3948 table_index: table_index as u16,
3949 row_id: id.clone(),
3950 op: if (index + offset) % 5 == 0 {
3951 Op::Delete
3952 } else {
3953 Op::Upsert
3954 },
3955 row_version: Some(index as i64 + 2),
3956 scopes: vec![],
3957 row: Some(
3958 encode_row_json(
3959 incremental.schema.table(&tables[table_index]).unwrap(),
3960 &id,
3961 &values,
3962 &incremental.encryption,
3963 )
3964 .unwrap(),
3965 ),
3966 }
3967 })
3968 .collect::<Vec<_>>();
3969 reference.overlay_dirty.set(true);
3970 incremental
3971 .apply_commit_frame(&tables, &changes, None)
3972 .unwrap();
3973 reference
3974 .apply_commit_frame(&tables, &changes, None)
3975 .unwrap();
3976 for table in &tables {
3977 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
3978 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
3979 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
3980 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
3981 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
3982 }
3983 }
3984 assert_eq!(incremental.local_revision(), reference.local_revision());
3985 assert_eq!(
3986 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3987 serde_json::to_value(reference.drain_change_batches()).unwrap()
3988 );
3989 }
3990 assert_eq!(
3991 incremental.overlay_rebuild_count.get(),
3992 if unique { 81 } else { 21 }
3993 );
3994 assert_eq!(reference.overlay_rebuild_count.get(), 81);
3995 }
3996 }
3997
3998 #[test]
3999 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
4000 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
4001 "columns":[{"name":"id", "type":"string", "nullable":false},
4002 {"name":"title", "type":"string", "nullable":false}],
4003 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
4004 let mut client =
4005 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
4006 for (id, title) in [("a", "original"), ("b", "occupied")] {
4007 client
4008 .write_base_row(
4009 "tasks",
4010 &vec![
4011 Some(ColumnValue::String(id.into())),
4012 Some(ColumnValue::String(title.into())),
4013 ],
4014 1,
4015 )
4016 .unwrap();
4017 }
4018 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4019 let pending = client
4020 .mutate(vec![Mutation::Upsert {
4021 table: "tasks".into(),
4022 values: Map::from_iter([
4023 ("id".into(), Value::from("a")),
4024 ("title".into(), Value::from("available")),
4025 ]),
4026 base_version: None,
4027 }])
4028 .unwrap();
4029 assert_eq!(
4030 client
4031 .conn
4032 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4033 .get::<_, String>(
4034 0
4035 ))
4036 .unwrap(),
4037 "available"
4038 );
4039 for (title, expected) in [("available", "original"), ("occupied", "available")] {
4040 let change = ssp2::model::Change {
4041 table_index: 0,
4042 row_id: "b".into(),
4043 op: Op::Upsert,
4044 row_version: Some(2),
4045 scopes: vec![],
4046 row: Some(
4047 encode_row_json(
4048 client.schema.table("tasks").unwrap(),
4049 "b",
4050 &Map::from_iter([
4051 ("id".into(), Value::from("b")),
4052 ("title".into(), Value::from(title)),
4053 ]),
4054 &client.encryption,
4055 )
4056 .unwrap(),
4057 ),
4058 };
4059 client
4060 .apply_commit_frame(&["tasks".into()], &[change], None)
4061 .unwrap();
4062 assert_eq!(
4063 client
4064 .conn
4065 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4066 .get::<_, String>(
4067 0
4068 ))
4069 .unwrap(),
4070 expected
4071 );
4072 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
4073 }
4074 assert_eq!(client.overlay_rebuild_count.get(), 3);
4075 }
4076
4077 fn measure_scoped_overlay(unchanged_rows: i64, rounds: i64) -> Vec<u128> {
4078 let schema = json!({ "version": 1, "tables": [
4079 { "name":"tasks", "primaryKey":"id", "columns":[
4080 {"name":"id","type":"string","nullable":false},
4081 {"name":"project_id","type":"string","nullable":false}],
4082 "scopes":[{"pattern":"project:{project_id}"}] },
4083 { "name":"catalogue", "primaryKey":"id", "columns":[
4084 {"name":"id","type":"string","nullable":false},
4085 {"name":"project_id","type":"string","nullable":false},
4086 {"name":"title","type":"string","nullable":false}],
4087 "scopes":[{"pattern":"project:{project_id}"}],
4088 "ftsIndexes":[{"name":"catalogue_fts","columns":["title"],"tokenize":"unicode61"}] }
4089 ] });
4090 let mut client =
4091 SyncClient::new("overlay-scope".into(), &schema, ClientLimits::default()).unwrap();
4092 if unchanged_rows > 0 {
4093 client.conn.execute_batch(&format!("WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i<{unchanged_rows}) INSERT INTO _syncular_base_catalogue SELECT 'c'||i,'p1','needle '||i,1 FROM n")).unwrap();
4094 }
4095 client.rebuild_overlay().expect("rebuild overlay");
4096 client.conn.execute_batch("CREATE TABLE audit(tbl TEXT); CREATE TRIGGER catalogue_changed AFTER DELETE ON catalogue BEGIN INSERT INTO audit VALUES ('catalogue'); END;").unwrap();
4097 let mut elapsed = Vec::new();
4098 for round in 1..=rounds {
4099 let pending = client
4100 .mutate(vec![Mutation::Upsert {
4101 table: "tasks".into(),
4102 values: Map::from_iter([
4103 ("id".into(), json!("t1")),
4104 ("project_id".into(), json!("p1")),
4105 ]),
4106 base_version: None,
4107 }])
4108 .unwrap();
4109 let started = std::time::Instant::now();
4110 client
4111 .write_base_row(
4112 "tasks",
4113 &vec![
4114 Some(ColumnValue::String("t1".into())),
4115 Some(ColumnValue::String("p1".into())),
4116 ],
4117 round,
4118 )
4119 .unwrap();
4120 let mut report = SyncReport::default();
4121 client
4122 .handle_push_results(
4123 &[Frame::PushResult {
4124 client_commit_id: pending.clone(),
4125 status: PushStatus::Applied,
4126 commit_seq: Some(round),
4127 results: vec![OpResult::Applied { op_index: 0 }],
4128 }],
4129 &HashSet::from([pending.as_str()]),
4130 &HashMap::new(),
4131 &mut report,
4132 Some(&pending),
4133 )
4134 .unwrap();
4135 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4136 elapsed.push(started.elapsed().as_nanos());
4137 }
4138 assert_eq!(
4139 client
4140 .conn
4141 .query_row("SELECT count(*) FROM audit", [], |r| r.get::<_, i64>(0))
4142 .unwrap(),
4143 0,
4144 "unchanged catalogue must never be copied"
4145 );
4146 assert_eq!(
4147 client
4148 .conn
4149 .query_row(
4150 "SELECT count(*) FROM catalogue_fts WHERE catalogue_fts MATCH 'needle'",
4151 [],
4152 |r| r.get::<_, i64>(0)
4153 )
4154 .unwrap(),
4155 unchanged_rows
4156 );
4157 assert_eq!(
4158 client.query("SELECT * FROM tasks", &[]).unwrap()[0]["_syncular_version"],
4159 json!(rounds)
4160 );
4161 elapsed
4162 }
4163
4164 #[test]
4165 fn changed_overlay_leaves_large_unchanged_table_and_fts_untouched() {
4166 let elapsed = measure_scoped_overlay(100_000, 1);
4167 eprintln!(
4168 "table-scoped overlay with 100k unchanged rows: {} ns",
4169 elapsed[0]
4170 );
4171 }
4172
4173 #[test]
4174 #[ignore = "isolated performance measurement; run with the shared heavy-check lock"]
4175 fn benchmark_scoped_overlay() {
4176 for rows in [0, 100_000, 200_000] {
4177 let mut elapsed = measure_scoped_overlay(rows, 30);
4178 elapsed.sort_unstable();
4179 eprintln!(
4180 "{}",
4181 json!({"unchangedRows": rows, "rounds": elapsed.len(),
4182 "medianNs": elapsed[elapsed.len()/2], "minNs": elapsed[0],
4183 "maxNs": elapsed[elapsed.len()-1], "unchangedDeletes": 0})
4184 );
4185 }
4186 }
4187
4188 #[test]
4189 fn released_context_discards_in_flight_ack_and_server_rows() {
4190 for barrier in [
4191 "preflight",
4192 "purge",
4193 "rebootstrap",
4194 "subscription",
4195 "failed-purge-replay",
4196 "failed-purge-publish",
4197 "failed-rebootstrap",
4198 ] {
4199 let mut client = client();
4200 client.set_meta(LOG_EPOCH_KEY, "epoch");
4201 client
4202 .subscribe(
4203 "tasks".into(),
4204 "tasks".into(),
4205 vec![("project_id".into(), vec!["p1".into()])],
4206 None,
4207 )
4208 .unwrap();
4209 let commit = client
4210 .mutate(vec![Mutation::Upsert {
4211 table: "tasks".into(),
4212 values: Map::from_iter([
4213 ("id".into(), json!("local")),
4214 ("project_id".into(), json!("p1")),
4215 ]),
4216 base_version: None,
4217 }])
4218 .unwrap();
4219 let prepared = client.prepare_sync_round(false).unwrap();
4220 client.set_transport_enabled(&mut CountingRealtimeTransport::default(), false);
4221 let payload = encode_row_json(
4222 &client.schema.tables[0],
4223 "server",
4224 &Map::from_iter([
4225 ("id".into(), json!("server")),
4226 ("project_id".into(), json!("p1")),
4227 ]),
4228 &client.encryption,
4229 )
4230 .unwrap();
4231 let completed = crate::CompletedSyncRound {
4232 prepared,
4233 exchange: crate::round::ExchangeResult::Reply {
4234 transport_failed: false,
4235 downloads: Default::default(),
4236 response: Ok(Message {
4237 wire_version: WIRE_VERSION,
4238 msg_kind: MsgKind::Response,
4239 frames: vec![
4240 Frame::RespHeader {
4241 required_schema_version: None,
4242 latest_schema_version: None,
4243 log_epoch: Some("epoch".into()),
4244 reset_required: Some(false),
4245 },
4246 Frame::PushResult {
4247 client_commit_id: commit.clone(),
4248 status: PushStatus::Applied,
4249 commit_seq: Some(1),
4250 results: vec![OpResult::Applied { op_index: 0 }],
4251 },
4252 Frame::SubStart {
4253 id: "tasks".into(),
4254 status: SubStatus::Active,
4255 reason_code: String::new(),
4256 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
4257 bootstrap: true,
4258 },
4259 Frame::Commit {
4260 commit_seq: 1,
4261 created_at_ms: 0,
4262 actor_id: "actor".into(),
4263 tables: vec!["tasks".into()],
4264 changes: vec![ssp2::model::Change {
4265 table_index: 0,
4266 row_id: "server".into(),
4267 op: Op::Upsert,
4268 row_version: Some(1),
4269 scopes: vec![("project_id".into(), "p1".into())],
4270 row: Some(payload),
4271 }],
4272 },
4273 Frame::SubEnd {
4274 next_cursor: 1,
4275 bootstrap_state: None,
4276 },
4277 ],
4278 }),
4279 },
4280 };
4281 match barrier {
4282 "preflight" => client.begin_security_preflight(),
4283 "purge" => {
4284 client
4285 .purge_local_data(&LocalDataPurgeInput {
4286 purge_id: "pending-round".into(),
4287 targets: vec![LocalDataPurgeTarget {
4288 table: "tasks".into(),
4289 selectors: BTreeMap::from([(
4290 "project_id".into(),
4291 vec!["p1".into()],
4292 )]),
4293 }],
4294 })
4295 .unwrap();
4296 }
4297 "rebootstrap" => {
4298 client
4299 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4300 rebootstrap_id: "pending-round".into(),
4301 })
4302 .unwrap();
4303 }
4304 "failed-purge-replay" | "failed-purge-publish" | "failed-rebootstrap" => {
4305 let trigger = if barrier == "failed-purge-replay" {
4306 "CREATE TRIGGER fail_local BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4307 } else {
4308 "CREATE TRIGGER fail_local BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4309 };
4310 client.conn.execute_batch(trigger).unwrap();
4311 let revision = client.local_revision();
4312 if barrier == "failed-rebootstrap" {
4313 assert!(client
4314 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4315 rebootstrap_id: "failed-round".into(),
4316 })
4317 .is_err());
4318 } else {
4319 assert!(client
4320 .purge_local_data(&LocalDataPurgeInput {
4321 purge_id: "failed-round".into(),
4322 targets: vec![LocalDataPurgeTarget {
4323 table: "tasks".into(),
4324 selectors: BTreeMap::from([(
4325 "project_id".into(),
4326 vec!["p1".into()]
4327 )]),
4328 }],
4329 })
4330 .is_err());
4331 }
4332 assert_eq!(client.local_revision(), revision);
4333 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
4334 client
4335 .conn
4336 .execute_batch("DROP TRIGGER fail_local")
4337 .unwrap();
4338 let applied = client.apply_sync_round(completed);
4339 assert!(
4340 matches!(
4341 applied,
4342 crate::AppliedSyncRound::Complete {
4343 outcome: SyncOutcome::Ok(_),
4344 ..
4345 }
4346 ),
4347 "{barrier}"
4348 );
4349 assert!(client.pending_commit_ids().is_empty());
4350 assert_eq!(
4351 client
4352 .query("SELECT id FROM tasks WHERE id = 'server'", &[])
4353 .unwrap()
4354 .len(),
4355 1
4356 );
4357 continue;
4358 }
4359 "subscription" => {
4360 client.unsubscribe("tasks");
4361 client
4362 .subscribe(
4363 "tasks".into(),
4364 "tasks".into(),
4365 vec![("project_id".into(), vec!["p2".into()])],
4366 None,
4367 )
4368 .unwrap();
4369 }
4370 _ => unreachable!(),
4371 }
4372 let pending_before_reply = client.pending_commit_ids();
4373 let revision = client.local_revision();
4374 let applied = client.apply_sync_round(completed);
4375 assert!(
4376 matches!(applied, crate::AppliedSyncRound::Complete { outcome: SyncOutcome::Failed { ref error_code, .. }, ref controls, .. } if error_code == "client.round_cancelled" && controls.is_empty())
4377 );
4378 assert_eq!(client.local_revision(), revision);
4379 assert!(client
4380 .conn
4381 .query_row(
4382 "SELECT NOT EXISTS(SELECT 1 FROM tasks WHERE id='server')",
4383 [],
4384 |r| r.get::<_, bool>(0)
4385 )
4386 .unwrap());
4387 assert_eq!(client.pending_commit_ids(), pending_before_reply);
4388 }
4389 }
4390
4391 #[test]
4392 fn sync_round_deadline_is_anchored_once_per_round_and_carried_across_continues() {
4393 struct DeadlineRecording {
4394 budget: std::time::Duration,
4395 seen: Vec<Option<std::time::Instant>>,
4396 }
4397 impl Transport for DeadlineRecording {
4398 fn round_deadline(&self) -> Option<std::time::Duration> {
4399 Some(self.budget)
4400 }
4401 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4402 self.seen.push(deadline);
4403 }
4404 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4405 Err(TransportError::new("transport.failed", "injected"))
4406 }
4407 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4408 self.sync(request)
4409 }
4410 fn download_segment(
4411 &mut self,
4412 _: &SegmentRequest,
4413 _: &mut dyn FnMut(u64),
4414 ) -> Result<Vec<u8>, TransportError> {
4415 Err(TransportError::new("transport.failed", "injected"))
4416 }
4417 fn blob_upload_grant(
4418 &mut self,
4419 _: &str,
4420 _: u64,
4421 _: Option<&str>,
4422 ) -> Result<crate::BlobUploadGrant, TransportError> {
4423 Ok(crate::BlobUploadGrant::Present)
4424 }
4425 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4426 Ok(())
4427 }
4428 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4429 Ok(())
4430 }
4431 fn realtime_close(&mut self) -> Result<(), TransportError> {
4432 Ok(())
4433 }
4434 }
4435
4436 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4437 {"name":"id","type":"string","nullable":false},
4438 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4439 let mut client =
4440 SyncClient::new("deadline-round".into(), &schema, Default::default()).unwrap();
4441 client.set_meta(LOG_EPOCH_KEY, "epoch");
4442 client
4443 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4444 .unwrap();
4445
4446 let mut transport = DeadlineRecording {
4447 budget: std::time::Duration::from_secs(30),
4448 seen: Vec::new(),
4449 };
4450 let prepared = client.prepare_sync_round(false).unwrap();
4451 assert_eq!(prepared.uploads.len(), 1, "one staged upload");
4452 let prepared = match client.apply_sync_round(prepared.exchange(&mut transport)) {
4454 crate::AppliedSyncRound::Continue(prepared) => prepared,
4455 crate::AppliedSyncRound::Complete { .. } => panic!("the upload must continue"),
4456 };
4457 let completed = prepared.exchange(&mut transport);
4459 let anchor = completed
4460 .prepared
4461 .round_deadline_at
4462 .expect("the upload round anchors a deadline");
4463 assert!(matches!(
4464 client.apply_sync_round(completed),
4465 crate::AppliedSyncRound::Complete { .. }
4466 ));
4467 assert_eq!(transport.seen.len(), 4, "one scope per exchange");
4468 assert_eq!(
4469 transport.seen[0],
4470 Some(anchor),
4471 "the upload anchors the round"
4472 );
4473 assert_eq!(
4474 transport.seen[1], None,
4475 "the upload exchange clears its scope"
4476 );
4477 assert_eq!(
4478 transport.seen[2],
4479 Some(anchor),
4480 "a continuation reuses the carried anchor instead of refreshing it"
4481 );
4482 assert_eq!(transport.seen[3], None, "the continuation clears its scope");
4483
4484 transport.budget = std::time::Duration::from_secs(60);
4487 let prepared = client.prepare_sync_round(false).unwrap();
4488 let completed = prepared.exchange(&mut transport);
4489 let fresh = completed
4490 .prepared
4491 .round_deadline_at
4492 .expect("the next round anchors a deadline");
4493 client.apply_sync_round(completed);
4494 assert!(
4495 fresh >= anchor + std::time::Duration::from_secs(30),
4496 "the next round anchors a fresh deadline from the new budget"
4497 );
4498 assert_eq!(transport.seen[4], Some(fresh));
4499 assert_eq!(
4500 transport.seen[5], None,
4501 "the new round clears its scope too"
4502 );
4503 }
4504
4505 #[derive(Default)]
4508 struct RoundDeadlineProbe {
4509 budget: Option<std::time::Duration>,
4510 budget_queries: std::cell::Cell<usize>,
4511 scoped: Vec<Option<std::time::Instant>>,
4512 }
4513
4514 impl Transport for RoundDeadlineProbe {
4515 fn round_deadline(&self) -> Option<std::time::Duration> {
4516 self.budget_queries.set(self.budget_queries.get() + 1);
4517 self.budget
4518 }
4519 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4520 self.scoped.push(deadline);
4521 }
4522 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4523 Err(TransportError::new("transport.failed", "injected"))
4524 }
4525 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4526 self.sync(request)
4527 }
4528 fn download_segment(
4529 &mut self,
4530 _: &SegmentRequest,
4531 _: &mut dyn FnMut(u64),
4532 ) -> Result<Vec<u8>, TransportError> {
4533 Err(TransportError::new("transport.failed", "injected"))
4534 }
4535 fn blob_upload_grant(
4536 &mut self,
4537 _: &str,
4538 _: u64,
4539 _: Option<&str>,
4540 ) -> Result<crate::BlobUploadGrant, TransportError> {
4541 Ok(crate::BlobUploadGrant::Present)
4542 }
4543 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4544 Ok(())
4545 }
4546 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4547 Ok(())
4548 }
4549 fn realtime_close(&mut self) -> Result<(), TransportError> {
4550 Ok(())
4551 }
4552 }
4553
4554 #[test]
4555 fn an_unusable_upload_body_does_not_anchor_the_round_deadline() {
4556 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4557 {"name":"id","type":"string","nullable":false},
4558 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4559 let mut client =
4560 SyncClient::new("deadline-unusable".into(), &schema, ClientLimits::default()).unwrap();
4561 client.set_meta(LOG_EPOCH_KEY, "epoch");
4562 client
4563 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4564 .unwrap();
4565 client
4566 .conn
4567 .execute_batch("DELETE FROM _syncular_blobs")
4568 .unwrap();
4569 let prepared = client.prepare_sync_round(false).unwrap();
4570 assert!(
4571 prepared.uploads.front().is_some_and(|u| u.bytes.is_err()),
4572 "the staged body must fail locally"
4573 );
4574 let mut transport = RoundDeadlineProbe {
4575 budget: Some(std::time::Duration::from_secs(30)),
4576 ..Default::default()
4577 };
4578 let completed = prepared.exchange(&mut transport);
4579 assert!(matches!(
4580 completed.exchange,
4581 crate::round::ExchangeResult::Upload { result: Err(_), .. }
4582 ));
4583 assert_eq!(
4584 transport.budget_queries.get(),
4585 0,
4586 "a round with no network work never asks for a budget"
4587 );
4588 assert_eq!(transport.scoped, vec![None]);
4589 }
4590
4591 #[test]
4592 fn a_transport_budget_the_clock_cannot_represent_fails_before_network() {
4593 struct HugeBudget;
4594 impl Transport for HugeBudget {
4595 fn round_deadline(&self) -> Option<std::time::Duration> {
4596 Some(std::time::Duration::MAX)
4597 }
4598 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4599 panic!("no network for an invalid budget")
4600 }
4601 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4602 self.sync(request)
4603 }
4604 fn download_segment(
4605 &mut self,
4606 _: &SegmentRequest,
4607 _: &mut dyn FnMut(u64),
4608 ) -> Result<Vec<u8>, TransportError> {
4609 panic!("no network for an invalid budget")
4610 }
4611 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4612 Ok(())
4613 }
4614 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4615 Ok(())
4616 }
4617 fn realtime_close(&mut self) -> Result<(), TransportError> {
4618 Ok(())
4619 }
4620 }
4621
4622 let mut client = client();
4623 client.set_meta(LOG_EPOCH_KEY, "epoch");
4624 let prepared = client.prepare_sync_round(false).unwrap();
4625 let outcome = match client.apply_sync_round(prepared.exchange(&mut HugeBudget)) {
4626 crate::AppliedSyncRound::Complete { outcome, .. } => outcome,
4627 crate::AppliedSyncRound::Continue(_) => {
4628 panic!("an unrepresentable budget must not continue")
4629 }
4630 };
4631 match outcome {
4632 SyncOutcome::Failed {
4633 error_code,
4634 message,
4635 ..
4636 } => {
4637 assert_eq!(error_code, "sync.invalid_request");
4638 assert_eq!(message, "transport round deadline is out of range");
4639 }
4640 _ => panic!("an unrepresentable budget must fail synchronously"),
4641 }
4642 }
4643
4644 #[test]
4645 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
4646 for failure in ["visible", "revision", "commit"] {
4647 let mut client = client();
4648 let id = client
4649 .mutate(vec![Mutation::Upsert {
4650 table: "tasks".into(),
4651 values: Map::from_iter([
4652 ("id".into(), Value::from("t1")),
4653 ("project_id".into(), Value::from("pending")),
4654 ]),
4655 base_version: None,
4656 }])
4657 .unwrap();
4658 client.drain_change_batches();
4659 let revision = client.local_revision();
4660 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
4661 let trigger = match failure {
4662 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
4663 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
4664 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
4665 };
4666 client.conn.execute_batch(&trigger).unwrap();
4667 let change = ssp2::model::Change {
4668 table_index: 0,
4669 row_id: "t1".into(),
4670 op: Op::Upsert,
4671 row_version: Some(2),
4672 scopes: vec![],
4673 row: Some(
4674 encode_row_json(
4675 client.schema.table("tasks").unwrap(),
4676 "t1",
4677 &Map::from_iter([
4678 ("id".into(), Value::from("t1")),
4679 ("project_id".into(), Value::from("remote")),
4680 ]),
4681 &client.encryption,
4682 )
4683 .unwrap(),
4684 ),
4685 };
4686 assert!(client
4687 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), None)
4688 .is_err());
4689 assert!(client.conn.is_autocommit());
4690 assert!(!client.overlay_dirty.get());
4691 assert_eq!(client.local_revision(), revision);
4692 assert!(client.drain_change_batches().is_empty());
4693 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
4694 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
4695 assert!(client
4696 .query("SELECT * FROM _syncular_base_tasks", &[])
4697 .unwrap()
4698 .is_empty());
4699 client
4700 .conn
4701 .execute_batch("DROP TRIGGER fail_pending")
4702 .unwrap();
4703 client
4704 .apply_commit_frame(&["tasks".into()], &[change], None)
4705 .unwrap();
4706 assert_eq!(client.local_revision(), revision + 1);
4707 assert_eq!(client.pending_commit_ids(), vec![id]);
4708 let mut expected = before[0].clone();
4711 expected.insert("_syncular_version".to_owned(), Value::from(2));
4712 assert_eq!(
4713 client.query("SELECT * FROM tasks", &[]).unwrap(),
4714 vec![expected]
4715 );
4716 assert_eq!(
4717 client
4718 .conn
4719 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
4720 .get::<_, String>(0))
4721 .unwrap(),
4722 "remote"
4723 );
4724 }
4725 }
4726
4727 #[test]
4728 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
4729 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
4730 "columns": [
4731 {"name":"id", "type":"string", "nullable":false},
4732 {"name":"project_id", "type":"string", "nullable":false},
4733 {"name":"title", "type":"string", "nullable":false}],
4734 "scopes": [{"pattern":"project:{project_id}"}],
4735 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
4736 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
4737 let mut incremental =
4738 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4739 let mut reference =
4740 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4741 for instance in [&mut incremental, &mut reference] {
4742 for index in 0..16 {
4743 instance
4744 .write_base_row(
4745 "tasks",
4746 &vec![
4747 Some(ColumnValue::String(format!("task{index}"))),
4748 Some(ColumnValue::String("p0".to_owned())),
4749 Some(ColumnValue::String(format!("base{index}"))),
4750 ],
4751 1,
4752 )
4753 .unwrap();
4754 }
4755 instance
4756 .rebuild_overlay_if_dirty()
4757 .expect("rebuild overlay");
4758 instance.overlay_rebuild_count.set(0);
4759 }
4760 for index in 0..100 {
4761 let mutations = (0..1 + index % 4)
4762 .map(|offset| {
4763 let id = format!("task{}", (index + offset) % 24);
4764 if (index + offset) % 7 == 0 {
4765 Mutation::Delete {
4766 table: "tasks".to_owned(),
4767 row_id: id,
4768 base_version: None,
4769 }
4770 } else {
4771 Mutation::Upsert {
4772 table: "tasks".to_owned(),
4773 values: Map::from_iter([
4774 ("id".to_owned(), Value::from(id)),
4775 (
4776 "project_id".to_owned(),
4777 Value::from(format!("p{}", index % 2)),
4778 ),
4779 (
4780 "title".to_owned(),
4781 Value::from(format!("needle{}", (index + offset) % 24)),
4782 ),
4783 ]),
4784 base_version: None,
4785 }
4786 }
4787 })
4788 .collect::<Vec<_>>();
4789 reference.overlay_dirty.set(true);
4790 incremental.mutate(mutations.clone()).unwrap();
4791 reference.mutate(mutations).unwrap();
4792 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
4793 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
4794 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
4795 }
4796 assert_eq!(incremental.local_revision(), reference.local_revision());
4797 assert_eq!(
4798 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4799 serde_json::to_value(reference.drain_change_batches()).unwrap()
4800 );
4801 }
4802 assert_eq!(incremental.outbox.len(), 100);
4803 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
4804 assert_eq!(reference.overlay_rebuild_count.get(), 100);
4805 }
4806
4807 #[test]
4808 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
4809 for fail_at in ["outbox", "revision"] {
4810 let mut client = client();
4811 let values = Map::from_iter([
4812 ("id".to_owned(), Value::from("t1")),
4813 ("project_id".to_owned(), Value::from("p1")),
4814 ]);
4815 let mutation = Mutation::Upsert {
4816 table: "tasks".to_owned(),
4817 values,
4818 base_version: None,
4819 };
4820 let trigger = if fail_at == "outbox" {
4821 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
4822 } else {
4823 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
4824 };
4825 client.conn.execute_batch(&trigger).unwrap();
4826 let revision = client.local_revision();
4827 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
4828 assert_eq!(client.local_revision(), revision);
4829 assert!(client.outbox.is_empty());
4830 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
4831 assert_eq!(
4832 client
4833 .conn
4834 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
4835 .get::<_, i64>(0))
4836 .unwrap(),
4837 0
4838 );
4839 assert!(client.drain_change_batches().is_empty());
4840 assert!(!client.overlay_dirty.get());
4841 client
4842 .conn
4843 .execute_batch("DROP TRIGGER fail_append")
4844 .unwrap();
4845 client.mutate(vec![mutation]).unwrap();
4846 assert_eq!(client.outbox.len(), 1);
4847 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
4848 }
4849 }
4850
4851 #[test]
4852 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
4853 let mut client = client();
4854 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
4855 let values = Map::from_iter([
4856 ("id".to_owned(), Value::from("t1")),
4857 ("project_id".to_owned(), Value::from("p1")),
4858 ]);
4859 let payload = encode_row_json(
4860 client.schema.table("tasks").unwrap(),
4861 "t1",
4862 &values,
4863 &client.encryption,
4864 )
4865 .unwrap();
4866 let first = ssp2::model::Change {
4867 table_index: 0,
4868 row_id: "t1".to_owned(),
4869 op: Op::Upsert,
4870 row_version: Some(1),
4871 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
4872 row: Some(payload),
4873 };
4874 let mut invalid = first.clone();
4875 invalid.table_index = 1;
4876 let revision = client.local_revision();
4877 client.begin_observation("failed_mirror").unwrap();
4878 assert!(client
4879 .apply_commit_changes(&tables, &[first, invalid])
4880 .is_err());
4881 client.rollback_observation("failed_mirror");
4882 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4883 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
4884 assert_eq!(
4885 client
4886 .conn
4887 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
4888 .get::<_, i64>(
4889 0
4890 ))
4891 .unwrap(),
4892 0
4893 );
4894 assert_eq!(client.local_revision(), revision);
4895 assert!(client.drain_change_batches().is_empty());
4896 }
4897
4898 #[test]
4899 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
4900 for failure in ["journal", "revision", "commit", "later-frame"] {
4901 let mut client = client();
4902 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4903 for id in ["first", "second"] {
4904 client
4905 .mutate(vec![Mutation::Upsert {
4906 table: "tasks".to_owned(),
4907 values: Map::from_iter([
4908 ("id".to_owned(), Value::from(id)),
4909 ("project_id".to_owned(), Value::from("p1")),
4910 ]),
4911 base_version: None,
4912 }])
4913 .unwrap();
4914 }
4915 let (_, meta) = client.build_request(false).unwrap();
4916 client
4918 .mutate(vec![Mutation::Upsert {
4919 table: "tasks".to_owned(),
4920 values: Map::from_iter([
4921 ("id".to_owned(), Value::from("later")),
4922 ("project_id".to_owned(), Value::from("p1")),
4923 ]),
4924 base_version: None,
4925 }])
4926 .unwrap();
4927 let ids = client.pending_commit_ids();
4928 let rows = client
4929 .query("SELECT * FROM tasks ORDER BY id", &[])
4930 .unwrap();
4931 let revision = client.local_revision();
4932 client.drain_change_batches();
4933 if failure != "later-frame" {
4934 client.conn.execute_batch(match failure {
4935 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
4936 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
4937 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
4938 }).unwrap();
4939 }
4940 let mut response = Message {
4941 wire_version: WIRE_VERSION,
4942 msg_kind: MsgKind::Response,
4943 frames: vec![Frame::RespHeader {
4944 required_schema_version: None,
4945 latest_schema_version: None,
4946 log_epoch: Some("epoch-1".to_owned()),
4947 reset_required: Some(false),
4948 }],
4949 };
4950 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
4951 response.frames.push(Frame::PushResult {
4952 client_commit_id: id.to_owned(),
4953 status: PushStatus::Applied,
4954 commit_seq: Some(1),
4955 results: vec![OpResult::Applied { op_index: 0 }],
4956 });
4957 }
4958 if failure == "later-frame" {
4959 response.frames.push(Frame::Error {
4960 code: "sync.invalid_request".to_owned(),
4961 message: "later error".to_owned(),
4962 category: "protocol".to_owned(),
4963 retryable: false,
4964 recommended_action: "retry".to_owned(),
4965 details: None,
4966 });
4967 }
4968 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
4969 let outcome = client.process_response(&mut transport, response.clone(), &meta);
4970 let expected = match failure {
4973 "later-frame" => "sync.invalid_request",
4974 "commit" => "storage.failed",
4975 _ => "client.outcome_persistence_failed",
4976 };
4977 assert!(
4978 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
4979 "{failure}: {outcome:?}"
4980 );
4981 if failure == "later-frame" {
4982 assert_eq!(client.pending_commit_ids(), ids[2..]);
4983 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
4984 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
4985 let batches = client.drain_change_batches();
4986 assert_eq!(batches.len(), 1);
4987 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
4988 assert_eq!(client.local_revision(), revision + 1);
4989 response.frames.pop();
4990 } else {
4991 assert_eq!(client.pending_commit_ids(), ids);
4992 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
4993 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
4994 assert_eq!(client.local_revision(), revision);
4995 assert!(client.drain_change_batches().is_empty());
4996 assert_eq!(
4997 client
4998 .query("SELECT * FROM tasks ORDER BY id", &[])
4999 .unwrap(),
5000 rows
5001 );
5002 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
5003 }
5004 for frame in &mut response.frames {
5005 if let Frame::PushResult { status, .. } = frame {
5006 *status = PushStatus::Cached;
5007 }
5008 }
5009 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
5010 else {
5011 panic!("retry failed");
5012 };
5013 assert_eq!(
5014 report.applied,
5015 if failure == "later-frame" {
5016 vec![]
5017 } else {
5018 ids[..2].to_vec()
5019 }
5020 );
5021 assert_eq!(client.pending_commit_ids(), ids[2..]);
5022 let batches = client.drain_change_batches();
5023 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
5024 if let Some(batch) = batches.first() {
5025 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
5026 }
5027 assert_eq!(client.local_revision(), revision + 1);
5028 }
5029 }
5030
5031 #[test]
5032 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
5033 for failure in ["revision", "journal", "commit"] {
5034 for status in ["applied", "cached", "rejected", "conflict"] {
5035 let mut client = client();
5036 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5037 for id in ["first", "later"] {
5038 client
5039 .mutate(vec![Mutation::Upsert {
5040 table: "tasks".to_owned(),
5041 values: Map::from_iter([
5042 ("id".to_owned(), Value::from(id)),
5043 ("project_id".to_owned(), Value::from("p1")),
5044 ]),
5045 base_version: None,
5046 }])
5047 .unwrap();
5048 }
5049 let (_, meta) = client.build_request(false).unwrap();
5050 let ids = client.pending_commit_ids();
5051 let rows = client
5052 .query("SELECT * FROM tasks ORDER BY id", &[])
5053 .unwrap();
5054 let revision = client.local_revision();
5055 client.drain_change_batches();
5056 let (push_status, results) = match status {
5057 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
5058 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
5059 "conflict" => (
5060 PushStatus::Rejected,
5061 vec![OpResult::Conflict {
5062 op_index: 0,
5063 code: "sync.version_conflict".to_owned(),
5064 message: "conflict".to_owned(),
5065 server_version: 1,
5066 server_row: encode_row_json(
5067 client.schema.table("tasks").unwrap(),
5068 "first",
5069 &Map::from_iter([
5070 ("id".to_owned(), Value::from("first")),
5071 ("project_id".to_owned(), Value::from("p1")),
5072 ]),
5073 &client.encryption,
5074 )
5075 .unwrap(),
5076 conflict_columns: Vec::new(),
5077 }],
5078 ),
5079 _ => (
5080 PushStatus::Rejected,
5081 vec![OpResult::Error {
5082 op_index: 0,
5083 code: "sync.validation_failed".to_owned(),
5084 message: "rejected".to_owned(),
5085 retryable: false,
5086 }],
5087 ),
5088 };
5089 let response = Message {
5090 wire_version: WIRE_VERSION,
5091 msg_kind: MsgKind::Response,
5092 frames: vec![
5093 Frame::RespHeader {
5094 required_schema_version: None,
5095 latest_schema_version: None,
5096 log_epoch: Some("epoch-1".to_owned()),
5097 reset_required: Some(false),
5098 },
5099 Frame::PushResult {
5100 client_commit_id: ids[0].clone(),
5101 status: push_status,
5102 commit_seq: Some(1),
5103 results,
5104 },
5105 ],
5106 };
5107 client.conn.execute_batch(match failure {
5108 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
5109 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
5110 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
5111 }).unwrap();
5112 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5113 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5114 assert_eq!(
5115 client.pending_commit_ids(),
5116 ids,
5117 "{failure}/{status}: pending identity"
5118 );
5119 assert_eq!(
5120 client.local_revision(),
5121 revision,
5122 "{failure}/{status}: revision"
5123 );
5124 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5125 assert!(client.conflicts().is_empty());
5126 assert!(client.rejections().is_empty());
5127 assert!(client.drain_change_batches().is_empty());
5128 assert_eq!(
5129 client
5130 .query("SELECT * FROM tasks ORDER BY id", &[])
5131 .unwrap(),
5132 rows
5133 );
5134 let expected = if failure == "commit" {
5137 "storage.failed"
5138 } else {
5139 "client.outcome_persistence_failed"
5140 };
5141 assert!(
5142 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5143 "{failure}/{status}: failed round"
5144 );
5145 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
5146 assert!(matches!(
5147 client.process_response(&mut transport, response, &meta),
5148 SyncOutcome::Ok(_)
5149 ));
5150 assert_eq!(client.pending_commit_ids(), ids[1..]);
5151 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5152 assert_eq!(client.drain_change_batches().len(), 1);
5153 }
5154 }
5155 }
5156
5157 #[test]
5158 fn mixed_push_results_reconcile_each_boundary_and_prune_once_per_response() {
5159 let mut client = client();
5160 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5161 for index in 0..4 {
5162 client
5163 .mutate(vec![Mutation::Upsert {
5164 table: "tasks".to_owned(),
5165 values: Map::from_iter([
5166 ("id".to_owned(), Value::from(format!("task-{index}"))),
5167 ("project_id".to_owned(), Value::from("project-1")),
5168 ]),
5169 base_version: None,
5170 }])
5171 .expect("queue commit");
5172 }
5173
5174 let (_, request_meta) = client.build_request(false).unwrap();
5175 let ids = &request_meta.pushed_ids;
5176 assert_eq!(ids.len(), 4);
5177 let response = Message {
5178 wire_version: WIRE_VERSION,
5179 msg_kind: MsgKind::Response,
5180 frames: vec![
5181 Frame::RespHeader {
5182 required_schema_version: None,
5183 latest_schema_version: None,
5184 log_epoch: Some("epoch-1".to_owned()),
5185 reset_required: Some(false),
5186 },
5187 Frame::PushResult {
5188 client_commit_id: ids[0].clone(),
5189 status: PushStatus::Applied,
5190 commit_seq: Some(1),
5191 results: vec![OpResult::Applied { op_index: 0 }],
5192 },
5193 Frame::PushResult {
5194 client_commit_id: ids[1].clone(),
5195 status: PushStatus::Cached,
5196 commit_seq: Some(2),
5197 results: vec![OpResult::Applied { op_index: 0 }],
5198 },
5199 Frame::PushResult {
5200 client_commit_id: ids[2].clone(),
5201 status: PushStatus::Rejected,
5202 commit_seq: None,
5203 results: vec![OpResult::Error {
5204 op_index: 0,
5205 code: "sync.validation_failed".to_owned(),
5206 message: "rejected".to_owned(),
5207 retryable: false,
5208 }],
5209 },
5210 Frame::PushResult {
5211 client_commit_id: ids[3].clone(),
5212 status: PushStatus::Rejected,
5213 commit_seq: None,
5214 results: vec![OpResult::Error {
5215 op_index: 0,
5216 code: "sync.idempotency_cache_miss".to_owned(),
5217 message: "retry".to_owned(),
5218 retryable: true,
5219 }],
5220 },
5221 ],
5222 };
5223 let mut transport =
5224 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
5225
5226 client.overlay_rebuild_count.set(0);
5227 client.outcome_prune_count.set(0);
5228 let outcome = client.process_response(&mut transport, response, &request_meta);
5229 let SyncOutcome::Ok(report) = outcome else {
5230 panic!("mixed push-result response failed");
5231 };
5232
5233 assert_eq!(report.applied, ids[..2]);
5234 assert_eq!(report.rejected, ids[2..3]);
5235 assert_eq!(report.retryable, ids[3..4]);
5236 assert_eq!(client.pending_commit_ids(), ids[3..4]);
5237 assert_eq!(
5238 client
5239 .query("SELECT id FROM tasks ORDER BY id", &[])
5240 .expect("query visible overlay"),
5241 ["task-0", "task-1", "task-3"]
5242 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
5243 .to_vec()
5244 );
5245 assert_eq!(client.overlay_rebuild_count.get(), 2);
5247 assert_eq!(client.outcome_prune_count.get(), 1);
5248 }
5249
5250 fn replay_schema(fts: bool) -> Value {
5251 json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
5252 {"name":"id","type":"string","nullable":false},
5253 {"name":"project_id","type":"string","nullable":false},
5254 {"name":"title","type":"string","nullable":false},
5255 {"name":"done","type":"boolean","nullable":false}
5256 ],"scopes":[{"pattern":"project:{project_id}"}], "ftsIndexes": if fts { json!([{"name":"tasks_fts","columns":["title"],"tokenize":"unicode61"}]) } else { json!([]) }}]})
5257 }
5258
5259 #[test]
5260 fn aborted_schema_and_epoch_resets_restore_memory_and_durable_intent() {
5261 for schema_reset in [false, true] {
5262 let mut schema = replay_schema(false);
5263 let mut client =
5264 SyncClient::new("reset-failure".into(), &schema, ClientLimits::default()).unwrap();
5265 let id = client
5266 .mutate(vec![Mutation::Upsert {
5267 table: "tasks".into(),
5268 values: Map::from_iter([
5269 ("id".into(), json!("queued")),
5270 ("project_id".into(), json!("p1")),
5271 ("title".into(), json!("old")),
5272 ("done".into(), json!(false)),
5273 ]),
5274 base_version: None,
5275 }])
5276 .unwrap();
5277 client.set_meta(LOG_EPOCH_KEY, "old");
5278 let revision = client.local_revision();
5279 let needed = client.sync_needed;
5280 client.drain_change_batches();
5281 client.drain_sync_intents();
5282 client.conn.execute_batch("CREATE TRIGGER refuse_revision BEFORE INSERT ON _syncular_meta WHEN NEW.key='localRevision' BEGIN SELECT RAISE(ABORT,'reset completion failed'); END;").unwrap();
5283 if schema_reset {
5284 schema["version"] = json!(2);
5285 schema["tables"][0]["columns"]
5286 .as_array_mut()
5287 .unwrap()
5288 .retain(|column| column["name"] != "title");
5289 client.schema = parse_schema_json(&schema).unwrap();
5290 assert!(client.run_schema_reset().is_err());
5291 } else {
5292 assert!(client.run_log_epoch_reset("new").is_err());
5293 }
5294 assert!(!client.upgrading);
5295 assert_eq!(client.sync_needed, needed);
5296 assert_eq!(client.local_revision(), revision);
5297 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5298 assert!(client.rejections.is_empty());
5299 assert!(client.commit_outcome(&id).unwrap().is_none());
5300 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("old"));
5301 assert_eq!(
5302 client
5303 .query("SELECT title FROM tasks WHERE id='queued'", &[])
5304 .unwrap()[0]["title"],
5305 "old"
5306 );
5307 assert_eq!(
5308 client
5309 .conn
5310 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5311 .get::<_, i64>(0))
5312 .unwrap(),
5313 1
5314 );
5315 assert!(client.drain_change_batches().is_empty());
5316 assert!(client.drain_sync_intents().is_empty());
5317 assert!(client.conn.is_autocommit());
5318 }
5319 }
5320
5321 #[test]
5322 fn cached_blob_cleanup_propagates_delete_failures() {
5323 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5324 {"name":"id","type":"string","nullable":false},
5325 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5326 let mut client =
5327 SyncClient::new("blob-cleanup".into(), &schema, ClientLimits::default()).unwrap();
5328 client.conn.execute_batch("CREATE TRIGGER refuse_blob_cleanup BEFORE DELETE ON _syncular_blobs BEGIN SELECT RAISE(ABORT,'blob cleanup failed'); END;").unwrap();
5329 client.conn.execute("INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,created_at_ms) VALUES ('test',X'01',1,0)", []).unwrap();
5330 client
5331 .subscribe("attachments".into(), "attachments".into(), vec![], None)
5332 .unwrap();
5333 client.subs[0].effective = Some(vec![]);
5334 client.persist_sub(&client.subs[0]).unwrap();
5335 let revision = client.local_revision();
5336 let (_, meta) = client.build_request(false).unwrap();
5337 let mut transport = CountingRealtimeTransport::default();
5338 let mut report = SyncReport::default();
5339 assert!(client
5340 .process_section(
5341 &mut transport,
5342 "attachments",
5343 SubStatus::Revoked,
5344 "sync.scope_revoked",
5345 vec![],
5346 vec![],
5347 Some((0, None)),
5348 &meta,
5349 &mut report
5350 )
5351 .is_err());
5352 assert_eq!(client.subs[0].state, SubState::Active);
5353 assert_eq!(client.local_revision(), revision);
5354 assert!(report.revoked.is_empty());
5355 assert!(client.conn.is_autocommit());
5356 client
5357 .conn
5358 .execute_batch("DROP TRIGGER refuse_blob_cleanup")
5359 .unwrap();
5360 assert!(client
5361 .process_section(
5362 &mut transport,
5363 "attachments",
5364 SubStatus::Revoked,
5365 "sync.scope_revoked",
5366 vec![],
5367 vec![],
5368 Some((0, None)),
5369 &meta,
5370 &mut report
5371 )
5372 .is_ok());
5373 assert_eq!(client.subs[0].state, SubState::Revoked);
5374 assert_eq!(
5375 client
5376 .conn
5377 .query_row("SELECT count(*) FROM _syncular_blobs", [], |row| row
5378 .get::<_, i64>(0))
5379 .unwrap(),
5380 0
5381 );
5382 }
5383
5384 #[test]
5385 fn overlay_replay_trigger_refuses_reopen_and_preserves_durable_intent() {
5386 let path =
5387 std::env::temp_dir().join(format!("syncular-replay-{}.db", uuid::Uuid::new_v4()));
5388 let schema = replay_schema(false);
5389 let mut client = SyncClient::open_path(
5390 "replay".into(),
5391 &schema,
5392 ClientLimits::default(),
5393 path.to_str().unwrap(),
5394 )
5395 .unwrap();
5396 let ids = ["early", "bad"].map(|id| {
5397 client
5398 .mutate(vec![Mutation::Upsert {
5399 table: "tasks".into(),
5400 values: Map::from_iter([
5401 ("id".into(), json!(id)),
5402 ("project_id".into(), json!("p1")),
5403 ("title".into(), json!(id)),
5404 ("done".into(), json!(false)),
5405 ]),
5406 base_version: None,
5407 }])
5408 .unwrap()
5409 });
5410 client.conn.execute_batch("DELETE FROM tasks; CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.id='bad' BEGIN SELECT RAISE(ABORT,'replay failed'); END;").unwrap();
5411 let revision = client.local_revision();
5412 drop(client);
5413 let error = SyncClient::open_path(
5414 "replay".into(),
5415 &schema,
5416 ClientLimits::default(),
5417 path.to_str().unwrap(),
5418 )
5419 .err()
5420 .expect("reopen must fail");
5421 assert!(error.contains("replay failed"));
5422 let conn = Connection::open(&path).unwrap();
5423 assert_eq!(
5424 conn.query_row("SELECT count(*) FROM tasks", [], |row| row.get::<_, i64>(0))
5425 .unwrap(),
5426 0
5427 );
5428 assert_eq!(
5429 conn.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5430 .get::<_, i64>(0))
5431 .unwrap(),
5432 2
5433 );
5434 assert_eq!(
5435 meta_get(&conn, LOCAL_REVISION_KEY),
5436 Some(revision.to_string())
5437 );
5438 conn.execute_batch("DROP TRIGGER fail_replay").unwrap();
5439 drop(conn);
5440 let reopened = SyncClient::open_path(
5441 "replay".into(),
5442 &schema,
5443 ClientLimits::default(),
5444 path.to_str().unwrap(),
5445 )
5446 .unwrap();
5447 assert_eq!(reopened.pending_commit_ids(), ids);
5448 assert_eq!(
5449 reopened
5450 .query("SELECT id FROM tasks ORDER BY id", &[])
5451 .unwrap()
5452 .len(),
5453 2
5454 );
5455 drop(reopened);
5456 std::fs::remove_file(path).unwrap();
5457 }
5458
5459 #[test]
5460 fn overlay_replay_failure_rolls_back_push_and_pull_apply_boundaries() {
5461 let schema = replay_schema(false);
5462 let mut client =
5463 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5464 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5465 client
5466 .write_base_row(
5467 "tasks",
5468 &vec![
5469 Some(ColumnValue::String("queued".into())),
5470 Some(ColumnValue::String("p1".into())),
5471 Some(ColumnValue::String("server".into())),
5472 Some(ColumnValue::Boolean(false)),
5473 ],
5474 1,
5475 )
5476 .unwrap();
5477 client.rebuild_overlay().unwrap();
5478 let id = client
5479 .patch(
5480 "tasks",
5481 "queued",
5482 Map::from_iter([("title".into(), json!("local"))]),
5483 Some(1),
5484 )
5485 .unwrap();
5486 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
5487 let base = client
5488 .query("SELECT * FROM _syncular_base_tasks", &[])
5489 .unwrap();
5490 let revision = client.local_revision();
5491 client.drain_change_batches();
5492 client.conn.execute_batch("CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.title='local' BEGIN SELECT RAISE(ABORT,'replay failed'); END").unwrap();
5493 let remote = vec![
5494 Some(ColumnValue::String("queued".into())),
5495 Some(ColumnValue::String("p1".into())),
5496 Some(ColumnValue::String("remote".into())),
5497 Some(ColumnValue::Boolean(false)),
5498 ];
5499 let bytes = crate::values::encode_row_json(
5500 client.schema.table("tasks").unwrap(),
5501 "queued",
5502 &Map::from_iter(
5503 client
5504 .schema
5505 .table("tasks")
5506 .unwrap()
5507 .columns
5508 .iter()
5509 .zip(&remote)
5510 .map(|(column, value)| (column.name.clone(), column_value_to_json(value))),
5511 ),
5512 &client.encryption,
5513 )
5514 .unwrap();
5515 let change = ssp2::model::Change {
5516 table_index: 0,
5517 op: Op::Upsert,
5518 row_id: "queued".into(),
5519 row_version: Some(2),
5520 scopes: vec![("project_id".into(), "p1".into())],
5521 row: Some(bytes),
5522 };
5523 assert!(client
5524 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), Some(2))
5525 .is_err());
5526 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5527 assert_eq!(
5528 client
5529 .query("SELECT * FROM _syncular_base_tasks", &[])
5530 .unwrap(),
5531 base
5532 );
5533 assert_eq!(client.local_revision(), revision);
5534 assert!(client.drain_change_batches().is_empty());
5535 let response = Message {
5536 wire_version: 2,
5537 msg_kind: MsgKind::Response,
5538 frames: vec![
5539 Frame::RespHeader {
5540 required_schema_version: None,
5541 latest_schema_version: None,
5542 log_epoch: Some("epoch-1".into()),
5543 reset_required: Some(false),
5544 },
5545 Frame::PushResult {
5546 client_commit_id: id.clone(),
5547 status: PushStatus::Applied,
5548 commit_seq: Some(2),
5549 results: vec![OpResult::Applied { op_index: 0 }],
5550 },
5551 ],
5552 };
5553 let (_, meta) = client.build_request(false).unwrap();
5554 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5555 assert!(
5556 matches!(client.process_response(&mut transport,response.clone(),&meta),SyncOutcome::Failed { error_code,.. } if error_code=="client.outcome_persistence_failed")
5557 );
5558 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5559 assert!(client.commit_outcome(&id).unwrap().is_none());
5560 assert_eq!(
5561 client
5562 .conn
5563 .query_row("SELECT count(*) FROM _syncular_acked_rows", [], |row| row
5564 .get::<_, i64>(
5565 0
5566 ))
5567 .unwrap(),
5568 0
5569 );
5570 assert_eq!(
5571 client
5572 .conn
5573 .query_row("SELECT count(*) FROM _syncular_row_deliveries", [], |row| {
5574 row.get::<_, i64>(0)
5575 })
5576 .unwrap(),
5577 0
5578 );
5579 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5580 assert_eq!(client.local_revision(), revision);
5581 assert!(client.drain_change_batches().is_empty());
5582 client
5583 .conn
5584 .execute_batch("DROP TRIGGER fail_replay")
5585 .unwrap();
5586 assert!(matches!(
5587 client.process_response(&mut transport, response, &meta),
5588 SyncOutcome::Ok(_)
5589 ));
5590 client
5591 .apply_commit_frame(&["tasks".into()], &[change], Some(2))
5592 .unwrap();
5593 assert!(client.pending_commit_ids().is_empty());
5594 assert_eq!(
5595 client.query("SELECT title FROM tasks", &[]).unwrap()[0]["title"],
5596 json!("remote")
5597 );
5598 }
5599
5600 #[test]
5601 fn overlay_replay_distinguishes_absence_from_sql_and_value_decode_errors() {
5602 let client = SyncClient::new(
5603 "replay".into(),
5604 &replay_schema(false),
5605 ClientLimits::default(),
5606 )
5607 .unwrap();
5608 let table = client.schema.table("tasks").unwrap();
5609 let patch = OutboxOp {
5610 table: "tasks".into(),
5611 row_id: "absent".into(),
5612 upsert: true,
5613 base_version: Some(1),
5614 values: Some(Map::from_iter([
5615 ("id".into(), json!("absent")),
5616 ("title".into(), json!("patch")),
5617 ])),
5618 };
5619 assert!(client.visible_row(table, "absent").unwrap().is_none());
5620 client
5621 .apply_outbox_ops([&patch])
5622 .expect("sparse replay over truly absent row");
5623 client
5624 .conn
5625 .execute_batch("INSERT INTO tasks VALUES ('bad','p1','title','invalid',1)")
5626 .unwrap();
5627 assert!(client
5628 .visible_row(table, "bad")
5629 .unwrap_err()
5630 .starts_with("sync.local_corrupt:"));
5631 let mut invalid = patch.clone();
5632 invalid.values = Some(Map::from_iter([
5633 ("id".into(), json!("bad-value")),
5634 ("project_id".into(), json!("p1")),
5635 ("title".into(), json!("title")),
5636 ("done".into(), json!({"bad":true})),
5637 ]));
5638 assert!(client
5639 .apply_outbox_ops([&invalid])
5640 .unwrap_err()
5641 .starts_with("sync.local_corrupt:"));
5642 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5643 assert!(
5644 client.apply_outbox_ops([&patch]).is_err(),
5645 "SQL failure must not count as absence"
5646 );
5647 }
5648
5649 #[test]
5650 fn overlay_replay_keeps_secondary_unique_deferral_and_lazy_base_precedence() {
5651 let mut schema = replay_schema(false);
5652 schema["tables"][0]["indexes"] =
5653 json!([{"name":"tasks_project_title", "columns":["project_id","title"],"unique":true}]);
5654 let mut client =
5655 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5656 let id = client
5657 .mutate(vec![Mutation::Upsert {
5658 table: "tasks".into(),
5659 values: Map::from_iter([
5660 ("id".into(), json!("queued")),
5661 ("project_id".into(), json!("p1")),
5662 ("title".into(), json!("occupied")),
5663 ("done".into(), json!(false)),
5664 ]),
5665 base_version: None,
5666 }])
5667 .unwrap();
5668 client
5669 .write_base_row(
5670 "tasks",
5671 &vec![
5672 Some(ColumnValue::String("server".into())),
5673 Some(ColumnValue::String("p1".into())),
5674 Some(ColumnValue::String("occupied".into())),
5675 Some(ColumnValue::Boolean(false)),
5676 ],
5677 1,
5678 )
5679 .unwrap();
5680 client.rebuild_overlay_if_dirty().unwrap();
5681 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5682 assert!(client
5683 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5684 .unwrap()
5685 .is_none());
5686 assert!(client.rejections().is_empty());
5687 client.delete_base_row("tasks", "server").unwrap();
5688 client.rebuild_overlay_if_dirty().unwrap();
5689 assert!(client
5690 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5691 .unwrap()
5692 .is_some());
5693 client
5694 .write_base_row(
5695 "tasks",
5696 &vec![
5697 Some(ColumnValue::String("base".into())),
5698 Some(ColumnValue::String("p1".into())),
5699 Some(ColumnValue::String("base-title".into())),
5700 Some(ColumnValue::Boolean(false)),
5701 ],
5702 1,
5703 )
5704 .unwrap();
5705 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5706 client
5707 .retain_failed_commit(
5708 "retained",
5709 &[OutboxOp {
5710 table: "tasks".into(),
5711 row_id: "base".into(),
5712 upsert: true,
5713 base_version: Some(1),
5714 values: Some(Map::from_iter([
5715 ("id".into(), json!("base")),
5716 ("title".into(), json!("patch")),
5717 ])),
5718 }],
5719 )
5720 .expect("an existing base must not read the unavailable visible table");
5721 }
5722
5723 #[test]
5724 fn overlay_replay_fts_and_savepoint_release_failures_roll_back_all_visible_changes() {
5725 for failure in ["fts", "release"] {
5726 let mut client = SyncClient::new(
5727 "replay".into(),
5728 &replay_schema(true),
5729 ClientLimits::default(),
5730 )
5731 .unwrap();
5732 client
5733 .mutate(vec![Mutation::Upsert {
5734 table: "tasks".into(),
5735 values: Map::from_iter([
5736 ("id".into(), json!("queued")),
5737 ("project_id".into(), json!("p1")),
5738 ("title".into(), json!("needle")),
5739 ("done".into(), json!(false)),
5740 ]),
5741 base_version: None,
5742 }])
5743 .unwrap();
5744 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
5745 let fts_before = client
5746 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5747 .unwrap();
5748 let triggers_before: Vec<String> = client
5749 .conn
5750 .prepare("SELECT sql FROM sqlite_master WHERE type='trigger' ORDER BY name")
5751 .unwrap()
5752 .query_map([], |row| row.get(0))
5753 .unwrap()
5754 .collect::<Result<_, _>>()
5755 .unwrap();
5756 if failure == "fts" {
5757 client.conn.execute_batch("CREATE TRIGGER fail_fts BEFORE INSERT ON _syncular_fts_tasks_fts BEGIN SELECT RAISE(ABORT,'fts replay failed'); END;").unwrap();
5758 } else {
5759 client.conn.execute_batch("PRAGMA foreign_keys=ON; CREATE TABLE parent(id TEXT PRIMARY KEY); CREATE TABLE deferred(id TEXT REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_release AFTER INSERT ON tasks BEGIN INSERT INTO deferred VALUES ('missing'); END;").unwrap();
5760 }
5761 client.overlay_dirty.set(true);
5762 assert!(client.rebuild_overlay().is_err());
5763 assert!(client.overlay_dirty.get());
5764 assert!(client.conn.is_autocommit());
5765 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
5766 assert_eq!(
5767 client
5768 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5769 .unwrap(),
5770 fts_before
5771 );
5772 let original_triggers: Vec<String> = client.conn.prepare("SELECT sql FROM sqlite_master WHERE type='trigger' AND name NOT LIKE 'fail_%' ORDER BY name").unwrap().query_map([], |row| row.get(0)).unwrap().collect::<Result<_,_>>().unwrap();
5773 assert_eq!(original_triggers, triggers_before);
5774 assert_eq!(client.pending_commit_ids().len(), 1);
5775 client
5776 .conn
5777 .execute_batch(if failure == "fts" {
5778 "DROP TRIGGER fail_fts"
5779 } else {
5780 "DROP TRIGGER fail_release"
5781 })
5782 .unwrap();
5783 client
5784 .rebuild_overlay()
5785 .expect("retry after removing the fault");
5786 }
5787 }
5788
5789 #[test]
5790 fn secondary_unique_collision_preserves_existing_synced_row() {
5791 let client = SyncClient::new(
5792 "unique-upsert-test".to_owned(),
5793 &json!({
5794 "version": 1,
5795 "tables": [{
5796 "name": "tasks",
5797 "primaryKey": "id",
5798 "columns": [
5799 { "name": "id", "type": "string", "nullable": false },
5800 { "name": "project_id", "type": "string", "nullable": false },
5801 { "name": "title", "type": "string", "nullable": false }
5802 ],
5803 "scopes": [{ "pattern": "project:{project_id}" }],
5804 "indexes": [{
5805 "name": "tasks_by_project_title",
5806 "columns": ["project_id", "title"],
5807 "unique": true
5808 }]
5809 }]
5810 }),
5811 ClientLimits::default(),
5812 )
5813 .expect("test client");
5814 let table = client.schema.table("tasks").expect("tasks table");
5815 let sql = client.insert_row_sql(&base_table("tasks"), table);
5816
5817 client
5818 .conn
5819 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
5820 .expect("insert first row");
5821 client
5822 .conn
5823 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
5824 .expect("update same primary key");
5825 client
5826 .conn
5827 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
5828 .expect("insert second row");
5829 assert!(client
5830 .conn
5831 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
5832 .is_err());
5833
5834 let rows = client
5835 .conn
5836 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
5837 .expect("prepare rows")
5838 .query_map([], |row| {
5839 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
5840 })
5841 .expect("query rows")
5842 .collect::<Result<Vec<_>, _>>()
5843 .expect("collect rows");
5844 assert_eq!(
5845 rows,
5846 vec![
5847 ("t1".to_owned(), "updated".to_owned()),
5848 ("t2".to_owned(), "original".to_owned())
5849 ]
5850 );
5851 }
5852
5853 #[test]
5854 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
5855 let path = std::env::temp_dir().join(format!(
5856 "syncular-startup-intent-{}.db",
5857 uuid::Uuid::new_v4()
5858 ));
5859 let schema = json!({
5860 "version": 1,
5861 "tables": [{
5862 "name": "tasks",
5863 "primaryKey": "id",
5864 "columns": [
5865 { "name": "id", "type": "string", "nullable": false },
5866 { "name": "project_id", "type": "string", "nullable": false }
5867 ],
5868 "scopes": [{ "pattern": "project:{project_id}" }]
5869 }]
5870 });
5871
5872 {
5873 let mut first = SyncClient::open_path_with_identity(
5874 None,
5875 &schema,
5876 ClientLimits::default(),
5877 path.to_str().expect("UTF-8 temp path"),
5878 )
5879 .expect("first open");
5880 first
5881 .set_window(
5882 &WindowBase {
5883 table: "tasks".to_owned(),
5884 variable: "project_id".to_owned(),
5885 fixed_scopes: Vec::new(),
5886 params: None,
5887 },
5888 &["persisted".to_owned()],
5889 )
5890 .expect("persist window");
5891 }
5892
5893 let mut reopened = SyncClient::open_path_with_identity(
5894 None,
5895 &schema,
5896 ClientLimits::default(),
5897 path.to_str().expect("UTF-8 temp path"),
5898 )
5899 .expect("reopen");
5900 assert!(reopened.sync_needed());
5901 assert!(matches!(
5902 reopened.drain_sync_intents().as_slice(),
5903 [SyncIntent::Interactive]
5904 ));
5905 drop(reopened);
5906 std::fs::remove_file(path).expect("remove temp database");
5907 }
5908
5909 #[test]
5910 fn reopening_preserves_immutable_subscription_identity_and_progress() {
5911 let path = std::env::temp_dir().join(format!(
5912 "syncular-subscription-identity-{}.db",
5913 uuid::Uuid::new_v4()
5914 ));
5915 let schema = json!({
5916 "version": 1,
5917 "tables": [
5918 {
5919 "name": "tasks",
5920 "primaryKey": "id",
5921 "columns": [
5922 { "name": "id", "type": "string", "nullable": false },
5923 { "name": "project_id", "type": "string", "nullable": false }
5924 ],
5925 "scopes": [{ "pattern": "project:{project_id}" }]
5926 },
5927 {
5928 "name": "docs",
5929 "primaryKey": "id",
5930 "columns": [
5931 { "name": "id", "type": "string", "nullable": false },
5932 { "name": "org_id", "type": "string", "nullable": false },
5933 { "name": "project_id", "type": "string", "nullable": false }
5934 ],
5935 "scopes": [
5936 { "pattern": "org:{org_id}" },
5937 { "pattern": "project:{projectId}", "column": "project_id" }
5938 ]
5939 }
5940 ]
5941 });
5942
5943 {
5944 let mut first = SyncClient::open_path_with_identity(
5945 None,
5946 &schema,
5947 ClientLimits::default(),
5948 path.to_str().expect("UTF-8 temp path"),
5949 )
5950 .expect("first open");
5951 first
5952 .subscribe(
5953 "stable-subscription".to_owned(),
5954 "tasks".to_owned(),
5955 vec![(
5956 "project_id".to_owned(),
5957 vec!["p2".to_owned(), "p1".to_owned()],
5958 )],
5959 Some(r#"{"view":"v1"}"#.to_owned()),
5960 )
5961 .expect("persist subscription");
5962 let persisted = {
5963 let subscription = first
5964 .subs
5965 .iter_mut()
5966 .find(|subscription| subscription.id == "stable-subscription")
5967 .expect("subscription");
5968 subscription.cursor = 41;
5969 subscription.bootstrap_state = Some("resume-token".to_owned());
5970 subscription.effective = Some(vec![(
5971 "project_id".to_owned(),
5972 vec!["p1".to_owned(), "p2".to_owned()],
5973 )]);
5974 subscription.synced_once = true;
5975 subscription.clone()
5976 };
5977 first.persist_sub(&persisted).unwrap();
5978 }
5979
5980 let mut reopened = SyncClient::open_path_with_identity(
5981 None,
5982 &schema,
5983 ClientLimits::default(),
5984 path.to_str().expect("UTF-8 temp path"),
5985 )
5986 .expect("reopen");
5987 let progress = reopened
5988 .subscription_state("stable-subscription")
5989 .expect("persisted state");
5990 assert_eq!(progress.cursor, 41);
5991 assert!(progress.has_resume_token);
5992
5993 reopened
5994 .subscribe(
5995 "stable-subscription".to_owned(),
5996 "tasks".to_owned(),
5997 vec![(
5998 "project_id".to_owned(),
5999 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
6000 )],
6001 Some(r#"{"view":"v1"}"#.to_owned()),
6002 )
6003 .expect("canonical intent is idempotent");
6004 assert_eq!(
6005 reopened
6006 .subscription_state("stable-subscription")
6007 .expect("unchanged state")
6008 .cursor,
6009 progress.cursor
6010 );
6011
6012 for (table, scopes, params) in [
6013 (
6014 "tasks",
6015 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6016 Some(r#"{"view":"v1"}"#.to_owned()),
6017 ),
6018 (
6019 "tasks",
6020 vec![(
6021 "project_id".to_owned(),
6022 vec!["p2".to_owned(), "p1".to_owned()],
6023 )],
6024 Some(r#"{"view":"v2"}"#.to_owned()),
6025 ),
6026 (
6027 "docs",
6028 vec![
6029 ("org_id".to_owned(), vec!["o1".to_owned()]),
6030 ("projectId".to_owned(), vec!["p1".to_owned()]),
6031 ],
6032 Some(r#"{"view":"v1"}"#.to_owned()),
6033 ),
6034 ] {
6035 let error = reopened
6036 .subscribe(
6037 "stable-subscription".to_owned(),
6038 table.to_owned(),
6039 scopes,
6040 params,
6041 )
6042 .expect_err("identity rebind must fail");
6043 assert!(error.starts_with("client.subscription_intent_mismatch:"));
6044 assert_eq!(
6045 reopened
6046 .subscription_state("stable-subscription")
6047 .expect("unchanged state")
6048 .cursor,
6049 progress.cursor
6050 );
6051 }
6052
6053 drop(reopened);
6054 std::fs::remove_file(path).expect("remove temp database");
6055 }
6056
6057 #[test]
6058 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
6059 let path = std::env::temp_dir().join(format!(
6060 "syncular-satisfied-schema-floor-{}.db",
6061 uuid::Uuid::new_v4()
6062 ));
6063 let schema = json!({
6064 "version": 23,
6065 "tables": [{
6066 "name": "tasks",
6067 "primaryKey": "id",
6068 "columns": [
6069 { "name": "id", "type": "string", "nullable": false },
6070 { "name": "project_id", "type": "string", "nullable": false }
6071 ],
6072 "scopes": [{ "pattern": "project:{project_id}" }]
6073 }]
6074 });
6075
6076 {
6077 let mut first = SyncClient::open_path_with_identity(
6078 None,
6079 &schema,
6080 ClientLimits::default(),
6081 path.to_str().expect("UTF-8 temp path"),
6082 )
6083 .expect("first open");
6084 first
6085 .subscribe(
6086 "tasks".to_owned(),
6087 "tasks".to_owned(),
6088 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6089 None,
6090 )
6091 .expect("persist subscription");
6092 first.set_schema_floor(Some(SchemaFloor {
6093 required_schema_version: Some(22),
6094 latest_schema_version: Some(22),
6095 }));
6096 }
6097
6098 let mut reopened = SyncClient::open_path_with_identity(
6099 None,
6100 &schema,
6101 ClientLimits::default(),
6102 path.to_str().expect("UTF-8 temp path"),
6103 )
6104 .expect("reopen");
6105 assert!(reopened.schema_floor().is_none());
6106 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
6107 assert!(reopened.sync_needed());
6108 assert!(matches!(
6109 reopened.drain_sync_intents().as_slice(),
6110 [SyncIntent::Interactive]
6111 ));
6112 drop(reopened);
6113 std::fs::remove_file(path).expect("remove temp database");
6114 }
6115
6116 #[test]
6117 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
6118 let path = std::env::temp_dir().join(format!(
6119 "syncular-unsatisfied-schema-floor-{}.db",
6120 uuid::Uuid::new_v4()
6121 ));
6122 let schema = json!({
6123 "version": 1,
6124 "tables": [{
6125 "name": "tasks",
6126 "primaryKey": "id",
6127 "columns": [
6128 { "name": "id", "type": "string", "nullable": false },
6129 { "name": "project_id", "type": "string", "nullable": false }
6130 ],
6131 "scopes": [{ "pattern": "project:{project_id}" }]
6132 }]
6133 });
6134
6135 {
6136 let mut first = SyncClient::open_path_with_identity(
6137 None,
6138 &schema,
6139 ClientLimits::default(),
6140 path.to_str().expect("UTF-8 temp path"),
6141 )
6142 .expect("first open");
6143 first
6144 .subscribe(
6145 "tasks".to_owned(),
6146 "tasks".to_owned(),
6147 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6148 None,
6149 )
6150 .expect("persist subscription");
6151 first.set_schema_floor(Some(SchemaFloor {
6152 required_schema_version: Some(2),
6153 latest_schema_version: Some(2),
6154 }));
6155 }
6156
6157 let reopened = SyncClient::open_path_with_identity(
6158 None,
6159 &schema,
6160 ClientLimits::default(),
6161 path.to_str().expect("UTF-8 temp path"),
6162 )
6163 .expect("reopen");
6164 assert_eq!(
6165 reopened.schema_floor(),
6166 Some(&SchemaFloor {
6167 required_schema_version: Some(2),
6168 latest_schema_version: Some(2),
6169 })
6170 );
6171 assert!(!reopened.sync_needed());
6172 drop(reopened);
6173 std::fs::remove_file(path).expect("remove temp database");
6174 }
6175
6176 #[test]
6177 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
6178 let path = std::env::temp_dir().join(format!(
6179 "syncular-indexed-column-bump-{}.db",
6180 uuid::Uuid::new_v4()
6181 ));
6182 let old_schema = json!({
6183 "version": 1,
6184 "tables": [
6185 {
6186 "name": "tasks",
6187 "primaryKey": "id",
6188 "columns": [
6189 { "name": "id", "type": "string", "nullable": false },
6190 { "name": "project_id", "type": "string", "nullable": false }
6191 ],
6192 "scopes": [{ "pattern": "project:{project_id}" }]
6193 },
6194 {
6195 "name": "legacy",
6196 "primaryKey": "id",
6197 "columns": [
6198 { "name": "id", "type": "string", "nullable": false },
6199 { "name": "project_id", "type": "string", "nullable": false }
6200 ],
6201 "scopes": [{ "pattern": "project:{project_id}" }]
6202 }
6203 ]
6204 });
6205 {
6206 let mut first = SyncClient::open_path_with_identity(
6207 None,
6208 &old_schema,
6209 ClientLimits::default(),
6210 path.to_str().expect("UTF-8 temp path"),
6211 )
6212 .expect("open old schema");
6213 first
6214 .subscribe(
6215 "legacy-sub".to_owned(),
6216 "legacy".to_owned(),
6217 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6218 None,
6219 )
6220 .expect("persist legacy subscription");
6221 }
6222
6223 let new_schema = json!({
6224 "version": 2,
6225 "tables": [{
6226 "name": "tasks",
6227 "primaryKey": "id",
6228 "columns": [
6229 { "name": "id", "type": "string", "nullable": false },
6230 { "name": "project_id", "type": "string", "nullable": false },
6231 { "name": "facility_membership_id", "type": "string", "nullable": true }
6232 ],
6233 "scopes": [{ "pattern": "project:{project_id}" }],
6234 "indexes": [{
6235 "name": "tasks_by_membership",
6236 "columns": ["project_id", "facility_membership_id"],
6237 "unique": false
6238 }]
6239 }]
6240 });
6241 let upgraded = SyncClient::open_path_with_identity(
6242 None,
6243 &new_schema,
6244 ClientLimits::default(),
6245 path.to_str().expect("UTF-8 temp path"),
6246 )
6247 .expect("open upgraded schema");
6248 let columns = upgraded
6249 .conn
6250 .prepare("PRAGMA table_info(tasks)")
6251 .expect("prepare columns")
6252 .query_map([], |row| row.get::<_, String>(1))
6253 .expect("query columns")
6254 .collect::<Result<Vec<_>, _>>()
6255 .expect("collect columns");
6256 assert!(columns.contains(&"facility_membership_id".to_owned()));
6257 let index_count: i64 = upgraded
6258 .conn
6259 .query_row(
6260 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
6261 [],
6262 |row| row.get(0),
6263 )
6264 .expect("query index");
6265 assert_eq!(index_count, 1);
6266 assert!(upgraded.subscription_state("legacy-sub").is_none());
6267 drop(upgraded);
6268 std::fs::remove_file(path).expect("remove temp database");
6269 }
6270
6271 #[test]
6272 fn migrates_pre_envelope_outcome_journal_additively() {
6273 let path = std::env::temp_dir().join(format!(
6274 "syncular-outcome-migration-{}.db",
6275 uuid::Uuid::new_v4()
6276 ));
6277 let conn = Connection::open(&path).expect("open legacy database");
6278 conn.execute_batch(
6279 "CREATE TABLE _syncular_commit_outcomes (
6280 seq INTEGER PRIMARY KEY AUTOINCREMENT,
6281 client_commit_id TEXT NOT NULL UNIQUE,
6282 status TEXT NOT NULL,
6283 recorded_at_ms INTEGER NOT NULL,
6284 results_json TEXT NOT NULL,
6285 resolution TEXT NOT NULL DEFAULT 'active',
6286 resolved_at_ms INTEGER,
6287 replacement_client_commit_id TEXT);",
6288 )
6289 .expect("create legacy outcome journal");
6290 drop(conn);
6291 let schema = json!({
6292 "version": 1,
6293 "tables": [{
6294 "name": "tasks",
6295 "primaryKey": "id",
6296 "columns": [
6297 { "name": "id", "type": "string", "nullable": false },
6298 { "name": "project_id", "type": "string", "nullable": false }
6299 ],
6300 "scopes": [{ "pattern": "project:{project_id}" }]
6301 }]
6302 });
6303 let client = SyncClient::open_path(
6304 "migration-native".to_owned(),
6305 &schema,
6306 ClientLimits::default(),
6307 path.to_str().expect("UTF-8 temp path"),
6308 )
6309 .expect("migrate database");
6310 let has_operations = client
6311 .conn
6312 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
6313 .expect("prepare table info")
6314 .query_map([], |row| row.get::<_, String>(1))
6315 .expect("query table info")
6316 .filter_map(Result::ok)
6317 .any(|column| column == "operations_json");
6318 assert!(has_operations);
6319 drop(client);
6320 std::fs::remove_file(path).expect("remove temp database");
6321 }
6322
6323 #[test]
6324 fn retained_rejection_survives_file_reopen_and_security_purge() {
6325 let path =
6326 std::env::temp_dir().join(format!("syncular-retained-{}.db", uuid::Uuid::new_v4()));
6327 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id", "columns": [
6328 { "name": "id", "type": "string", "nullable": false },
6329 { "name": "project_id", "type": "string", "nullable": false },
6330 { "name": "title", "type": "string", "nullable": false }
6331 ], "scopes": [{ "pattern": "project:{project_id}" }] }] });
6332 {
6333 let mut first = SyncClient::open_path(
6334 "retained-native".to_owned(),
6335 &schema,
6336 ClientLimits::default(),
6337 path.to_str().unwrap(),
6338 )
6339 .unwrap();
6340 first
6341 .write_base_row(
6342 "tasks",
6343 &vec![
6344 Some(ColumnValue::String("t1".into())),
6345 Some(ColumnValue::String("p1".into())),
6346 Some(ColumnValue::String("server".into())),
6347 ],
6348 1,
6349 )
6350 .unwrap();
6351 first.rebuild_overlay().expect("rebuild overlay");
6352 let operation = OutboxOp {
6353 table: "tasks".into(),
6354 row_id: "t1".into(),
6355 upsert: true,
6356 base_version: Some(1),
6357 values: Some(Map::from_iter([
6358 ("id".into(), json!("t1")),
6359 ("title".into(), json!("mine")),
6360 ])),
6361 };
6362 first
6363 .retain_failed_commit("retained", std::slice::from_ref(&operation))
6364 .unwrap();
6365 first
6366 .persist_commit_outcome(
6367 "retained",
6368 CommitOutcomeStatus::Rejected,
6369 &[CommitOperationOutcome::Error {
6370 rejection: RejectionRecord {
6371 client_commit_id: "retained".into(),
6372 op_index: 0,
6373 code: "validation.denied".into(),
6374 message: "denied".into(),
6375 retryable: false,
6376 details: None,
6377 operation: None,
6378 },
6379 }],
6380 Some(&[operation]),
6381 )
6382 .unwrap();
6383 first.rebuild_overlay().expect("rebuild overlay");
6384 }
6385 {
6386 let mut reopened = SyncClient::open_path(
6387 "retained-native".to_owned(),
6388 &schema,
6389 ClientLimits::default(),
6390 path.to_str().unwrap(),
6391 )
6392 .unwrap();
6393 let row = reopened
6394 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6395 .expect("read visible row")
6396 .unwrap();
6397 assert_eq!(row.0[2], Some(ColumnValue::String("mine".into())));
6398 assert_eq!(
6399 reopened
6400 .commit_outcome("retained")
6401 .unwrap()
6402 .unwrap()
6403 .retained_rows
6404 .unwrap()[0]
6405 .server_row
6406 .as_ref()
6407 .unwrap()["title"],
6408 json!("server")
6409 );
6410 reopened
6411 .conn
6412 .execute("DELETE FROM _syncular_base_tasks WHERE id = 't1'", [])
6413 .unwrap();
6414 reopened.rebuild_overlay().expect("rebuild overlay");
6415 assert!(reopened
6416 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6417 .expect("read visible row")
6418 .is_none());
6419 let retained = reopened.commit_outcome("retained").unwrap().unwrap();
6420 assert!(retained.retained_rows.is_some());
6421 assert_eq!(
6422 retained.operations.unwrap()[0].values.as_ref().unwrap()["title"],
6423 json!("mine")
6424 );
6425 reopened
6426 .purge_local_data(&LocalDataPurgeInput {
6427 purge_id: "protected-purge".into(),
6428 targets: vec![LocalDataPurgeTarget {
6429 table: "tasks".into(),
6430 selectors: BTreeMap::from_iter([("project_id".into(), vec!["p1".into()])]),
6431 }],
6432 })
6433 .unwrap();
6434 assert!(reopened
6435 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6436 .expect("read visible row")
6437 .is_none());
6438 assert!(reopened
6439 .commit_outcome("retained")
6440 .unwrap()
6441 .unwrap()
6442 .retained_rows
6443 .is_none());
6444 }
6445 std::fs::remove_file(path).unwrap();
6446 }
6447
6448 #[test]
6449 fn durable_conflict_outcome_and_resolution_survive_reopen() {
6450 let path = std::env::temp_dir().join(format!(
6451 "syncular-durable-outcome-{}.db",
6452 uuid::Uuid::new_v4()
6453 ));
6454 let schema = json!({
6455 "version": 1,
6456 "tables": [{
6457 "name": "tasks",
6458 "primaryKey": "id",
6459 "columns": [
6460 { "name": "id", "type": "string", "nullable": false },
6461 { "name": "project_id", "type": "string", "nullable": false }
6462 ],
6463 "scopes": [{ "pattern": "project:{project_id}" }]
6464 }]
6465 });
6466 let conflict = ConflictRecord {
6467 client_commit_id: "losing-commit".to_owned(),
6468 op_index: 0,
6469 table: "tasks".to_owned(),
6470 row_id: "t1".to_owned(),
6471 code: "sync.version_conflict".to_owned(),
6472 message: "stale base version".to_owned(),
6473 server_version: 2,
6474 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6475 conflict_columns: vec!["project_id".to_owned()],
6476 operation: Some(CommitOperation {
6477 table: "tasks".to_owned(),
6478 row_id: "t1".to_owned(),
6479 op: "upsert".to_owned(),
6480 base_version: Some(1),
6481 values: Some(Map::from_iter([
6482 ("id".into(), json!("t1")),
6483 ("project_id".into(), json!("p1")),
6484 ])),
6485 }),
6486 };
6487 let failed_operations = vec![
6488 OutboxOp {
6489 upsert: true,
6490 table: "tasks".to_owned(),
6491 row_id: "t1".to_owned(),
6492 base_version: Some(1),
6493 values: Some(Map::from_iter([
6494 ("id".into(), json!("t1")),
6495 ("project_id".into(), json!("p1")),
6496 ])),
6497 },
6498 OutboxOp {
6499 upsert: true,
6500 table: "tasks".to_owned(),
6501 row_id: "status-event-1".to_owned(),
6502 base_version: Some(0),
6503 values: Some(Map::from_iter([
6504 ("id".into(), json!("status-event-1")),
6505 ("project_id".into(), json!("p1")),
6506 ])),
6507 },
6508 ];
6509
6510 {
6511 let mut first = SyncClient::open_path(
6512 "durable-native".to_owned(),
6513 &schema,
6514 ClientLimits::default(),
6515 path.to_str().expect("UTF-8 temp path"),
6516 )
6517 .expect("first open");
6518 first
6519 .begin_observation("test_outcome")
6520 .expect("begin outcome");
6521 first
6522 .persist_commit_outcome(
6523 "losing-commit",
6524 CommitOutcomeStatus::Conflict,
6525 &[CommitOperationOutcome::Conflict {
6526 conflict: conflict.clone(),
6527 }],
6528 Some(&failed_operations),
6529 )
6530 .expect("persist outcome");
6531 first.conflicts.push(conflict);
6532 first
6533 .finish_observation(
6534 "test_outcome",
6535 ChangeAccumulator {
6536 conflicts: true,
6537 outcomes: true,
6538 ..ChangeAccumulator::default()
6539 },
6540 )
6541 .expect("commit outcome");
6542 }
6543
6544 {
6545 let mut reopened = SyncClient::open_path(
6546 "durable-native".to_owned(),
6547 &schema,
6548 ClientLimits::default(),
6549 path.to_str().expect("UTF-8 temp path"),
6550 )
6551 .expect("reopen");
6552 assert_eq!(reopened.conflicts().len(), 1);
6553 assert_eq!(
6554 reopened.conflicts()[0].conflict_columns,
6555 vec!["project_id".to_owned()],
6556 "conflictColumns survives the journal round trip"
6557 );
6558 let outcome = reopened
6559 .commit_outcome("losing-commit")
6560 .expect("read outcome")
6561 .expect("outcome");
6562 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6563 let operations = outcome.operations.expect("aggregate envelope");
6564 assert_eq!(operations.len(), 2);
6565 assert_eq!(operations[1].row_id, "status-event-1");
6566 let resolved = reopened
6567 .resolve_commit_outcome(ResolveCommitOutcomeInput {
6568 client_commit_id: "losing-commit".to_owned(),
6569 resolution: CommitOutcomeResolution::ResolvedKeepServer,
6570 replacement_client_commit_id: None,
6571 })
6572 .expect("resolve");
6573 assert_eq!(
6574 resolved.resolution,
6575 CommitOutcomeResolution::ResolvedKeepServer
6576 );
6577 assert!(reopened.conflicts().is_empty());
6578 }
6579
6580 let reopened = SyncClient::open_path(
6581 "durable-native".to_owned(),
6582 &schema,
6583 ClientLimits::default(),
6584 path.to_str().expect("UTF-8 temp path"),
6585 )
6586 .expect("second reopen");
6587 assert!(reopened.conflicts().is_empty());
6588 assert_eq!(
6589 reopened
6590 .commit_outcome("losing-commit")
6591 .expect("read outcome")
6592 .expect("outcome")
6593 .resolution,
6594 CommitOutcomeResolution::ResolvedKeepServer
6595 );
6596 drop(reopened);
6597 std::fs::remove_file(path).expect("remove temp database");
6598 }
6599
6600 #[test]
6601 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
6602 let path =
6603 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6604 let schema = json!({
6605 "version": 1,
6606 "tables": [{
6607 "name": "tasks",
6608 "primaryKey": "id",
6609 "columns": [
6610 { "name": "id", "type": "string", "nullable": false },
6611 { "name": "project_id", "type": "string", "nullable": false }
6612 ],
6613 "scopes": [{ "pattern": "project:{project_id}" }]
6614 }]
6615 });
6616 let mut client = SyncClient::open_path_with_identity(
6617 Some("sidecar-client".to_owned()),
6618 &schema,
6619 ClientLimits::default(),
6620 path.to_str().expect("UTF-8 temp path"),
6621 )
6622 .expect("open owner");
6623 let base = WindowBase {
6624 table: "tasks".to_owned(),
6625 variable: "project_id".to_owned(),
6626 fixed_scopes: Vec::new(),
6627 params: None,
6628 };
6629 client
6630 .set_window(&base, &["one".to_owned()])
6631 .expect("set window");
6632 client
6633 .mutate(vec![Mutation::Upsert {
6634 table: "tasks".to_owned(),
6635 values: Map::from_iter([
6636 ("id".to_owned(), Value::from("t1")),
6637 ("project_id".to_owned(), Value::from("one")),
6638 ]),
6639 base_version: None,
6640 }])
6641 .expect("local mutate");
6642
6643 let coverage = [WindowCoverage {
6644 base,
6645 units: vec!["one".to_owned(), "missing".to_owned()],
6646 }];
6647 let owner = client
6648 .query_snapshot(
6649 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6650 &[],
6651 &coverage,
6652 None,
6653 )
6654 .expect("owner snapshot");
6655 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6656 let sidecar = reader
6657 .query_snapshot(
6658 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6659 &[],
6660 &coverage,
6661 )
6662 .expect("sidecar snapshot");
6663
6664 assert_eq!(sidecar.revision, owner.revision);
6665 assert_eq!(sidecar.rows, owner.rows);
6666 assert_eq!(
6667 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
6668 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
6669 );
6670 assert_eq!(sidecar.revision, "2");
6671 assert_eq!(sidecar.rows[0]["id"], "t1");
6672 assert_eq!(sidecar.rows[0]["server_version"], -1);
6673 assert!(!sidecar.coverage.complete);
6674 assert_eq!(sidecar.coverage.pending.len(), 1);
6675 assert_eq!(sidecar.coverage.missing.len(), 1);
6676
6677 drop(reader);
6678 drop(client);
6679 std::fs::remove_file(path).expect("remove temp database");
6680 }
6681
6682 #[test]
6683 fn file_snapshot_reader_reports_catchup_delivery_and_multi_statement() {
6684 let path =
6685 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6686 let schema = json!({
6687 "version": 1,
6688 "tables": [{
6689 "name": "tasks",
6690 "primaryKey": "id",
6691 "columns": [
6692 { "name": "id", "type": "string", "nullable": false },
6693 { "name": "project_id", "type": "string", "nullable": false }
6694 ],
6695 "scopes": [{ "pattern": "project:{project_id}" }]
6696 }]
6697 });
6698 let mut client = SyncClient::open_path_with_identity(
6699 Some("sidecar-catchup".to_owned()),
6700 &schema,
6701 ClientLimits::default(),
6702 path.to_str().expect("UTF-8 temp path"),
6703 )
6704 .expect("open owner");
6705 client
6706 .subscribe(
6707 "s1".into(),
6708 "tasks".into(),
6709 vec![("project_id".into(), vec!["one".into()])],
6710 None,
6711 )
6712 .expect("subscribe");
6713 let commit_id = client
6714 .mutate(vec![Mutation::Upsert {
6715 table: "tasks".to_owned(),
6716 values: Map::from_iter([
6717 ("id".to_owned(), Value::from("t1")),
6718 ("project_id".to_owned(), Value::from("one")),
6719 ]),
6720 base_version: None,
6721 }])
6722 .expect("mutate");
6723 let revision = client.local_revision().to_string();
6724
6725 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6726 assert_eq!(reader.local_revision().expect("revision"), revision);
6727
6728 let read = reader
6729 .snapshot_read(&SnapshotReadRequest {
6730 statements: vec![
6731 SnapshotStatement {
6732 sql: "SELECT 1 AS n",
6733 params: &[],
6734 },
6735 SnapshotStatement {
6736 sql: "SELECT id FROM tasks ORDER BY id",
6737 params: &[],
6738 },
6739 ],
6740 coverage: &[],
6741 subscriptions: vec!["s1".to_owned(), "absent".to_owned()],
6742 commit_ids: vec![commit_id.clone(), "never-known".to_owned()],
6743 })
6744 .expect("snapshot read");
6745 assert_eq!(read.revision, revision);
6746 assert_eq!(read.queries.len(), 2);
6747 assert_eq!(read.queries[0][0]["n"], 1);
6748 assert_eq!(read.queries[1][0]["id"], "t1");
6749 match &read.subscriptions[0] {
6750 SubscriptionCatchup::Known {
6751 status,
6752 cursor,
6753 bootstrap_complete,
6754 known_pending_pages,
6755 ..
6756 } => {
6757 assert_eq!(status, "active");
6758 assert_eq!(*cursor, -1);
6759 assert!(!bootstrap_complete);
6760 assert!(known_pending_pages);
6761 }
6762 other => panic!("expected a known subscription: {other:?}"),
6763 }
6764 assert!(matches!(
6765 &read.subscriptions[1],
6766 SubscriptionCatchup::Unknown { id } if id.as_str() == "absent"
6767 ));
6768 assert!(matches!(
6769 &read.deliveries[0],
6770 CommitDelivery::Pending { client_commit_id } if client_commit_id == &commit_id
6771 ));
6772 assert!(matches!(
6773 &read.deliveries[1],
6774 CommitDelivery::Unknown { client_commit_id } if client_commit_id.as_str() == "never-known"
6775 ));
6776
6777 assert!(reader
6779 .query_snapshot("SELECT * FROM does_not_exist", &[], &[])
6780 .is_err());
6781 assert_eq!(
6782 reader.local_revision().expect("revision after rollback"),
6783 revision
6784 );
6785 assert!(reader
6787 .query_snapshot("DELETE FROM tasks", &[], &[])
6788 .is_err());
6789
6790 client
6792 .conn
6793 .execute(
6794 "UPDATE _syncular_meta SET value = '+1' WHERE key = 'localRevision'",
6795 [],
6796 )
6797 .expect("corrupt revision");
6798 let failure = reader.local_revision().expect_err("corrupt revision fails");
6799 assert_eq!(failure.code, Some("sync.local_corrupt"));
6800
6801 drop(reader);
6802 drop(client);
6803 std::fs::remove_file(path).expect("remove temp database");
6804 }
6805
6806 #[test]
6807 fn snapshot_delivery_is_the_persisted_journal_view_without_retained_rows() {
6808 let path =
6809 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6810 let schema = json!({
6811 "version": 1,
6812 "tables": [{
6813 "name": "tasks",
6814 "primaryKey": "id",
6815 "columns": [
6816 { "name": "id", "type": "string", "nullable": false },
6817 { "name": "project_id", "type": "string", "nullable": false }
6818 ],
6819 "scopes": [{ "pattern": "project:{project_id}" }]
6820 }]
6821 });
6822 let mut client = SyncClient::open_path_with_identity(
6823 Some("sidecar-delivery".to_owned()),
6824 &schema,
6825 ClientLimits::default(),
6826 path.to_str().expect("UTF-8 temp path"),
6827 )
6828 .expect("open owner");
6829 client.begin_observation("test_delivery").expect("observe");
6830 let operations = [OutboxOp {
6831 upsert: true,
6832 table: "tasks".to_owned(),
6833 row_id: "t1".to_owned(),
6834 base_version: Some(1),
6835 values: Some(Map::from_iter([
6836 ("id".into(), json!("t1")),
6837 ("project_id".into(), json!("p1")),
6838 ])),
6839 }];
6840 client
6841 .persist_commit_outcome(
6842 "conflict-commit",
6843 CommitOutcomeStatus::Conflict,
6844 &[CommitOperationOutcome::Conflict {
6845 conflict: ConflictRecord {
6846 client_commit_id: "conflict-commit".to_owned(),
6847 op_index: 0,
6848 table: "tasks".to_owned(),
6849 row_id: "t1".to_owned(),
6850 code: "sync.version_conflict".to_owned(),
6851 message: "stale base version".to_owned(),
6852 server_version: 2,
6853 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6854 conflict_columns: vec!["project_id".to_owned()],
6855 operation: None,
6856 },
6857 }],
6858 Some(&operations),
6859 )
6860 .expect("persist conflict");
6861 client
6862 .persist_commit_outcome(
6863 "rejected-commit",
6864 CommitOutcomeStatus::Rejected,
6865 &[CommitOperationOutcome::Error {
6866 rejection: RejectionRecord {
6867 client_commit_id: "rejected-commit".to_owned(),
6868 op_index: 0,
6869 code: "sync.unknown_table".to_owned(),
6870 message: "unknown table".to_owned(),
6871 retryable: false,
6872 details: None,
6873 operation: None,
6874 },
6875 }],
6876 Some(&operations),
6877 )
6878 .expect("persist rejection");
6879 client
6880 .finish_observation(
6881 "test_delivery",
6882 ChangeAccumulator {
6883 conflicts: true,
6884 outcomes: true,
6885 ..ChangeAccumulator::default()
6886 },
6887 )
6888 .expect("commit outcomes");
6889
6890 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6891 match reader.commit_delivery("conflict-commit").expect("conflict") {
6892 CommitDelivery::Known { outcome, .. } => {
6893 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6894 assert!(matches!(
6895 outcome.results.as_slice(),
6896 [CommitOperationOutcome::Conflict { conflict }]
6897 if conflict.code == "sync.version_conflict"
6898 && conflict.server_version == 2
6899 ));
6900 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
6901 assert!(
6902 outcome.retained_rows.is_none(),
6903 "the sidecar reports the persisted view, not owner-derived retained rows"
6904 );
6905 }
6906 other => panic!("expected a known conflict: {other:?}"),
6907 }
6908 match reader
6909 .commit_delivery("rejected-commit")
6910 .expect("rejection")
6911 {
6912 CommitDelivery::Known { outcome, .. } => {
6913 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
6914 assert!(matches!(
6915 outcome.results.as_slice(),
6916 [CommitOperationOutcome::Error { rejection }]
6917 if rejection.code == "sync.unknown_table"
6918 ));
6919 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
6920 assert!(outcome.retained_rows.is_none());
6921 }
6922 other => panic!("expected a known rejection: {other:?}"),
6923 }
6924 drop(reader);
6925 drop(client);
6926 std::fs::remove_file(path).expect("remove temp database");
6927 }
6928
6929 #[test]
6930 fn snapshot_read_isolates_a_concurrent_writer_commit() {
6931 use std::sync::mpsc::channel;
6932 let path = std::env::temp_dir().join(format!(
6933 "syncular-read-concurrent-{}.db",
6934 uuid::Uuid::new_v4()
6935 ));
6936 let schema = json!({
6937 "version": 1,
6938 "tables": [{
6939 "name": "tasks",
6940 "primaryKey": "id",
6941 "columns": [
6942 { "name": "id", "type": "string", "nullable": false },
6943 { "name": "project_id", "type": "string", "nullable": false }
6944 ],
6945 "scopes": [{ "pattern": "project:{project_id}" }]
6946 }]
6947 });
6948 let mut client = SyncClient::open_path_with_identity(
6949 Some("sidecar-concurrent".to_owned()),
6950 &schema,
6951 ClientLimits::default(),
6952 path.to_str().expect("UTF-8 temp path"),
6953 )
6954 .expect("open owner");
6955 client
6956 .mutate(vec![Mutation::Upsert {
6957 table: "tasks".to_owned(),
6958 values: Map::from_iter([
6959 ("id".to_owned(), Value::from("t1")),
6960 ("project_id".to_owned(), Value::from("one")),
6961 ]),
6962 base_version: None,
6963 }])
6964 .expect("seed row");
6965 let revision_before = client.local_revision().to_string();
6966
6967 let (ready_tx, ready_rx) = channel::<()>();
6970 let (done_tx, done_rx) = channel::<()>();
6971 let writer_path = path.to_string_lossy().into_owned();
6972 let writer_schema = schema.clone();
6973 let writer = std::thread::spawn(move || {
6974 let mut writer = SyncClient::open_path_with_identity(
6975 Some("sidecar-concurrent".to_owned()),
6976 &writer_schema,
6977 ClientLimits::default(),
6978 &writer_path,
6979 )
6980 .expect("open writer");
6981 ready_rx.recv().expect("reader signalled");
6982 writer
6983 .mutate(vec![Mutation::Upsert {
6984 table: "tasks".to_owned(),
6985 values: Map::from_iter([
6986 ("id".to_owned(), Value::from("t2")),
6987 ("project_id".to_owned(), Value::from("one")),
6988 ]),
6989 base_version: None,
6990 }])
6991 .expect("writer commit");
6992 let _ = done_tx.send(());
6993 });
6994 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6995 SNAPSHOT_READ_HOOK.with(|slot| {
6996 *slot.borrow_mut() = Some(Box::new(move || {
6997 ready_tx.send(()).expect("signal writer");
6998 done_rx.recv().expect("writer finished");
6999 }));
7000 });
7001 let read = reader
7002 .snapshot_read(&SnapshotReadRequest {
7003 statements: vec![
7004 SnapshotStatement {
7005 sql: "SELECT count(*) AS n FROM tasks",
7006 params: &[],
7007 },
7008 SnapshotStatement {
7009 sql: "SELECT count(*) AS n FROM tasks",
7010 params: &[],
7011 },
7012 ],
7013 coverage: &[],
7014 subscriptions: Vec::new(),
7015 commit_ids: Vec::new(),
7016 })
7017 .expect("snapshot read");
7018 assert_eq!(
7019 read.queries[0][0]["n"], 1,
7020 "first statement sees the pre-commit row"
7021 );
7022 assert_eq!(
7023 read.queries[1][0]["n"], 1,
7024 "a later statement keeps the one read snapshot"
7025 );
7026 assert_eq!(
7027 read.revision, revision_before,
7028 "the read reports the snapshot revision, not the writer's"
7029 );
7030 writer.join().expect("join writer");
7031
7032 assert_ne!(
7033 reader.local_revision().expect("revision after commit"),
7034 revision_before,
7035 "the next read observes the committed revision"
7036 );
7037 let after = reader
7038 .query_snapshot("SELECT count(*) AS n FROM tasks", &[], &[])
7039 .expect("count after commit");
7040 assert_eq!(
7041 after.rows[0]["n"], 2,
7042 "the committed row is visible afterwards"
7043 );
7044
7045 drop(reader);
7046 drop(client);
7047 std::fs::remove_file(path).expect("remove temp database");
7048 }
7049
7050 #[test]
7051 fn snapshot_read_rejects_every_non_canonical_revision() {
7052 let path = std::env::temp_dir().join(format!(
7053 "syncular-read-revision-{}.db",
7054 uuid::Uuid::new_v4()
7055 ));
7056 let schema = json!({
7057 "version": 1,
7058 "tables": [{
7059 "name": "tasks",
7060 "primaryKey": "id",
7061 "columns": [
7062 { "name": "id", "type": "string", "nullable": false },
7063 { "name": "project_id", "type": "string", "nullable": false }
7064 ],
7065 "scopes": [{ "pattern": "project:{project_id}" }]
7066 }]
7067 });
7068 let client = SyncClient::open_path_with_identity(
7069 Some("sidecar-revision".to_owned()),
7070 &schema,
7071 ClientLimits::default(),
7072 path.to_str().expect("UTF-8 temp path"),
7073 )
7074 .expect("open owner");
7075 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7076 for bad in [
7077 "+1",
7078 "01",
7079 "-1",
7080 " 1",
7081 "1 ",
7082 "18446744073709551616",
7083 "1.0",
7084 "abc",
7085 "",
7086 ] {
7087 client
7088 .conn
7089 .execute(
7090 "UPDATE _syncular_meta SET value = ?1 WHERE key = 'localRevision'",
7091 rusqlite::params![bad],
7092 )
7093 .expect("corrupt revision");
7094 let failure = reader.local_revision().expect_err("corrupt marker fails");
7095 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad:?}");
7096 }
7097 for value in ["X'31'", "CAST(X'FF' AS TEXT)"] {
7098 client
7099 .conn
7100 .execute_batch(&format!(
7101 "UPDATE _syncular_meta SET value={value} WHERE key='localRevision'"
7102 ))
7103 .expect("corrupt stored revision type");
7104 assert_eq!(
7105 reader.local_revision().expect_err("invalid SQL field").code,
7106 Some("sync.local_corrupt")
7107 );
7108 }
7109 client
7111 .conn
7112 .execute("DELETE FROM _syncular_meta WHERE key = 'localRevision'", [])
7113 .expect("delete marker");
7114 assert_eq!(reader.local_revision().expect("legacy zero"), "0");
7115
7116 drop(reader);
7117 drop(client);
7118 std::fs::remove_file(path).expect("remove temp database");
7119 }
7120
7121 #[test]
7122 fn snapshot_read_rejects_corrupt_subscription_state() {
7123 let path = std::env::temp_dir().join(format!(
7124 "syncular-read-substate-{}.db",
7125 uuid::Uuid::new_v4()
7126 ));
7127 let schema = json!({
7128 "version": 1,
7129 "tables": [{
7130 "name": "tasks",
7131 "primaryKey": "id",
7132 "columns": [
7133 { "name": "id", "type": "string", "nullable": false },
7134 { "name": "project_id", "type": "string", "nullable": false }
7135 ],
7136 "scopes": [{ "pattern": "project:{project_id}" }]
7137 }]
7138 });
7139 let mut client = SyncClient::open_path_with_identity(
7140 Some("sidecar-substate".to_owned()),
7141 &schema,
7142 ClientLimits::default(),
7143 path.to_str().expect("UTF-8 temp path"),
7144 )
7145 .expect("open owner");
7146 client
7147 .subscribe(
7148 "s1".into(),
7149 "tasks".into(),
7150 vec![("project_id".into(), vec!["one".into()])],
7151 None,
7152 )
7153 .expect("subscribe");
7154 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7155 for bad in [
7156 "not json".to_owned(),
7157 "{}".to_owned(),
7158 json!({ "status": "bogus", "cursor": 0 }).to_string(),
7159 json!({ "status": "active", "cursor": "x" }).to_string(),
7160 json!({ "status": "active", "cursor": 0, "bootstrapState": 7 }).to_string(),
7161 json!({ "status": "active", "cursor": 0, "reasonCode": 7 }).to_string(),
7162 ] {
7163 client
7164 .conn
7165 .execute(
7166 "UPDATE _syncular_subscriptions SET state_json = ?1 WHERE id = 's1'",
7167 rusqlite::params![bad],
7168 )
7169 .expect("corrupt subscription state");
7170 let failure = reader
7171 .subscription_catchup("s1")
7172 .expect_err("corrupt state fails");
7173 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad}");
7174 }
7175
7176 for assignment in [
7177 "tbl=X'7461736b73'",
7178 "state_json=X'7b7d'",
7179 "state_json=CAST(X'FF' AS TEXT)",
7180 ] {
7181 client.conn.execute_batch(&format!(
7182 "UPDATE _syncular_subscriptions SET tbl='tasks',state_json='{{\"status\":\"active\",\"cursor\":0}}'; UPDATE _syncular_subscriptions SET {assignment} WHERE id='s1'"
7183 )).expect("corrupt stored subscription type");
7184 assert_eq!(
7185 reader
7186 .subscription_catchup("s1")
7187 .expect_err("invalid SQL field")
7188 .code,
7189 Some("sync.local_corrupt")
7190 );
7191 }
7192
7193 drop(reader);
7194 drop(client);
7195 std::fs::remove_file(path).expect("remove temp database");
7196 }
7197
7198 #[test]
7199 fn snapshot_read_rolls_back_a_later_invalid_statement_and_guards_every_write() {
7200 let path = std::env::temp_dir().join(format!(
7201 "syncular-read-rollback-{}.db",
7202 uuid::Uuid::new_v4()
7203 ));
7204 let schema = json!({
7205 "version": 1,
7206 "tables": [{
7207 "name": "tasks",
7208 "primaryKey": "id",
7209 "columns": [
7210 { "name": "id", "type": "string", "nullable": false },
7211 { "name": "project_id", "type": "string", "nullable": false }
7212 ],
7213 "scopes": [{ "pattern": "project:{project_id}" }]
7214 }]
7215 });
7216 let mut client = SyncClient::open_path_with_identity(
7217 Some("sidecar-rollback".to_owned()),
7218 &schema,
7219 ClientLimits::default(),
7220 path.to_str().expect("UTF-8 temp path"),
7221 )
7222 .expect("open owner");
7223 let commit_id = client
7224 .mutate(vec![Mutation::Upsert {
7225 table: "tasks".to_owned(),
7226 values: Map::from_iter([
7227 ("id".to_owned(), Value::from("t1")),
7228 ("project_id".to_owned(), Value::from("one")),
7229 ]),
7230 base_version: None,
7231 }])
7232 .expect("seed row");
7233 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7234 let revision = reader.local_revision().expect("revision");
7235
7236 let failure = reader
7239 .snapshot_read(&SnapshotReadRequest {
7240 statements: vec![
7241 SnapshotStatement {
7242 sql: "SELECT id FROM tasks",
7243 params: &[],
7244 },
7245 SnapshotStatement {
7246 sql: "SELECT * FROM does_not_exist",
7247 params: &[],
7248 },
7249 ],
7250 coverage: &[],
7251 subscriptions: Vec::new(),
7252 commit_ids: vec![commit_id.clone()],
7253 })
7254 .expect_err("second statement fails");
7255 assert_eq!(failure.code, None);
7256 assert_eq!(
7257 reader.local_revision().expect("revision after rollback"),
7258 revision,
7259 "a rolled-back read consumes no revision"
7260 );
7261 assert!(
7262 !matches!(
7263 reader
7264 .commit_delivery(&commit_id)
7265 .expect("delivery after rollback"),
7266 CommitDelivery::Unknown { .. }
7267 ),
7268 "the same reader serves the next read after releasing the connection"
7269 );
7270
7271 let guard = reader
7273 .snapshot_read(&SnapshotReadRequest {
7274 statements: vec![SnapshotStatement {
7275 sql: "DELETE FROM tasks",
7276 params: &[],
7277 }],
7278 coverage: &[],
7279 subscriptions: Vec::new(),
7280 commit_ids: Vec::new(),
7281 })
7282 .expect_err("write guard");
7283 assert!(guard.message.contains("read-only"), "{}", guard.message);
7284
7285 drop(reader);
7286 drop(client);
7287 std::fs::remove_file(path).expect("remove temp database");
7288 }
7289
7290 #[test]
7291 fn owned_query_failures_classify_order_bound_and_clear() {
7292 let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7293 rusqlite::ffi::Error::new(266),
7294 Some("private SQLite prose".to_owned()),
7295 ));
7296 assert_eq!(io.code, Some("client.storage_io"));
7297 assert_eq!(io.sqlite_code, Some(266));
7298 assert_eq!(io.message, "local SQLite storage I/O failed");
7299 let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7300 rusqlite::ffi::Error::new(26),
7301 None,
7302 ));
7303 assert_eq!(corrupt.code, Some("client.storage_corrupt"));
7304 assert!(!io.retryable());
7305 assert!(!corrupt.retryable());
7306
7307 for (raw, extended) in [(5, 5), (5 | (1 << 8), 261), (6, 6), (6 | (1 << 8), 262)] {
7310 let busy = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7311 rusqlite::ffi::Error::new(raw),
7312 Some("database is locked".to_owned()),
7313 ));
7314 assert_eq!(busy.code, Some("client.storage_busy"));
7315 assert_eq!(busy.sqlite_code, Some(extended));
7316 assert_eq!(busy.message, "local SQLite storage is busy");
7317 assert!(busy.retryable());
7318 }
7319 let unrelated = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7320 rusqlite::ffi::Error::new(1),
7321 Some("generic SQLite prose".to_owned()),
7322 ));
7323 assert_eq!(unrelated.code, None);
7324 assert!(!unrelated.retryable());
7325
7326 let mut client = client();
7327 let generic = QueryReadFailure::from("private query prose".to_owned());
7328 client.record_query_read(
7329 QueryOwner {
7330 id: "first",
7331 tables: &["tasks", "tasks"],
7332 },
7333 Some(&generic),
7334 );
7335 client.record_query_read(
7336 QueryOwner {
7337 id: "second",
7338 tables: &["tasks"],
7339 },
7340 Some(&generic),
7341 );
7342 client.record_query_read(
7343 QueryOwner {
7344 id: "first",
7345 tables: &["docs"],
7346 },
7347 Some(&io),
7348 );
7349 let failures = &client
7350 .diagnostics_snapshot(&Default::default())
7351 .expect("diagnostics")
7352 .query_failures;
7353 assert_eq!(
7354 failures
7355 .iter()
7356 .map(|entry| entry.id.as_str())
7357 .collect::<Vec<_>>(),
7358 vec!["second", "first"]
7359 );
7360 assert_eq!(failures[1].tables, vec!["docs"]);
7361 assert_eq!(failures[1].code, "client.storage_io");
7362
7363 for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
7364 client.record_query_read(
7365 QueryOwner {
7366 id: &format!("query-{index}"),
7367 tables: &["tasks"],
7368 },
7369 Some(&generic),
7370 );
7371 }
7372 let failures = &client
7373 .diagnostics_snapshot(&Default::default())
7374 .expect("bounded diagnostics")
7375 .query_failures;
7376 assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
7377 assert_eq!(failures.last().expect("last failure").id, "query-256");
7378
7379 client
7380 .query_snapshot(
7381 "SELECT id FROM tasks",
7382 &[],
7383 &[],
7384 Some(QueryOwner {
7385 id: "query-256",
7386 tables: &["tasks"],
7387 }),
7388 )
7389 .expect("successful owned read");
7390 assert!(client
7391 .diagnostics_snapshot(&Default::default())
7392 .expect("cleared diagnostics")
7393 .query_failures
7394 .iter()
7395 .all(|entry| entry.id != "query-256"));
7396 }
7397
7398 #[test]
7399 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
7400 let mut client = client();
7401 let image = Connection::open_in_memory().unwrap();
7402 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
7403 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7404 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
7405 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7406 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
7407 let table = client.schema.table("tasks").unwrap().clone();
7408 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7409 assert!(client
7410 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
7411 .is_err());
7412 assert!(client.conn.is_autocommit());
7413 assert_eq!(
7414 client
7415 .query("SELECT count(*) AS n FROM tasks", &[])
7416 .unwrap()[0]["n"],
7417 1024
7418 );
7419 assert_eq!(client.drain_change_batches().len(), 1);
7420 image
7421 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
7422 .unwrap();
7423 assert!(matches!(
7424 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
7425 Ok(2050)
7426 ));
7427 assert_eq!(
7428 client
7429 .query("SELECT count(*) AS n FROM tasks", &[])
7430 .unwrap()[0]["n"],
7431 2050
7432 );
7433 assert_eq!(client.drain_change_batches().len(), 3);
7434 }
7435
7436 #[test]
7437 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
7438 let mut client = client();
7439 let base = WindowBase {
7440 table: "tasks".into(),
7441 variable: "project_id".into(),
7442 fixed_scopes: Vec::new(),
7443 params: None,
7444 };
7445 client
7446 .set_window(&base, &["p1".into(), "p2".into()])
7447 .unwrap();
7448 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7449 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7450 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
7451 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7452 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7453 client.drain_change_batches();
7454 assert!(client.set_window(&base, &["p2".into()]).is_err());
7455 assert!(client.conn.is_autocommit());
7456 assert_eq!(
7457 client
7458 .query("SELECT count(*) AS n FROM tasks", &[])
7459 .unwrap()[0]["n"],
7460 1027
7461 );
7462 assert_eq!(client.window_state(&base).units, vec!["p2"]);
7463 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
7464 assert_eq!(client.drain_change_batches().len(), 1);
7465 client
7466 .conn
7467 .execute_batch("DROP TRIGGER interrupt_evict")
7468 .unwrap();
7469 client.drain_pending_evictions().unwrap();
7470 assert!(client.load_pending_evictions().unwrap().is_empty());
7471 assert_eq!(
7472 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
7473 "held"
7474 );
7475 assert_eq!(client.drain_change_batches().len(), 2);
7476 }
7477
7478 #[test]
7479 fn eviction_marker_failures_roll_back_window_changes() {
7480 let mut client = client();
7481 let base = WindowBase {
7482 table: "tasks".into(),
7483 variable: "project_id".into(),
7484 fixed_scopes: Vec::new(),
7485 params: None,
7486 };
7487 client.set_window(&base, &["p1".into()]).unwrap();
7488 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7489 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7490 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7491 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7492 let revision = client.local_revision();
7493 assert!(client.set_window(&base, &[]).is_err());
7494 assert!(client.conn.is_autocommit());
7495 assert_eq!(client.local_revision(), revision);
7496 assert_eq!(
7497 client
7498 .query("SELECT count(*) AS n FROM tasks", &[])
7499 .unwrap()[0]["n"],
7500 2050
7501 );
7502 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7503 client.conn.execute_batch("DROP TRIGGER fail_marker;
7504 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7505 assert!(client.set_window(&base, &[]).is_err());
7506 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
7507 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7508 assert!(client.drain_pending_evictions().is_err());
7509 assert!(client.conn.is_autocommit());
7510 assert_eq!(
7511 client
7512 .query("SELECT count(*) AS n FROM tasks", &[])
7513 .unwrap()[0]["n"],
7514 2
7515 );
7516 assert!(client.set_window(&base, &["p1".into()]).is_err());
7517 assert!(client.conn.is_autocommit());
7518 assert!(client.window_state(&base).units.is_empty());
7519 assert!(client.subs.is_empty());
7520 assert_eq!(
7521 client
7522 .query(
7523 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
7524 &[]
7525 )
7526 .unwrap()[0]["n"],
7527 1
7528 );
7529 client
7530 .conn
7531 .execute_batch("DROP TRIGGER fail_marker")
7532 .unwrap();
7533 client.set_window(&base, &["p1".into()]).unwrap();
7534 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7535 }
7536
7537 #[test]
7538 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
7539 let mut client = client();
7540 client
7541 .conn
7542 .execute(
7543 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
7544 [],
7545 )
7546 .unwrap();
7547 assert_eq!(
7548 client.drain_pending_evictions().unwrap_err(),
7549 "sync.local_corrupt: invalid pending eviction scopes"
7550 );
7551 assert!(client.conn.is_autocommit());
7552 assert!(client.enqueue_startup_sync_if_needed().is_err());
7553 }
7554
7555 #[test]
7556 fn image_chunks_reject_null_primary_keys() {
7557 let mut client = client();
7558 let image = Connection::open_in_memory().unwrap();
7559 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7560 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
7561 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
7562 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7563 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
7564 let table = client.schema.table("tasks").unwrap().clone();
7565 assert!(client
7566 .apply_sqlite_image(
7567 &image,
7568 &table,
7569 true,
7570 &[("project_id".into(), vec!["p1".into()])],
7571 1025,
7572 7,
7573 "digest"
7574 )
7575 .is_err());
7576 assert!(client.conn.is_autocommit());
7577 assert_eq!(
7578 client
7579 .query("SELECT count(*) AS n FROM tasks", &[])
7580 .unwrap()[0]["n"],
7581 0
7582 );
7583 }
7584
7585 #[test]
7586 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
7587 for images in [false, true] {
7588 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7589 "name":name,"primaryKey":"id","columns":[
7590 {"name":"id","type":"string","nullable":false},
7591 {"name":"project_id","type":"string","nullable":false},
7592 {"name":"title","type":"string","nullable":false}],
7593 "scopes":[{"pattern":"project:{project_id}"}],
7594 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
7595 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7596 })))});
7597 let mut actual =
7598 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
7599 let mut reference =
7600 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
7601 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7602 for client in [&mut actual, &mut reference] {
7603 client
7604 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
7605 .unwrap();
7606 client.subs[0].effective = Some(scopes.clone());
7607 for (table, id, title) in [
7608 ("tasks", "a", "original"),
7609 ("tasks", "b", "occupied"),
7610 ("tasks", "c", "deleted"),
7611 ("notes", "n", "untouched"),
7612 ] {
7613 client
7614 .write_base_row(
7615 table,
7616 &vec![
7617 Some(ColumnValue::String(id.into())),
7618 Some(ColumnValue::String("p1".into())),
7619 Some(ColumnValue::String(title.into())),
7620 ],
7621 1,
7622 )
7623 .unwrap();
7624 }
7625 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7626 client
7627 .mutate(vec![
7628 Mutation::Upsert {
7629 table: "tasks".into(),
7630 values: Map::from_iter([
7631 ("id".into(), Value::from("a")),
7632 ("project_id".into(), Value::from("p1")),
7633 ("title".into(), Value::from("available")),
7634 ]),
7635 base_version: None,
7636 },
7637 Mutation::Delete {
7638 table: "tasks".into(),
7639 row_id: "c".into(),
7640 base_version: None,
7641 },
7642 Mutation::Upsert {
7643 table: "tasks".into(),
7644 values: Map::from_iter([
7645 ("id".into(), Value::from("local")),
7646 ("project_id".into(), Value::from("p1")),
7647 ("title".into(), Value::from("local value")),
7648 ]),
7649 base_version: None,
7650 },
7651 Mutation::Upsert {
7652 table: "notes".into(),
7653 values: Map::from_iter([
7654 ("id".into(), Value::from("n")),
7655 ("project_id".into(), Value::from("p1")),
7656 ("title".into(), Value::from("pending note")),
7657 ]),
7658 base_version: None,
7659 },
7660 ])
7661 .unwrap();
7662 }
7663 let pending = actual.pending_commit_ids();
7664 actual.overlay_rebuild_count.set(0);
7665 for (id, title) in [
7666 ("b", "available"),
7667 ("b", "occupied"),
7668 ("a", "server original"),
7669 ("d", "local value"),
7670 ("d", "released"),
7671 ("c", "redelivered"),
7672 ] {
7673 let table = actual.schema.table("tasks").unwrap().clone();
7674 let values = vec![
7675 Some(ColumnValue::String(id.into())),
7676 Some(ColumnValue::String("p1".into())),
7677 Some(ColumnValue::String(title.into())),
7678 ];
7679 if images {
7680 let image = Connection::open_in_memory().unwrap();
7681 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7682 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7683 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7684 image
7685 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
7686 .unwrap();
7687 assert!(matches!(
7688 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
7689 Ok(1)
7690 ));
7691 } else {
7692 let segment = RowsSegment {
7693 table: "tasks".into(),
7694 schema_version: 1,
7695 columns: table.wire_columns.clone(),
7696 blocks: vec![vec![ssp2::segment::SegmentRow {
7697 server_version: 2,
7698 values: values.clone(),
7699 }]],
7700 };
7701 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
7702 }
7703 reference.write_base_row("tasks", &values, 2).unwrap();
7704 reference.rebuild_overlay().expect("rebuild overlay");
7705 for table in ["tasks", "notes"] {
7706 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
7707 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
7708 }
7709 }
7710 assert_eq!(actual.pending_commit_ids(), pending);
7711 }
7712 assert_eq!(
7713 actual.overlay_rebuild_count.get(),
7714 0,
7715 "each chunk must leave unrelated tables alone"
7716 );
7717 let before = actual
7718 .query("SELECT * FROM tasks ORDER BY id", &[])
7719 .unwrap();
7720 let fts_before = actual
7721 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7722 .unwrap();
7723 let revision = actual.local_revision();
7724 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
7725 let table = actual.schema.table("tasks").unwrap().clone();
7726 let segment = RowsSegment {
7727 table: "tasks".into(),
7728 schema_version: 1,
7729 columns: table.wire_columns.clone(),
7730 blocks: vec![vec![ssp2::segment::SegmentRow {
7731 server_version: 3,
7732 values: vec![
7733 Some(ColumnValue::String("b".into())),
7734 Some(ColumnValue::String("p1".into())),
7735 Some(ColumnValue::String("fresh".into())),
7736 ],
7737 }]],
7738 };
7739 assert!(actual.apply_segment(0, &segment, false).is_err());
7740 assert!(actual.conn.is_autocommit());
7741 assert_eq!(
7742 actual
7743 .query("SELECT * FROM tasks ORDER BY id", &[])
7744 .unwrap(),
7745 before
7746 );
7747 assert_eq!(
7748 actual
7749 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7750 .unwrap(),
7751 fts_before
7752 );
7753 assert_eq!(actual.local_revision(), revision);
7754 assert_eq!(actual.pending_commit_ids(), pending);
7755 }
7756 }
7757
7758 #[test]
7759 fn import_reconciliation_preserves_typed_pending_primary_keys() {
7760 for (kind, a, b) in [
7761 ("string", json!("a"), json!("b")),
7762 ("integer", json!(1), json!(2)),
7763 ("boolean", json!(true), json!(false)),
7764 ("json", json!("1"), json!("2")),
7765 ] {
7766 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
7767 for images in [false, true] {
7768 let mut client =
7769 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
7770 let table = client.schema.table("tasks").unwrap().clone();
7771 client
7772 .subscribe("tasks".into(), "tasks".into(), vec![], None)
7773 .unwrap();
7774 for (id, title) in [(&a, "original"), (&b, "occupied")] {
7775 client
7776 .write_base_row(
7777 "tasks",
7778 &vec![
7779 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
7780 Some(ColumnValue::String(title.into())),
7781 ],
7782 1,
7783 )
7784 .unwrap();
7785 }
7786 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7787 client
7788 .mutate(vec![Mutation::Upsert {
7789 table: "tasks".into(),
7790 values: Map::from_iter([
7791 ("id".into(), a.clone()),
7792 ("title".into(), json!("available")),
7793 ]),
7794 base_version: None,
7795 }])
7796 .unwrap();
7797 client.overlay_rebuild_count.set(0);
7798 for (title, expected) in [("available", "original"), ("occupied", "available")] {
7799 let values = vec![
7800 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
7801 Some(ColumnValue::String(title.into())),
7802 ];
7803 if images {
7804 let image = Connection::open_in_memory().unwrap();
7805 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7806 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7807 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7808 image
7809 .execute(
7810 "INSERT INTO tasks VALUES (?1,?2,2)",
7811 rusqlite::params![RowParam::Cell(&values[0]), title],
7812 )
7813 .unwrap();
7814 assert!(
7815 matches!(
7816 client.apply_sqlite_image(
7817 &image,
7818 &table,
7819 false,
7820 &[],
7821 1,
7822 7,
7823 "digest"
7824 ),
7825 Ok(1)
7826 ),
7827 "{kind}, images={images}"
7828 );
7829 } else {
7830 let segment = RowsSegment {
7831 table: "tasks".into(),
7832 schema_version: 1,
7833 columns: table.wire_columns.clone(),
7834 blocks: vec![vec![ssp2::segment::SegmentRow {
7835 server_version: 2,
7836 values,
7837 }]],
7838 };
7839 assert!(
7840 matches!(client.apply_segment(0, &segment, false), Ok(1)),
7841 "{kind}"
7842 );
7843 }
7844 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
7845 let actual: String = client
7846 .conn
7847 .query_row(
7848 "SELECT title FROM tasks WHERE id=?1",
7849 [RowParam::Cell(&key)],
7850 |row| row.get(0),
7851 )
7852 .unwrap();
7853 assert_eq!(actual, expected, "{kind}, images={images}");
7854 }
7855 assert_eq!(client.overlay_rebuild_count.get(), 0);
7856 }
7857 }
7858 }
7859
7860 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
7861 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7862 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
7863 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
7864 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7865 })))});
7866 let mut client =
7867 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
7868 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
7869 client.rebuild_overlay().expect("rebuild overlay");
7870 client
7871 .mutate(vec![Mutation::Upsert {
7872 table: "tasks".into(),
7873 values: Map::from_iter([
7874 ("id".into(), json!("offline")),
7875 ("project_id".into(), json!("p1")),
7876 ("title".into(), json!("pending edit")),
7877 ]),
7878 base_version: None,
7879 }])
7880 .unwrap();
7881 let image = Connection::open_in_memory().unwrap();
7882 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7883 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
7884 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
7885 image
7886 .execute(
7887 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
7888 [rows],
7889 )
7890 .unwrap();
7891 client.overlay_rebuild_count.set(0);
7892 let table = client.schema.table("tasks").unwrap().clone();
7893 (client, image, table)
7894 }
7895
7896 #[test]
7897 fn sqlite_full_image_import_retains_the_first_code_and_recovers() {
7898 let (mut client, image, table) = unique_import_fixture(1);
7899 image
7900 .execute("UPDATE tasks SET title = ?1", ["full ".repeat(32768)])
7901 .unwrap();
7902 client
7903 .conn
7904 .execute_batch("PRAGMA max_page_count = 1")
7905 .unwrap();
7906 let effective = [("project_id".into(), vec!["p1".into()])];
7907 assert!(client
7908 .apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest")
7909 .is_err());
7910 let failure = client.storage_failure.borrow().clone().unwrap();
7911 assert_eq!(failure.code, Some("client.storage_full"));
7912 assert_eq!(failure.sqlite_code, Some(13));
7913 assert!(!failure.message.contains("rollback"));
7914 assert!(client.conn.is_autocommit());
7915 client
7916 .conn
7917 .execute_batch("PRAGMA max_page_count = 1073741823")
7918 .unwrap();
7919 assert!(matches!(
7920 client.apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest"),
7921 Ok(1)
7922 ));
7923 assert!(client.conn.is_autocommit());
7924 }
7925
7926 #[test]
7927 fn authoring_storage_failures_are_structured_and_scoped() {
7928 let path = std::env::temp_dir().join(format!(
7929 "syncular-authoring-failure-{}.db",
7930 uuid::Uuid::new_v4()
7931 ));
7932 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
7933 {"name":"id","type":"string","nullable":false},
7934 {"name":"project_id","type":"string","nullable":false},
7935 {"name":"title","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
7936 let mut client = SyncClient::open_path(
7937 "authoring".into(),
7938 &schema,
7939 ClientLimits::default(),
7940 path.to_str().unwrap(),
7941 )
7942 .expect("open client");
7943 client.create_synced_tables().expect("create synced tables");
7944 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
7946
7947 let upsert = |id: &str| {
7948 vec![Mutation::Upsert {
7949 table: "tasks".into(),
7950 values: Map::from_iter([
7951 ("id".into(), json!(id)),
7952 ("project_id".into(), json!("p1")),
7953 ("title".into(), json!("first")),
7954 ]),
7955 base_version: None,
7956 }]
7957 };
7958
7959 let locker = Connection::open(&path).expect("second connection");
7960 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
7961 let busy = client.mutate(upsert("t1")).expect_err("busy write");
7962 assert_eq!(busy.code, "client.storage_busy");
7963 assert!(busy.retryable);
7964 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
7965 assert!(client.pending_commit_ids().is_empty());
7966 assert_eq!(client.local_revision(), 0);
7967
7968 let invalid = client.mutate(Vec::new()).expect_err("empty commit");
7970 assert_eq!(invalid.code, "sync.invalid_request");
7971 assert_eq!(invalid.message, "the authoring request is invalid");
7972 assert!(
7973 invalid.details.as_ref().unwrap()["legacyCause"]
7974 .as_str()
7975 .unwrap()
7976 .contains("at least one operation"),
7977 "{invalid:?}"
7978 );
7979
7980 locker.execute_batch("ROLLBACK").unwrap();
7981 let id = client.mutate(upsert("t1")).expect("retry after busy");
7982 assert_eq!(client.pending_commit_ids(), vec![id]);
7983 assert_eq!(client.local_revision(), 1);
7984
7985 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
7987 let busy_again = client.mutate(upsert("t2")).expect_err("busy write again");
7988 assert_eq!(busy_again.code, "client.storage_busy");
7989 assert!(busy_again.retryable);
7990 locker.execute_batch("ROLLBACK").unwrap();
7991
7992 let full = ClientError::from(QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7994 rusqlite::ffi::Error::new(13),
7995 Some("database or disk is full".to_owned()),
7996 )));
7997 assert_eq!(full.code, "client.storage_full");
7998 assert!(!full.retryable);
7999 assert_eq!(full.details.as_ref().unwrap()["sqliteCode"], 13);
8000
8001 drop(locker);
8002 drop(client);
8003 let _ = std::fs::remove_file(&path);
8004 }
8005
8006 #[test]
8007 fn caller_value_failures_are_invalid_request_and_leave_no_outbox_entry() {
8008 let path = std::env::temp_dir().join(format!(
8009 "syncular-caller-values-{}.db",
8010 uuid::Uuid::new_v4()
8011 ));
8012 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8013 {"name":"id","type":"string","nullable":false},
8014 {"name":"project_id","type":"string","nullable":false},
8015 {"name":"title","type":"string","nullable":false},
8016 {"name":"priority","type":"integer","nullable":false},
8017 {"name":"payload","type":"bytes","nullable":false},
8018 {"name":"extra","type":"bytes","nullable":true}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8019 let mut client = SyncClient::open_path(
8020 "caller-values".into(),
8021 &schema,
8022 ClientLimits::default(),
8023 path.to_str().unwrap(),
8024 )
8025 .expect("open client");
8026 client.create_synced_tables().expect("create synced tables");
8027
8028 let values = |id: &str| {
8029 Map::from_iter([
8030 ("id".to_owned(), json!(id)),
8031 ("project_id".to_owned(), json!("p1")),
8032 ("title".to_owned(), json!("first")),
8033 ("priority".to_owned(), json!(1)),
8034 ("payload".to_owned(), json!({"$bytes": "0a"})),
8035 ])
8036 };
8037 let upsert = |values: Map<String, Value>| {
8038 vec![Mutation::Upsert {
8039 table: "tasks".into(),
8040 values,
8041 base_version: None,
8042 }]
8043 };
8044 let seed = client.mutate(upsert(values("t1"))).expect("seed row");
8045 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
8046
8047 for (label, override_value, cause) in [
8052 ("unknown column", ("nope", json!(1)), "unknown column"),
8053 (
8054 "internal sync column",
8055 ("_sync_version", json!(1)),
8056 "internal sync column",
8057 ),
8058 ("wrong value type", ("title", json!(7)), "expected a string"),
8059 (
8060 "malformed bytes envelope",
8061 ("payload", json!({"$bytes": "zz"})),
8062 "expected exactly",
8063 ),
8064 (
8065 "odd-length bytes envelope",
8066 ("payload", json!({"$bytes": "abc"})),
8067 "expected exactly",
8068 ),
8069 (
8070 "plus-sign bytes envelope",
8071 ("payload", json!({"$bytes": "+a"})),
8072 "expected exactly",
8073 ),
8074 (
8075 "bytes envelope with a second key",
8076 ("payload", json!({"$bytes": "0a", "extra": 1})),
8077 "expected exactly",
8078 ),
8079 (
8080 "no bytes envelope",
8081 ("payload", json!({"nested": true})),
8082 "expected exactly",
8083 ),
8084 ] {
8085 let mut full = values("t2");
8086 full.insert(override_value.0.to_owned(), override_value.1);
8087 let error = client.mutate(upsert(full)).expect_err(label);
8088 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8089 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8090 assert!(!error.retryable, "{label}");
8091 let legacy = error.details.as_ref().unwrap()["legacyCause"]
8092 .as_str()
8093 .unwrap();
8094 assert!(
8095 legacy.starts_with("sync.invalid_request: ") && legacy.contains(cause),
8096 "{label}: {legacy}"
8097 );
8098 assert_eq!(client.pending_commit_ids(), vec![seed.clone()], "{label}");
8099 assert_eq!(client.local_revision(), 1, "{label}");
8100 assert_eq!(
8101 client.query("SELECT * FROM tasks", &[]).unwrap(),
8102 rows,
8103 "{label}"
8104 );
8105 }
8106
8107 let mut missing = values("t3");
8110 missing.remove("payload");
8111 let missing_error = client.mutate(upsert(missing)).expect_err("missing column");
8112 assert_eq!(missing_error.code, "sync.invalid_request");
8113 assert!(missing_error.details.as_ref().unwrap()["legacyCause"]
8114 .as_str()
8115 .unwrap()
8116 .contains("is not nullable"));
8117
8118 let mut bad_row_id = values("t4");
8121 bad_row_id.insert("id".to_owned(), json!({"nested": true}));
8122 let row_id_error = client.mutate(upsert(bad_row_id)).expect_err("bad rowId");
8123 assert_eq!(row_id_error.code, "sync.invalid_request");
8124
8125 for (label, partial, cause) in [
8127 (
8128 "patch wrong value type",
8129 ("priority", json!("high")),
8130 "expected an integer",
8131 ),
8132 ("patch unknown column", ("nope", json!(1)), "unknown column"),
8133 ] {
8134 let error = client
8135 .patch(
8136 "tasks",
8137 "t1",
8138 Map::from_iter([(partial.0.to_owned(), partial.1)]),
8139 None,
8140 )
8141 .expect_err(label);
8142 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8143 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8144 assert!(
8145 error.details.as_ref().unwrap()["legacyCause"]
8146 .as_str()
8147 .unwrap()
8148 .contains(cause),
8149 "{label}"
8150 );
8151 }
8152 assert_eq!(client.pending_commit_ids(), vec![seed.clone()]);
8153 assert_eq!(client.local_revision(), 1);
8154 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
8155
8156 for (label, payload) in [
8159 ("explicit null", json!(null)),
8160 ("empty envelope", json!({"$bytes": ""})),
8161 ("uppercase envelope", json!({"$bytes": "0AFF"})),
8162 ] {
8163 let mut accepted = values("t-ok");
8164 accepted.insert("extra".to_owned(), payload);
8165 let encoded = client.mutate(upsert(accepted)).unwrap_or_else(|error| {
8166 panic!("{label} must be accepted: {error:?}");
8167 });
8168 assert!(!encoded.is_empty(), "{label}");
8169 }
8170
8171 let next = client
8173 .mutate(upsert(values("t5")))
8174 .expect("valid after failures");
8175 assert_ne!(next, seed);
8176
8177 drop(client);
8178 let _ = std::fs::remove_file(&path);
8179 }
8180
8181 #[test]
8182 fn a_legacy_outbox_value_the_codec_refuses_is_dropped_instead_of_wedging() {
8183 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8184 {"name":"id","type":"string","nullable":false},
8185 {"name":"project_id","type":"string","nullable":false},
8186 {"name":"payload","type":"bytes","nullable":true}],
8187 "scopes":[{"pattern":"project:{project_id}"}]}]});
8188 let open = |path: &std::path::Path| {
8189 SyncClient::open_path(
8190 "legacy-values".into(),
8191 &schema,
8192 ClientLimits::default(),
8193 path.to_str().unwrap(),
8194 )
8195 .expect("open client")
8196 };
8197 let row = |id: &str| {
8198 Map::from_iter([
8199 ("id".to_owned(), json!(id)),
8200 ("project_id".to_owned(), json!("p1")),
8201 ("payload".to_owned(), json!({"$bytes": "0a"})),
8202 ])
8203 };
8204 let upsert = |values| {
8205 vec![Mutation::Upsert {
8206 table: "tasks".into(),
8207 values,
8208 base_version: None,
8209 }]
8210 };
8211 for (label, stored_cell, representable) in [
8212 ("plus-sign envelope", json!({"$bytes": "+a"}), false),
8213 ("non-hex envelope", json!({"$bytes": "zz"}), false),
8214 ("odd-length envelope", json!({"$bytes": "abc"}), false),
8215 ("no envelope", json!({"nested": true}), false),
8216 ("wrong declared type", json!("high"), true),
8220 ("stored null for a non-nullable column", json!(null), true),
8221 ] {
8222 let path = std::env::temp_dir().join(format!(
8223 "syncular-legacy-values-{}-{}.db",
8224 label.replace(' ', "-"),
8225 uuid::Uuid::new_v4()
8226 ));
8227 let mut client = open(&path);
8228 client.create_synced_tables().expect("create synced tables");
8229 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8230 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8231 drop(client);
8232
8233 let connection = Connection::open(&path).expect("raw connection");
8235 let (column, cell) = if stored_cell.is_null() {
8236 ("project_id", stored_cell.clone())
8237 } else {
8238 ("payload", stored_cell.clone())
8239 };
8240 let mut planted_values = json!({
8241 "id": "t1", "project_id": "p1", "payload": {"$bytes": "0a"}
8242 });
8243 planted_values[column] = cell;
8244 let planted = json!([{
8245 "op": "upsert",
8246 "table": "tasks",
8247 "rowId": "t1",
8248 "baseVersion": null,
8249 "values": planted_values
8250 }]);
8251 connection
8252 .execute(
8253 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8254 rusqlite::params![planted.to_string(), stale],
8255 )
8256 .expect("plant legacy intent");
8257 drop(connection);
8258
8259 let reopened = open(&path);
8262 assert_eq!(
8263 reopened.pending_commit_ids(),
8264 vec![successor.clone()],
8265 "{label}"
8266 );
8267 let rejection = reopened
8268 .rejections()
8269 .iter()
8270 .find(|rejection| rejection.client_commit_id == stale)
8271 .unwrap_or_else(|| panic!("{label}: rejection"));
8272 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8273 assert_eq!(
8274 rejection.message, "the persisted commit carries values the current codec refuses",
8275 "{label}"
8276 );
8277 assert_eq!(rejection.operation.is_some(), representable, "{label}");
8278 assert_eq!(
8279 rejection
8280 .details
8281 .as_ref()
8282 .and_then(|details| details.reason.as_deref()),
8283 Some(INVALID_STORED_VALUES_REASON),
8284 "{label}"
8285 );
8286 let journal = reopened
8288 .commit_outcome(&stale)
8289 .expect("read outcome")
8290 .expect("journal entry");
8291 assert!(matches!(journal.status, CommitOutcomeStatus::Rejected));
8292 drop(reopened);
8293 let mut reopened = open(&path);
8294 let reread = reopened
8295 .commit_outcome(&stale)
8296 .expect("read outcome")
8297 .expect("journal entry");
8298 assert!(matches!(reread.status, CommitOutcomeStatus::Rejected));
8299
8300 assert_eq!(
8303 reopened
8304 .query("SELECT id FROM tasks ORDER BY id", &[])
8305 .unwrap(),
8306 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8307 "{label}"
8308 );
8309 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8310 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8311 assert!(!reopened
8312 .drop_unencodable_outbox(false, None)
8313 .expect("idempotent prepass"));
8314 let mut transport = CountingRealtimeTransport::default();
8315 let outcome = reopened.sync(&mut transport);
8316 assert!(matches!(outcome, SyncOutcome::Failed { .. }), "{outcome:?}");
8317 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8318 let next = reopened.mutate(upsert(row("t3"))).expect("after the drop");
8319 assert_eq!(reopened.pending_commit_ids(), vec![successor, next]);
8320 assert_eq!(reopened.pending_payloads().len(), 2, "{label}");
8321 drop(reopened);
8322 let _ = std::fs::remove_file(&path);
8323 }
8324 }
8325
8326 #[test]
8327 fn two_legacy_values_on_the_same_row_recover_in_fifo_order() {
8328 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8329 {"name":"id","type":"string","nullable":false},
8330 {"name":"project_id","type":"string","nullable":false},
8331 {"name":"payload","type":"bytes","nullable":true}],
8332 "scopes":[{"pattern":"project:{project_id}"}]}]});
8333 for (label, column, planted) in [
8334 ("malformed envelope", "payload", json!({"$bytes": "zz"})),
8335 (
8336 "stored null for a non-nullable column",
8337 "project_id",
8338 json!(null),
8339 ),
8340 ("wrong declared type", "payload", json!(7)),
8341 ] {
8342 let path = std::env::temp_dir().join(format!(
8343 "syncular-shared-row-{}-{}.db",
8344 label.replace(' ', "-"),
8345 uuid::Uuid::new_v4()
8346 ));
8347 let open = |path: &std::path::Path| {
8348 SyncClient::open_path(
8349 "shared-row".into(),
8350 &schema,
8351 ClientLimits::default(),
8352 path.to_str().unwrap(),
8353 )
8354 .expect("open client")
8355 };
8356 let row = |id: &str| {
8357 Map::from_iter([
8358 ("id".to_owned(), json!(id)),
8359 ("project_id".to_owned(), json!("p1")),
8360 ("payload".to_owned(), json!({"$bytes": "0a"})),
8361 ])
8362 };
8363 let upsert = |values| {
8364 vec![Mutation::Upsert {
8365 table: "tasks".into(),
8366 values,
8367 base_version: None,
8368 }]
8369 };
8370 let mut client = open(&path);
8371 client.create_synced_tables().expect("create synced tables");
8372 let older = client.mutate(upsert(row("t1"))).expect("older commit");
8373 let newer = client.mutate(upsert(row("t1"))).expect("newer commit");
8374 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8375 drop(client);
8376
8377 let connection = Connection::open(&path).expect("raw connection");
8378 for commit_id in [&older, &newer] {
8379 let mut values = row("t1");
8380 values.insert(column.to_owned(), planted.clone());
8381 let planted_ops = json!([{
8382 "op": "upsert",
8383 "table": "tasks",
8384 "rowId": "t1",
8385 "baseVersion": null,
8386 "values": values
8387 }]);
8388 connection
8389 .execute(
8390 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8391 rusqlite::params![planted_ops.to_string(), commit_id],
8392 )
8393 .expect("plant legacy intent");
8394 }
8395 drop(connection);
8396
8397 let mut reopened = open(&path);
8399 assert_eq!(
8400 reopened.pending_commit_ids(),
8401 vec![successor.clone()],
8402 "{label}"
8403 );
8404 assert_eq!(
8405 reopened
8406 .rejections()
8407 .iter()
8408 .map(|rejection| rejection.client_commit_id.clone())
8409 .collect::<Vec<_>>(),
8410 vec![older.clone(), newer.clone()],
8411 "{label}"
8412 );
8413 assert!(
8414 reopened
8415 .rejections()
8416 .iter()
8417 .all(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE
8418 && rejection.details.as_ref().and_then(|d| d.reason.as_deref())
8419 == Some(INVALID_STORED_VALUES_REASON)),
8420 "{label}"
8421 );
8422 for commit_id in [&older, &newer] {
8423 assert!(
8424 matches!(
8425 reopened
8426 .commit_outcome(commit_id)
8427 .expect("outcome")
8428 .map(|o| o.status),
8429 Some(CommitOutcomeStatus::Rejected)
8430 ),
8431 "{label}"
8432 );
8433 }
8434 assert_eq!(
8435 reopened
8436 .query("SELECT id FROM tasks ORDER BY id", &[])
8437 .unwrap(),
8438 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8439 "{label}"
8440 );
8441 let revision = reopened.local_revision();
8442 let batches = reopened.drain_change_batches();
8443 assert!(
8444 batches.iter().any(|batch| batch.rejections_changed
8445 && batch.outcomes_changed
8446 && batch.tables.iter().any(|change| change.table == "tasks")
8447 && batch.revision == revision.to_string()),
8448 "{label}: {batches:?}"
8449 );
8450 drop(reopened);
8451
8452 let mut stable = open(&path);
8455 assert_eq!(stable.local_revision(), revision, "{label}");
8456 assert!(stable.drain_change_batches().is_empty(), "{label}");
8457 assert_eq!(stable.pending_commit_ids(), vec![successor], "{label}");
8458 drop(stable);
8459 let _ = std::fs::remove_file(&path);
8460 }
8461 }
8462
8463 #[test]
8464 fn recovery_omits_a_legacy_operation_the_journal_reader_rejects() {
8465 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8466 {"name":"id","type":"string","nullable":false},
8467 {"name":"project_id","type":"string","nullable":false},
8468 {"name":"payload","type":"bytes","nullable":true}],
8469 "scopes":[{"pattern":"project:{project_id}"}]}]});
8470 for (label, second_op) in [
8471 (
8472 "delete carrying values",
8473 json!({"op": "delete", "table": "tasks", "rowId": "t9", "values": {}}),
8474 ),
8475 (
8476 "baseVersion beyond the safe bound",
8477 json!({"op": "upsert", "table": "tasks", "rowId": "t9",
8478 "baseVersion": MAX_JS_SAFE_INTEGER + 1,
8479 "values": {"id": "t9", "project_id": "p1", "payload": {"$bytes": "0a"}}}),
8480 ),
8481 ] {
8482 let path = std::env::temp_dir().join(format!(
8483 "syncular-journal-shape-{}-{}.db",
8484 label.replace(' ', "-"),
8485 uuid::Uuid::new_v4()
8486 ));
8487 let open = |path: &std::path::Path| {
8488 SyncClient::open_path(
8489 "journal-shape".into(),
8490 &schema,
8491 ClientLimits::default(),
8492 path.to_str().unwrap(),
8493 )
8494 .expect("open client")
8495 };
8496 let values = |id: &str| {
8497 Map::from_iter([
8498 ("id".to_owned(), json!(id)),
8499 ("project_id".to_owned(), json!("p1")),
8500 ("payload".to_owned(), json!({"$bytes": "0a"})),
8501 ])
8502 };
8503 let upsert = |values| {
8504 vec![Mutation::Upsert {
8505 table: "tasks".into(),
8506 values,
8507 base_version: None,
8508 }]
8509 };
8510 let mut client = open(&path);
8511 client.create_synced_tables().expect("create synced tables");
8512 let stale = client.mutate(upsert(values("t1"))).expect("stale commit");
8513 drop(client);
8514
8515 let connection = Connection::open(&path).expect("raw connection");
8516 let planted = json!([
8520 {
8521 "op": "upsert",
8522 "table": "tasks",
8523 "rowId": "t1",
8524 "baseVersion": null,
8525 "values": {"id": "t1", "project_id": null,
8526 "payload": {"$bytes": "0a"}}
8527 },
8528 second_op
8529 ]);
8530 connection
8531 .execute(
8532 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8533 rusqlite::params![planted.to_string(), stale],
8534 )
8535 .expect("plant legacy intent");
8536 drop(connection);
8537
8538 let reopened = open(&path);
8539 let journal = reopened
8540 .commit_outcome(&stale)
8541 .unwrap_or_else(|error| panic!("{label}: journal read: {error}"))
8542 .unwrap_or_else(|| panic!("{label}: journal entry"));
8543 assert!(
8544 matches!(journal.status, CommitOutcomeStatus::Rejected),
8545 "{label}"
8546 );
8547 assert!(journal.operations.is_none(), "{label}");
8548 assert!(
8549 reopened
8550 .commit_outcomes(CommitOutcomeQuery::default())
8551 .is_ok(),
8552 "{label}: outcome list query"
8553 );
8554 assert!(reopened.pending_commit_ids().is_empty(), "{label}");
8555 drop(reopened);
8556 let _ = std::fs::remove_file(&path);
8557 }
8558 }
8559
8560 #[test]
8561 fn legacy_values_recover_at_the_recreate_and_bump_reset_boundaries() {
8562 let schema = |version: i32, extra: bool| {
8563 let mut columns = vec![
8564 json!({"name":"id","type":"string","nullable":false}),
8565 json!({"name":"project_id","type":"string","nullable":false}),
8566 json!({"name":"payload","type":"bytes","nullable":true}),
8567 ];
8568 if extra {
8569 columns.push(json!({"name":"later","type":"string","nullable":true}));
8570 }
8571 json!({"version":version,"tables":[{"name":"tasks","primaryKey":"id","columns":columns,
8572 "scopes":[{"pattern":"project:{project_id}"}]}]})
8573 };
8574 for (label, bump) in [
8575 ("same-version recreation", false),
8576 ("version-bump reset", true),
8577 ("version-only bump reset", true),
8578 ] {
8579 let path = std::env::temp_dir().join(format!(
8580 "syncular-legacy-reset-{}-{}.db",
8581 label.replace(' ', "-"),
8582 uuid::Uuid::new_v4()
8583 ));
8584 let v1 = schema(1, false);
8585 let open = |schema: &Value, path: &std::path::Path| {
8586 SyncClient::open_path(
8587 "legacy-reset".into(),
8588 schema,
8589 ClientLimits::default(),
8590 path.to_str().unwrap(),
8591 )
8592 .expect("open client")
8593 };
8594 let row = |id: &str| {
8595 Map::from_iter([
8596 ("id".to_owned(), json!(id)),
8597 ("project_id".to_owned(), json!("p1")),
8598 ("payload".to_owned(), json!({"$bytes": "0a"})),
8599 ])
8600 };
8601 let upsert = |values| {
8602 vec![Mutation::Upsert {
8603 table: "tasks".into(),
8604 values,
8605 base_version: None,
8606 }]
8607 };
8608 let mut client = open(&v1, &path);
8609 client.create_synced_tables().expect("create synced tables");
8610 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8611 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8612 drop(client);
8613
8614 let connection = Connection::open(&path).expect("raw connection");
8615 let planted = json!([{
8616 "op": "upsert",
8617 "table": "tasks",
8618 "rowId": "t1",
8619 "baseVersion": null,
8620 "values": {"id": "t1", "project_id": 7, "payload": {"$bytes": "0a"}}
8621 }]);
8622 connection
8623 .execute(
8624 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8625 rusqlite::params![planted.to_string(), stale],
8626 )
8627 .expect("plant legacy intent");
8628 drop(connection);
8629
8630 let target = if bump {
8631 schema(2, label == "version-bump reset")
8632 } else {
8633 v1.clone()
8634 };
8635 let reopened = open(&target, &path);
8636 let rejection = reopened
8637 .rejections()
8638 .iter()
8639 .find(|rejection| rejection.client_commit_id == stale)
8640 .unwrap_or_else(|| panic!("{label}: rejection"));
8641 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8642 assert_eq!(
8643 rejection
8644 .details
8645 .as_ref()
8646 .and_then(|details| details.reason.as_deref()),
8647 Some(INVALID_STORED_VALUES_REASON),
8648 "{label}"
8649 );
8650 assert!(rejection.operation.is_some(), "{label}");
8651 assert_eq!(
8652 reopened.pending_commit_ids(),
8653 vec![successor.clone()],
8654 "{label}"
8655 );
8656 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8661 if label == "version-bump reset" {
8662 assert_eq!(
8663 reopened
8664 .query("SELECT id FROM tasks ORDER BY id", &[])
8665 .unwrap(),
8666 Vec::<Map<String, Value>>::new(),
8667 "{label}"
8668 );
8669 let mut next = reopened;
8673 let revision = next.local_revision();
8674 let batches = next.drain_change_batches();
8675 assert!(
8676 batches.iter().any(|batch| batch.rejections_changed
8677 && batch.outcomes_changed
8678 && batch.tables.iter().any(|change| change.table == "tasks")
8679 && batch.revision == revision.to_string()),
8680 "{label}: {batches:?}"
8681 );
8682 next.mutate(upsert(Map::from_iter([
8683 ("id".to_owned(), json!("t3")),
8684 ("project_id".to_owned(), json!("p1")),
8685 ("payload".to_owned(), json!({"$bytes": "0b"})),
8686 ("later".to_owned(), json!("v2")),
8687 ])))
8688 .expect("v2 authoring after the reset");
8689 assert_eq!(next.pending_commit_ids().len(), 2, "{label}");
8690 assert_eq!(next.pending_payloads().len(), 2, "{label}");
8691 drop(next);
8692 } else {
8693 assert_eq!(
8694 reopened
8695 .query("SELECT id FROM tasks ORDER BY id", &[])
8696 .unwrap(),
8697 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8698 "{label}"
8699 );
8700 drop(reopened);
8701 }
8702
8703 if !bump {
8706 let mut recreated = open(&v1, &path);
8707 let fresh = recreated
8708 .mutate(upsert(row("t3")))
8709 .expect("fresh commit while open");
8710 recreated.drain_change_batches();
8711 let planted_fresh = json!([{
8712 "op": "upsert",
8713 "table": "tasks",
8714 "rowId": "t3",
8715 "baseVersion": null,
8716 "values": {"id": "t3", "project_id": "p1",
8717 "payload": {"$bytes": "zz"}}
8718 }]);
8719 let connection = Connection::open(&path).expect("raw connection");
8720 let affected = connection
8721 .execute(
8722 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8723 rusqlite::params![planted_fresh.to_string(), fresh],
8724 )
8725 .expect("plant fresh legacy intent");
8726 assert_eq!(
8727 affected, 1,
8728 "{label}: the in-place plant must hit the queued commit"
8729 );
8730 drop(connection);
8731 let revision_before = recreated.local_revision();
8732 recreated
8733 .recreate_with_schema(&v1)
8734 .unwrap_or_else(|error| panic!("{label}: recreation failed: {error}"));
8735 assert_eq!(recreated.pending_commit_ids(), vec![successor], "{label}");
8736 let fresh_rejection = recreated
8737 .rejections()
8738 .iter()
8739 .find(|rejection| rejection.client_commit_id == fresh)
8740 .expect("the in-place recovery rejection");
8741 assert_eq!(fresh_rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8742 assert_eq!(
8743 fresh_rejection
8744 .details
8745 .as_ref()
8746 .and_then(|details| details.reason.as_deref()),
8747 Some(INVALID_STORED_VALUES_REASON),
8748 "{label}"
8749 );
8750 assert!(
8751 matches!(
8752 recreated
8753 .commit_outcome(&fresh)
8754 .expect("outcome read")
8755 .map(|outcome| outcome.status),
8756 Some(CommitOutcomeStatus::Rejected)
8757 ),
8758 "{label}"
8759 );
8760 assert!(recreated.local_revision() > revision_before, "{label}");
8761 let revision = recreated.local_revision();
8762 let batches = recreated.drain_change_batches();
8763 assert_eq!(batches.len(), 1, "{label}: one recovery batch");
8764 assert!(
8765 batches.iter().any(|batch| batch.rejections_changed
8766 && batch.outcomes_changed
8767 && batch.tables.iter().any(|change| change.table == "tasks")
8768 && batch.revision == revision.to_string()),
8769 "{label}: {batches:?}"
8770 );
8771 assert_eq!(recreated.pending_payloads().len(), 1, "{label}");
8772 assert_eq!(
8773 recreated
8774 .query("SELECT id FROM tasks ORDER BY id", &[])
8775 .unwrap(),
8776 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8777 "{label}: the recovered commit leaves no projection"
8778 );
8779 drop(recreated);
8780 }
8781 let _ = std::fs::remove_file(&path);
8782 }
8783 }
8784
8785 #[test]
8786 fn legacy_values_recover_over_a_synced_authoritative_row() {
8787 let path = std::env::temp_dir().join(format!(
8788 "syncular-authoritative-row-{}.db",
8789 uuid::Uuid::new_v4()
8790 ));
8791 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8792 {"name":"id","type":"string","nullable":false},
8793 {"name":"project_id","type":"string","nullable":false},
8794 {"name":"title","type":"string","nullable":false},
8795 {"name":"payload","type":"bytes","nullable":true}],
8796 "scopes":[{"pattern":"project:{project_id}"}]}]});
8797 let open = |path: &std::path::Path| {
8798 SyncClient::open_path(
8799 "authoritative-row".into(),
8800 &schema,
8801 ClientLimits::default(),
8802 path.to_str().unwrap(),
8803 )
8804 .expect("open client")
8805 };
8806 let row = |title: &str| {
8807 Map::from_iter([
8808 ("id".to_owned(), json!("t1")),
8809 ("project_id".to_owned(), json!("p1")),
8810 ("title".to_owned(), json!(title)),
8811 ("payload".to_owned(), json!({"$bytes": "0a"})),
8812 ])
8813 };
8814 let upsert = |values| {
8815 vec![Mutation::Upsert {
8816 table: "tasks".into(),
8817 values,
8818 base_version: None,
8819 }]
8820 };
8821 let mut client = open(&path);
8822 client.create_synced_tables().expect("create synced tables");
8823 client
8826 .conn
8827 .execute_batch(
8828 "INSERT INTO _syncular_base_tasks VALUES ('t1','p1','authoritative',x'0a',1);
8829 INSERT INTO tasks VALUES ('t1','p1','authoritative',x'0a',1);",
8830 )
8831 .expect("seed authoritative row");
8832 let older = client.mutate(upsert(row("older"))).expect("older");
8833 let newer = client.mutate(upsert(row("newer"))).expect("newer");
8834 let successor = client.mutate(upsert(row("successor"))).expect("successor");
8835 drop(client);
8836
8837 let connection = Connection::open(&path).expect("raw connection");
8838 for commit_id in [&older, &newer] {
8839 let planted = json!([{
8840 "op": "upsert",
8841 "table": "tasks",
8842 "rowId": "t1",
8843 "baseVersion": null,
8844 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
8845 "payload": {"$bytes": "zz"}}
8846 }]);
8847 connection
8848 .execute(
8849 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8850 rusqlite::params![planted.to_string(), commit_id],
8851 )
8852 .expect("plant legacy intent");
8853 }
8854 drop(connection);
8855
8856 let reopened = open(&path);
8857 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8858 assert_eq!(
8859 reopened
8860 .rejections()
8861 .iter()
8862 .map(|rejection| rejection.client_commit_id.clone())
8863 .collect::<Vec<_>>(),
8864 vec![older.clone(), newer.clone()]
8865 );
8866 assert_eq!(
8869 reopened.query("SELECT id,title,_syncular_version FROM tasks", &[]),
8870 Ok(vec![Map::from_iter([
8871 ("id".to_owned(), json!("t1")),
8872 ("title".to_owned(), json!("successor")),
8873 ("_syncular_version".to_owned(), json!(1)),
8874 ])])
8875 );
8876
8877 drop(reopened);
8880 let connection = Connection::open(&path).expect("raw connection");
8881 connection
8882 .execute(
8883 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8884 rusqlite::params![
8885 json!([{
8886 "op": "upsert",
8887 "table": "tasks",
8888 "rowId": "t1",
8889 "baseVersion": null,
8890 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
8891 "payload": {"$bytes": "zz"}}
8892 }])
8893 .to_string(),
8894 successor
8895 ],
8896 )
8897 .expect("plant successor intent");
8898 drop(connection);
8899 let late = open(&path);
8900 assert!(late.pending_commit_ids().is_empty());
8901 assert_eq!(
8902 late.query("SELECT id,title,_syncular_version FROM tasks", &[]),
8903 Ok(vec![Map::from_iter([
8904 ("id".to_owned(), json!("t1")),
8905 ("title".to_owned(), json!("authoritative")),
8906 ("_syncular_version".to_owned(), json!(1)),
8907 ])])
8908 );
8909 drop(late);
8910 let _ = std::fs::remove_file(&path);
8911 }
8912
8913 #[test]
8914 fn a_failed_value_recovery_commit_restores_every_side_effect() {
8915 let path = std::env::temp_dir().join(format!(
8916 "syncular-value-recovery-commit-{}.db",
8917 uuid::Uuid::new_v4()
8918 ));
8919 let path_str = path.to_str().expect("path");
8920 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8921 {"name":"id","type":"string","nullable":false},
8922 {"name":"project_id","type":"string","nullable":false},
8923 {"name":"payload","type":"bytes","nullable":true}],
8924 "scopes":[{"pattern":"project:{project_id}"}]}]});
8925 let conn = Connection::open(path_str).expect("replica connection");
8928 let mut client = SyncClient::with_connection(
8929 "value-recovery-commit".into(),
8930 &schema,
8931 ClientLimits::default(),
8932 conn,
8933 )
8934 .expect("replica");
8935 assert_eq!(
8936 client
8937 .conn
8938 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
8939 .expect("journal mode"),
8940 "delete"
8941 );
8942 client.create_synced_tables().expect("create synced tables");
8943 let stale = client
8944 .mutate(vec![Mutation::Upsert {
8945 table: "tasks".into(),
8946 values: Map::from_iter([
8947 ("id".to_owned(), json!("t1")),
8948 ("project_id".to_owned(), json!("p1")),
8949 ("payload".to_owned(), json!({"$bytes": "0a"})),
8950 ]),
8951 base_version: None,
8952 }])
8953 .expect("queued write");
8954 client.drain_change_batches();
8955 let affected = client
8956 .conn
8957 .execute(
8958 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8959 rusqlite::params![
8960 json!([{
8961 "op": "upsert",
8962 "table": "tasks",
8963 "rowId": "t1",
8964 "baseVersion": null,
8965 "values": {"id": "t1", "project_id": "p1",
8966 "payload": {"$bytes": "zz"}}
8967 }])
8968 .to_string(),
8969 stale
8970 ],
8971 )
8972 .expect("plant legacy intent");
8973 assert_eq!(affected, 1, "the planted row must exist");
8974 let revision = client.local_revision();
8975 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
8976 let prior_outbox = client.pending_commit_ids();
8977 let prior_rejections = client.rejections().len();
8978 let prior_changes = client.change_queue.len();
8979 let prior_intents = client.sync_intent_queue.len();
8980 let prior_failed = client.failed_commits.len();
8981 let reader = Connection::open(path_str).expect("reader");
8984 reader
8985 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
8986 .expect("reader snapshot");
8987 client
8988 .conn
8989 .busy_timeout(std::time::Duration::ZERO)
8990 .expect("busy timeout");
8991 let error = client
8992 .recreate_with_schema(&schema)
8993 .expect_err("a blocked COMMIT must refuse the recovery");
8994 assert!(error.starts_with("client.storage_busy:"), "{error}");
8995 assert!(client.conn.is_autocommit(), "the transaction is released");
8996 assert_eq!(client.local_revision(), revision);
8997 assert_eq!(client.pending_commit_ids(), prior_outbox);
8998 assert_eq!(client.rejections().len(), prior_rejections);
8999 assert_eq!(client.change_queue.len(), prior_changes);
9000 assert_eq!(client.sync_intent_queue.len(), prior_intents);
9001 assert_eq!(client.failed_commits.len(), prior_failed);
9002 assert_eq!(
9003 client.query("SELECT * FROM tasks", &[]).expect("rows"),
9004 rows
9005 );
9006 assert!(
9007 client
9008 .commit_outcome(&stale)
9009 .expect("outcome read")
9010 .is_none(),
9011 "the rolled-back recovery kept no outcome"
9012 );
9013 reader.execute_batch("ROLLBACK").expect("release reader");
9015 client
9016 .recreate_with_schema(&schema)
9017 .expect("retry after the blocked commit");
9018 assert!(client.pending_commit_ids().is_empty());
9019 assert_eq!(client.rejections().len(), prior_rejections + 1);
9020 assert!(client.local_revision() > revision);
9021 let batches = client.drain_change_batches();
9022 assert!(
9023 batches.iter().any(|batch| batch.rejections_changed
9024 && batch.outcomes_changed
9025 && batch.tables.iter().any(|change| change.table == "tasks")),
9026 "{batches:?}"
9027 );
9028 drop(client);
9029 let _ = std::fs::remove_file(&path);
9030 }
9031
9032 #[cfg(feature = "e2ee")]
9033 #[test]
9034 fn a_failed_recovery_restores_retained_failed_commits() {
9035 let path = std::env::temp_dir().join(format!(
9036 "syncular-value-recovery-retention-{}.db",
9037 uuid::Uuid::new_v4()
9038 ));
9039 let path_str = path.to_str().expect("path");
9040 let schema = json!({"version":1,"tables":[{"name":"secrets","primaryKey":"id","columns":[
9041 {"name":"id","type":"string","nullable":false},
9042 {"name":"project_id","type":"string","nullable":false},
9043 {"name":"note","type":"bytes","nullable":true,"encrypted":true,"declaredType":"string"}],
9044 "scopes":[{"pattern":"project:{project_id}"}]}]});
9045 let conn = Connection::open(path_str).expect("replica connection");
9046 let mut client = SyncClient::with_connection(
9047 "value-recovery-retention".into(),
9048 &schema,
9049 ClientLimits::default(),
9050 conn,
9051 )
9052 .expect("replica");
9053 client.create_synced_tables().expect("create synced tables");
9054 client.set_retain_failed_commits(true).expect("retention");
9055 client.set_encryption(crate::values::EncryptionConfig::default());
9057 client
9058 .mutate(vec![Mutation::Upsert {
9059 table: "secrets".into(),
9060 values: Map::from_iter([
9061 ("id".to_owned(), json!("s1")),
9062 ("project_id".to_owned(), json!("p1")),
9063 ("note".to_owned(), json!("secret")),
9064 ]),
9065 base_version: None,
9066 }])
9067 .expect("queued write");
9068 client.drain_change_batches();
9069 let revision = client.local_revision();
9070 let prior_outbox = client.pending_commit_ids();
9071 let prior_rejections = client.rejections().len();
9072 let prior_failed = client.failed_commits.len();
9073 let reader = Connection::open(path_str).expect("reader");
9074 reader
9075 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9076 .expect("reader snapshot");
9077 client
9078 .conn
9079 .busy_timeout(std::time::Duration::ZERO)
9080 .expect("busy timeout");
9081 let error = client
9082 .drop_unencodable_outbox(true, None)
9083 .expect_err("a blocked COMMIT must refuse the recovery");
9084 assert!(error.starts_with("client.storage_busy:"), "{error}");
9085 assert_eq!(client.failed_commits.len(), prior_failed);
9086 assert_eq!(client.pending_commit_ids(), prior_outbox);
9087 assert_eq!(client.rejections().len(), prior_rejections);
9088 assert_eq!(client.local_revision(), revision);
9089 reader.execute_batch("ROLLBACK").expect("release reader");
9091 client
9092 .drop_unencodable_outbox(true, None)
9093 .expect("retry after the blocked commit");
9094 assert_eq!(client.failed_commits.len(), prior_failed + 1);
9095 assert!(client.pending_commit_ids().is_empty());
9096 assert_eq!(client.rejections().len(), prior_rejections + 1);
9097 drop(client);
9098 let _ = std::fs::remove_file(&path);
9099 }
9100
9101 #[test]
9102 fn client_error_maps_legacy_codes_and_static_messages() {
9103 let code_only = ClientError::from("sync.unknown_table".to_owned());
9104 assert_eq!(code_only.code, "sync.unknown_table");
9105 assert_eq!(code_only.message, "the commit targets an unknown table");
9106 assert!(code_only.details.is_none());
9107
9108 let dynamic = ClientError::from(
9109 "sync.invalid_request: table \"tasks\": patch cannot write scope column".to_owned(),
9110 );
9111 assert_eq!(dynamic.code, "sync.invalid_request");
9112 assert_eq!(dynamic.message, "the authoring request is invalid");
9113 assert!(dynamic.details.as_ref().unwrap()["legacyCause"]
9114 .as_str()
9115 .unwrap()
9116 .contains("patch cannot write scope column"));
9117
9118 let unclassified =
9119 ClientError::from("a commit must contain at least one operation".to_owned());
9120 assert_eq!(unclassified.code, "client.failed");
9121 assert_eq!(unclassified.message, "the local authoring operation failed");
9122 assert!(unclassified.details.as_ref().unwrap()["legacyCause"]
9123 .as_str()
9124 .unwrap()
9125 .contains("at least one operation"));
9126
9127 let metadata = ClientError::from(QueryReadFailure {
9129 code: None,
9130 sqlite_code: Some(1),
9131 sqlite_message: Some("generic SQLite prose".to_owned()),
9132 rollback_failure: Some(Box::new(QueryReadFailure::from(
9133 rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(1), None),
9134 ))),
9135 message: "unclassified read".to_owned(),
9136 });
9137 assert_eq!(metadata.code, "client.failed");
9138 assert_eq!(metadata.message, "the local authoring operation failed");
9139 let details = metadata.details.as_ref().unwrap();
9140 assert_eq!(details["sqliteCode"], 1);
9141 assert_eq!(details["sqliteMessage"], "generic SQLite prose");
9142 assert_eq!(details["rollbackFailure"]["sqliteCode"], 1);
9143 assert_eq!(details["legacyCause"], "unclassified read");
9144 }
9145
9146 #[test]
9147 fn schema_marker_read_keeps_storage_classification() {
9148 let busy = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9149 rusqlite::ffi::Error::new(5),
9150 Some("database is locked".to_owned()),
9151 ));
9152 assert!(busy.starts_with("client.storage_busy:"), "{busy}");
9153 let invalid = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9154 rusqlite::ffi::Error::new(1),
9155 Some("unreadable".to_owned()),
9156 ));
9157 assert_eq!(
9158 invalid,
9159 "sync.local_corrupt: persisted local schema marker is unreadable or invalid"
9160 );
9161
9162 let path =
9165 std::env::temp_dir().join(format!("syncular-marker-busy-{}.db", uuid::Uuid::new_v4()));
9166 let setup = Connection::open(&path).expect("setup connection");
9167 setup
9168 .execute_batch(
9169 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value TEXT);\n INSERT INTO _syncular_meta VALUES ('localSchemaVersion','1');",
9170 )
9171 .unwrap();
9172 let locker = Connection::open(&path).expect("locker connection");
9173 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9174 let reader = Connection::open(&path).expect("reader connection");
9175 reader.busy_timeout(std::time::Duration::ZERO).unwrap();
9176 let locked = read_local_schema_version(&reader, 1).expect_err("locked marker read");
9177 assert!(locked.starts_with("client.storage_busy:"), "{locked}");
9178 locker.execute_batch("ROLLBACK").unwrap();
9179 assert_eq!(read_local_schema_version(&reader, 1).unwrap(), Some(1));
9180 let rows = setup
9181 .query_row("SELECT count(*) FROM _syncular_meta", [], |row| {
9182 row.get::<_, i64>(0)
9183 })
9184 .unwrap();
9185 assert_eq!(rows, 1, "the guard performed no writes");
9186 drop(reader);
9187 drop(locker);
9188 drop(setup);
9189 let _ = std::fs::remove_file(&path);
9190 }
9191
9192 #[cfg(feature = "crdt-yjs")]
9193 #[test]
9194 fn crdt_authoring_pre_read_preserves_storage_classification() {
9195 let path =
9196 std::env::temp_dir().join(format!("syncular-crdt-busy-{}.db", uuid::Uuid::new_v4()));
9197 let schema = json!({"version":1,"tables":[{"name":"notes","primaryKey":"id","columns":[
9198 {"name":"id","type":"string","nullable":false},
9199 {"name":"project_id","type":"string","nullable":false},
9200 {"name":"doc","type":"crdt","nullable":true,"crdtType":"yjs-doc"}],
9201 "scopes":[{"pattern":"project:{project_id}"}]}]});
9202 let mut client = SyncClient::open_path(
9203 "crdt-busy".into(),
9204 &schema,
9205 ClientLimits::default(),
9206 path.to_str().unwrap(),
9207 )
9208 .expect("open client");
9209 client.create_synced_tables().expect("create synced tables");
9210 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
9211
9212 let locker = Connection::open(&path).expect("locker connection");
9213 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9214 let busy = client
9215 .crdt_apply_update("notes", "n1", "doc", &[])
9216 .expect_err("busy crdt pre-read");
9217 assert_eq!(busy.code, "client.storage_busy");
9218 assert!(busy.retryable);
9219 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
9220 assert!(client.pending_commit_ids().is_empty());
9221 assert_eq!(client.local_revision(), 0);
9222
9223 locker.execute_batch("ROLLBACK").unwrap();
9224 drop(locker);
9225 drop(client);
9226 let _ = std::fs::remove_file(&path);
9227 }
9228
9229 #[test]
9230 fn large_unique_import_copies_no_unrelated_tables() {
9231 let (mut client, image, table) = unique_import_fixture(10240);
9232 let pending = client.pending_commit_ids();
9233 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
9234 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
9235 assert!(matches!(
9236 client.apply_sqlite_image(
9237 &image,
9238 &table,
9239 true,
9240 &[("project_id".into(), vec!["p1".into()])],
9241 10240,
9242 7,
9243 "digest"
9244 ),
9245 Ok(10240)
9246 ));
9247 assert_eq!(client.overlay_rebuild_count.get(), 0);
9248 assert_eq!(client.pending_commit_ids(), pending);
9249 assert_eq!(
9250 client
9251 .query("SELECT count(*) AS n FROM tasks", &[])
9252 .unwrap()[0]["n"],
9253 10241
9254 );
9255 assert_eq!(
9256 client
9257 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9258 .unwrap()[0]["n"],
9259 10241
9260 );
9261 assert_eq!(
9262 client
9263 .query("SELECT count(*) AS n FROM notes_fts", &[])
9264 .unwrap()[0]["n"],
9265 8192
9266 );
9267 }
9268
9269 #[test]
9270 #[ignore = "manual scaling benchmark; see bench/README.md"]
9271 fn benchmark_unique_image_import() {
9272 let mut samples = Vec::new();
9273 for rows in [10240, 51200, 102400] {
9274 for trial in 0..4 {
9275 let (mut client, image, table) = unique_import_fixture(rows);
9276 let start = std::time::Instant::now();
9277 assert!(
9278 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
9279 );
9280 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
9281 assert_eq!(
9282 client
9283 .query("SELECT count(*) AS n FROM tasks", &[])
9284 .unwrap()[0]["n"],
9285 rows + 1
9286 );
9287 assert_eq!(
9288 client
9289 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9290 .unwrap()[0]["n"],
9291 rows + 1
9292 );
9293 assert_eq!(
9294 client
9295 .query("SELECT count(*) AS n FROM notes_fts", &[])
9296 .unwrap()[0]["n"],
9297 8192
9298 );
9299 assert_eq!(client.pending_commit_ids().len(), 1);
9300 if trial > 0 {
9301 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
9302 }
9303 }
9304 }
9305 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
9306 let output = serde_json::to_string_pretty(&artifact).unwrap();
9307 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
9308 use std::io::Write;
9309 let mut file = std::fs::OpenOptions::new()
9310 .write(true)
9311 .create_new(true)
9312 .open(path)
9313 .unwrap();
9314 file.write_all(output.as_bytes()).unwrap();
9315 } else {
9316 println!("{output}");
9317 }
9318 }
9319
9320 #[test]
9321 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
9322 let schema = json!({
9323 "version": 1,
9324 "tables": [{
9325 "name": "catalogue_codes",
9326 "primaryKey": "id",
9327 "columns": [
9328 { "name": "id", "type": "string", "nullable": false },
9329 { "name": "release_id", "type": "string", "nullable": false },
9330 { "name": "code", "type": "string", "nullable": false },
9331 { "name": "title", "type": "string", "nullable": false }
9332 ],
9333 "scopes": [{ "pattern": "release:{release_id}" }],
9334 "ftsIndexes": [{
9335 "name": "catalogue_codes_fts",
9336 "columns": ["code", "title"],
9337 "tokenize": "unicode61 remove_diacritics 2"
9338 }]
9339 }]
9340 });
9341 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
9342 .expect("FTS5 client");
9343 let insert_trigger: String = client
9344 .conn
9345 .query_row(
9346 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
9347 [],
9348 |row| row.get(0),
9349 )
9350 .expect("insert trigger");
9351 let replace_guard: String = client
9352 .conn
9353 .query_row(
9354 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
9355 [],
9356 |row| row.get(0),
9357 )
9358 .expect("replace guard");
9359 assert!(!insert_trigger.contains("DELETE FROM"));
9360 assert!(replace_guard.contains("BEFORE INSERT"));
9361 assert!(replace_guard.contains("WHEN EXISTS"));
9362 let search = |client: &SyncClient, query: &str| {
9363 client
9364 .query(
9365 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
9366 &[Value::from(query)],
9367 )
9368 .expect("FTS query")
9369 };
9370
9371 client
9372 .mutate(vec![Mutation::Upsert {
9373 table: "catalogue_codes".to_owned(),
9374 values: Map::from_iter([
9375 ("id".to_owned(), Value::from("c1")),
9376 ("release_id".to_owned(), Value::from("r1")),
9377 ("code".to_owned(), Value::from("A01")),
9378 ("title".to_owned(), Value::from("Cholera")),
9379 ]),
9380 base_version: None,
9381 }])
9382 .expect("insert code");
9383 assert_eq!(search(&client, "cholera").len(), 1);
9384 let prior = client
9385 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9386 .unwrap();
9387 client
9388 .conn
9389 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
9390 .unwrap();
9391 client.create_synced_tables().unwrap();
9392 assert_eq!(
9393 client
9394 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9395 .unwrap(),
9396 prior
9397 );
9398 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
9399 let rendered = serde_json::to_string(&plan).unwrap();
9400 assert!(rendered.contains("COVERING INDEX"));
9401 assert!(rendered.contains("INDEX 0:="));
9402 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
9403 assert_eq!(search(&client, "cholera").len(), 1);
9404
9405 client
9406 .mutate(vec![Mutation::Upsert {
9407 table: "catalogue_codes".to_owned(),
9408 values: Map::from_iter([
9409 ("id".to_owned(), Value::from("c1")),
9410 ("release_id".to_owned(), Value::from("r1")),
9411 ("code".to_owned(), Value::from("A01")),
9412 ("title".to_owned(), Value::from("Enteric infection")),
9413 ]),
9414 base_version: None,
9415 }])
9416 .expect("update code");
9417 assert!(search(&client, "cholera").is_empty());
9418 assert_eq!(search(&client, "enteric").len(), 1);
9419
9420 client
9421 .mutate(vec![Mutation::Delete {
9422 table: "catalogue_codes".to_owned(),
9423 row_id: "c1".to_owned(),
9424 base_version: None,
9425 }])
9426 .expect("delete code");
9427 assert!(search(&client, "enteric").is_empty());
9428 }
9429
9430 #[test]
9431 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
9432 let schema = json!({
9433 "version": 1,
9434 "tables": [{
9435 "name": "patient_notes",
9436 "primaryKey": "id",
9437 "columns": [
9438 { "name": "id", "type": "string", "nullable": false },
9439 { "name": "practice_id", "type": "string", "nullable": false },
9440 { "name": "encryption_key_id", "type": "string", "nullable": false },
9441 { "name": "title", "type": "string", "nullable": false }
9442 ],
9443 "scopes": [{ "pattern": "practice:{practice_id}" }],
9444 "ftsIndexes": [{
9445 "name": "patient_notes_fts",
9446 "columns": ["title"],
9447 "tokenize": "unicode61 remove_diacritics 2"
9448 }]
9449 }]
9450 });
9451 let mut client = SyncClient::new(
9452 "local-purge-test".to_owned(),
9453 &schema,
9454 ClientLimits::default(),
9455 )
9456 .expect("local purge client");
9457 client
9458 .conn
9459 .execute(
9460 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
9461 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
9462 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
9463 [],
9464 )
9465 .expect("seed base rows");
9466 client.overlay_dirty.set(true);
9467 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
9468
9469 let note = |id: &str, key_id: &str, title: &str| {
9470 Map::from_iter([
9471 ("id".to_owned(), Value::from(id)),
9472 ("practice_id".to_owned(), Value::from("practice-1")),
9473 ("encryption_key_id".to_owned(), Value::from(key_id)),
9474 ("title".to_owned(), Value::from(title)),
9475 ])
9476 };
9477 let doomed = client
9478 .mutate(vec![
9479 Mutation::Upsert {
9480 table: "patient_notes".to_owned(),
9481 values: note("target", "key-revoked", "Target changed"),
9482 base_version: None,
9483 },
9484 Mutation::Upsert {
9485 table: "patient_notes".to_owned(),
9486 values: note("unrelated", "key-held", "Unrelated changed"),
9487 base_version: None,
9488 },
9489 ])
9490 .expect("doomed commit");
9491 let kept = client
9492 .mutate(vec![Mutation::Upsert {
9493 table: "patient_notes".to_owned(),
9494 values: note("kept", "key-held", "Kept optimistic"),
9495 base_version: None,
9496 }])
9497 .expect("kept commit");
9498 client.drain_change_batches();
9499
9500 let input = LocalDataPurgeInput {
9501 purge_id: "purge-001".to_owned(),
9502 targets: vec![LocalDataPurgeTarget {
9503 table: "patient_notes".to_owned(),
9504 selectors: BTreeMap::from([(
9505 "encryption_key_id".to_owned(),
9506 vec!["key-revoked".to_owned()],
9507 )]),
9508 }],
9509 };
9510 assert_eq!(
9511 client.purge_local_data(&input).expect("apply purge"),
9512 LocalDataPurgeResult {
9513 already_applied: false,
9514 purged_rows: 1,
9515 dropped_commits: 1,
9516 }
9517 );
9518 let rows = client
9519 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
9520 .expect("visible rows");
9521 assert_eq!(rows.len(), 2);
9522 assert_eq!(rows[0]["id"], "kept");
9523 assert_eq!(rows[1]["id"], "unrelated");
9524 assert_eq!(rows[1]["title"], "Unrelated original");
9525 let fts = client
9526 .query(
9527 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
9528 &[],
9529 )
9530 .expect("fts query");
9531 assert!(fts.is_empty());
9532 assert_eq!(client.outbox.len(), 1);
9533 assert_eq!(client.outbox[0].client_commit_id, kept);
9534 let outcome = client
9535 .commit_outcome(&doomed)
9536 .expect("read doomed outcome")
9537 .expect("doomed outcome");
9538 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
9539 match &outcome.results[0] {
9540 CommitOperationOutcome::Error { rejection } => {
9541 assert_eq!(rejection.code, "client.local_data_purged");
9542 assert_eq!(rejection.client_commit_id, doomed);
9543 }
9544 other => panic!("expected local purge rejection, got {other:?}"),
9545 }
9546 assert_eq!(client.drain_change_batches().len(), 1);
9547 assert_eq!(
9548 client.purge_local_data(&input).expect("retry purge"),
9549 LocalDataPurgeResult {
9550 already_applied: true,
9551 purged_rows: 0,
9552 dropped_commits: 0,
9553 }
9554 );
9555 let conflicting = LocalDataPurgeInput {
9556 purge_id: input.purge_id.clone(),
9557 targets: vec![LocalDataPurgeTarget {
9558 table: "patient_notes".to_owned(),
9559 selectors: BTreeMap::from([(
9560 "encryption_key_id".to_owned(),
9561 vec!["key-held".to_owned()],
9562 )]),
9563 }],
9564 };
9565 assert!(client
9566 .purge_local_data(&conflicting)
9567 .expect_err("id collision must fail")
9568 .contains("already used with a different plan"));
9569 }
9570}
9571
9572impl SubState {
9573 fn name(self) -> &'static str {
9574 match self {
9575 SubState::Active => "active",
9576 SubState::Revoked => "revoked",
9577 SubState::Failed => "failed",
9578 }
9579 }
9580
9581 fn parse(value: &str) -> Self {
9582 match value {
9583 "revoked" => Self::Revoked,
9584 "failed" => Self::Failed,
9585 _ => Self::Active,
9586 }
9587 }
9588}
9589
9590#[derive(Debug, Clone)]
9591struct Subscription {
9592 id: String,
9593 table: String,
9594 requested: Vec<(String, Vec<String>)>,
9595 params: Option<String>,
9596 cursor: i64,
9597 bootstrap_state: Option<String>,
9599 state: SubState,
9600 reason_code: Option<String>,
9601 effective: Option<Vec<(String, Vec<String>)>>,
9604 synced_once: bool,
9605}
9606
9607#[cfg(test)]
9608mod subscription_rearm_tests {
9609 use super::*;
9610 use serde_json::json;
9611
9612 fn client() -> SyncClient {
9613 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9614 {"name":"id","type":"string","nullable":false},
9615 {"name":"project_id","type":"string","nullable":false}],
9616 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[],"ftsIndexes":[]}]});
9617 SyncClient::new("rearm".into(), &schema, ClientLimits::default()).expect("client")
9618 }
9619
9620 fn scopes() -> Vec<(String, Vec<String>)> {
9621 vec![("project_id".to_owned(), vec!["p1".to_owned()])]
9622 }
9623
9624 fn store_revoked(client: &mut SyncClient) {
9626 client
9627 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9628 .expect("subscribe");
9629 let sub = &mut client.subs[0];
9630 sub.cursor = 41;
9631 sub.bootstrap_state = Some("resume".to_owned());
9632 sub.effective = Some(scopes());
9633 sub.state = SubState::Revoked;
9634 sub.reason_code = Some("sync.scope_revoked".to_owned());
9635 client.persist_sub(&client.subs[0]).expect("persist");
9636 client.sync_needed = false;
9637 client.sync_intent_queue.clear();
9638 }
9639
9640 #[test]
9641 fn resubscribing_the_same_intent_rearms_a_revoked_registration() {
9642 let mut client = client();
9643 store_revoked(&mut client);
9644
9645 client
9646 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9647 .expect("re-arm");
9648
9649 let view = client.subscription_state("sub").expect("state");
9650 assert_eq!(view.status, "active");
9651 assert_eq!(view.cursor, -1);
9652 assert!(!view.has_resume_token);
9653 assert_eq!(view.effective_scopes, None);
9654 assert_eq!(view.reason_code, None);
9655 assert!(client.sync_needed, "a re-arm asks for a pull");
9656 assert!(matches!(
9657 client.drain_sync_intents().as_slice(),
9658 [SyncIntent::Interactive]
9659 ));
9660 let stored: String = client
9661 .conn
9662 .query_row(
9663 "SELECT state_json FROM _syncular_subscriptions WHERE id = 'sub'",
9664 [],
9665 |row| row.get(0),
9666 )
9667 .expect("persisted row");
9668 let stored: Value = serde_json::from_str(&stored).expect("stored json");
9669 assert_eq!(stored["status"], json!("active"));
9670 assert_eq!(stored["cursor"], json!(-1));
9671 assert_eq!(stored["bootstrapState"], Value::Null);
9672 assert_eq!(stored["effectiveScopes"], Value::Null);
9673 assert_eq!(stored["reasonCode"], Value::Null);
9674 }
9675
9676 #[test]
9677 fn a_failed_registration_is_not_rearmed() {
9678 let mut client = client();
9679 client
9680 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9681 .expect("subscribe");
9682 client.subs[0].state = SubState::Failed;
9683 client.subs[0].reason_code = Some("sync.scope_revoked".to_owned());
9684 client.persist_sub(&client.subs[0]).expect("persist");
9685 client.sync_needed = false;
9686 client.sync_intent_queue.clear();
9687
9688 client
9689 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9690 .expect("subscribe");
9691
9692 let view = client.subscription_state("sub").expect("state");
9693 assert_eq!(view.status, "failed");
9694 assert_eq!(view.reason_code.as_deref(), Some("sync.scope_revoked"));
9695 assert!(!client.sync_needed);
9696 assert!(client.drain_sync_intents().is_empty());
9697 }
9698}
9699
9700#[derive(Debug, Clone)]
9701struct OutboxOp {
9702 upsert: bool,
9703 table: String,
9704 row_id: String,
9705 base_version: Option<i64>,
9706 values: Option<Map<String, Value>>,
9710}
9711
9712impl From<&OutboxOp> for CommitOperation {
9713 fn from(operation: &OutboxOp) -> Self {
9714 Self {
9715 table: operation.table.clone(),
9716 row_id: operation.row_id.clone(),
9717 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
9718 base_version: operation.base_version,
9719 values: operation.values.clone(),
9720 }
9721 }
9722}
9723
9724#[derive(Debug, Clone)]
9725struct OutboxCommit {
9726 client_commit_id: String,
9727 ops: Vec<OutboxOp>,
9728}
9729
9730fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>)> {
9734 commit
9735 .ops
9736 .iter()
9737 .filter(|operation| operation.upsert)
9738 .map(|operation| {
9739 (
9740 operation.table.as_str(),
9741 operation
9742 .values
9743 .as_ref()
9744 .map(|values| values.keys().map(String::as_str).collect())
9745 .unwrap_or_default(),
9746 )
9747 })
9748 .collect()
9749}
9750
9751type FailedCommit = (OutboxCommit, Map<String, Value>);
9752
9753#[derive(Debug, Clone)]
9754struct CompiledLocalDataPurgeTarget {
9755 table: String,
9756 selectors: Vec<(String, Vec<String>)>,
9757}
9758
9759struct StoredCommitOutcomeRow {
9760 sequence: i64,
9761 client_commit_id: String,
9762 status: String,
9763 recorded_at_ms: i64,
9764 results_json: String,
9765 operations_json: Option<String>,
9766 resolution: String,
9767 resolved_at_ms: Option<i64>,
9768 replacement_client_commit_id: Option<String>,
9769}
9770
9771impl StoredCommitOutcomeRow {
9772 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
9773 Ok(Self {
9774 sequence: row.get(0)?,
9775 client_commit_id: row.get(1)?,
9776 status: row.get(2)?,
9777 recorded_at_ms: row.get(3)?,
9778 results_json: row.get(4)?,
9779 operations_json: row.get(5)?,
9780 resolution: row.get(6)?,
9781 resolved_at_ms: row.get(7)?,
9782 replacement_client_commit_id: row.get(8)?,
9783 })
9784 }
9785}
9786
9787enum SectionError {
9790 FailClosed,
9791 Abort(String, String),
9792}
9793
9794#[derive(Debug)]
9795pub(crate) struct RequestMeta {
9796 pushed_ids: Vec<String>,
9797 fresh: Vec<(String, bool)>,
9800 pub(crate) accept: u8,
9801 deferred_commits: usize,
9805}
9806
9807#[derive(Default)]
9808struct ChangeAccumulator {
9809 tables: BTreeMap<String, Option<BTreeSet<String>>>,
9811 windows: BTreeMap<(String, String), BTreeSet<String>>,
9812 status: bool,
9813 conflicts: bool,
9814 rejections: bool,
9815 outcomes: bool,
9816}
9817
9818impl ChangeAccumulator {
9819 fn table(&mut self, table: &str) {
9820 self.tables.insert(table.to_owned(), None);
9821 }
9822
9823 fn scope(&mut self, table: &str, key: String) {
9824 match self.tables.get_mut(table) {
9825 Some(None) => {}
9826 Some(Some(keys)) => {
9827 keys.insert(key);
9828 }
9829 None => {
9830 self.tables
9831 .insert(table.to_owned(), Some(BTreeSet::from([key])));
9832 }
9833 }
9834 }
9835
9836 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
9837 self.windows
9838 .entry((base_key.to_owned(), table.to_owned()))
9839 .or_default()
9840 .insert(unit.to_owned());
9841 }
9842
9843 fn touched(&self) -> bool {
9844 !self.tables.is_empty()
9845 || !self.windows.is_empty()
9846 || self.status
9847 || self.conflicts
9848 || self.rejections
9849 || self.outcomes
9850 }
9851}
9852
9853const PUSH_OPS_PER_REQUEST: usize = 500;
9859const MAX_LOCAL_PURGE_TARGETS: usize = 64;
9860const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
9861const MAX_LOCAL_PURGE_VALUES: usize = 128;
9862const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
9863pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
9864
9865struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
9867
9868impl OverlayDirty {
9869 fn clean() -> Self {
9870 Self(RefCell::new(Some(BTreeSet::new())))
9871 }
9872 fn get(&self) -> bool {
9873 self.0
9874 .borrow()
9875 .as_ref()
9876 .is_none_or(|tables| !tables.is_empty())
9877 }
9878 fn set(&self, dirty: bool) {
9879 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
9880 }
9881 fn table(&self, table: &str) {
9882 if let Some(tables) = self.0.borrow_mut().as_mut() {
9883 tables.insert(table.to_owned());
9884 }
9885 }
9886 fn snapshot(&self) -> Option<BTreeSet<String>> {
9887 self.0.borrow().clone()
9888 }
9889 fn restore(&self, tables: Option<BTreeSet<String>>) {
9890 *self.0.borrow_mut() = tables;
9891 }
9892}
9893
9894pub struct SyncClient {
9895 #[cfg(feature = "bench-internals")]
9896 benchmark_phases: Recorder,
9897 conn: Connection,
9898 schema: ClientSchema,
9899 client_id: String,
9900 limits: ClientLimits,
9901 subs: Vec<Subscription>,
9902 outbox: Vec<OutboxCommit>,
9903 failed_commits: Vec<FailedCommit>,
9904 retain_failed_commits: bool,
9905 conflicts: Vec<ConflictRecord>,
9906 rejections: Vec<RejectionRecord>,
9907 schema_floor: Option<SchemaFloor>,
9908 lease_state: Option<LeaseState>,
9910 stopped: bool,
9912 upgrading: bool,
9914 sync_needed: bool,
9916 realtime_policy: RealtimePolicy,
9919 realtime_state: RealtimeState,
9920 realtime_reason_code: Option<String>,
9921 realtime_retry_delay_ms: Option<u64>,
9922 presence: HashMap<String, HashMap<String, PresencePeer>>,
9924 now_ms: Option<i64>,
9927 encryption: crate::values::EncryptionConfig,
9931 security_preflight: bool,
9934 transport_enabled: bool,
9935 previous_version: Option<PreviousVersionContextConfig>,
9939 insert_sql: RefCell<HashMap<String, String>>,
9945 overlay_dirty: OverlayDirty,
9948 storage_failure: RefCell<Option<QueryReadFailure>>,
9950 #[cfg(test)]
9953 overlay_rebuild_count: Cell<usize>,
9954 #[cfg(test)]
9955 acknowledged_replay_count: Cell<usize>,
9956 #[cfg(test)]
9959 outcome_prune_count: Cell<usize>,
9960 progress: ProgressObserver,
9962 change_queue: VecDeque<ClientChangeBatch>,
9963 sync_intent_queue: VecDeque<SyncIntent>,
9964 retry_delay_ms: u64,
9966 round_retry_delay_ms: Option<u64>,
9968 active_round: Option<uuid::Uuid>,
9969 last_round: Option<DiagnosticLastRound>,
9970 last_change: Option<DiagnosticLastChange>,
9971 query_failures: Vec<DiagnosticQueryFailure>,
9973 coalesced: Option<CoalescedApply>,
9975 coalesced_failure: Option<String>,
9978}
9979
9980struct CoalescedApply {
9984 rows: usize,
9985 subs: Vec<Subscription>,
9986 outbox: Vec<OutboxCommit>,
9987 failed_commits: Vec<FailedCommit>,
9988 conflicts: Vec<ConflictRecord>,
9989 rejections: Vec<RejectionRecord>,
9990 schema_floor: Option<SchemaFloor>,
9991 lease_state: Option<LeaseState>,
9992 stopped: bool,
9993 upgrading: bool,
9994 sync_needed: bool,
9995 overlay_dirty: Option<BTreeSet<String>>,
9996 change_queue: usize,
9997 sync_intent_queue: usize,
9998 last_change: Option<DiagnosticLastChange>,
9999}
10000
10001const COALESCED_APPLY_ROWS: usize = 1024;
10002
10003pub(crate) fn quote_ident(name: &str) -> String {
10004 format!("\"{}\"", name.replace('"', "\"\""))
10005}
10006pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
10010 conn.query_row(
10011 "SELECT value FROM _syncular_meta WHERE key = ?1",
10012 rusqlite::params![key],
10013 |row| row.get::<_, String>(0),
10014 )
10015 .ok()
10016}
10017
10018pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
10019 let _ = conn.execute(
10020 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
10021 rusqlite::params![key, value],
10022 );
10023}
10024
10025pub(crate) fn meta_delete(conn: &Connection, key: &str) {
10026 let _ = conn.execute(
10027 "DELETE FROM _syncular_meta WHERE key = ?1",
10028 rusqlite::params![key],
10029 );
10030}
10031
10032fn row_id_predicate(table: &TableSchema) -> String {
10034 let key = quote_ident(&table.primary_key);
10035 let text_match = format!("CAST({key} AS TEXT) = ?1");
10036 match table.columns[table.pk_index].ty {
10037 ColumnType::String | ColumnType::Json => {
10038 format!("{key} = ?1 AND {text_match}")
10039 }
10040 ColumnType::Integer | ColumnType::Boolean => {
10044 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
10045 }
10046 _ => text_match,
10049 }
10050}
10051
10052fn is_local_operation_code_like(value: &str) -> bool {
10053 let bytes = value.as_bytes();
10054 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
10055 && bytes
10056 .iter()
10057 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
10058}
10059
10060fn base_table(name: &str) -> String {
10061 quote_ident(&format!("_syncular_base_{name}"))
10062}
10063
10064type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
10069
10070fn window_base_key(base: &WindowBase) -> String {
10071 format!(
10072 "{}\0{}\0{}",
10073 base.table,
10074 base.variable,
10075 canonical_scope_json(&base.fixed_scopes)
10076 )
10077}
10078
10079fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
10081 let mut scopes = base.fixed_scopes.clone();
10082 scopes.retain(|(k, _)| k != &base.variable);
10083 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
10084 scopes
10085}
10086
10087fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
10091 let canonical = canonical_scope_json(&unit_scopes(base, unit));
10092 let digest = Sha256::digest(canonical.as_bytes());
10093 let hex = bytes_to_hex(&digest);
10094 format!("w:{}:{}", base.table, &hex[..16])
10095}
10096
10097fn visible_table(name: &str) -> String {
10098 quote_ident(name)
10099}
10100
10101const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
10102
10103fn is_synced_table_name(name: &str) -> bool {
10106 if name.starts_with("sqlite_") {
10107 return false;
10108 }
10109 if name.starts_with("_syncular_base_") {
10110 return true; }
10112 !name.starts_with("_syncular_")
10114}
10115
10116fn blob_id_for(bytes: &[u8]) -> String {
10118 let digest = Sha256::digest(bytes);
10119 format!("sha256:{}", bytes_to_hex(&digest))
10120}
10121
10122enum RowParam<'a> {
10126 Cell(&'a Option<ColumnValue>),
10127 Version(i64),
10128}
10129
10130impl rusqlite::ToSql for RowParam<'_> {
10131 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
10132 Ok(match self {
10133 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
10134 RowParam::Cell(cell) => match cell {
10135 None => ToSqlOutput::Owned(SqlValue::Null),
10136 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
10137 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
10138 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
10139 Some(ColumnValue::Boolean(b)) => {
10140 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
10141 }
10142 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
10143 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
10144 }
10145 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
10147 ToSqlOutput::Borrowed(ValueRef::Blob(b))
10148 }
10149 },
10150 })
10151 }
10152}
10153
10154fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
10156 match param.to_sql().map_err(|error| error.to_string())? {
10157 ToSqlOutput::Owned(value) => Ok(value),
10158 ToSqlOutput::Borrowed(value) => {
10159 SqlValue::try_from(value).map_err(|error| error.to_string())
10160 }
10161 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
10162 }
10163}
10164
10165fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
10173 use ssp2::segment::ColumnType;
10174 let mismatch = || {
10175 Err(format!(
10176 "image column {:?} holds a value of the wrong type",
10177 column.name
10178 ))
10179 };
10180 match value {
10181 ValueRef::Null => {
10182 if !column.nullable {
10183 return Err(format!(
10184 "image column {:?} is NULL but not nullable",
10185 column.name
10186 ));
10187 }
10188 Ok(ToSqlOutput::Owned(SqlValue::Null))
10189 }
10190 ValueRef::Integer(i) => match column.ty {
10191 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
10192 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
10193 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
10194 _ => mismatch(),
10195 },
10196 ValueRef::Real(_) => match column.ty {
10197 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
10198 _ => mismatch(),
10199 },
10200 ValueRef::Text(t) => {
10201 std::str::from_utf8(t)
10202 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
10203 match column.ty {
10204 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
10205 Ok(ToSqlOutput::Borrowed(value))
10206 }
10207 _ => mismatch(),
10208 }
10209 }
10210 ValueRef::Blob(_) => match column.ty {
10211 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
10213 _ => mismatch(),
10214 },
10215 }
10216}
10217
10218pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
10219 use rusqlite::types::ValueRef;
10220 match value {
10221 ValueRef::Null => Value::Null,
10222 ValueRef::Integer(i) => match column.ty {
10223 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
10224 ssp2::segment::ColumnType::Float => {
10225 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
10226 }
10227 _ => Value::from(i),
10228 },
10229 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10230 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10231 ValueRef::Blob(b) => {
10232 let mut map = Map::new();
10233 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10234 Value::Object(map)
10235 }
10236 }
10237}
10238
10239fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
10243 Ok(match value {
10244 Value::Null => SqlValue::Null,
10245 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
10246 Value::Number(n) => {
10247 if let Some(i) = n.as_i64() {
10248 SqlValue::Integer(i)
10249 } else if let Some(f) = n.as_f64() {
10250 SqlValue::Real(f)
10251 } else {
10252 return Err(format!("query param number {n} is out of range"));
10253 }
10254 }
10255 Value::String(s) => SqlValue::Text(s.clone()),
10256 Value::Object(_) => {
10257 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
10258 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
10259 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
10260 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
10261 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
10262 })?)
10263 } else {
10264 return Err(
10265 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
10266 .to_owned(),
10267 );
10268 }
10269 }
10270 Value::Array(_) => return Err("query array params are not supported".to_owned()),
10271 })
10272}
10273
10274fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
10279 use rusqlite::types::ValueRef;
10280 match value {
10281 ValueRef::Null => Value::Null,
10282 ValueRef::Integer(i) => {
10283 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
10287 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
10288 Value::from(i)
10289 } else {
10290 let mut map = Map::new();
10291 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
10292 Value::Object(map)
10293 }
10294 }
10295 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10296 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10297 ValueRef::Blob(b) => {
10298 let mut map = Map::new();
10299 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10300 Value::Object(map)
10301 }
10302 }
10303}
10304
10305fn query_connection(
10307 conn: &Connection,
10308 sql: &str,
10309 params: &[QueryValue],
10310) -> Result<Vec<QueryRow>, QueryReadFailure> {
10311 crate::query_guard::assert_read_only_query(sql)?;
10312 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
10313 let bound: Vec<SqlValue> = params
10314 .iter()
10315 .map(json_param_to_sql)
10316 .collect::<Result<_, _>>()?;
10317 let mut stmt = conn.prepare(&lowered_sql)?;
10318 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
10319 let bound_refs: Vec<&dyn rusqlite::ToSql> =
10320 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
10321 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
10322 let mut out = Vec::new();
10323 while let Some(row) = sql_rows.next()? {
10324 let mut record = Map::new();
10325 for (i, name) in column_names.iter().enumerate() {
10326 let value = row.get_ref(i)?;
10327 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
10328 }
10329 out.push(record);
10330 }
10331 Ok(out)
10332}
10333
10334fn persisted_window_state(
10335 conn: &Connection,
10336 base: &WindowBase,
10337) -> Result<WindowState, QueryReadFailure> {
10338 let mut stmt = conn.prepare(
10339 "SELECT windows.unit, subscriptions.state_json
10340 FROM _syncular_windows AS windows
10341 JOIN _syncular_subscriptions AS subscriptions
10342 ON subscriptions.id = windows.sub_id
10343 WHERE windows.base = ?1
10344 ORDER BY windows.unit ASC",
10345 )?;
10346 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
10347 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
10348 })?;
10349 let mut units = Vec::new();
10350 let mut pending = Vec::new();
10351 for row in rows {
10352 let (unit, raw) = row?;
10353 let state: Value = serde_json::from_str(&raw)
10354 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
10355 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
10356 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
10357 || state
10358 .get("bootstrapState")
10359 .is_some_and(|value| !value.is_null());
10360 if is_pending {
10361 pending.push(unit.clone());
10362 }
10363 units.push(unit);
10364 }
10365 Ok(WindowState { units, pending })
10366}
10367
10368#[derive(Debug, Clone, Copy)]
10370pub struct SnapshotStatement<'a> {
10371 pub sql: &'a str,
10372 pub params: &'a [Value],
10373}
10374
10375#[derive(Debug, Clone, Default)]
10379pub struct SnapshotReadRequest<'a> {
10380 pub statements: Vec<SnapshotStatement<'a>>,
10381 pub coverage: &'a [WindowCoverage],
10382 pub subscriptions: Vec<String>,
10383 pub commit_ids: Vec<String>,
10384}
10385
10386#[cfg(test)]
10387thread_local! {
10388 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
10392 std::cell::RefCell::new(None);
10393}
10394
10395#[cfg(test)]
10396fn run_snapshot_read_hook() {
10397 SNAPSHOT_READ_HOOK.with(|slot| {
10398 if let Some(hook) = slot.borrow_mut().take() {
10399 hook();
10400 }
10401 });
10402}
10403
10404fn snapshot_read_connection(
10409 conn: &Connection,
10410 request: &SnapshotReadRequest<'_>,
10411) -> Result<SnapshotRead, QueryReadFailure> {
10412 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
10413 let corrupt = |message: &str| QueryReadFailure {
10414 code: Some("sync.local_corrupt"),
10415 sqlite_code: None,
10416 sqlite_message: None,
10417 rollback_failure: None,
10418 message: message.to_owned(),
10419 };
10420 let stored_failure = |error: rusqlite::Error, message: &str| match error {
10421 rusqlite::Error::Utf8Error(..)
10422 | rusqlite::Error::InvalidColumnType(..)
10423 | rusqlite::Error::FromSqlConversionFailure(..)
10424 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
10425 _ => QueryReadFailure::from(error),
10426 };
10427 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
10428 let revision: Option<String> = conn
10431 .query_row(
10432 "SELECT value FROM _syncular_meta WHERE key = ?1",
10433 rusqlite::params![LOCAL_REVISION_KEY],
10434 |row| row.get(0),
10435 )
10436 .optional()
10437 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
10438 let revision = match revision {
10439 None => "0".to_owned(),
10440 Some(value)
10441 if value == "0"
10442 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10443 && value.bytes().all(|b| b.is_ascii_digit())
10444 && value.parse::<u64>().is_ok()) =>
10445 {
10446 value
10447 }
10448 Some(_) => return Err(corrupt("persisted local revision is invalid")),
10449 };
10450 let mut queries = Vec::with_capacity(request.statements.len());
10451 for statement in &request.statements {
10452 queries.push(query_connection(conn, statement.sql, statement.params)?);
10453 #[cfg(test)]
10454 run_snapshot_read_hook();
10455 }
10456 let mut pending = Vec::new();
10457 let mut missing = Vec::new();
10458 for requested in request.coverage {
10459 let base_key = window_base_key(&requested.base);
10460 let state = persisted_window_state(conn, &requested.base)?;
10461 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
10462 let reference = WindowUnitRef {
10463 base_key: base_key.clone(),
10464 unit: unit.clone(),
10465 };
10466 if !state.units.iter().any(|held| held == &unit) {
10467 missing.push(reference);
10468 } else if state.pending.iter().any(|held| held == &unit) {
10469 pending.push(reference);
10470 }
10471 }
10472 }
10473 let mut subscriptions = Vec::with_capacity(request.subscriptions.len());
10474 for id in &request.subscriptions {
10475 let row: Option<(String, String)> = conn
10476 .query_row(
10477 "SELECT tbl, state_json FROM _syncular_subscriptions WHERE id = ?1",
10478 rusqlite::params![id],
10479 |row| Ok((row.get(0)?, row.get(1)?)),
10480 )
10481 .optional()
10482 .map_err(|error| {
10483 stored_failure(error, "persisted subscription state is invalid")
10484 })?;
10485 let Some((table, raw)) = row else {
10486 subscriptions.push(SubscriptionCatchup::Unknown { id: id.clone() });
10487 continue;
10488 };
10489 let state: Value = serde_json::from_str(&raw)
10492 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10493 let status = match state.get("status").and_then(Value::as_str) {
10494 Some(value @ ("active" | "revoked" | "failed")) => value,
10495 _ => return Err(corrupt("persisted subscription state is invalid")),
10496 };
10497 let Some(cursor) = state.get("cursor").and_then(Value::as_i64) else {
10498 return Err(corrupt("persisted subscription state is invalid"));
10499 };
10500 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
10501 return Err(corrupt("persisted subscription state is invalid"));
10502 }
10503 let effective_scopes = state
10504 .get("effectiveScopes")
10505 .filter(|value| !value.is_null());
10506 if let Some(scopes) = effective_scopes {
10507 json_to_scope_map(scopes)
10508 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10509 }
10510 let has_resume_token = match state.get("bootstrapState") {
10511 None | Some(Value::Null) => false,
10512 Some(Value::String(_)) => true,
10513 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10514 };
10515 let reason_code = match state.get("reasonCode") {
10518 None | Some(Value::Null) => None,
10519 Some(Value::String(value)) => Some(value.clone()),
10520 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10521 };
10522 let active = status == "active";
10523 subscriptions.push(SubscriptionCatchup::Known {
10524 id: id.clone(),
10525 table,
10526 status: status.to_owned(),
10527 cursor,
10528 has_resume_token,
10529 bootstrap_complete: active && cursor >= 0 && !has_resume_token,
10530 known_pending_pages: active && (has_resume_token || cursor < 0),
10531 effective_scopes: effective_scopes.cloned(),
10532 reason_code,
10533 });
10534 }
10535 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
10536 for id in &request.commit_ids {
10537 let pending: bool = conn.query_row(
10538 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
10539 rusqlite::params![id],
10540 |row| row.get(0),
10541 )?;
10542 if pending {
10543 deliveries.push(CommitDelivery::Pending {
10544 client_commit_id: id.clone(),
10545 });
10546 continue;
10547 }
10548 let row = conn
10549 .query_row(
10550 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
10551 resolution, resolved_at_ms, replacement_client_commit_id
10552 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
10553 rusqlite::params![id],
10554 StoredCommitOutcomeRow::from_row,
10555 )
10556 .optional()
10557 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
10558 deliveries.push(match row {
10559 Some(row) => CommitDelivery::Known {
10560 client_commit_id: id.clone(),
10561 outcome: SyncClient::outcome_from_row(row)
10562 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
10563 },
10564 None => CommitDelivery::Unknown {
10565 client_commit_id: id.clone(),
10566 },
10567 });
10568 }
10569 Ok(SnapshotRead {
10570 revision,
10571 queries,
10572 coverage: CoverageSnapshot {
10573 complete: pending.is_empty() && missing.is_empty(),
10574 pending,
10575 missing,
10576 },
10577 subscriptions,
10578 deliveries,
10579 })
10580 })();
10581 match result {
10582 Ok(read) => {
10583 conn.execute_batch("RELEASE syncular_snapshot_read")?;
10584 Ok(read)
10585 }
10586 Err(mut error) => {
10587 if let Err(rollback) = conn
10588 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
10589 {
10590 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
10591 }
10592 Err(error)
10593 }
10594 }
10595}
10596
10597fn snapshot_connection(
10598 conn: &Connection,
10599 sql: &str,
10600 params: &[Value],
10601 coverage: &[WindowCoverage],
10602) -> Result<QuerySnapshot, QueryReadFailure> {
10603 let read = snapshot_read_connection(
10604 conn,
10605 &SnapshotReadRequest {
10606 statements: vec![SnapshotStatement { sql, params }],
10607 coverage,
10608 subscriptions: Vec::new(),
10609 commit_ids: Vec::new(),
10610 },
10611 )?;
10612 Ok(QuerySnapshot {
10613 revision: read.revision,
10614 rows: read
10615 .queries
10616 .into_iter()
10617 .next()
10618 .expect("single-query snapshot returns one result"),
10619 coverage: read.coverage,
10620 })
10621}
10622
10623pub struct FileQuerySnapshotReader {
10628 path: String,
10629 conn: Option<Connection>,
10630}
10631
10632impl FileQuerySnapshotReader {
10633 #[must_use]
10634 pub fn new(path: impl Into<String>) -> Self {
10635 Self {
10636 path: path.into(),
10637 conn: None,
10638 }
10639 }
10640
10641 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
10642 if self.conn.is_none() {
10643 let conn = Connection::open_with_flags(
10644 &self.path,
10645 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
10646 )
10647 .map_err(QueryReadFailure::from)?;
10648 conn.busy_timeout(std::time::Duration::from_millis(250))
10649 .map_err(QueryReadFailure::from)?;
10650 self.conn = Some(conn);
10651 }
10652 self.conn
10653 .as_ref()
10654 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
10655 }
10656
10657 pub fn snapshot_read(
10660 &mut self,
10661 request: &SnapshotReadRequest<'_>,
10662 ) -> Result<SnapshotRead, QueryReadFailure> {
10663 snapshot_read_connection(self.connection()?, request)
10664 }
10665
10666 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
10669 Ok(self
10670 .snapshot_read(&SnapshotReadRequest::default())?
10671 .revision)
10672 }
10673
10674 pub fn subscription_catchup(
10676 &mut self,
10677 id: &str,
10678 ) -> Result<SubscriptionCatchup, QueryReadFailure> {
10679 let read = self.snapshot_read(&SnapshotReadRequest {
10680 subscriptions: vec![id.to_owned()],
10681 ..Default::default()
10682 })?;
10683 Ok(read
10684 .subscriptions
10685 .into_iter()
10686 .next()
10687 .expect("one requested catch-up"))
10688 }
10689
10690 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
10693 let read = self.snapshot_read(&SnapshotReadRequest {
10694 commit_ids: vec![id.to_owned()],
10695 ..Default::default()
10696 })?;
10697 Ok(read
10698 .deliveries
10699 .into_iter()
10700 .next()
10701 .expect("one requested delivery"))
10702 }
10703
10704 pub fn query_snapshot(
10707 &mut self,
10708 sql: &str,
10709 params: &[Value],
10710 coverage: &[WindowCoverage],
10711 ) -> Result<QuerySnapshot, QueryReadFailure> {
10712 snapshot_connection(self.connection()?, sql, params, coverage)
10713 }
10714}
10715
10716fn validate_authority_reads(
10717 schema: &ClientSchema,
10718 declarations: &[crate::api::AuthorityReadDeclaration],
10719) -> Result<(), String> {
10720 let mut seen = BTreeSet::new();
10721 for read in declarations {
10722 let table = schema
10723 .table(&read.table)
10724 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
10725 let plain = |name: &str| {
10726 table.columns.iter().enumerate().any(|(i, c)| {
10727 c.name == name
10728 && name != "authority_version"
10729 && !name.starts_with("_sync")
10730 && !table.encrypted_columns.iter().any(|e| e.index == i)
10731 && matches!(
10732 c.ty,
10733 ssp2::segment::ColumnType::String
10734 | ssp2::segment::ColumnType::Integer
10735 | ssp2::segment::ColumnType::Float
10736 | ssp2::segment::ColumnType::Boolean
10737 | ssp2::segment::ColumnType::Json
10738 )
10739 })
10740 };
10741 if !seen.insert(&read.table)
10742 || read.columns.is_empty()
10743 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
10744 || !read.columns.contains(&table.primary_key)
10745 || !read.columns.iter().all(|c| plain(c))
10746 || read.scopes.is_empty()
10747 || !read.scopes.iter().all(|(v, values)| {
10748 table
10749 .scope_column(v)
10750 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
10751 && !values.is_empty()
10752 && values.iter().all(|v| !v.is_empty() && v != "*")
10753 })
10754 {
10755 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
10756 }
10757 }
10758 Ok(())
10759}
10760
10761fn schema_marker_read_failure(error: rusqlite::Error) -> String {
10766 let failure = QueryReadFailure::from(error);
10767 if failure.code.is_some() {
10768 failure.to_string()
10769 } else {
10770 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
10771 }
10772}
10773
10774fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
10778 if limits.outcome_retention_max_entries == Some(0) {
10779 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
10780 }
10781 if [
10782 limits.max_push_commits_per_request,
10783 limits.max_push_operations_per_request,
10784 limits.max_push_request_bytes,
10785 ]
10786 .into_iter()
10787 .flatten()
10788 .any(|value| value == 0 || value > u32::MAX as usize)
10789 {
10790 return Err(
10791 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
10792 );
10793 }
10794 Ok(())
10795}
10796
10797fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
10798 let corrupt =
10799 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
10800 let table_type: Option<String> = conn
10801 .query_row(
10802 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
10803 [],
10804 |row| row.get(0),
10805 )
10806 .optional()
10807 .map_err(schema_marker_read_failure)?;
10808 let Some(table_type) = table_type else {
10809 return Ok(None);
10810 };
10811 if table_type != "table" {
10812 return Err(corrupt());
10813 }
10814 let mut statement = conn
10818 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
10819 .map_err(schema_marker_read_failure)?;
10820 let mut rows = statement
10821 .query([LOCAL_SCHEMA_VERSION_KEY])
10822 .map_err(schema_marker_read_failure)?;
10823 let value: Option<String> = rows
10824 .next()
10825 .map_err(schema_marker_read_failure)?
10826 .map(|row| row.get(0))
10827 .transpose()
10828 .map_err(schema_marker_read_failure)?;
10829 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
10830 return Err(corrupt());
10831 }
10832 let Some(value) = value else {
10833 let has_descriptor = conn
10834 .query_row(
10835 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
10836 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
10837 |_| Ok(()),
10838 )
10839 .optional()
10840 .map_err(schema_marker_read_failure)?
10841 .is_some();
10842 if has_descriptor {
10843 return Err(corrupt());
10844 }
10845 return Ok(None);
10846 };
10847 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10848 || !value.bytes().all(|b| b.is_ascii_digit())
10849 {
10850 return Err(corrupt());
10851 }
10852 let version = value.parse::<i32>().map_err(|_| corrupt())?;
10853 if version > requested {
10854 return Err(
10855 "client.schema_downgrade: persisted local schema is newer than the requested schema"
10856 .to_owned(),
10857 );
10858 }
10859 Ok(Some(version))
10860}
10861
10862impl SyncClient {
10863 pub fn new_with_identity(
10864 client_id: Option<String>,
10865 schema_json: &Value,
10866 limits: ClientLimits,
10867 ) -> Result<Self, String> {
10868 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
10869 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
10870 Self::with_connection(resolved, schema_json, limits, conn)
10871 }
10872
10873 pub fn new(
10874 client_id: String,
10875 schema_json: &Value,
10876 limits: ClientLimits,
10877 ) -> Result<Self, String> {
10878 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
10879 Self::with_connection(client_id, schema_json, limits, conn)
10880 }
10881
10882 pub fn open_path(
10888 client_id: String,
10889 schema_json: &Value,
10890 limits: ClientLimits,
10891 path: &str,
10892 ) -> Result<Self, String> {
10893 validate_client_limits(&limits)?;
10894 parse_schema_json(schema_json)?;
10895 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
10896 Self::with_connection(client_id, schema_json, limits, conn)
10897 }
10898
10899 pub fn open_path_with_identity(
10900 client_id: Option<String>,
10901 schema_json: &Value,
10902 limits: ClientLimits,
10903 path: &str,
10904 ) -> Result<Self, String> {
10905 validate_client_limits(&limits)?;
10906 parse_schema_json(schema_json)?;
10909 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
10910 conn.busy_timeout(std::time::Duration::from_millis(250))
10918 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
10919 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
10920 let journal_mode: String = client
10921 .conn
10922 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
10923 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
10924 if journal_mode != "wal" {
10925 return Err(
10926 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
10927 );
10928 }
10929 Ok(client)
10930 }
10931
10932 #[must_use]
10933 pub fn client_id(&self) -> &str {
10934 &self.client_id
10935 }
10936
10937 pub fn with_connection(
10944 client_id: String,
10945 schema_json: &Value,
10946 limits: ClientLimits,
10947 conn: Connection,
10948 ) -> Result<Self, String> {
10949 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
10950 }
10951
10952 fn with_connection_identity(
10955 client_id: Option<String>,
10956 schema_json: &Value,
10957 limits: ClientLimits,
10958 conn: Connection,
10959 ) -> Result<Self, String> {
10960 validate_client_limits(&limits)?;
10961 let schema = parse_schema_json(schema_json)?;
10962 validate_authority_reads(&schema, &limits.authority_reads)?;
10963 let previous_version = match limits.previous_version_context {
10966 Some(config) => {
10967 config.validate()?;
10968 Some(config)
10969 }
10970 None => None,
10971 };
10972 let mut client = SyncClient {
10973 #[cfg(feature = "bench-internals")]
10974 benchmark_phases: Recorder::default(),
10975 conn,
10976 schema,
10977 client_id: client_id
10978 .clone()
10979 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
10980 limits,
10981 subs: Vec::new(),
10982 outbox: Vec::new(),
10983 failed_commits: Vec::new(),
10984 retain_failed_commits: false,
10985 conflicts: Vec::new(),
10986 rejections: Vec::new(),
10987 schema_floor: None,
10988 lease_state: None,
10989 stopped: false,
10990 upgrading: false,
10991 sync_needed: false,
10992 realtime_policy: RealtimePolicy::Optional,
10993 realtime_state: RealtimeState::Disconnected,
10994 realtime_reason_code: None,
10995 realtime_retry_delay_ms: None,
10996 presence: HashMap::new(),
10997 now_ms: None,
10998 encryption: crate::values::EncryptionConfig::default(),
10999 security_preflight: false,
11000 transport_enabled: true,
11001 previous_version,
11002 insert_sql: RefCell::new(HashMap::new()),
11003 overlay_dirty: OverlayDirty::clean(),
11004 storage_failure: RefCell::new(None),
11005 #[cfg(test)]
11006 overlay_rebuild_count: Cell::new(0),
11007 #[cfg(test)]
11008 acknowledged_replay_count: Cell::new(0),
11009 #[cfg(test)]
11010 outcome_prune_count: Cell::new(0),
11011 progress: ProgressObserver::default(),
11012 change_queue: VecDeque::new(),
11013 sync_intent_queue: VecDeque::new(),
11014 retry_delay_ms: 250,
11015 round_retry_delay_ms: None,
11016 active_round: None,
11017 last_round: None,
11018 last_change: None,
11019 query_failures: Vec::new(),
11020 coalesced: None,
11021 coalesced_failure: None,
11022 };
11023 client
11027 .conn
11028 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
11029 client
11035 .conn
11036 .execute_batch("BEGIN IMMEDIATE")
11037 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
11038 let startup = (|| -> Result<(), String> {
11039 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
11042 client.create_bookkeeping_tables()?;
11045 match client.get_meta(CLIENT_ID_KEY) {
11046 Some(existing) => match client_id {
11047 Some(requested) if existing != requested => {
11048 return Err(format!(
11049 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
11050 ))
11051 }
11052 _ => client.client_id = existing,
11053 },
11054 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
11055 }
11056 client.restore_persisted_state()?;
11057 client.failed_commits = client.load_failed_commits()?;
11058 match marker {
11059 None => {
11060 client.create_synced_tables()?;
11061 client.persist_schema_version()?;
11062 client.save_subscription_scope_schema();
11063 }
11064 Some(version) if version == client.schema.version => {
11065 client.create_synced_tables()?;
11066 set_local_schema_descriptor(&client.conn, &client.schema)
11070 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
11071 client.save_subscription_scope_schema();
11072 }
11073 Some(_) => client.run_schema_reset()?,
11074 }
11075 client.reconcile_previous_version_at_boot()?;
11081 client.clear_satisfied_persisted_schema_floor();
11082 client.prune_unknown_subscriptions(false)?;
11083 client.drop_unencodable_outbox(false, None)?;
11092 if marker == Some(client.schema.version)
11093 && (!client.outbox.is_empty()
11094 || !client.failed_commits.is_empty()
11095 || client.has_acknowledged_rows())
11096 {
11097 client.overlay_dirty.set(true);
11101 client.rebuild_overlay()?;
11102 }
11103 client.enqueue_startup_sync_if_needed()?;
11110 Ok(())
11111 })();
11112 match startup {
11113 Ok(()) => match client.conn.execute_batch("COMMIT") {
11114 Ok(()) => Ok(client),
11115 Err(error) => {
11116 let failure = Self::sqlite_failure(&client.storage_failure, error);
11117 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
11118 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
11120 }
11121 Err(failure)
11122 }
11123 },
11124 Err(error) => {
11125 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
11126 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
11128 }
11129 Err(error)
11130 }
11131 }
11132 }
11133
11134 pub fn set_now_ms(&mut self, now_ms: i64) {
11137 self.now_ms = Some(now_ms);
11138 }
11139
11140 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
11144 self.encryption = encryption;
11145 }
11146
11147 #[must_use]
11148 pub fn security_lifecycle(&self) -> &'static str {
11149 if self.security_preflight {
11150 "preflight"
11151 } else {
11152 "active"
11153 }
11154 }
11155
11156 #[must_use]
11157 pub fn security_preflight(&self) -> bool {
11158 self.security_preflight
11159 }
11160
11161 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
11163 if self.transport_enabled != enabled {
11164 self.transport_enabled = enabled;
11165 self.sync_intent_queue.clear();
11166 self.sync_intent_queue
11167 .push_back(if enabled && !self.security_preflight {
11168 SyncIntent::Interactive
11169 } else {
11170 SyncIntent::None
11171 });
11172 }
11173 if !enabled && self.active_round.is_none() {
11175 self.disconnect_realtime(transport);
11176 }
11177 }
11178
11179 pub fn transport_enabled(&self) -> bool {
11180 self.transport_enabled
11181 }
11182
11183 pub fn begin_security_preflight(&mut self) {
11190 self.seal_security_on_teardown();
11191 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
11194 }
11195
11196 pub fn seal_security_on_teardown(&mut self) {
11206 self.cancel_sync_round();
11207 self.security_preflight = true;
11208 self.encryption = crate::values::EncryptionConfig::default();
11209 self.sync_intent_queue.clear();
11210 }
11211
11212 pub fn activate_security(
11214 &mut self,
11215 encryption: crate::values::EncryptionConfig,
11216 ) -> Result<(), String> {
11217 if !self.security_preflight {
11218 return Err(
11219 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
11220 );
11221 }
11222 self.enqueue_startup_sync_if_needed()?;
11223 self.encryption = encryption;
11224 self.security_preflight = false;
11225 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
11226 Ok(())
11227 }
11228
11229 fn clock_now_ms(&self) -> i64 {
11230 self.now_ms.unwrap_or_else(crate::wall_clock_ms)
11231 }
11232
11233 fn create_bookkeeping_tables(&self) -> Result<(), String> {
11234 self.conn
11236 .execute_batch(
11237 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
11238 seq INTEGER PRIMARY KEY AUTOINCREMENT,
11239 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
11240 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
11241 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
11242 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
11243 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
11244 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
11245 PRIMARY KEY(commit_id, idx));
11246 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
11247 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
11248 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
11249 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
11250 seq INTEGER PRIMARY KEY AUTOINCREMENT,
11251 client_commit_id TEXT NOT NULL UNIQUE,
11252 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
11253 recorded_at_ms INTEGER NOT NULL,
11254 results_json TEXT NOT NULL,
11255 operations_json TEXT,
11256 resolution TEXT NOT NULL DEFAULT 'active'
11257 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
11258 resolved_at_ms INTEGER,
11259 replacement_client_commit_id TEXT);
11260 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
11261 ON _syncular_commit_outcomes(resolution, seq);
11262 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
11263 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
11264 CREATE TABLE IF NOT EXISTS _syncular_meta (
11265 key TEXT PRIMARY KEY, value TEXT NOT NULL);
11266 CREATE TABLE IF NOT EXISTS _syncular_windows (
11267 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
11268 PRIMARY KEY (base, unit));
11269 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
11270 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
11271 effective_scopes TEXT NOT NULL);
11272 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
11273 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
11274 PRIMARY KEY(commit_id, blob_id));
11275 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
11276 ON _syncular_blob_commit_refs(blob_id);",
11277 )
11278 .map_err(|e| e.to_string())?;
11279 let _ = self
11282 .conn
11283 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
11284 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
11285 self.set_meta(LOCAL_REVISION_KEY, "0");
11286 }
11287 if self.schema_has_blobs() {
11290 let exists = self
11291 .conn
11292 .query_row(
11293 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
11294 [],
11295 |row| row.get::<_, bool>(0),
11296 )
11297 .map_err(|e| e.to_string())?;
11298 if exists {
11299 let mut statement = self
11300 .conn
11301 .prepare("PRAGMA table_info(_syncular_blobs)")
11302 .map_err(|e| e.to_string())?;
11303 let mut columns = statement
11304 .query_map([], |row| row.get::<_, String>(1))
11305 .map_err(|e| e.to_string())?
11306 .collect::<Result<Vec<_>, _>>()
11307 .map_err(|e| e.to_string())?;
11308 columns.sort();
11309 let mut expected = vec![
11310 "blob_id",
11311 "byte_length",
11312 "bytes",
11313 "created_at_ms",
11314 "media_type",
11315 ];
11316 expected.sort_unstable();
11317 if columns != expected {
11318 return Err(
11319 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
11320 );
11321 }
11322 } else {
11323 self.conn
11324 .execute_batch(
11325 "CREATE TABLE _syncular_blobs(
11326 blob_id TEXT PRIMARY KEY,
11327 bytes BLOB NOT NULL,
11328 byte_length INTEGER NOT NULL,
11329 media_type TEXT,
11330 created_at_ms INTEGER NOT NULL);",
11331 )
11332 .map_err(|e| e.to_string())?;
11333 }
11334 self.conn
11335 .execute_batch(
11336 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
11337 blob_id TEXT PRIMARY KEY,
11338 media_type TEXT,
11339 created_at_ms INTEGER NOT NULL);",
11340 )
11341 .map_err(|e| e.to_string())?;
11342 }
11343 Ok(())
11344 }
11345
11346 fn schema_has_blobs(&self) -> bool {
11348 self.schema
11349 .tables
11350 .iter()
11351 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
11352 }
11353
11354 fn get_meta(&self, key: &str) -> Option<String> {
11357 meta_get(&self.conn, key)
11358 }
11359
11360 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
11361 self.conn
11362 .query_row(
11363 "SELECT value FROM _syncular_meta WHERE key = ?1",
11364 rusqlite::params![key],
11365 |row| row.get::<_, String>(0),
11366 )
11367 .optional()
11368 .map_err(|_| {
11369 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11370 })
11371 }
11372
11373 fn persist_schema_version(&self) -> Result<(), String> {
11376 self.conn
11377 .execute(
11378 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
11379 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
11380 )
11381 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11382 set_local_schema_descriptor(&self.conn, &self.schema)
11383 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11384 }
11385
11386 fn set_meta(&self, key: &str, value: &str) {
11387 meta_set(&self.conn, key, value);
11388 }
11389
11390 fn delete_meta(&self, key: &str) {
11391 meta_delete(&self.conn, key);
11392 }
11393
11394 fn load_outbox(&self) -> Result<Vec<OutboxCommit>, String> {
11396 let mut stmt = self
11397 .conn
11398 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
11399 .map_err(|error| error.to_string())?;
11400 let rows = stmt
11401 .query_map([], |row| {
11402 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11403 })
11404 .map_err(|error| error.to_string())?;
11405 let mut commits = Vec::new();
11406 for row in rows {
11407 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
11408 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
11409 format!("invalid persisted outbox {client_commit_id:?}: {error}")
11410 })?;
11411 let mut ops = Vec::with_capacity(entries.len());
11412 for entry in entries {
11413 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
11414 ops.push(OutboxOp {
11415 upsert: op == "upsert",
11416 table: entry
11417 .get("table")
11418 .and_then(Value::as_str)
11419 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
11420 .to_owned(),
11421 row_id: entry
11422 .get("rowId")
11423 .and_then(Value::as_str)
11424 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
11425 .to_owned(),
11426 base_version: entry.get("baseVersion").and_then(Value::as_i64),
11427 values: entry.get("values").and_then(Value::as_object).cloned(),
11430 });
11431 }
11432 commits.push(OutboxCommit {
11433 client_commit_id,
11434 ops,
11435 });
11436 }
11437 Ok(commits)
11438 }
11439
11440 fn restore_persisted_state(&mut self) -> Result<(), String> {
11441 self.subs = {
11442 let mut stmt = self
11443 .conn
11444 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
11445 .map_err(|error| error.to_string())?;
11446 let rows = stmt
11447 .query_map([], |row| {
11448 Ok((
11449 row.get::<_, String>(0)?,
11450 row.get::<_, String>(1)?,
11451 row.get::<_, String>(2)?,
11452 ))
11453 })
11454 .map_err(|error| error.to_string())?;
11455 let mut subscriptions = Vec::new();
11456 for row in rows {
11457 let (id, table, raw) = row.map_err(|error| error.to_string())?;
11458 let state: Value = serde_json::from_str(&raw)
11459 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
11460 let requested = json_to_scope_map(
11461 state.get("requested").unwrap_or(&Value::Object(Map::new())),
11462 )?;
11463 let effective = state
11464 .get("effectiveScopes")
11465 .filter(|value| !value.is_null())
11466 .map(json_to_scope_map)
11467 .transpose()?;
11468 subscriptions.push(Subscription {
11469 id,
11470 table,
11471 requested,
11472 params: state
11473 .get("params")
11474 .and_then(Value::as_str)
11475 .map(str::to_owned),
11476 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
11477 bootstrap_state: state
11478 .get("bootstrapState")
11479 .and_then(Value::as_str)
11480 .map(str::to_owned),
11481 state: SubState::parse(
11482 state
11483 .get("status")
11484 .and_then(Value::as_str)
11485 .unwrap_or("active"),
11486 ),
11487 reason_code: state
11488 .get("reasonCode")
11489 .and_then(Value::as_str)
11490 .map(str::to_owned),
11491 effective,
11492 synced_once: state
11493 .get("syncedOnce")
11494 .and_then(Value::as_bool)
11495 .unwrap_or(false),
11496 });
11497 }
11498 subscriptions
11499 };
11500
11501 self.outbox = self.load_outbox()?;
11502
11503 self.prune_commit_outcomes()?;
11504 let active = self.commit_outcomes(CommitOutcomeQuery {
11505 active_only: true,
11506 ..CommitOutcomeQuery::default()
11507 })?;
11508 self.conflicts = active
11509 .iter()
11510 .flat_map(|outcome| outcome.results.iter())
11511 .filter_map(|result| match result {
11512 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
11513 _ => None,
11514 })
11515 .collect();
11516 self.rejections = active
11517 .iter()
11518 .flat_map(|outcome| outcome.results.iter())
11519 .filter_map(|result| match result {
11520 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
11521 _ => None,
11522 })
11523 .collect();
11524
11525 self.lease_state = self
11526 .get_meta(LEASE_STATE_KEY)
11527 .map(|raw| serde_json::from_str(&raw))
11528 .transpose()
11529 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
11530 self.schema_floor = self
11531 .get_meta(SCHEMA_FLOOR_KEY)
11532 .map(|raw| serde_json::from_str(&raw))
11533 .transpose()
11534 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
11535 self.stopped = self.schema_floor.is_some();
11536 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
11539 self.security_preflight = true;
11540 }
11541 Ok(())
11542 }
11543
11544 fn save_subscription_scope_schema(&self) {
11545 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
11546 .schema
11547 .tables
11548 .iter()
11549 .map(|table| {
11550 (
11551 table.name.clone(),
11552 table
11553 .scope_variables
11554 .iter()
11555 .map(|scope| {
11556 (
11557 scope.variable.clone(),
11558 [scope.prefix.clone(), scope.column.clone()],
11559 )
11560 })
11561 .collect(),
11562 )
11563 })
11564 .collect();
11565 self.set_meta(
11566 "subscriptionScopeSchema",
11567 &serde_json::to_string(&declarations).expect("scope schema serialization"),
11568 );
11569 }
11570
11571 fn prune_unknown_subscriptions(&mut self, bump: bool) -> Result<(), String> {
11573 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
11574 self.get_meta("subscriptionScopeSchema")
11575 .map(|raw| serde_json::from_str(&raw))
11576 .transpose()
11577 .map_err(|error| format!("invalid persisted scope schema: {error}"))?
11578 } else {
11579 None
11580 };
11581 let compatible = |table_name: &str, requested: &[(String, Vec<String>)]| {
11582 let Some(table) = self.schema.table(table_name) else {
11583 return false;
11584 };
11585 if requested
11586 .iter()
11587 .any(|(variable, _)| table.scope_column(variable).is_none())
11588 {
11589 return false;
11590 }
11591 if !bump {
11592 return true;
11593 }
11594 let Some(old) = previous.as_ref().and_then(|tables| tables.get(table_name)) else {
11595 return false;
11596 };
11597 let variables: BTreeSet<_> = if requested.is_empty() {
11598 old.keys()
11599 .cloned()
11600 .chain(
11601 table
11602 .scope_variables
11603 .iter()
11604 .map(|scope| scope.variable.clone()),
11605 )
11606 .collect()
11607 } else {
11608 requested
11609 .iter()
11610 .map(|(variable, _)| variable.clone())
11611 .collect()
11612 };
11613 variables.iter().all(|variable| {
11614 let current = table
11615 .scope_variables
11616 .iter()
11617 .find(|scope| &scope.variable == variable);
11618 match (old.get(variable), current) {
11619 (Some(old), Some(current)) => {
11620 old == &[current.prefix.clone(), current.column.clone()]
11621 }
11622 _ => false,
11623 }
11624 })
11625 };
11626 let mut stale_ids: BTreeSet<_> = self
11627 .subs
11628 .iter()
11629 .filter(|sub| !compatible(&sub.table, &sub.requested))
11630 .map(|sub| sub.id.clone())
11631 .collect();
11632 for (id, table, effective) in self.load_pending_evictions()? {
11633 if !compatible(&table, &effective) {
11634 stale_ids.insert(id);
11635 }
11636 }
11637 for id in &stale_ids {
11638 for sql in [
11639 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
11640 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
11641 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
11642 ] {
11643 self.conn
11644 .execute(sql, rusqlite::params![id])
11645 .map_err(|error| error.to_string())?;
11646 }
11647 }
11648 self.subs.retain(|sub| !stale_ids.contains(&sub.id));
11649 Ok(())
11650 }
11651
11652 #[must_use]
11653 pub fn local_revision(&self) -> u64 {
11654 self.get_meta(LOCAL_REVISION_KEY)
11655 .and_then(|value| value.parse().ok())
11656 .unwrap_or(0)
11657 }
11658
11659 pub fn snapshot_read(
11665 &mut self,
11666 request: &SnapshotReadRequest<'_>,
11667 owner: Option<QueryOwner<'_>>,
11668 ) -> Result<SnapshotRead, QueryReadFailure> {
11669 let result = snapshot_read_connection(&self.conn, request);
11670 if let Some(owner) = owner {
11671 self.record_query_read(owner, result.as_ref().err());
11672 }
11673 result
11674 }
11675
11676 #[must_use]
11677 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
11678 SyncStatusSnapshot {
11679 current_schema_version: self.schema.version,
11680 outbox: self.outbox.len(),
11681 upgrading: self.upgrading,
11682 lease_state: self.lease_state.clone(),
11683 schema_floor: self.schema_floor.clone(),
11684 sync_needed: self.sync_needed,
11685 previous_version_context: self.previous_version_status(),
11686 }
11687 }
11688
11689 fn previous_version_status(&self) -> PreviousVersionStatus {
11692 let Some(replica_path) = self.previous_version_replica_path() else {
11693 return PreviousVersionStatus {
11694 present: false,
11695 created_at_ms: None,
11696 };
11697 };
11698 match crate::previous_version::read_previous_version_record(&replica_path) {
11699 Ok(Some(record)) => PreviousVersionStatus {
11700 present: true,
11701 created_at_ms: Some(record.created_at_ms),
11702 },
11703 _ => PreviousVersionStatus {
11704 present: std::path::Path::new(
11705 &crate::previous_version::previous_version_container_path(&replica_path),
11706 )
11707 .exists(),
11708 created_at_ms: None,
11709 },
11710 }
11711 }
11712
11713 pub fn previous_version_snapshot(
11719 &self,
11720 spec: &PreviousVersionReadSpec,
11721 ) -> Result<PreviousVersionSnapshot, String> {
11722 let now_ms = self.clock_now_ms();
11723 crate::previous_version::previous_version_snapshot(
11724 &self.conn,
11725 self.previous_version_replica_path().as_deref(),
11726 self.previous_version.as_ref(),
11727 PreviousVersionLifecycle {
11728 lease_inactive: self.previous_version_lease_inactive(now_ms),
11729 scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
11730 coverage_complete: self.previous_version_coverage_complete(),
11731 },
11732 now_ms,
11733 self.schema.version,
11734 spec,
11735 )
11736 }
11737
11738 #[must_use]
11740 pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
11741 crate::previous_version::stored_previous_version_audit(&self.conn)
11742 }
11743
11744 pub fn previous_version_discard(
11748 &self,
11749 ) -> crate::previous_version::PreviousVersionDiscardOutcome {
11750 self.drop_previous_version()
11751 }
11752
11753 fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
11757 let absent = crate::previous_version::PreviousVersionDiscardOutcome {
11758 present: false,
11759 discarded: false,
11760 };
11761 let Some(replica_path) = self.previous_version_replica_path() else {
11762 return absent;
11763 };
11764 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
11765 .unwrap_or(absent)
11766 }
11767
11768 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
11770 let Some(lease) = &self.lease_state else {
11771 return false;
11772 };
11773 if lease.error_code.is_some() {
11774 return true;
11775 }
11776 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
11777 }
11778
11779 fn previous_version_coverage_complete(&self) -> bool {
11783 let active = self
11784 .subs
11785 .iter()
11786 .filter(|sub| sub.state == SubState::Active)
11787 .collect::<Vec<_>>();
11788 if active.is_empty() {
11789 return false;
11790 }
11791 active
11792 .iter()
11793 .all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
11794 }
11795
11796 fn previous_version_lifetime_check(&self) {
11800 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
11801 return;
11802 };
11803 let now_ms = self.clock_now_ms();
11804 if self.previous_version_lease_inactive(now_ms) {
11805 let _ = self.drop_previous_version();
11806 return;
11807 }
11808 let Some(replica_path) = self.previous_version_replica_path() else {
11809 return;
11810 };
11811 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
11812 else {
11813 return;
11814 };
11815 if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
11816 let _ = self.drop_previous_version();
11817 return;
11818 }
11819 if now_ms - record.created_at_ms > config.max_age_ms {
11820 let _ = self.drop_previous_version();
11821 return;
11822 }
11823 if self.previous_version_coverage_complete() {
11824 let _ = self.drop_previous_version();
11825 }
11826 }
11827
11828 pub fn diagnostics_snapshot(
11829 &self,
11830 request: &ClientDiagnosticsRequest,
11831 ) -> Result<ClientDiagnosticsSnapshot, String> {
11832 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
11833 return Err(format!(
11834 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
11835 ));
11836 }
11837 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
11838 for sub in &self.subs {
11839 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
11840 let complete =
11841 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
11842 let state = match sub.state {
11843 SubState::Revoked => "revoked",
11844 SubState::Failed => "failed",
11845 SubState::Active if reset => "reset",
11846 SubState::Active if complete => "complete",
11847 SubState::Active => "bootstrapping",
11848 };
11849 subscriptions.insert(
11850 sub.id.clone(),
11851 DiagnosticSubscription {
11852 id: sub.id.clone(),
11853 table: sub.table.clone(),
11854 state: state.to_owned(),
11855 complete,
11856 cursor: Some(sub.cursor),
11857 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
11858 },
11859 );
11860 }
11861 for expected in &request.expected_subscriptions {
11862 if expected.id.is_empty() || expected.table.is_empty() {
11863 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
11864 }
11865 if subscriptions
11866 .get(&expected.id)
11867 .is_some_and(|registered| registered.table != expected.table)
11868 {
11869 subscriptions.insert(
11870 expected.id.clone(),
11871 DiagnosticSubscription {
11872 id: expected.id.clone(),
11873 table: expected.table.clone(),
11874 state: "failed".to_owned(),
11875 complete: false,
11876 cursor: None,
11877 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
11878 },
11879 );
11880 } else {
11881 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
11882 DiagnosticSubscription {
11883 id: expected.id.clone(),
11884 table: expected.table.clone(),
11885 state: "unregistered".to_owned(),
11886 complete: false,
11887 cursor: None,
11888 reason_code: None,
11889 }
11890 });
11891 }
11892 }
11893 let mut ordered_subscriptions = Vec::new();
11894 let mut included = BTreeSet::new();
11895 for expected in &request.expected_subscriptions {
11896 if included.insert(expected.id.clone()) {
11897 if let Some(subscription) = subscriptions.get(&expected.id) {
11898 ordered_subscriptions.push(subscription.clone());
11899 }
11900 }
11901 }
11902 for (id, subscription) in &subscriptions {
11903 if included.insert(id.clone()) {
11904 ordered_subscriptions.push(subscription.clone());
11905 }
11906 }
11907 let subscriptions_truncated =
11908 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
11909 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
11910 let captured_at_ms = self.clock_now_ms();
11911 let lease = if let Some(error_code) = self
11912 .lease_state
11913 .as_ref()
11914 .and_then(|state| state.error_code.clone())
11915 {
11916 ClientDiagnosticsLease {
11917 state: "stopped".to_owned(),
11918 expires_at_ms: self
11919 .lease_state
11920 .as_ref()
11921 .and_then(|state| state.expires_at_ms),
11922 error_code: Some(Self::diagnostic_code(&error_code)),
11923 }
11924 } else if let Some(expires_at_ms) = self
11925 .lease_state
11926 .as_ref()
11927 .and_then(|state| state.expires_at_ms)
11928 {
11929 ClientDiagnosticsLease {
11930 state: if expires_at_ms <= captured_at_ms {
11931 "expired".to_owned()
11932 } else {
11933 "active".to_owned()
11934 },
11935 expires_at_ms: Some(expires_at_ms),
11936 error_code: None,
11937 }
11938 } else {
11939 ClientDiagnosticsLease {
11940 state: "none".to_owned(),
11941 expires_at_ms: None,
11942 error_code: None,
11943 }
11944 };
11945 let connectivity = match self.last_round.as_ref() {
11946 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
11947 Some(round) if round.status == "succeeded" => "online",
11948 Some(round)
11949 if round.status == "failed"
11950 && round
11951 .error_code
11952 .as_deref()
11953 .is_some_and(Self::retryable_transport_code) =>
11954 {
11955 "offline"
11956 }
11957 _ => "unknown",
11958 };
11959 Ok(ClientDiagnosticsSnapshot {
11960 version: CLIENT_DIAGNOSTICS_VERSION,
11961 captured_at_ms,
11962 host: ClientDiagnosticsHost {
11963 kind: "direct".to_owned(),
11964 role: "single".to_owned(),
11965 connectivity: connectivity.to_owned(),
11966 realtime: self.realtime_state,
11967 realtime_policy: self.realtime_policy,
11968 realtime_reason_code: self.realtime_reason_code.clone(),
11969 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
11970 },
11971 security_lifecycle: self.security_lifecycle().to_owned(),
11972 schema: ClientDiagnosticsSchema {
11973 current_version: self.schema.version,
11974 upgrading: self.upgrading,
11975 required_version: self
11976 .schema_floor
11977 .as_ref()
11978 .and_then(|floor| floor.required_schema_version),
11979 latest_version: self
11980 .schema_floor
11981 .as_ref()
11982 .and_then(|floor| floor.latest_schema_version),
11983 },
11984 replica: ClientDiagnosticsReplica {
11985 local_revision: self.local_revision().to_string(),
11986 sync_needed: self.sync_needed,
11987 pending_outbox: self.outbox.len(),
11988 },
11989 lease,
11990 subscriptions: ordered_subscriptions,
11991 subscriptions_truncated,
11992 last_round: self.last_round.clone(),
11993 last_change: self.last_change.clone(),
11994 storage: self.diagnostics_storage(),
11995 query_failures: self.query_failures.clone(),
11996 })
11997 }
11998
11999 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
12000 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
12001 let page_count: i64 = self
12002 .conn
12003 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
12004 let page_size: i64 = self
12005 .conn
12006 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
12007 let outbox_bytes: i64 = self
12009 .conn
12010 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
12011 .query_row([], |row| row.get(0))?;
12012 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
12013 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
12014 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
12015 let blob_bytes = if self.schema_has_blobs() {
12016 self.conn
12017 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
12018 .query_row([], |row| row.get(0))?
12019 } else {
12020 0
12021 };
12022 let pressure = self
12023 .limits
12024 .blob_cache_max_bytes
12025 .is_some_and(|limit| blob_bytes > limit);
12026 Ok(ClientDiagnosticsStorage {
12027 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
12028 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
12029 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
12030 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
12031 retained_outcome_entries: Some(outcome_entries.max(0)),
12032 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
12033 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
12034 })
12035 };
12036 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
12037 status: "unreadable".to_owned(),
12038 database_bytes_approx: None,
12039 pending_outbox_bytes_approx: None,
12040 retained_outcome_bytes_approx: None,
12041 retained_outcome_entries: None,
12042 blob_cache_bytes_approx: None,
12043 pressure_reason_code: None,
12044 })
12045 }
12046
12047 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
12048 self.change_queue.drain(..).collect()
12049 }
12050
12051 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
12052 let intents = self.sync_intent_queue.drain(..).collect();
12053 if self.transport_enabled {
12054 intents
12055 } else {
12056 vec![SyncIntent::None]
12057 }
12058 }
12059
12060 fn schedule_background_retry(&mut self) -> u64 {
12061 let delay_ms = self.retry_delay_ms;
12062 self.sync_intent_queue
12063 .push_back(SyncIntent::Background { delay_ms });
12064 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
12065 self.round_retry_delay_ms = Some(delay_ms);
12066 delay_ms
12067 }
12068
12069 fn reset_background_retry(&mut self) {
12070 self.retry_delay_ms = 250;
12071 }
12072
12073 fn retryable_transport_code(code: &str) -> bool {
12074 code == "transport.failed"
12075 || code == "transport.unavailable"
12076 || code == "sync.transport_failed"
12077 }
12078
12079 fn retryable_failure_code(code: &str) -> bool {
12084 Self::retryable_transport_code(code)
12085 || matches!(
12086 code,
12087 "sync.auth_required"
12088 | "sync.auth_lease_required"
12089 | "sync.auth_lease_revoked"
12090 | "sync.internal_error"
12091 | "sync.idempotency_cache_miss"
12092 | "sync.schema_not_ready"
12093 | "sync.segment_expired"
12094 | "sync.rate_limited"
12095 | "sync.websocket_connection_limit"
12096 )
12097 }
12098
12099 fn diagnostic_code(code: &str) -> String {
12100 let valid = !code.is_empty()
12101 && code.len() <= 96
12102 && code.contains('.')
12103 && code
12104 .bytes()
12105 .next()
12106 .is_some_and(|byte| byte.is_ascii_lowercase())
12107 && code.bytes().all(|byte| {
12108 byte.is_ascii_lowercase()
12109 || byte.is_ascii_digit()
12110 || matches!(byte, b'.' | b'_' | b'-')
12111 });
12112 if valid {
12113 code.to_owned()
12114 } else {
12115 "client.unknown_failure".to_owned()
12116 }
12117 }
12118
12119 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
12120 if self.sync_needed != value {
12121 if self.begin_observation("syncular_status").is_ok() {
12122 self.sync_needed = value;
12123 let batch = ChangeAccumulator {
12124 status: true,
12125 ..ChangeAccumulator::default()
12126 };
12127 if self.finish_observation("syncular_status", batch).is_err() {
12128 self.rollback_observation("syncular_status");
12129 }
12130 } else {
12131 self.sync_needed = value;
12132 }
12133 }
12134 if value && interactive {
12135 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12136 }
12137 }
12138
12139 fn sqlite_failure(
12140 storage_failure: &RefCell<Option<QueryReadFailure>>,
12141 error: rusqlite::Error,
12142 ) -> String {
12143 let message = error.to_string();
12144 let failure = QueryReadFailure::from(error);
12145 let code = failure.code;
12146 if code.is_some() && storage_failure.borrow().is_none() {
12147 *storage_failure.borrow_mut() = Some(failure);
12148 }
12149 match code {
12150 Some(code) => format!("{code}: {message}"),
12151 None => message,
12152 }
12153 }
12154
12155 fn begin_observation(&self, name: &str) -> Result<(), String> {
12156 if let Some(failure) = &self.coalesced_failure {
12157 return Err(failure.clone());
12158 }
12159 if self.coalesced.is_some() && self.conn.is_autocommit() {
12162 return Err("the coalesced apply transaction was rolled back".to_owned());
12163 }
12164 self.conn
12165 .execute_batch(&format!("SAVEPOINT {name}"))
12166 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12167 }
12168
12169 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
12172 if self.coalesced.is_some() || !self.conn.is_autocommit() {
12173 return Ok(());
12174 }
12175 self.conn
12176 .execute_batch("BEGIN")
12177 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12178 self.coalesced = Some(CoalescedApply {
12179 rows: 0,
12180 subs: self.subs.clone(),
12181 outbox: self.outbox.clone(),
12182 failed_commits: self.failed_commits.clone(),
12183 conflicts: self.conflicts.clone(),
12184 rejections: self.rejections.clone(),
12185 schema_floor: self.schema_floor.clone(),
12186 lease_state: self.lease_state.clone(),
12187 stopped: self.stopped,
12188 upgrading: self.upgrading,
12189 sync_needed: self.sync_needed,
12190 overlay_dirty: self.overlay_dirty.snapshot(),
12191 change_queue: self.change_queue.len(),
12192 sync_intent_queue: self.sync_intent_queue.len(),
12193 last_change: self.last_change.clone(),
12194 });
12195 Ok(())
12196 }
12197
12198 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
12201 let Some(open) = &self.coalesced else {
12202 return Ok(());
12203 };
12204 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
12205 self.commit_coalesced_apply()?;
12206 self.begin_coalesced_apply()?;
12207 }
12208 if let Some(open) = &mut self.coalesced {
12209 open.rows += rows;
12210 }
12211 Ok(())
12212 }
12213
12214 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
12219 let Some(open) = self.coalesced.take() else {
12220 return Ok(());
12221 };
12222 let committed = if self.conn.is_autocommit() {
12223 Err("the coalesced apply transaction was rolled back".to_owned())
12224 } else {
12225 self.conn
12226 .execute_batch("COMMIT")
12227 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12228 };
12229 let Err(error) = committed else {
12230 return Ok(());
12231 };
12232 if !self.conn.is_autocommit() {
12233 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12234 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12235 }
12236 }
12237 self.subs = open.subs;
12238 self.outbox = open.outbox;
12239 self.failed_commits = open.failed_commits;
12240 self.conflicts = open.conflicts;
12241 self.rejections = open.rejections;
12242 self.schema_floor = open.schema_floor;
12243 self.lease_state = open.lease_state;
12244 self.stopped = open.stopped;
12245 self.upgrading = open.upgrading;
12246 self.sync_needed = open.sync_needed;
12247 self.overlay_dirty.restore(open.overlay_dirty);
12248 self.change_queue.truncate(open.change_queue);
12249 self.sync_intent_queue.truncate(open.sync_intent_queue);
12250 self.last_change = open.last_change;
12251 self.coalesced_failure = Some(error.clone());
12252 Err(error)
12253 }
12254
12255 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
12257 let committed = self.commit_coalesced_apply();
12258 match self.coalesced_failure.take() {
12259 Some(failure) => Err(failure),
12260 None => committed,
12261 }
12262 }
12263
12264 fn rollback_observation(&self, name: &str) {
12265 if let Err(error) = self
12266 .conn
12267 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
12268 {
12269 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
12270 if failure.rollback_failure.is_none() {
12271 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
12272 }
12273 }
12274 }
12275 }
12276
12277 fn finish_observation(
12278 &mut self,
12279 name: &str,
12280 mut batch: ChangeAccumulator,
12281 ) -> Result<(), String> {
12282 #[cfg(feature = "bench-internals")]
12283 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
12284 if self.failed_commits.iter().any(|(commit, _)| {
12285 commit
12286 .ops
12287 .iter()
12288 .any(|operation| batch.tables.contains_key(&operation.table))
12289 }) {
12290 batch.outcomes = true;
12291 }
12292 if !batch.touched() {
12293 self.conn
12294 .execute_batch(&format!("RELEASE {name}"))
12295 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12296 return Ok(());
12297 }
12298 let revision = self
12299 .local_revision()
12300 .checked_add(1)
12301 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
12302 self.conn
12303 .execute(
12304 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
12305 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
12306 )
12307 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12308 let status = batch.status.then(|| self.status_snapshot());
12309 let event = ClientChangeBatch {
12310 revision: revision.to_string(),
12311 tables: batch
12312 .tables
12313 .into_iter()
12314 .map(|(table, scope_keys)| TableChange {
12315 table,
12316 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
12317 })
12318 .collect(),
12319 windows: batch
12320 .windows
12321 .into_iter()
12322 .map(|((base_key, table), units)| WindowChange {
12323 base_key,
12324 table,
12325 units: units.into_iter().collect(),
12326 })
12327 .collect(),
12328 status,
12329 conflicts_changed: batch.conflicts,
12330 rejections_changed: batch.rejections,
12331 outcomes_changed: batch.outcomes,
12332 };
12333 self.conn
12334 .execute_batch(&format!("RELEASE {name}"))
12335 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12336 let mut diagnostic_tables = event
12337 .tables
12338 .iter()
12339 .map(|entry| entry.table.clone())
12340 .collect::<BTreeSet<_>>()
12341 .into_iter()
12342 .collect::<Vec<_>>();
12343 let mut diagnostic_windows = event
12344 .windows
12345 .iter()
12346 .map(|entry| entry.table.clone())
12347 .collect::<BTreeSet<_>>()
12348 .into_iter()
12349 .collect::<Vec<_>>();
12350 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
12351 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
12352 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
12353 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
12354 self.last_change = Some(DiagnosticLastChange {
12355 revision: event.revision.clone(),
12356 recorded_at_ms: self.clock_now_ms(),
12357 tables: diagnostic_tables,
12358 windows: diagnostic_windows,
12359 domains_truncated,
12360 status_changed: event.status.is_some(),
12361 conflicts_changed: event.conflicts_changed,
12362 rejections_changed: event.rejections_changed,
12363 outcomes_changed: event.outcomes_changed,
12364 });
12365 self.change_queue.push_back(event);
12366 Ok(())
12367 }
12368
12369 fn record_scope_map(
12370 &self,
12371 batch: &mut ChangeAccumulator,
12372 table_name: &str,
12373 scopes: &[(String, Vec<String>)],
12374 ) {
12375 let Some(table) = self.schema.table(table_name) else {
12376 return;
12377 };
12378 for (variable, values) in scopes {
12379 let Some(scope) = table
12380 .scope_variables
12381 .iter()
12382 .find(|scope| &scope.variable == variable)
12383 else {
12384 continue;
12385 };
12386 for value in values {
12387 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12388 }
12389 }
12390 }
12391
12392 fn record_row_scopes(
12394 &self,
12395 batch: &mut ChangeAccumulator,
12396 table_name: &str,
12397 row_id: &str,
12398 base: bool,
12399 ) -> bool {
12400 let Some(table) = self.schema.table(table_name) else {
12401 return false;
12402 };
12403 if table.scope_variables.is_empty() {
12404 return false;
12405 }
12406 let columns = table
12407 .scope_variables
12408 .iter()
12409 .map(|scope| quote_ident(&scope.column))
12410 .collect::<Vec<_>>()
12411 .join(", ");
12412 let full_table = if base {
12413 base_table(table_name)
12414 } else {
12415 visible_table(table_name)
12416 };
12417 let sql = format!(
12418 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
12419 row_id_predicate(table)
12420 );
12421 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
12422 return false;
12423 };
12424 let values = stmt.query_row(rusqlite::params![row_id], |row| {
12425 let mut values = Vec::with_capacity(table.scope_variables.len());
12426 for index in 0..table.scope_variables.len() {
12427 values.push(row.get::<_, Option<String>>(index)?);
12428 }
12429 Ok(values)
12430 });
12431 let Ok(values) = values else {
12432 return false;
12433 };
12434 let mut recorded = false;
12435 for (scope, value) in table.scope_variables.iter().zip(values) {
12436 if let Some(value) = value {
12437 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12438 recorded = true;
12439 }
12440 }
12441 recorded
12442 }
12443
12444 fn record_commit_changes(
12445 &self,
12446 batch: &mut ChangeAccumulator,
12447 tables: &[String],
12448 changes: &[ssp2::model::Change],
12449 ) {
12450 #[cfg(feature = "bench-internals")]
12451 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
12452 for change in changes {
12453 let Some(table_name) = tables.get(change.table_index as usize) else {
12454 continue;
12455 };
12456 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
12457 if let Some(table) = self.schema.table(table_name) {
12458 for (variable, value) in &change.scopes {
12459 if let Some(scope) = table
12460 .scope_variables
12461 .iter()
12462 .find(|scope| &scope.variable == variable)
12463 {
12464 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12465 precise = true;
12466 }
12467 }
12468 }
12469 if !precise {
12470 batch.table(table_name);
12471 }
12472 }
12473 }
12474
12475 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
12476 if effective.is_empty() {
12477 return false;
12478 }
12479 let Some(table) = self.schema.table(table_name) else {
12480 return false;
12481 };
12482 let mut clauses = Vec::new();
12483 let mut params = Vec::new();
12484 for (variable, values) in effective {
12485 let Some(column) = table.scope_column(variable) else {
12486 return false;
12487 };
12488 if values.is_empty() {
12489 return false;
12490 }
12491 let placeholders = values
12492 .iter()
12493 .map(|value| {
12494 params.push(SqlValue::Text(value.clone()));
12495 "?"
12496 })
12497 .collect::<Vec<_>>()
12498 .join(", ");
12499 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
12500 }
12501 let sql = format!(
12502 "SELECT 1 FROM {} WHERE {} LIMIT 1",
12503 base_table(table_name),
12504 clauses.join(" AND ")
12505 );
12506 self.conn
12507 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
12508 .is_ok()
12509 }
12510
12511 pub fn upgrading(&self) -> bool {
12513 self.upgrading
12514 }
12515
12516 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
12524 let new_schema = parse_schema_json(schema_json)?;
12525 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
12526 self.conn
12527 .execute_batch("BEGIN IMMEDIATE")
12528 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12529 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
12530 let prior_subs = self.subs.clone();
12536 let prior_outbox = self.outbox.clone();
12537 let prior_rejections = self.rejections.clone();
12538 let prior_upgrading = self.upgrading;
12539 let prior_stopped = self.stopped;
12540 let prior_schema_floor = self.schema_floor.clone();
12541 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12542 let prior_sync_needed = self.sync_needed;
12543 let prior_active_round = self.active_round;
12544 let prior_changes = self.change_queue.len();
12545 let prior_intents = self.sync_intent_queue.len();
12546 let prior_last_change = self.last_change.clone();
12547 let guarded = (|| -> Result<(), String> {
12548 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
12553 self.create_bookkeeping_tables()?;
12554 match marker {
12555 Some(version) if version < self.schema.version => self.run_schema_reset()?,
12556 None => {
12557 self.create_synced_tables()?;
12558 self.persist_schema_version()?;
12559 }
12560 _ => {}
12561 }
12562 self.outbox = self.load_outbox()?;
12567 self.drop_unencodable_outbox(false, None)?;
12572 if marker == Some(self.schema.version)
12573 && (!self.outbox.is_empty()
12574 || !self.failed_commits.is_empty()
12575 || self.has_acknowledged_rows())
12576 {
12577 self.overlay_dirty.set(true);
12578 self.rebuild_overlay()?;
12579 }
12580 self.prune_unknown_subscriptions(false)?;
12581 self.save_subscription_scope_schema();
12582 self.enqueue_startup_sync_if_needed()?;
12586 Ok(())
12587 })();
12588 let outcome = match guarded {
12589 Ok(()) => self
12590 .conn
12591 .execute_batch("COMMIT")
12592 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
12593 Err(error) => Err(error),
12594 };
12595 if let Err(error) = outcome {
12596 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12597 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12599 }
12600 self.schema = prior_schema;
12601 self.subs = prior_subs;
12602 self.outbox = prior_outbox;
12603 self.rejections = prior_rejections;
12604 self.upgrading = prior_upgrading;
12605 self.stopped = prior_stopped;
12606 self.schema_floor = prior_schema_floor;
12607 self.overlay_dirty.restore(prior_overlay_dirty);
12608 self.sync_needed = prior_sync_needed;
12609 self.active_round = prior_active_round;
12610 self.change_queue.truncate(prior_changes);
12611 self.sync_intent_queue.truncate(prior_intents);
12612 self.last_change = prior_last_change;
12613 self.insert_sql.borrow_mut().clear();
12615 return Err(error);
12616 }
12617 Ok(())
12618 }
12619
12620 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
12621 let startup_work = !self.stopped
12622 && (!self.outbox.is_empty()
12623 || self.has_acknowledged_rows()
12624 || !self.load_pending_evictions()?.is_empty()
12625 || self.subs.iter().any(|sub| sub.state == SubState::Active));
12626 if startup_work {
12627 self.sync_needed = true;
12628 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12629 }
12630 Ok(())
12631 }
12632
12633 fn clear_satisfied_persisted_schema_floor(&mut self) {
12640 let satisfied = self
12641 .schema_floor
12642 .as_ref()
12643 .and_then(|floor| floor.required_schema_version)
12644 .is_some_and(|required| self.schema.version >= required);
12645 if !satisfied {
12646 return;
12647 }
12648 self.schema_floor = None;
12649 self.stopped = false;
12650 self.delete_meta(SCHEMA_FLOOR_KEY);
12651 }
12652
12653 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
12658 let Some(replica_path) = self.previous_version_replica_path() else {
12659 return Ok(());
12660 };
12661 let max_age_ms = self
12662 .previous_version
12663 .filter(|config| config.enabled)
12664 .map(|config| config.max_age_ms);
12665 reconcile_previous_version_at_boot(
12666 &self.conn,
12667 &replica_path,
12668 self.schema.version,
12669 max_age_ms,
12670 self.clock_now_ms(),
12671 )
12672 }
12673
12674 fn previous_version_replica_path(&self) -> Option<String> {
12678 let path = self.conn.path()?;
12679 if path.is_empty() || path.starts_with("file:") {
12680 return None;
12681 }
12682 Some(path.to_owned())
12683 }
12684
12685 fn run_schema_reset(&mut self) -> Result<(), String> {
12691 self.begin_observation("syncular_schema_reset")?;
12692 let prior_subs = self.subs.clone();
12693 let prior_outbox = self.outbox.clone();
12694 let prior_rejections = self.rejections.clone();
12695 let prior_upgrading = self.upgrading;
12696 let prior_stopped = self.stopped;
12697 let prior_schema_floor = self.schema_floor.clone();
12698 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12699 let prior_sync_needed = self.sync_needed;
12700 let prior_active_round = self.active_round;
12701 let mut batch = ChangeAccumulator::default();
12702 let result = self
12703 .run_schema_reset_observed(&mut batch, true)
12704 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
12705 if let Err(error) = result {
12706 self.rollback_observation("syncular_schema_reset");
12707 self.subs = prior_subs;
12708 self.outbox = prior_outbox;
12709 self.rejections = prior_rejections;
12710 self.upgrading = prior_upgrading;
12711 self.stopped = prior_stopped;
12712 self.schema_floor = prior_schema_floor;
12713 self.overlay_dirty.restore(prior_overlay_dirty);
12714 self.sync_needed = prior_sync_needed;
12715 self.active_round = prior_active_round;
12716 return Err(error);
12717 }
12718 Ok(())
12719 }
12720
12721 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
12722 if self.get_meta(LOG_EPOCH_KEY).is_none() {
12723 self.begin_observation("syncular_epoch_acquisition")?;
12724 let previous_sync_needed = self.sync_needed;
12725 if let Err(error) = self.conn.execute(
12726 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12727 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12728 ) {
12729 self.rollback_observation("syncular_epoch_acquisition");
12730 return Err(Self::sqlite_failure(&self.storage_failure, error));
12731 }
12732 self.sync_needed = true;
12733 let batch = ChangeAccumulator {
12734 status: true,
12735 ..ChangeAccumulator::default()
12736 };
12737 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
12738 self.rollback_observation("syncular_epoch_acquisition");
12739 self.sync_needed = previous_sync_needed;
12740 return Err(error);
12741 }
12742 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12743 return Ok(Vec::new());
12744 }
12745 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
12746 self.begin_observation("syncular_log_epoch_reset")?;
12747 let prior_subs = self.subs.clone();
12748 let prior_outbox = self.outbox.clone();
12749 let prior_rejections = self.rejections.clone();
12750 let prior_upgrading = self.upgrading;
12751 let prior_stopped = self.stopped;
12752 let prior_schema_floor = self.schema_floor.clone();
12753 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12754 let prior_sync_needed = self.sync_needed;
12755 let prior_active_round = self.active_round;
12756 let mut batch = ChangeAccumulator::default();
12757 let result = self
12758 .run_schema_reset_observed(&mut batch, false)
12759 .and_then(|()| {
12760 self.conn
12761 .execute(
12762 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12763 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12764 )
12765 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12766 self.sync_needed = true;
12767 batch.status = true;
12768 self.finish_observation("syncular_log_epoch_reset", batch)
12769 });
12770 if let Err(error) = result {
12771 self.rollback_observation("syncular_log_epoch_reset");
12772 self.subs = prior_subs;
12773 self.outbox = prior_outbox;
12774 self.rejections = prior_rejections;
12775 self.upgrading = prior_upgrading;
12776 self.stopped = prior_stopped;
12777 self.schema_floor = prior_schema_floor;
12778 self.overlay_dirty.restore(prior_overlay_dirty);
12779 self.sync_needed = prior_sync_needed;
12780 self.active_round = prior_active_round;
12781 return Err(error);
12782 }
12783 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12784 Ok(resets)
12785 }
12786
12787 fn run_schema_reset_observed(
12788 &mut self,
12789 batch: &mut ChangeAccumulator,
12790 capture: bool,
12791 ) -> Result<(), String> {
12792 read_local_schema_version(&self.conn, self.schema.version)?;
12798 self.conn
12799 .execute_batch(
12800 "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
12801 )
12802 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12803 self.cancel_sync_round();
12804 self.upgrading = true;
12805 batch.status = true;
12806 if self.previous_version.is_some_and(|config| config.enabled) {
12814 let previous_version = self
12815 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
12816 .and_then(|value| value.parse::<i32>().ok())
12817 .unwrap_or(self.schema.version);
12818 let pending = self
12819 .outbox
12820 .iter()
12821 .map(|commit| PendingCommitAudit {
12822 commit_id: &commit.client_commit_id,
12823 operations: commit_audit_operations(commit),
12824 })
12825 .collect::<Vec<_>>();
12826 crate::previous_version::write_previous_version_audit(
12827 &self.conn,
12828 &crate::previous_version::build_previous_version_audit(
12829 &self.schema,
12830 &pending,
12831 previous_version,
12832 self.schema.version,
12833 self.clock_now_ms(),
12834 ),
12835 );
12836 }
12837 if let Some(replica_path) = self.previous_version_replica_path() {
12842 sweep_previous_version_container(&self.conn, &replica_path)?;
12843 if capture {
12844 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
12845 let previous_version = self
12846 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
12847 .and_then(|value| value.parse::<i32>().ok())
12848 .unwrap_or(self.schema.version);
12849 capture_previous_version_from_replica(
12850 &self.conn,
12851 &replica_path,
12852 previous_version,
12853 self.schema.version,
12854 &config,
12855 self.clock_now_ms(),
12856 )?;
12857 }
12858 }
12859 }
12860 for table in &self.schema.tables {
12861 batch.table(&table.name);
12862 }
12863 for (base_key, unit, table) in self.load_registered_window_units() {
12864 batch.window(&base_key, &table, &unit);
12865 }
12866 if capture {
12867 self.prune_unknown_subscriptions(true)?;
12868 }
12869 self.insert_sql.borrow_mut().clear();
12871 self.overlay_dirty.set(true);
12872 let virtual_tables: Vec<String> = {
12880 let mut stmt = self
12881 .conn
12882 .prepare(
12883 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
12884 )
12885 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12886 let rows = stmt
12887 .query_map([], |row| row.get::<_, String>(0))
12888 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12889 rows.filter_map(Result::ok)
12890 .filter(|name| is_synced_table_name(name))
12891 .collect()
12892 };
12893 for name in virtual_tables {
12894 self.conn
12895 .execute_batch(&format!(
12896 "DROP TABLE IF EXISTS {}",
12897 quote_ident(&format!("_syncular_fts_{name}"))
12898 ))
12899 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12900 self.conn
12901 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
12902 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12903 }
12904 let existing: Vec<String> = {
12905 let mut stmt = self
12906 .conn
12907 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
12908 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12909 let rows = stmt
12910 .query_map([], |row| row.get::<_, String>(0))
12911 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12912 rows.filter_map(Result::ok)
12913 .filter(|name| is_synced_table_name(name))
12914 .collect()
12915 };
12916 for name in &existing {
12917 if let Some(table) = name.strip_prefix("_syncular_base_") {
12918 batch.table(table);
12919 } else if !name.starts_with("_syncular_") {
12920 batch.table(name);
12921 }
12922 }
12923 for name in existing {
12924 self.conn
12925 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
12926 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12927 }
12928 self.create_synced_tables()?;
12930 for sub in &mut self.subs {
12932 sub.cursor = -1;
12933 sub.bootstrap_state = None;
12934 sub.effective = None;
12935 sub.state = SubState::Active;
12936 sub.reason_code = None;
12937 sub.synced_once = false;
12938 }
12939 let subs = self.subs.clone();
12940 for sub in &subs {
12941 self.persist_sub(sub)?;
12942 }
12943 self.stopped = false;
12945 self.schema_floor = None;
12946 self.delete_meta(SCHEMA_FLOOR_KEY);
12947 self.persist_schema_version()?;
12949 self.save_subscription_scope_schema();
12950 self.drop_unencodable_outbox(false, Some(batch))?;
12961 self.rebuild_overlay()?;
12963 Ok(())
12964 }
12965
12966 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
12973 let incompatible = self
12974 .outbox
12975 .iter()
12976 .filter(|commit| {
12977 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
12978 })
12979 .cloned()
12980 .collect::<Vec<_>>();
12981 if incompatible.is_empty() {
12982 return Ok(false);
12983 }
12984 let prior_outbox = self.outbox.clone();
12990 let prior_rejections = self.rejections.clone();
12991 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12992 let owns_transaction = self.conn.is_autocommit();
12993 self.begin_observation("syncular_outbox_drop")?;
12994 let mut batch = ChangeAccumulator::default();
12995 let dropped = (|| -> Result<(), String> {
12996 let mut rejections = Vec::new();
12997 for commit in &incompatible {
12998 let results = commit
12999 .ops
13000 .iter()
13001 .enumerate()
13002 .map(|(op_index, operation)| {
13003 let rejection = RejectionRecord {
13004 client_commit_id: commit.client_commit_id.clone(),
13005 op_index: op_index as i32,
13006 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
13007 message: "the persisted commit cannot encode under the current schema"
13008 .to_owned(),
13009 retryable: false,
13010 details: None,
13011 operation: Some(CommitOperation::from(operation)),
13012 };
13013 rejections.push(rejection.clone());
13014 CommitOperationOutcome::Error { rejection }
13015 })
13016 .collect::<Vec<_>>();
13017 self.persist_commit_outcome(
13018 &commit.client_commit_id,
13019 CommitOutcomeStatus::Rejected,
13020 &results,
13021 Some(&commit.ops),
13022 )?;
13023 self.delete_outbox_persisted(&commit.client_commit_id)?;
13024 for operation in &commit.ops {
13027 if self.schema.table(&operation.table).is_some() {
13028 batch.table(&operation.table);
13029 }
13030 }
13031 }
13032 self.prune_commit_outcomes()?;
13033 let incompatible_ids = incompatible
13034 .iter()
13035 .map(|commit| commit.client_commit_id.as_str())
13036 .collect::<BTreeSet<_>>();
13037 self.outbox
13038 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
13039 self.rejections.extend(rejections);
13040 self.overlay_dirty.set(true);
13043 self.rebuild_overlay()?;
13044 batch.status = true;
13045 batch.rejections = true;
13046 batch.outcomes = true;
13047 Ok(())
13048 })();
13049 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
13050 if let Err(error) = result {
13051 if owns_transaction {
13055 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
13056 let mut failure = self.storage_failure.borrow_mut();
13057 let failure =
13058 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
13059 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
13060 }
13061 } else {
13062 self.rollback_observation("syncular_outbox_drop");
13063 }
13064 self.outbox = prior_outbox;
13065 self.rejections = prior_rejections;
13066 self.overlay_dirty.restore(prior_overlay_dirty);
13067 return Err(error);
13068 }
13069 Ok(true)
13070 }
13071
13072 fn create_synced_tables(&self) -> Result<(), String> {
13075 for table in &self.schema.tables {
13076 for (full, index_prefix) in [
13080 (base_table(&table.name), "_syncular_base_"),
13081 (visible_table(&table.name), ""),
13082 ] {
13083 let mut cols: Vec<String> =
13084 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
13085 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
13086 let sql = format!(
13087 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
13088 cols.join(", "),
13089 quote_ident(&table.primary_key)
13090 );
13091 self.conn
13092 .execute(&sql, [])
13093 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13094 for index in &table.indexes {
13098 let unique = if index.unique { "UNIQUE " } else { "" };
13099 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
13100 let cols_sql = index
13101 .columns
13102 .iter()
13103 .map(|c| quote_ident(c))
13104 .collect::<Vec<_>>()
13105 .join(", ");
13106 let index_sql = format!(
13107 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
13108 );
13109 self.conn
13110 .execute(&index_sql, [])
13111 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13112 }
13113 }
13114 }
13115 for table in &self.schema.tables {
13118 for index in &table.fts_indexes {
13119 self.create_fts_projection(table, index)?;
13120 }
13121 }
13122 Ok(())
13123 }
13124
13125 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
13126 let count: i64 = self
13127 .conn
13128 .query_row(
13129 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
13130 rusqlite::params![index.name],
13131 |row| row.get(0),
13132 )
13133 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13134 Ok(count > 0)
13135 }
13136
13137 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
13138 for suffix in ["ai", "ad", "au"] {
13139 self.conn
13140 .execute(
13141 &format!(
13142 "DROP TRIGGER IF EXISTS {}",
13143 quote_ident(&format!("{}_{suffix}", index.name))
13144 ),
13145 [],
13146 )
13147 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13148 }
13149 Ok(())
13150 }
13151
13152 fn create_fts_triggers(
13153 &self,
13154 table: &TableSchema,
13155 index: &FtsIndexSchema,
13156 ) -> Result<(), String> {
13157 let fts = quote_ident(&index.name);
13158 let source = visible_table(&table.name);
13159 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
13160 let pk = quote_ident(&table.primary_key);
13161 let projection_columns = std::iter::once(source_id.clone())
13162 .chain(index.columns.iter().map(|column| quote_ident(column)))
13163 .collect::<Vec<_>>()
13164 .join(", ");
13165 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
13166 .chain(
13167 index
13168 .columns
13169 .iter()
13170 .map(|column| format!("new.{}", quote_ident(column))),
13171 )
13172 .collect::<Vec<_>>()
13173 .join(", ");
13174 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13175 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
13176 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
13177 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
13178 let bi = quote_ident(&format!("{}_bi", index.name));
13179 let ai = quote_ident(&format!("{}_ai", index.name));
13180 let ad = quote_ident(&format!("{}_ad", index.name));
13181 let au = quote_ident(&format!("{}_au", index.name));
13182 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
13183 let sql = format!(
13184 "DROP TRIGGER IF EXISTS {bi};
13185 DROP TRIGGER IF EXISTS {ai};
13186 DROP TRIGGER IF EXISTS {ad};
13187 DROP TRIGGER IF EXISTS {au};
13188 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
13189 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
13190 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
13191 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
13192 );
13193 self.conn
13194 .execute_batch(&sql)
13195 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13196 }
13197
13198 fn rebuild_fts_projection(
13199 &self,
13200 table: &TableSchema,
13201 index: &FtsIndexSchema,
13202 ) -> Result<(), String> {
13203 let fts = quote_ident(&index.name);
13204 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
13205 let pk = quote_ident(&table.primary_key);
13206 let indexed_columns = index
13207 .columns
13208 .iter()
13209 .map(|column| quote_ident(column))
13210 .collect::<Vec<_>>();
13211 let projection_columns = std::iter::once(source_id.clone())
13212 .chain(indexed_columns.iter().cloned())
13213 .collect::<Vec<_>>()
13214 .join(", ");
13215 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13216 let sql = format!(
13217 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
13218 columns = indexed_columns.join(", "),
13219 source = visible_table(&table.name),
13220 );
13221 self.conn
13222 .execute_batch(&sql)
13223 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13224 }
13225
13226 fn create_fts_projection(
13227 &self,
13228 table: &TableSchema,
13229 index: &FtsIndexSchema,
13230 ) -> Result<(), String> {
13231 self.begin_observation("syncular_fts_schema")?;
13232 let result = (|| {
13233 let existed = self.fts_projection_exists(index)?;
13234 let tokenizer = index.tokenize.replace('\'', "''");
13235 let columns = index
13236 .columns
13237 .iter()
13238 .map(|column| quote_ident(column))
13239 .collect::<Vec<_>>()
13240 .join(", ");
13241 let sql = format!(
13242 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
13243 fts = quote_ident(&index.name),
13244 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
13245 );
13246 self.conn.execute(&sql, []).map_err(|error| {
13247 format!(
13248 "cannot create local FTS5 projection {:?}: {error}",
13249 index.name
13250 )
13251 })?;
13252 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13253 let mapped: bool = self
13254 .conn
13255 .query_row(
13256 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
13257 [format!("_syncular_fts_{}", index.name)],
13258 |row| row.get(0),
13259 )
13260 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13261 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13262 if !mapped && existed {
13263 self.conn
13264 .execute_batch(&format!(
13265 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
13266 quote_ident(FTS_SOURCE_ID_COLUMN),
13267 quote_ident(&index.name)
13268 ))
13269 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13270 }
13271 self.create_fts_triggers(table, index)?;
13272 if !existed {
13273 self.rebuild_fts_projection(table, index)?;
13274 }
13275 Ok(())
13276 })();
13277 if let Err(error) = result {
13278 self.rollback_observation("syncular_fts_schema");
13279 return Err(error);
13280 }
13281 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
13282 self.rollback_observation("syncular_fts_schema");
13283 return Err(error.to_string());
13284 }
13285 Ok(())
13286 }
13287
13288 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
13291 let state = serde_json::json!({
13292 "requested": scope_map_to_json(&sub.requested),
13293 "params": sub.params,
13294 "cursor": sub.cursor,
13295 "bootstrapState": sub.bootstrap_state,
13296 "status": sub.state.name(),
13297 "reasonCode": sub.reason_code,
13298 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
13299 "syncedOnce": sub.synced_once,
13300 });
13301 self.conn.execute(
13302 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
13303 rusqlite::params![sub.id, sub.table, state.to_string()],
13304 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13305 }
13306
13307 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
13308 let ops: Vec<Value> = commit
13309 .ops
13310 .iter()
13311 .map(|op| {
13312 serde_json::json!({
13313 "op": if op.upsert { "upsert" } else { "delete" },
13314 "table": op.table,
13315 "rowId": op.row_id,
13316 "baseVersion": op.base_version,
13317 "values": op.values.clone().map(Value::Object),
13318 })
13319 })
13320 .collect();
13321 self.conn
13322 .execute(
13323 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
13324 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
13325 )
13326 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13327 for operation in &commit.ops {
13328 let Some(values) = &operation.values else {
13329 continue;
13330 };
13331 let Some(table) = self.schema.table(&operation.table) else {
13332 continue;
13333 };
13334 for column in table
13335 .columns
13336 .iter()
13337 .filter(|column| column.ty == ColumnType::BlobRef)
13338 {
13339 let Some(Value::String(raw)) = values.get(&column.name) else {
13340 continue;
13341 };
13342 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
13343 continue;
13344 };
13345 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
13346 continue;
13347 };
13348 self.conn
13349 .execute(
13350 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
13351 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
13352 rusqlite::params![commit.client_commit_id, blob_id],
13353 )
13354 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13355 }
13356 }
13357 Ok(())
13358 }
13359
13360 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
13361 if let Some(commit) = self
13362 .outbox
13363 .iter()
13364 .find(|commit| commit.client_commit_id == client_commit_id)
13365 {
13366 for operation in &commit.ops {
13367 self.overlay_dirty.table(&operation.table);
13368 }
13369 }
13370
13371 self.conn
13372 .execute(
13373 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
13374 rusqlite::params![client_commit_id],
13375 )
13376 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13377 self.conn
13378 .execute(
13379 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
13380 rusqlite::params![client_commit_id],
13381 )
13382 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13383 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
13384 .map(|_| ())
13385 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13386 }
13387
13388 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
13389 match status {
13390 CommitOutcomeStatus::Applied => "applied",
13391 CommitOutcomeStatus::Cached => "cached",
13392 CommitOutcomeStatus::Conflict => "conflict",
13393 CommitOutcomeStatus::Rejected => "rejected",
13394 }
13395 }
13396
13397 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
13398 match resolution {
13399 CommitOutcomeResolution::Active => "active",
13400 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
13401 CommitOutcomeResolution::Superseded => "superseded",
13402 CommitOutcomeResolution::Dismissed => "dismissed",
13403 }
13404 }
13405
13406 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
13407 match value {
13408 "applied" => Ok(CommitOutcomeStatus::Applied),
13409 "cached" => Ok(CommitOutcomeStatus::Cached),
13410 "conflict" => Ok(CommitOutcomeStatus::Conflict),
13411 "rejected" => Ok(CommitOutcomeStatus::Rejected),
13412 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
13413 }
13414 }
13415
13416 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
13417 match value {
13418 "active" => Ok(CommitOutcomeResolution::Active),
13419 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
13420 "superseded" => Ok(CommitOutcomeResolution::Superseded),
13421 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
13422 _ => Err(format!(
13423 "invalid persisted commit outcome resolution {value:?}"
13424 )),
13425 }
13426 }
13427
13428 fn persist_commit_outcome(
13429 &self,
13430 client_commit_id: &str,
13431 status: CommitOutcomeStatus,
13432 results: &[CommitOperationOutcome],
13433 operations: Option<&[OutboxOp]>,
13434 ) -> Result<(), String> {
13435 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
13436 let operations_json = operations
13437 .map(|items| {
13438 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
13439 })
13440 .transpose()
13441 .map_err(|error| error.to_string())?;
13442 self.conn
13443 .execute(
13444 "INSERT INTO _syncular_commit_outcomes (
13445 client_commit_id, status, recorded_at_ms, results_json,
13446 operations_json, resolution
13447 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
13448 rusqlite::params![
13449 client_commit_id,
13450 Self::outcome_status_name(status),
13451 self.clock_now_ms(),
13452 results_json,
13453 operations_json
13454 ],
13455 )
13456 .map(|_| ())
13457 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13458 }
13459
13460 fn stored_cell_is_representable(cell: &Value) -> bool {
13463 cell.is_null()
13464 || cell.is_string()
13465 || cell.is_boolean()
13466 || cell.is_number()
13467 || is_canonical_bytes_envelope(cell)
13468 }
13469
13470 fn stored_row_is_representable(value: &Value) -> bool {
13471 value
13472 .as_object()
13473 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
13474 }
13475
13476 fn stored_operation_is_representable(value: &Value) -> bool {
13477 value.get("table").is_some_and(Value::is_string)
13478 && value.get("rowId").is_some_and(Value::is_string)
13479 && value.get("baseVersion").is_none_or(|v| {
13480 v.as_i64()
13481 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
13482 })
13483 && match value.get("op").and_then(Value::as_str) {
13484 Some("upsert") => value
13485 .get("values")
13486 .is_some_and(Self::stored_row_is_representable),
13487 Some("delete") => value.get("values").is_none(),
13488 _ => false,
13489 }
13490 }
13491
13492 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
13496 serde_json::to_value(CommitOperation::from(operation))
13497 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
13498 }
13499
13500 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
13501 let StoredCommitOutcomeRow {
13502 sequence,
13503 client_commit_id,
13504 status,
13505 recorded_at_ms,
13506 results_json,
13507 operations_json,
13508 resolution,
13509 resolved_at_ms,
13510 replacement_client_commit_id,
13511 } = row;
13512 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
13513 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
13514 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
13515 {
13516 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
13517 }
13518 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
13519 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
13520 for result in &stored {
13521 let nested = match result.get("status").and_then(Value::as_str) {
13522 Some("conflict") => result.get("conflict"),
13523 Some("error") => result.get("rejection"),
13524 _ => None,
13525 };
13526 if nested
13527 .and_then(|record| record.get("operation"))
13528 .is_some_and(|op| !Self::stored_operation_is_representable(op))
13529 || (result.get("status").and_then(Value::as_str) == Some("conflict")
13530 && nested
13531 .and_then(|record| record.get("serverRow"))
13532 .is_some_and(|row| !Self::stored_row_is_representable(row)))
13533 {
13534 return Err(invalid());
13535 }
13536 }
13537 let results: Vec<CommitOperationOutcome> =
13538 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
13539 let has_conflict = results
13540 .iter()
13541 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
13542 let has_error = results
13543 .iter()
13544 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
13545 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
13546 let redacted = results.is_empty() && operations_json.is_none();
13547 if match status {
13548 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
13549 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
13550 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
13551 } {
13552 return Err(invalid());
13553 }
13554 let operations: Option<Vec<CommitOperation>> = operations_json
13555 .map(|raw| {
13556 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
13557 if !stored.iter().all(Self::stored_operation_is_representable) {
13558 return Err(invalid());
13559 }
13560 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
13561 })
13562 .transpose()?;
13563 for result in &results {
13564 match result {
13565 CommitOperationOutcome::Conflict { conflict }
13566 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
13567 {
13568 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
13569 }
13570 CommitOperationOutcome::Error { rejection } => {
13571 if let Some(details) = &rejection.details {
13572 let encoded = serde_json::to_string(details).map_err(|_| {
13573 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13574 })?;
13575 RejectionDetails::parse(&encoded).map_err(|_| {
13576 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13577 })?;
13578 }
13579 }
13580 _ => {}
13581 }
13582 }
13583 Ok(CommitOutcome {
13584 sequence,
13585 client_commit_id,
13586 status,
13587 recorded_at_ms,
13588 results,
13589 operations,
13590 retained_rows: None,
13591 resolution: Self::parse_outcome_resolution(&resolution)?,
13592 resolved_at_ms,
13593 replacement_client_commit_id,
13594 })
13595 }
13596
13597 fn unique_conflicts(
13599 &self,
13600 table: &TableSchema,
13601 values: &Map<String, Value>,
13602 base: bool,
13603 ) -> Result<Vec<RetainedUniqueConflict>, String> {
13604 let source = if base {
13605 base_table(&table.name)
13606 } else {
13607 visible_table(&table.name)
13608 };
13609 let mut conflicts = Vec::new();
13610 for index in table.indexes.iter().filter(|index| index.unique) {
13611 let mut params = vec![json_param_to_sql(
13612 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
13613 )?];
13614 for name in &index.columns {
13615 let column = table
13616 .columns
13617 .iter()
13618 .find(|column| column.name == *name)
13619 .ok_or("sync.local_corrupt")?;
13620 let cell = json_to_column_value(column, values.get(name))?;
13621 params.push(owned_sql_value(RowParam::Cell(&cell))?);
13622 }
13623 let sql = format!(
13624 "SELECT * FROM {source} WHERE {} != ? AND {}",
13625 quote_ident(&table.primary_key),
13626 index
13627 .columns
13628 .iter()
13629 .map(|column| format!("{} = ?", quote_ident(column)))
13630 .collect::<Vec<_>>()
13631 .join(" AND ")
13632 );
13633 let mut statement = self
13634 .conn
13635 .prepare_cached(&sql)
13636 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13637 let records = statement
13638 .query_map(rusqlite::params_from_iter(params), |record| {
13639 let values = table
13640 .columns
13641 .iter()
13642 .enumerate()
13643 .map(|(index, column)| {
13644 let value = sql_ref_to_json(column, record.get_ref(index)?);
13645 json_to_column_value(column, Some(&value)).map_err(|message| {
13646 rusqlite::Error::ToSqlConversionFailure(message.into())
13647 })
13648 })
13649 .collect::<Result<Row, _>>()?;
13650 Ok((values, record.get::<_, i64>(table.columns.len())?))
13651 })
13652 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13653 for record in records {
13654 let (row, version) =
13655 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13656 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
13657 conflicts.push(RetainedUniqueConflict {
13658 index: index.name.clone(),
13659 columns: index.columns.clone(),
13660 row_id,
13661 server_row: table
13662 .columns
13663 .iter()
13664 .zip(&row)
13665 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
13666 .collect(),
13667 server_version: version,
13668 });
13669 }
13670 }
13671 Ok(conflicts)
13672 }
13673
13674 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
13675 if let Some((commit, initial)) = self
13676 .failed_commits
13677 .iter()
13678 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
13679 {
13680 let mut rows = Vec::new();
13681 for operation in &commit.ops {
13682 let table = self
13683 .schema
13684 .table(&operation.table)
13685 .ok_or_else(|| "sync.unknown_table".to_owned())?;
13686 let key =
13687 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
13688 .map_err(|_| "sync.local_corrupt".to_owned())?;
13689 let base = self.stored_row(table, &operation.row_id, true)?;
13690 rows.push(RetainedCommitRow {
13691 table: operation.table.clone(),
13692 row_id: operation.row_id.clone(),
13693 unique_conflicts: if operation.upsert {
13694 self.unique_conflicts(
13695 table,
13696 initial
13697 .get(&key)
13698 .and_then(Value::as_object)
13699 .or(operation.values.as_ref())
13700 .ok_or("sync.local_corrupt")?,
13701 true,
13702 )?
13703 } else {
13704 Vec::new()
13705 },
13706 local_row: if operation.upsert {
13707 initial
13708 .get(&key)
13709 .and_then(Value::as_object)
13710 .cloned()
13711 .or_else(|| operation.values.clone())
13712 } else {
13713 None
13714 },
13715 server_row: base.as_ref().map(|(values, _)| {
13716 table
13717 .columns
13718 .iter()
13719 .zip(values)
13720 .map(|(column, value)| {
13721 (column.name.clone(), column_value_to_json(value))
13722 })
13723 .collect()
13724 }),
13725 server_version: base.map(|(_, version)| version),
13726 });
13727 }
13728 outcome.retained_rows = Some(rows);
13729 }
13730 Ok(outcome)
13731 }
13732
13733 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
13734 let row = self
13735 .conn
13736 .query_row(
13737 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
13738 resolution, resolved_at_ms, replacement_client_commit_id
13739 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
13740 rusqlite::params![client_commit_id],
13741 |row| {
13742 Ok(StoredCommitOutcomeRow {
13743 sequence: row.get(0)?,
13744 client_commit_id: row.get(1)?,
13745 status: row.get(2)?,
13746 recorded_at_ms: row.get(3)?,
13747 results_json: row.get(4)?,
13748 operations_json: row.get(5)?,
13749 resolution: row.get(6)?,
13750 resolved_at_ms: row.get(7)?,
13751 replacement_client_commit_id: row.get(8)?,
13752 })
13753 },
13754 )
13755 .optional()
13756 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13757 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
13758 .transpose()
13759 }
13760
13761 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
13762 if query.limit == Some(0) {
13763 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
13764 }
13765 let mut sql = String::from(
13766 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
13767 resolution, resolved_at_ms, replacement_client_commit_id
13768 FROM _syncular_commit_outcomes",
13769 );
13770 if query.active_only {
13771 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
13772 }
13773 sql.push_str(" ORDER BY seq DESC");
13774 if let Some(limit) = query.limit {
13775 sql.push_str(&format!(" LIMIT {limit}"));
13776 }
13777 let mut stmt = self
13778 .conn
13779 .prepare(&sql)
13780 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13781 let rows = stmt
13782 .query_map([], |row| {
13783 Ok(StoredCommitOutcomeRow {
13784 sequence: row.get(0)?,
13785 client_commit_id: row.get(1)?,
13786 status: row.get(2)?,
13787 recorded_at_ms: row.get(3)?,
13788 results_json: row.get(4)?,
13789 operations_json: row.get(5)?,
13790 resolution: row.get(6)?,
13791 resolved_at_ms: row.get(7)?,
13792 replacement_client_commit_id: row.get(8)?,
13793 })
13794 })
13795 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13796 let mut outcomes = Vec::new();
13797 for row in rows {
13798 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
13799 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
13800 )?)?);
13801 }
13802 Ok(outcomes)
13803 }
13804
13805 fn prune_commit_outcomes(&self) -> Result<(), String> {
13806 #[cfg(test)]
13807 self.outcome_prune_count
13808 .set(self.outcome_prune_count.get() + 1);
13809 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
13810 let count = self
13811 .conn
13812 .query_row(
13813 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
13814 [],
13815 |row| row.get::<_, i64>(0),
13816 )
13817 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
13818 as usize;
13819 let excess = count.saturating_sub(max_entries);
13820 if excess == 0 {
13821 return Ok(());
13822 }
13823 let mut stmt = self
13824 .conn
13825 .prepare(
13826 "SELECT seq FROM _syncular_commit_outcomes
13827 WHERE status IN ('applied', 'cached') OR resolution != 'active'
13828 ORDER BY seq ASC LIMIT ?1",
13829 )
13830 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13831 let rows = stmt
13832 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
13833 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13834 let sequences = rows
13835 .collect::<Result<Vec<_>, _>>()
13836 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13837 drop(stmt);
13838 for sequence in sequences {
13839 self.conn
13840 .execute(
13841 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
13842 rusqlite::params![sequence],
13843 )
13844 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13845 }
13846 Ok(())
13847 }
13848
13849 pub fn subscribe(
13852 &mut self,
13853 id: String,
13854 table: String,
13855 scopes: Vec<(String, Vec<String>)>,
13856 params: Option<String>,
13857 ) -> Result<(), String> {
13858 if self.schema.table(&table).is_none() {
13859 return Err(format!("unknown table {table:?}"));
13860 }
13861 let registered_at = self
13862 .subs
13863 .iter()
13864 .position(|subscription| subscription.id == id);
13865 if let Some(index) = registered_at {
13866 let existing = &self.subs[index];
13867 let same_intent = existing.table == table
13868 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
13869 && existing.params == params;
13870 if !same_intent {
13871 return Err(
13872 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
13873 .to_owned(),
13874 );
13875 }
13876 if existing.state != SubState::Revoked {
13881 return Ok(());
13882 }
13883 }
13884 let sub = Subscription {
13885 id: id.clone(),
13886 table,
13887 requested: scopes,
13888 params,
13889 cursor: -1,
13890 bootstrap_state: None,
13891 state: SubState::Active,
13892 reason_code: None,
13893 effective: None,
13894 synced_once: false,
13895 };
13896 self.persist_sub(&sub)?;
13897 match registered_at {
13898 Some(index) => self.subs[index] = sub,
13899 None => self.subs.push(sub),
13900 }
13901 if registered_at.is_some() {
13905 self.set_sync_needed(true, true);
13906 }
13907 Ok(())
13908 }
13909
13910 pub fn subscribe_many(&mut self, requests: Vec<SubscriptionRequest>) -> Result<(), String> {
13916 let prior_subs = self.subs.clone();
13917 self.begin_observation("syncular_subscribe")?;
13918 let registered = (|| {
13919 for request in requests {
13920 self.subscribe(request.id, request.table, request.scopes, request.params)?;
13921 }
13922 self.conn
13923 .execute_batch("RELEASE syncular_subscribe")
13924 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13925 })();
13926 if let Err(error) = registered {
13927 self.rollback_observation("syncular_subscribe");
13928 self.subs = prior_subs;
13929 return Err(error);
13930 }
13931 Ok(())
13932 }
13933
13934 pub fn unsubscribe(&mut self, id: &str) {
13935 if self.subs.iter().any(|s| s.id == id) {
13936 self.cancel_sync_round();
13937 }
13938 self.subs.retain(|s| s.id != id);
13939 let _ = self.conn.execute(
13940 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
13941 rusqlite::params![id],
13942 );
13943 }
13944
13945 pub fn set_window(
13953 &mut self,
13954 base: &WindowBase,
13955 units: &[String],
13956 ) -> Result<CommandEffects, String> {
13957 let table = self
13958 .schema
13959 .table(&base.table)
13960 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
13961 if table.scope_column(&base.variable).is_none() {
13962 return Err(format!(
13963 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
13964 base.table, base.variable
13965 ));
13966 }
13967 let base_key = window_base_key(base);
13968 let wanted: std::collections::HashSet<&String> = units.iter().collect();
13969 let live = self.load_window_units(&base_key);
13970 self.begin_observation("syncular_window")?;
13971 let mut batch = ChangeAccumulator::default();
13972 let mut changed = false;
13973
13974 let prior_subs = self.subs.clone();
13975 let widened = (|| {
13976 for unit in units {
13978 if live.iter().any(|(u, _)| u == unit) {
13979 continue;
13980 }
13981 let sub_id = derive_sub_id(base, unit);
13982 self.delete_pending_evict(&sub_id)?;
13983 self.insert_window_unit(&base_key, unit, &sub_id)?;
13984 self.subscribe(
13985 sub_id,
13986 base.table.clone(),
13987 unit_scopes(base, unit),
13988 base.params.clone(),
13989 )?;
13990 batch.window(&base_key, &base.table, unit);
13991 changed = true;
13992 }
13993
13994 self.finish_observation("syncular_window", batch)
13995 })();
13996 if let Err(error) = widened {
13997 self.rollback_observation("syncular_window");
13998 self.subs = prior_subs;
13999 return Err(error);
14000 }
14001 for (unit, sub_id) in live {
14002 if wanted.contains(&unit) {
14003 continue;
14004 }
14005 self.rebuild_overlay_if_dirty()?;
14006 let prior_subs = self.subs.clone();
14007 self.begin_observation("syncular_window")?;
14008 let effective = self
14009 .subs
14010 .iter()
14011 .find(|sub| sub.id == sub_id)
14012 .and_then(|sub| sub.effective.clone())
14013 .unwrap_or_else(|| unit_scopes(base, &unit));
14014 let mut batch = ChangeAccumulator::default();
14015 self.record_scope_map(&mut batch, &base.table, &effective);
14016 batch.window(&base_key, &base.table, &unit);
14017 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
14018 let remaining = match result {
14019 Ok(remaining) => remaining,
14020 Err(error) => {
14021 self.rollback_observation("syncular_window");
14022 self.subs = prior_subs;
14023 return Err(error);
14024 }
14025 };
14026 if let Err(error) = self.finish_observation("syncular_window", batch) {
14027 self.rollback_observation("syncular_window");
14028 self.subs = prior_subs;
14029 return Err(error);
14030 }
14031 if remaining {
14032 self.drain_pending_evictions()?;
14033 }
14034 changed = true;
14035 }
14036 Ok(if changed {
14037 CommandEffects::interactive()
14038 } else {
14039 CommandEffects::none()
14040 })
14041 }
14042
14043 pub fn window_state(&self, base: &WindowBase) -> WindowState {
14048 let mut units = Vec::new();
14049 let mut pending = Vec::new();
14050 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
14051 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
14052 Some(sub) => {
14053 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
14054 }
14055 None => true,
14056 };
14057 if is_pending {
14058 pending.push(unit.clone());
14059 }
14060 units.push(unit);
14061 }
14062 WindowState { units, pending }
14063 }
14064
14065 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
14066 let mut stmt = match self
14067 .conn
14068 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
14069 {
14070 Ok(stmt) => stmt,
14071 Err(_) => return Vec::new(),
14072 };
14073 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
14074 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
14075 });
14076 match rows {
14077 Ok(rows) => rows.filter_map(Result::ok).collect(),
14078 Err(_) => Vec::new(),
14079 }
14080 }
14081
14082 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
14083 let mut stmt = match self.conn.prepare(
14084 "SELECT windows.base, windows.unit, subscriptions.tbl
14085 FROM _syncular_windows AS windows
14086 JOIN _syncular_subscriptions AS subscriptions
14087 ON subscriptions.id = windows.sub_id
14088 ORDER BY windows.base, windows.unit",
14089 ) {
14090 Ok(stmt) => stmt,
14091 Err(_) => return Vec::new(),
14092 };
14093 let rows = stmt.query_map([], |row| {
14094 Ok((
14095 row.get::<_, String>(0)?,
14096 row.get::<_, String>(1)?,
14097 row.get::<_, String>(2)?,
14098 ))
14099 });
14100 match rows {
14101 Ok(rows) => rows.filter_map(Result::ok).collect(),
14102 Err(_) => Vec::new(),
14103 }
14104 }
14105
14106 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
14107 self.conn
14108 .query_row(
14109 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
14110 rusqlite::params![sub_id],
14111 |row| Ok((row.get(0)?, row.get(1)?)),
14112 )
14113 .ok()
14114 }
14115
14116 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
14117 self.conn
14118 .execute(
14119 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
14120 rusqlite::params![base_key, unit, sub_id],
14121 )
14122 .map(|_| ())
14123 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14124 }
14125
14126 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
14127 self.conn
14128 .execute(
14129 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
14130 rusqlite::params![base_key, unit],
14131 )
14132 .map(|_| ())
14133 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14134 }
14135
14136 fn evict_unit(
14143 &mut self,
14144 base_key: &str,
14145 base: &WindowBase,
14146 unit: &str,
14147 sub_id: &str,
14148 ) -> Result<bool, String> {
14149 let effective = self
14150 .subs
14151 .iter()
14152 .find(|s| s.id == sub_id)
14153 .and_then(|s| s.effective.clone())
14154 .unwrap_or_else(|| unit_scopes(base, unit));
14155 let pinned = self.pinned_row_ids(&base.table);
14156 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
14157 self.delete_window_unit(base_key, unit)?;
14158 self.conn
14159 .execute(
14160 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
14161 [sub_id],
14162 )
14163 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14164 self.subs.retain(|sub| sub.id != sub_id);
14165 if deferred {
14166 self.save_pending_evict(sub_id, &base.table, &effective)?;
14167 } else {
14168 self.delete_pending_evict(sub_id)?;
14169 }
14170 Ok(remaining)
14171 }
14172
14173 fn evict_scope_rows(
14177 &self,
14178 table_name: &str,
14179 effective: &[(String, Vec<String>)],
14180 pinned: &std::collections::HashSet<String>,
14181 ) -> Result<(bool, bool), String> {
14182 if effective.is_empty() {
14183 return Ok((false, false));
14184 }
14185 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
14186 let mut clauses = Vec::new();
14187 let mut params: Vec<SqlValue> = Vec::new();
14188 for (variable, values) in effective {
14189 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
14190 if values.is_empty() {
14191 return Ok((false, false));
14192 }
14193 let holes = values
14194 .iter()
14195 .map(|v| {
14196 params.push(SqlValue::Text(v.clone()));
14197 "?"
14198 })
14199 .collect::<Vec<_>>()
14200 .join(", ");
14201 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
14202 }
14203 let scope = clauses.join(" AND ");
14204 let scope_params = params.len();
14205 let pk = quote_ident(&table.primary_key);
14206 if !pinned.is_empty() {
14207 let holes = pinned
14208 .iter()
14209 .map(|id| {
14210 params.push(SqlValue::Text(id.clone()));
14211 "?"
14212 })
14213 .collect::<Vec<_>>()
14214 .join(", ");
14215 clauses.push(format!("{pk} NOT IN ({holes})"));
14216 }
14217 let predicate = clauses.join(" AND ");
14218 let base = base_table(table_name);
14219 let visible = visible_table(table_name);
14220 let mut ids: Vec<SqlValue> = Vec::new();
14221 for source in [&base, &visible] {
14222 let mut stmt = self
14223 .conn
14224 .prepare_cached(&format!(
14225 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
14226 ))
14227 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14228 ids = stmt
14229 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
14230 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
14231 .collect::<Result<_, _>>()
14232 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14233 if !ids.is_empty() {
14234 break;
14235 }
14236 }
14237 if !ids.is_empty() {
14238 for id in &ids {
14239 self.conn
14240 .execute(
14241 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
14242 rusqlite::params![table_name, id],
14243 )
14244 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14245 }
14246 let holes = vec!["?"; ids.len()].join(", ");
14247 for source in [&base, &visible] {
14248 self.conn
14249 .execute(
14250 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
14251 rusqlite::params_from_iter(&ids),
14252 )
14253 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14254 }
14255 }
14256 let mut remaining = false;
14257 let mut deferred = false;
14258 for source in [&base, &visible] {
14259 remaining |= self
14260 .conn
14261 .query_row(
14262 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
14263 rusqlite::params_from_iter(¶ms),
14264 |row| row.get::<_, bool>(0),
14265 )
14266 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14267 deferred |= self
14268 .conn
14269 .query_row(
14270 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
14271 rusqlite::params_from_iter(¶ms[..scope_params]),
14272 |row| row.get::<_, bool>(0),
14273 )
14274 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14275 }
14276 Ok((remaining, deferred))
14277 }
14278
14279 fn drain_pending_evictions(&mut self) -> Result<(), String> {
14281 let pending = self.load_pending_evictions()?;
14282 if pending.is_empty() {
14283 return Ok(());
14284 }
14285 self.rebuild_overlay_if_dirty()?;
14286 for (sub_id, table_name, effective) in pending {
14287 if self.schema.table(&table_name).is_none() {
14288 self.delete_pending_evict(&sub_id)?;
14289 continue;
14290 }
14291 loop {
14292 std::thread::yield_now();
14293 self.begin_observation("syncular_evict")?;
14294 let pinned = self.pinned_row_ids(&table_name);
14295 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
14296 let (remaining, deferred) = match result {
14297 Ok(result) => result,
14298 Err(error) => {
14299 self.rollback_observation("syncular_evict");
14300 return Err(error);
14301 }
14302 };
14303 if !deferred {
14304 if let Err(error) = self.delete_pending_evict(&sub_id) {
14305 self.rollback_observation("syncular_evict");
14306 return Err(error);
14307 }
14308 }
14309 let mut batch = ChangeAccumulator::default();
14310 self.record_scope_map(&mut batch, &table_name, &effective);
14311 if let Err(error) = self.finish_observation("syncular_evict", batch) {
14312 self.rollback_observation("syncular_evict");
14313 return Err(error);
14314 }
14315 if !remaining {
14316 break;
14317 }
14318 }
14319 }
14320 Ok(())
14321 }
14322
14323 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
14326 let mut pinned = std::collections::HashSet::new();
14327 for commit in self
14328 .outbox
14329 .iter()
14330 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
14331 {
14332 for op in &commit.ops {
14333 if op.table == table {
14334 pinned.insert(op.row_id.clone());
14335 }
14336 }
14337 }
14338 pinned
14339 }
14340
14341 fn save_pending_evict(
14342 &self,
14343 sub_id: &str,
14344 table: &str,
14345 effective: &[(String, Vec<String>)],
14346 ) -> Result<(), String> {
14347 self.conn.execute(
14348 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
14349 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
14350 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14351 }
14352
14353 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
14354 self.conn
14355 .execute(
14356 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
14357 rusqlite::params![sub_id],
14358 )
14359 .map(|_| ())
14360 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14361 }
14362
14363 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
14364 let mut stmt = self
14365 .conn
14366 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
14367 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14368 let rows = stmt
14369 .query_map([], |row| {
14370 Ok((
14371 row.get::<_, String>(0)?,
14372 row.get::<_, String>(1)?,
14373 row.get::<_, String>(2)?,
14374 ))
14375 })
14376 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14377 rows.map(|row| {
14378 let (sub_id, table, json) =
14379 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14380 let value: Value = serde_json::from_str(&json)
14381 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14382 let effective = json_to_scope_map(&value)
14383 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14384 Ok((sub_id, table, effective))
14385 })
14386 .collect()
14387 }
14388
14389 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
14391 self.retain_failed_commits = enabled;
14392 for (commit, _) in &self.failed_commits {
14393 for operation in &commit.ops {
14394 self.overlay_dirty.table(&operation.table);
14395 }
14396 }
14397 self.failed_commits = self.load_failed_commits()?;
14398 for (commit, _) in &self.failed_commits {
14399 for operation in &commit.ops {
14400 self.overlay_dirty.table(&operation.table);
14401 }
14402 }
14403 Ok(())
14404 }
14405
14406 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
14407 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
14408 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14409 let rows = statement
14410 .query_map([], |row| {
14411 Ok((
14412 row.get::<_, String>(0)?,
14413 row.get::<_, String>(1)?,
14414 row.get::<_, String>(2)?,
14415 ))
14416 })
14417 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14418 rows.map(|row| {
14419 let (id, operations, initial) =
14420 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14421 let operations: Vec<CommitOperation> =
14422 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
14423 let initial =
14424 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
14425 Ok((
14426 OutboxCommit {
14427 client_commit_id: id,
14428 ops: operations
14429 .into_iter()
14430 .map(|op| OutboxOp {
14431 upsert: op.op == "upsert",
14432 table: op.table,
14433 row_id: op.row_id,
14434 base_version: op.base_version,
14435 values: op.values,
14436 })
14437 .collect(),
14438 },
14439 initial,
14440 ))
14441 })
14442 .collect()
14443 }
14444
14445 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
14446 let mut initial = Map::new();
14447 for operation in operations {
14448 let table = self
14449 .schema
14450 .table(&operation.table)
14451 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14452 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
14453 Some(row) => Some(row),
14454 None => self.visible_row(table, &operation.row_id)?,
14455 };
14456 if let Some((values, _)) = initial_row {
14457 let mut intended: Map<String, Value> = table
14458 .columns
14459 .iter()
14460 .zip(values.iter())
14461 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
14462 .collect();
14463 if let Some(patch) = &operation.values {
14464 intended.extend(patch.clone());
14465 }
14466 initial.insert(
14467 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14468 .map_err(|_| "sync.local_corrupt".to_owned())?,
14469 Value::Object(intended),
14470 );
14471 }
14472 }
14473 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
14474 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
14475 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
14476 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14477 self.failed_commits.push((
14478 OutboxCommit {
14479 client_commit_id: id.to_owned(),
14480 ops: operations.to_vec(),
14481 },
14482 initial,
14483 ));
14484 Ok(())
14485 }
14486
14487 fn drop_failed_commits(
14488 &mut self,
14489 matches: impl Fn(&str, &Map<String, Value>) -> bool,
14490 ) -> Result<Vec<OutboxCommit>, String> {
14491 let mut doomed = self.drop_acknowledged_rows(&matches)?;
14492 for (commit, initial) in &self.failed_commits {
14493 for operation in &commit.ops {
14494 let key =
14495 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14496 .map_err(|_| "sync.local_corrupt".to_owned())?;
14497 if initial
14498 .get(&key)
14499 .and_then(Value::as_object)
14500 .is_some_and(|values| matches(&operation.table, values))
14501 {
14502 doomed.push(commit.clone());
14503 break;
14504 }
14505 }
14506 }
14507 for commit in &doomed {
14508 self.conn
14509 .execute(
14510 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
14511 rusqlite::params![commit.client_commit_id],
14512 )
14513 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14514 }
14515 for commit in &doomed {
14516 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14517 }
14518 self.conflicts.retain(|record| {
14519 !doomed
14520 .iter()
14521 .any(|commit| commit.client_commit_id == record.client_commit_id)
14522 });
14523 self.rejections.retain(|record| {
14524 !doomed
14525 .iter()
14526 .any(|commit| commit.client_commit_id == record.client_commit_id)
14527 });
14528 self.failed_commits.retain(|(commit, _)| {
14529 !doomed
14530 .iter()
14531 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
14532 });
14533 for commit in &doomed {
14534 for operation in &commit.ops {
14535 self.overlay_dirty.table(&operation.table);
14536 }
14537 }
14538 Ok(doomed)
14539 }
14540
14541 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
14545 let prior = self.storage_failure.borrow_mut().take();
14546 let outcome = (|| -> Result<String, String> {
14547 if mutations.is_empty() {
14548 return Err(
14549 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
14550 .to_owned(),
14551 );
14552 }
14553 let mut ops = Vec::with_capacity(mutations.len());
14554 for mutation in mutations {
14555 match mutation {
14556 Mutation::Patch {
14557 table,
14558 values,
14559 base_version,
14560 } => {
14561 let schema_table = self
14562 .schema
14563 .table(&table)
14564 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14565 let values = normalize_values_casing(schema_table, values)?;
14566 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14567 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
14568 }
14569 Mutation::Upsert {
14570 table,
14571 values,
14572 base_version,
14573 } => {
14574 let schema_table = self.schema.table(&table).ok_or_else(|| {
14575 format!("sync.unknown_table: unknown table {table:?}")
14576 })?;
14577 let values = normalize_values_casing(schema_table, values)?;
14581 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14582 let values = full_row_values(schema_table, values)?;
14585 self.validate_author_encode(schema_table, &row_id, &values)?;
14591 ops.push(OutboxOp {
14592 upsert: true,
14593 table,
14594 row_id,
14595 base_version,
14596 values: Some(values),
14597 });
14598 }
14599 Mutation::Delete {
14600 table,
14601 row_id,
14602 base_version,
14603 } => {
14604 if self.schema.table(&table).is_none() {
14605 return Err(format!("sync.unknown_table: unknown table {table:?}"));
14606 }
14607 ops.push(OutboxOp {
14608 upsert: false,
14609 table,
14610 row_id,
14611 base_version,
14612 values: None,
14613 });
14614 }
14615 }
14616 }
14617 self.record_outbox_commit(ops)
14618 })();
14619 self.finish_authoring(prior, outcome)
14620 }
14621
14622 fn finish_authoring(
14628 &self,
14629 prior: Option<QueryReadFailure>,
14630 outcome: Result<String, String>,
14631 ) -> Result<String, ClientError> {
14632 let fresh = self.storage_failure.borrow_mut().take();
14633 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
14634 match outcome {
14635 Ok(id) => Ok(id),
14636 Err(message) => Err(fresh
14637 .map(ClientError::from)
14638 .unwrap_or_else(|| ClientError::from(message))),
14639 }
14640 }
14641
14642 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
14643 let commit = OutboxCommit {
14644 client_commit_id: uuid::Uuid::new_v4().to_string(),
14645 ops,
14646 };
14647 self.begin_observation("syncular_mutation")?;
14648 let mut batch = ChangeAccumulator::default();
14649 for op in &commit.ops {
14650 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
14651 if let Some(values) = &op.values {
14652 if let Some(table) = self.schema.table(&op.table) {
14653 for scope in &table.scope_variables {
14654 if let Some(Value::String(value)) = values.get(&scope.column) {
14655 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
14656 precise = true;
14657 }
14658 }
14659 }
14660 }
14661 if !precise {
14662 batch.table(&op.table);
14663 }
14664 }
14665 if let Err(error) = self.persist_outbox_insert(&commit) {
14666 self.rollback_observation("syncular_mutation");
14667 return Err(error);
14668 }
14669 let was_dirty = self.overlay_dirty.snapshot();
14670 if let Err(error) = self.rebuild_overlay_if_dirty() {
14675 self.rollback_observation("syncular_mutation");
14676 self.overlay_dirty.restore(was_dirty.clone());
14677 return Err(error);
14678 }
14679 if let Some(error) = commit
14680 .ops
14681 .iter()
14682 .find_map(|op| self.apply_outbox_op(op).err())
14683 {
14684 self.rollback_observation("syncular_mutation");
14685 self.overlay_dirty.restore(was_dirty.clone());
14686 return Err(error);
14687 }
14688 let id = commit.client_commit_id.clone();
14689 self.outbox.push(commit);
14690 batch.status = true;
14691 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
14692 self.rollback_observation("syncular_mutation");
14693 self.outbox.pop();
14694 self.overlay_dirty.restore(was_dirty.clone());
14695 return Err(error);
14696 }
14697 Ok(id)
14698 }
14699
14700 pub fn patch(
14706 &mut self,
14707 table: &str,
14708 row_id: &str,
14709 partial: Map<String, Value>,
14710 base_version: Option<i64>,
14711 ) -> Result<String, ClientError> {
14712 let prior = self.storage_failure.borrow_mut().take();
14713 let outcome = self.perform_patch(table, row_id, partial, base_version);
14714 self.finish_authoring(prior, outcome)
14715 }
14716
14717 fn perform_patch(
14720 &mut self,
14721 table: &str,
14722 row_id: &str,
14723 partial: Map<String, Value>,
14724 base_version: Option<i64>,
14725 ) -> Result<String, String> {
14726 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
14727 self.record_outbox_commit(vec![operation])
14728 }
14729
14730 fn prepare_patch(
14731 &self,
14732 table: &str,
14733 row_id: &str,
14734 partial: Map<String, Value>,
14735 base_version: Option<i64>,
14736 ) -> Result<OutboxOp, String> {
14737 let schema_table = self
14738 .schema
14739 .table(table)
14740 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
14741 let partial = normalize_values_casing(schema_table, partial)?;
14742 if let Some(pk) = partial.get(&schema_table.primary_key) {
14743 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
14744 return Err(format!(
14745 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
14746 ));
14747 }
14748 }
14749 let exists: bool = self
14752 .conn
14753 .query_row(
14754 &format!(
14755 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
14756 visible_table(table),
14757 row_id_predicate(schema_table)
14758 ),
14759 rusqlite::params![row_id],
14760 |row| row.get(0),
14761 )
14762 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14763 if !exists {
14764 return Err("sync.row_missing: a sparse patch requires a local row".to_owned());
14765 }
14766 for scope in &schema_table.scope_variables {
14776 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
14777 continue;
14778 }
14779 let Some(column) = schema_table
14780 .columns
14781 .iter()
14782 .find(|column| column.name == scope.column)
14783 else {
14784 continue;
14785 };
14786 let stored = self
14787 .conn
14788 .query_row(
14789 &format!(
14790 "SELECT {} FROM {} WHERE {}",
14791 quote_ident(&scope.column),
14792 visible_table(&schema_table.name),
14793 row_id_predicate(schema_table)
14794 ),
14795 rusqlite::params![row_id],
14796 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
14797 )
14798 .optional()
14799 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14800 let incoming = json_to_column_value(column, partial.get(&scope.column))
14805 .ok()
14806 .map(|value| column_value_to_json(&value));
14807 let equal = stored.as_ref().is_some_and(|stored| {
14811 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
14812 });
14813 if !equal {
14814 return Err(format!(
14815 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
14816 scope.column
14817 ));
14818 }
14819 }
14820 let mut values = partial;
14821 for scope in &schema_table.scope_variables {
14824 values.remove(&scope.column);
14825 }
14826 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
14827 ColumnType::Integer => match row_id.parse::<i64>() {
14829 Ok(v)
14830 if v.to_string() == row_id
14831 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
14832 .contains(&v) =>
14833 {
14834 Value::from(v)
14835 }
14836 _ => {
14837 return Err(format!(
14838 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
14839 ))
14840 }
14841 },
14842 ColumnType::Boolean => match row_id {
14843 "true" => Value::Bool(true),
14844 "false" => Value::Bool(false),
14845 _ => {
14846 return Err(format!(
14847 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
14848 ))
14849 }
14850 },
14851 _ => Value::from(row_id.to_owned()),
14852 };
14853 values.insert(schema_table.primary_key.clone(), pk_value);
14854 self.validate_author_encode(schema_table, row_id, &values)?;
14860 Ok(OutboxOp {
14861 upsert: true,
14862 table: table.to_owned(),
14863 row_id: row_id.to_owned(),
14864 base_version,
14865 values: Some(values),
14866 })
14867 }
14868
14869 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
14874 let selector = self.encryption.key_id_columns.get(&table.name)?;
14875 let index = table
14876 .columns
14877 .iter()
14878 .position(|column| &column.name == selector)?;
14879 let sql = format!(
14880 "SELECT {} FROM {} WHERE {}",
14881 quote_ident(selector),
14882 visible_table(&table.name),
14883 row_id_predicate(table)
14884 );
14885 let key: Option<String> = self
14886 .conn
14887 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
14888 .ok()?;
14889 let key = key.filter(|key| !key.is_empty())?;
14890 let mut fallback: Row = vec![None; table.columns.len()];
14891 fallback[index] = Some(ColumnValue::String(key));
14892 Some(fallback)
14893 }
14894
14895 fn validate_author_encode(
14900 &self,
14901 table: &TableSchema,
14902 row_id: &str,
14903 values: &Map<String, Value>,
14904 ) -> Result<(), String> {
14905 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
14906 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
14907 result => result.map(|_| ()),
14908 }
14909 }
14910
14911 fn drop_unencodable_outbox(
14925 &mut self,
14926 with_encryption: bool,
14927 batch: Option<&mut ChangeAccumulator>,
14928 ) -> Result<bool, String> {
14929 let failures = self
14930 .outbox
14931 .iter()
14932 .filter_map(|commit| {
14933 commit
14934 .ops
14935 .iter()
14936 .find_map(|op| {
14937 let values = op.values.as_ref().filter(|_| op.upsert)?;
14938 let table = self.schema.table(&op.table)?;
14939 let fallback = self.stored_key_fallback(table, &op.row_id);
14940 match encode_sparse_row_json(
14941 table,
14942 &op.row_id,
14943 values,
14944 &self.encryption,
14945 fallback.as_ref(),
14946 ) {
14947 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
14948 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
14949 }
14950 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
14955 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
14956 OUTBOX_INCOMPATIBLE_CODE,
14957 )),
14958 _ => None,
14959 }
14960 })
14961 .map(|(error, code)| (commit.clone(), error, code))
14962 })
14963 .collect::<Vec<_>>();
14964 if failures.is_empty() {
14965 return Ok(false);
14966 }
14967 let prior_outbox = self.outbox.clone();
14973 let prior_rejections = self.rejections.clone();
14974 let prior_failed_commits = self.failed_commits.clone();
14978 let prior_changes = self.change_queue.len();
14979 let prior_intents = self.sync_intent_queue.len();
14980 let prior_last_change = self.last_change.clone();
14981 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14982 let owns_observation = batch.is_none();
14983 let owns_transaction = self.conn.is_autocommit();
14984 let mut domains = ChangeAccumulator::default();
14985 if owns_observation {
14986 self.begin_observation("syncular_value_recovery")?;
14987 }
14988 let dropped = (|| -> Result<(), String> {
14989 for (commit, message, code) in failures {
14990 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
14991 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
14992 }
14993 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
14994 let rejection = RejectionRecord {
14995 client_commit_id: commit.client_commit_id.clone(),
14996 op_index: 0,
14997 code: code.to_owned(),
14998 message,
14999 retryable: false,
15000 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
15001 field_paths: None,
15002 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
15003 required_action: None,
15004 references: None,
15005 }),
15006 operation: representable
15007 .then(|| commit.ops.first())
15008 .flatten()
15009 .map(CommitOperation::from),
15010 };
15011 self.persist_commit_outcome(
15012 &commit.client_commit_id,
15013 CommitOutcomeStatus::Rejected,
15014 &[CommitOperationOutcome::Error {
15015 rejection: rejection.clone(),
15016 }],
15017 representable.then_some(commit.ops.as_slice()),
15018 )?;
15019 self.delete_outbox_persisted(&commit.client_commit_id)?;
15020 self.outbox
15021 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
15022 self.rejections.push(rejection);
15023 for operation in &commit.ops {
15024 domains.table(operation.table.as_str());
15025 }
15026 }
15027 domains.status = true;
15028 domains.rejections = true;
15029 domains.outcomes = true;
15030 self.prune_commit_outcomes()?;
15031 self.overlay_dirty.set(true);
15033 self.rebuild_overlay()?;
15034 Ok(())
15035 })();
15036 let result = match batch {
15037 Some(batch) => {
15038 if dropped.is_ok() {
15039 for table in domains.tables.keys() {
15040 batch.table(table.as_str());
15041 }
15042 batch.status |= domains.status;
15043 batch.rejections |= domains.rejections;
15044 batch.outcomes |= domains.outcomes;
15045 }
15046 dropped
15047 }
15048 None => {
15049 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
15050 }
15051 };
15052 if let Err(error) = result {
15053 if owns_observation {
15054 if owns_transaction {
15055 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
15056 let mut failure = self.storage_failure.borrow_mut();
15057 let failure =
15058 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
15059 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
15060 }
15061 } else {
15062 self.rollback_observation("syncular_value_recovery");
15063 }
15064 }
15065 self.outbox = prior_outbox;
15066 self.rejections = prior_rejections;
15067 self.failed_commits = prior_failed_commits;
15068 self.change_queue.truncate(prior_changes);
15069 self.sync_intent_queue.truncate(prior_intents);
15070 self.last_change = prior_last_change;
15071 self.overlay_dirty.restore(prior_overlay_dirty);
15072 return Err(error);
15073 }
15074 Ok(true)
15075 }
15076
15077 pub fn pending_commit_ids(&self) -> Vec<String> {
15078 self.outbox
15079 .iter()
15080 .map(|c| c.client_commit_id.clone())
15081 .collect()
15082 }
15083
15084 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
15088 let mut payloads = Vec::new();
15089 for commit in &self.outbox {
15090 for op in &commit.ops {
15091 let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
15092 continue;
15093 };
15094 let Some(table) = self.schema.table(&op.table) else {
15095 continue;
15096 };
15097 let fallback = self.stored_key_fallback(table, &op.row_id);
15098 if let Ok(payload) = encode_sparse_row_json(
15099 table,
15100 &op.row_id,
15101 values,
15102 &self.encryption,
15103 fallback.as_ref(),
15104 ) {
15105 payloads.push(payload);
15106 }
15107 }
15108 }
15109 payloads
15110 }
15111
15112 pub fn conflicts(&self) -> &[ConflictRecord] {
15113 &self.conflicts
15114 }
15115
15116 pub fn rejections(&self) -> &[RejectionRecord] {
15117 &self.rejections
15118 }
15119
15120 pub fn resolve_commit_outcome(
15121 &mut self,
15122 input: ResolveCommitOutcomeInput,
15123 ) -> Result<CommitOutcome, String> {
15124 let current = self
15125 .commit_outcome(&input.client_commit_id)?
15126 .ok_or_else(|| {
15127 format!(
15128 "sync.outcome_not_found: no durable outcome exists for {:?}",
15129 input.client_commit_id
15130 )
15131 })?;
15132 if current.resolution != CommitOutcomeResolution::Active {
15133 return Ok(current);
15134 }
15135 if input.resolution == CommitOutcomeResolution::Active {
15136 return Err("sync.invalid_request: resolution must leave active state".to_owned());
15137 }
15138 match input.resolution {
15139 CommitOutcomeResolution::Superseded => {
15140 let replacement = input
15141 .replacement_client_commit_id
15142 .as_deref()
15143 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
15144 .ok_or_else(|| {
15145 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
15146 .to_owned()
15147 })?;
15148 let _ = replacement;
15149 }
15150 _ if input.replacement_client_commit_id.is_some() => {
15151 return Err(
15152 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
15153 .to_owned(),
15154 );
15155 }
15156 _ => {}
15157 }
15158 let allowed = match current.status {
15159 CommitOutcomeStatus::Conflict => matches!(
15160 input.resolution,
15161 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
15162 ),
15163 CommitOutcomeStatus::Rejected => {
15164 matches!(
15165 input.resolution,
15166 CommitOutcomeResolution::Superseded
15167 | CommitOutcomeResolution::ResolvedKeepServer
15168 )
15169 }
15170 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
15171 input.resolution == CommitOutcomeResolution::Dismissed
15172 }
15173 };
15174 if !allowed {
15175 return Err(format!(
15176 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
15177 input.resolution, current.status
15178 ));
15179 }
15180
15181 self.begin_observation("syncular_outcome_resolution")?;
15182 let previous_failed = self.failed_commits.clone();
15183 let previous_conflicts = self.conflicts.clone();
15184 let previous_rejections = self.rejections.clone();
15185 let result = (|| {
15186 self.conn
15187 .execute(
15188 "UPDATE _syncular_commit_outcomes
15189 SET resolution = ?1, resolved_at_ms = ?2,
15190 replacement_client_commit_id = ?3
15191 WHERE client_commit_id = ?4 AND resolution = 'active'",
15192 rusqlite::params![
15193 Self::outcome_resolution_name(input.resolution),
15194 self.clock_now_ms(),
15195 input.replacement_client_commit_id,
15196 input.client_commit_id
15197 ],
15198 )
15199 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15200 let resolved = self
15201 .commit_outcome(¤t.client_commit_id)?
15202 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
15203 self.conn
15204 .execute(
15205 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
15206 rusqlite::params![input.client_commit_id],
15207 )
15208 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15209 for (commit, _) in &self.failed_commits {
15210 if commit.client_commit_id == input.client_commit_id {
15211 for operation in &commit.ops {
15212 self.overlay_dirty.table(&operation.table);
15213 }
15214 }
15215 }
15216 self.failed_commits
15217 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
15218 self.prune_commit_outcomes()?;
15219 Ok(resolved)
15220 })();
15221 let resolved = match result {
15222 Ok(outcome) => outcome,
15223 Err(error) => {
15224 self.rollback_observation("syncular_outcome_resolution");
15225 self.failed_commits = previous_failed;
15226 self.conflicts = previous_conflicts;
15227 self.rejections = previous_rejections;
15228 return Err(error);
15229 }
15230 };
15231 self.conflicts
15232 .retain(|record| record.client_commit_id != current.client_commit_id);
15233 self.rejections
15234 .retain(|record| record.client_commit_id != current.client_commit_id);
15235 let mut batch = ChangeAccumulator {
15236 conflicts: current.status == CommitOutcomeStatus::Conflict,
15237 rejections: current.status == CommitOutcomeStatus::Rejected,
15238 outcomes: true,
15239 ..ChangeAccumulator::default()
15240 };
15241 if let Some(operations) = ¤t.operations {
15242 for operation in operations {
15243 batch.table(&operation.table);
15244 }
15245 }
15246 if let Err(error) = self
15247 .rebuild_overlay_if_dirty()
15248 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
15249 {
15250 self.rollback_observation("syncular_outcome_resolution");
15251 self.failed_commits = previous_failed;
15252 self.conflicts = previous_conflicts;
15253 self.rejections = previous_rejections;
15254 return Err(error);
15255 }
15256 Ok(resolved)
15257 }
15258
15259 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
15260 self.schema_floor.as_ref()
15261 }
15262
15263 pub fn lease_state(&self) -> Option<&LeaseState> {
15265 self.lease_state.as_ref()
15266 }
15267
15268 fn record_lease_error(&mut self, code: &str) {
15271 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
15272 return;
15273 }
15274 let mut next = self.lease_state.clone().unwrap_or_default();
15275 next.error_code = Some(code.to_owned());
15276 self.set_lease_state(Some(next));
15277 }
15278
15279 fn set_lease_state(&mut self, next: Option<LeaseState>) {
15280 if self.lease_state == next {
15281 return;
15282 }
15283 if self.begin_observation("syncular_lease").is_err() {
15284 return;
15285 }
15286 self.lease_state = next;
15287 if let Some(lease) = &self.lease_state {
15288 if let Ok(json) = serde_json::to_string(lease) {
15289 self.set_meta(LEASE_STATE_KEY, &json);
15290 }
15291 } else {
15292 self.delete_meta(LEASE_STATE_KEY);
15293 }
15294 let batch = ChangeAccumulator {
15295 status: true,
15296 ..ChangeAccumulator::default()
15297 };
15298 if self.finish_observation("syncular_lease", batch).is_err() {
15299 self.rollback_observation("syncular_lease");
15300 }
15301 }
15302
15303 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
15304 if self.schema_floor == next {
15305 return;
15306 }
15307 if self.begin_observation("syncular_schema_floor").is_err() {
15308 return;
15309 }
15310 self.schema_floor = next;
15311 self.stopped = self.schema_floor.is_some();
15312 if let Some(floor) = &self.schema_floor {
15313 if let Ok(json) = serde_json::to_string(floor) {
15314 self.set_meta(SCHEMA_FLOOR_KEY, &json);
15315 }
15316 } else {
15317 self.delete_meta(SCHEMA_FLOOR_KEY);
15318 }
15319 let batch = ChangeAccumulator {
15320 status: true,
15321 ..ChangeAccumulator::default()
15322 };
15323 if self
15324 .finish_observation("syncular_schema_floor", batch)
15325 .is_err()
15326 {
15327 self.rollback_observation("syncular_schema_floor");
15328 }
15329 }
15330
15331 fn set_upgrading(&mut self, value: bool) {
15332 if self.upgrading == value {
15333 return;
15334 }
15335 if self.begin_observation("syncular_upgrading").is_err() {
15336 return;
15337 }
15338 self.upgrading = value;
15339 let batch = ChangeAccumulator {
15340 status: true,
15341 ..ChangeAccumulator::default()
15342 };
15343 if self
15344 .finish_observation("syncular_upgrading", batch)
15345 .is_err()
15346 {
15347 self.rollback_observation("syncular_upgrading");
15348 }
15349 }
15350
15351 pub fn sync_needed(&self) -> bool {
15352 self.sync_needed
15353 }
15354
15355 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
15356 let sub = self.subs.iter().find(|s| s.id == id)?;
15357 Some(SubscriptionStateView {
15358 id: sub.id.clone(),
15359 table: sub.table.clone(),
15360 status: sub.state.name().to_owned(),
15361 cursor: sub.cursor,
15362 has_resume_token: sub.bootstrap_state.is_some(),
15363 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
15364 reason_code: sub.reason_code.clone(),
15365 })
15366 }
15367
15368 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
15369 let schema_table = self
15370 .schema
15371 .table(table)
15372 .ok_or_else(|| format!("unknown table {table:?}"))?;
15373 let sql = format!(
15374 "SELECT * FROM {} ORDER BY {} ASC",
15375 visible_table(table),
15376 quote_ident(&schema_table.primary_key)
15377 );
15378 let mut stmt = self
15379 .conn
15380 .prepare(&sql)
15381 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15382 let mut rows = stmt
15383 .query([])
15384 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15385 let mut out = Vec::new();
15386 while let Some(row) = rows
15387 .next()
15388 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15389 {
15390 let mut values = Map::new();
15391 for (i, column) in schema_table.columns.iter().enumerate() {
15392 let value = row
15393 .get_ref(i)
15394 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15395 values.insert(column.name.clone(), sql_ref_to_json(column, value));
15396 }
15397 let version: i64 = row
15398 .get(schema_table.columns.len())
15399 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15400 let row_id = match values.get(&schema_table.primary_key) {
15401 Some(Value::String(s)) => s.clone(),
15402 Some(Value::Number(n)) => n.to_string(),
15403 Some(Value::Bool(b)) => b.to_string(),
15404 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
15405 };
15406 out.push(RowState {
15407 row_id,
15408 version,
15409 values,
15410 });
15411 }
15412 Ok(out)
15413 }
15414
15415 #[cfg(feature = "crdt-yjs")]
15431 fn crdt_column_bytes(
15432 &self,
15433 table: &str,
15434 row_id: &str,
15435 column: &str,
15436 ) -> Result<Option<Vec<u8>>, String> {
15437 let schema_table = self
15438 .schema
15439 .table(table)
15440 .ok_or_else(|| format!("unknown table {table:?}"))?;
15441 let col = schema_table
15442 .columns
15443 .iter()
15444 .find(|c| c.name == column)
15445 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
15446 if col.ty != ColumnType::Crdt {
15447 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
15448 }
15449 let sql = format!(
15450 "SELECT {} FROM {} WHERE {}",
15451 quote_ident(column),
15452 visible_table(table),
15453 row_id_predicate(schema_table)
15454 );
15455 let bytes: Option<Vec<u8>> = self
15456 .conn
15457 .prepare_cached(&sql)
15458 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15459 .query_row(rusqlite::params![row_id], |row| {
15460 row.get::<_, Option<Vec<u8>>>(0)
15461 })
15462 .map_err(|e| match e {
15463 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
15464 other => Self::sqlite_failure(&self.storage_failure, other),
15465 })?;
15466 Ok(bytes)
15467 }
15468
15469 #[cfg(feature = "crdt-yjs")]
15473 pub fn crdt_text(
15474 &self,
15475 table: &str,
15476 row_id: &str,
15477 column: &str,
15478 name: &str,
15479 ) -> Result<String, String> {
15480 let bytes = self
15481 .crdt_column_bytes(table, row_id, column)?
15482 .unwrap_or_default();
15483 crate::crdt::text(&bytes, name)
15484 }
15485
15486 #[cfg(feature = "crdt-yjs")]
15490 pub fn crdt_insert_text(
15491 &mut self,
15492 table: &str,
15493 row_id: &str,
15494 column: &str,
15495 name: &str,
15496 index: u32,
15497 value: &str,
15498 ) -> Result<String, ClientError> {
15499 let prior = self.storage_failure.borrow_mut().take();
15500 let outcome = (|| -> Result<String, String> {
15501 let current = self
15502 .crdt_column_bytes(table, row_id, column)?
15503 .unwrap_or_default();
15504 let update = crate::crdt::insert_text(¤t, name, index, value)?;
15505 self.perform_crdt_push_update(table, row_id, column, &update)
15506 })();
15507 self.finish_authoring(prior, outcome)
15508 }
15509
15510 #[cfg(feature = "crdt-yjs")]
15514 pub fn crdt_delete_text(
15515 &mut self,
15516 table: &str,
15517 row_id: &str,
15518 column: &str,
15519 name: &str,
15520 index: u32,
15521 len: u32,
15522 ) -> Result<String, ClientError> {
15523 let prior = self.storage_failure.borrow_mut().take();
15524 let outcome = (|| -> Result<String, String> {
15525 let current = self
15526 .crdt_column_bytes(table, row_id, column)?
15527 .unwrap_or_default();
15528 let update = crate::crdt::delete_text(¤t, name, index, len)?;
15529 self.perform_crdt_push_update(table, row_id, column, &update)
15530 })();
15531 self.finish_authoring(prior, outcome)
15532 }
15533
15534 #[cfg(feature = "crdt-yjs")]
15539 pub fn crdt_apply_update(
15540 &mut self,
15541 table: &str,
15542 row_id: &str,
15543 column: &str,
15544 update: &[u8],
15545 ) -> Result<String, ClientError> {
15546 let prior = self.storage_failure.borrow_mut().take();
15547 let outcome = (|| -> Result<String, String> {
15548 let current = self
15549 .crdt_column_bytes(table, row_id, column)?
15550 .unwrap_or_default();
15551 let next = crate::crdt::apply_update(¤t, update)?;
15552 self.perform_crdt_push_update(table, row_id, column, &next)
15553 })();
15554 self.finish_authoring(prior, outcome)
15555 }
15556
15557 #[cfg(feature = "crdt-yjs")]
15564 fn perform_crdt_push_update(
15565 &mut self,
15566 table: &str,
15567 row_id: &str,
15568 column: &str,
15569 crdt_bytes: &[u8],
15570 ) -> Result<String, String> {
15571 let mut bytes_obj = Map::new();
15572 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
15573 let mut partial = Map::new();
15574 partial.insert(column.to_owned(), Value::Object(bytes_obj));
15575 self.perform_patch(table, row_id, partial, None)
15576 }
15577
15578 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
15593 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
15594 }
15595
15596 #[cfg(feature = "bench-internals")]
15598 #[doc(hidden)]
15599 pub fn benchmark_connection(&mut self) -> &mut Connection {
15600 &mut self.conn
15601 }
15602
15603 #[cfg(feature = "bench-internals")]
15605 #[doc(hidden)]
15606 pub fn benchmark_phases(
15607 &self,
15608 enabled: Option<bool>,
15609 reset: bool,
15610 ) -> Result<Option<Value>, String> {
15611 self.benchmark_phases.configure(enabled, reset)?;
15612 Ok(self.benchmark_phases.snapshot())
15613 }
15614
15615 pub fn authority_snapshot(&self) -> Result<Value, String> {
15617 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
15618 if self.limits.authority_reads.is_empty() {
15619 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
15620 }
15621 self.conn
15622 .execute_batch("SAVEPOINT authority_snapshot")
15623 .map_err(|e| e.to_string())?;
15624 let result = (|| {
15625 let revision = self
15626 .get_meta(LOCAL_REVISION_KEY)
15627 .ok_or("sync.local_corrupt: authority revision is missing")?;
15628 let parsed_revision = revision
15629 .parse::<u64>()
15630 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
15631 if parsed_revision.to_string() != revision {
15632 return Err("sync.local_corrupt: authority revision is invalid".into());
15633 }
15634 let mut tables = Vec::new();
15635 for read in &self.limits.authority_reads {
15636 let table = self
15637 .schema
15638 .table(&read.table)
15639 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
15640 let scalar = |column: &str, value: &Value| -> Result<String, String> {
15641 let ty = table
15642 .columns
15643 .iter()
15644 .find(|c| c.name == column)
15645 .expect("validated authority column")
15646 .ty;
15647 if ty == ColumnType::Boolean {
15648 if let Some(value) = value.as_bool() {
15649 return Ok(value.to_string());
15650 }
15651 return match value.as_i64() {
15652 Some(0) => Ok("false".into()),
15653 Some(1) => Ok("true".into()),
15654 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
15655 };
15656 }
15657 match value {
15658 Value::String(s) => Ok(s.clone()),
15659 Value::Number(n) => Ok(n.to_string()),
15660 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
15661 .get("$bigint")
15662 .and_then(Value::as_str)
15663 .map(str::to_owned)
15664 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
15665 _ => {
15666 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
15667 }
15668 }
15669 };
15670 let matches = |row: &QueryRow| -> Result<bool, String> {
15671 for (variable, values) in &read.scopes {
15672 let column = table
15673 .scope_column(variable)
15674 .expect("validated authority scope");
15675 let value = row
15676 .get(column)
15677 .ok_or("sync.local_corrupt: authority scope column is missing")?;
15678 if value.is_null() || !values.contains(&scalar(column, value)?) {
15679 return Ok(false);
15680 }
15681 }
15682 Ok(true)
15683 };
15684 let selected = read
15685 .columns
15686 .iter()
15687 .map(|c| quote_ident(c))
15688 .collect::<Vec<_>>()
15689 .join(",");
15690 let mut intent = BTreeSet::new();
15691 let mut relevant = BTreeSet::new();
15692 for row in query_connection(&self.conn,
15693 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
15694 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
15695 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
15696 intent.insert(id.clone());
15697 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
15698 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
15699 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
15700 }
15701 }
15702 let mut rows = BTreeMap::new();
15703 for full in [base_table(&read.table), visible_table(&read.table)] {
15704 for mut values in query_connection(
15705 &self.conn,
15706 &format!(
15707 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
15708 ),
15709 &[],
15710 )
15711 .map_err(|e| e.to_string())?
15712 {
15713 if !matches(&values)? {
15714 continue;
15715 }
15716 let id = scalar(
15717 &table.primary_key,
15718 values
15719 .get(&table.primary_key)
15720 .expect("selected primary key"),
15721 )?;
15722 relevant.insert(id.clone());
15723 if full == base_table(&read.table) {
15724 let version = values
15725 .remove("authority_version")
15726 .expect("selected version");
15727 let version = version
15728 .as_i64()
15729 .ok_or("sync.local_corrupt: authority server version is invalid")?;
15730 if version >= 0 {
15731 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
15732 }
15733 }
15734 }
15735 }
15736 intent.retain(|id| relevant.contains(id));
15737 let rows = rows.into_values().collect::<Vec<_>>();
15738 let mut persisted = Vec::new();
15739 for row in query_connection(
15740 &self.conn,
15741 "SELECT state_json FROM _syncular_subscriptions WHERE tbl=? ORDER BY rowid",
15742 &[Value::String(read.table.clone())],
15743 )
15744 .map_err(|e| e.message)?
15745 {
15746 let state: Value = serde_json::from_str(
15747 row.get("state_json")
15748 .and_then(Value::as_str)
15749 .ok_or("sync.local_corrupt: authority subscription state is missing")?,
15750 )
15751 .map_err(|_| "sync.local_corrupt: authority subscription state is invalid")?;
15752 let effective = state.get("effectiveScopes").and_then(Value::as_object);
15753 let status = state
15754 .get("status")
15755 .and_then(Value::as_str)
15756 .ok_or("sync.local_corrupt: authority subscription status is missing")?;
15757 let cursor = state
15758 .get("cursor")
15759 .and_then(Value::as_i64)
15760 .ok_or("sync.local_corrupt: authority subscription cursor is missing")?;
15761 let complete = effective.is_some_and(|s| s.len() == read.scopes.len())
15762 && status == "active"
15763 && cursor >= 0
15764 && state.get("bootstrapState").is_none_or(Value::is_null)
15765 && state.get("params").is_none_or(Value::is_null);
15766 let exposed = effective.map(|s| {
15767 read.scopes
15768 .keys()
15769 .map(|v| (v.clone(), s.get(v).cloned().unwrap_or(json!([]))))
15770 .collect::<Map<_, _>>()
15771 });
15772 let requested = read
15773 .scopes
15774 .keys()
15775 .map(|v| {
15776 (
15777 v.clone(),
15778 state
15779 .get("requested")
15780 .and_then(|s| s.get(v))
15781 .cloned()
15782 .unwrap_or(json!([])),
15783 )
15784 })
15785 .collect::<Map<_, _>>();
15786 persisted.push(json!({"requestedScopes": requested, "status": status, "cursor": cursor, "effectiveScopes": exposed, "complete": complete}));
15787 }
15788 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
15789 match selectors.split_first() {
15790 None => !candidates.is_empty(),
15791 Some(((variable, values), rest)) => values.iter().all(|value| {
15792 covered(
15793 rest,
15794 &candidates
15795 .iter()
15796 .copied()
15797 .filter(|p| {
15798 p["effectiveScopes"][variable.as_str()]
15799 .as_array()
15800 .is_some_and(|held| {
15801 held.contains(&Value::String(value.clone()))
15802 || held.contains(&Value::String("*".into()))
15803 })
15804 })
15805 .collect::<Vec<_>>(),
15806 )
15807 }),
15808 }
15809 }
15810 let coverage = if covered(
15811 &read.scopes.iter().collect::<Vec<_>>(),
15812 &persisted
15813 .iter()
15814 .filter(|p| p["complete"] == true)
15815 .collect::<Vec<_>>(),
15816 ) {
15817 "complete"
15818 } else if persisted.iter().any(|p| {
15819 p["status"] == "active"
15820 && read.scopes.iter().all(|(v, values)| {
15821 values.iter().any(|value| {
15822 p["requestedScopes"][v].as_array().is_some_and(|held| {
15823 held.contains(&Value::String(value.clone()))
15824 || held.contains(&Value::String("*".into()))
15825 })
15826 })
15827 })
15828 }) {
15829 "pending"
15830 } else {
15831 "missing"
15832 };
15833 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
15834 }
15835 Ok(
15836 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
15837 )
15838 })();
15839 match result {
15840 Ok(snapshot) => {
15841 self.conn
15842 .execute_batch("RELEASE authority_snapshot")
15843 .map_err(|e| e.to_string())?;
15844 Ok(snapshot)
15845 }
15846 Err(error) => {
15847 self.conn
15848 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
15849 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
15850 Err(error)
15851 }
15852 }
15853 }
15854
15855 pub fn query_snapshot(
15859 &mut self,
15860 sql: &str,
15861 params: &[QueryValue],
15862 coverage: &[WindowCoverage],
15863 owner: Option<QueryOwner<'_>>,
15864 ) -> Result<QuerySnapshot, String> {
15865 let result = snapshot_connection(&self.conn, sql, params, coverage);
15866 if let Some(owner) = owner {
15867 self.record_query_read(owner, result.as_ref().err());
15868 }
15869 result.map_err(|failure| failure.to_string())
15870 }
15871
15872 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
15876 let position = self
15877 .query_failures
15878 .iter()
15879 .position(|entry| entry.id == owner.id);
15880 let Some(failure) = failure else {
15881 if let Some(index) = position {
15882 self.query_failures.remove(index);
15883 }
15884 return;
15885 };
15886 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
15887 .into_iter()
15888 .map(str::to_owned)
15889 .collect();
15890 let code = failure.code.unwrap_or("client.query_failed").to_owned();
15891 if position.is_some_and(|index| {
15892 let entry = &self.query_failures[index];
15893 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
15894 }) {
15895 return;
15896 }
15897 if let Some(index) = position {
15898 self.query_failures.remove(index);
15899 }
15900 self.query_failures.push(DiagnosticQueryFailure {
15901 id: owner.id.to_owned(),
15902 tables,
15903 code,
15904 sqlite_code: failure.sqlite_code,
15905 at_ms: self.clock_now_ms(),
15906 });
15907 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
15908 self.query_failures.remove(0);
15909 }
15910 }
15911
15912 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
15915 #[cfg(feature = "bench-internals")]
15916 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
15917 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
15918 let max_commits = self.limits.max_push_commits_per_request;
15919 let max_ops = self
15920 .limits
15921 .max_push_operations_per_request
15922 .unwrap_or(PUSH_OPS_PER_REQUEST);
15923 let byte_cap = self.limits.max_push_request_bytes;
15924 let accept = self.limits.accept.unwrap_or(if url_capable {
15927 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
15928 } else {
15929 DEFAULT_ACCEPT
15930 });
15931 let header = Frame::ReqHeader {
15932 client_id: self.client_id.clone(),
15933 schema_version: self.schema.version,
15934 log_epoch: log_epoch.clone(),
15935 };
15936 let pull = Frame::PullHeader {
15937 limit_commits: self.limits.limit_commits.unwrap_or(0),
15938 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
15939 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
15940 accept,
15941 };
15942 let mut fresh = Vec::new();
15943 let mut sub_frames = Vec::new();
15944 for sub in &self.subs {
15945 if sub.state != SubState::Active {
15946 continue;
15947 }
15948 let mut scopes = sub.requested.clone();
15949 sort_scope_map(&mut scopes);
15950 sub_frames.push(Frame::Subscription {
15951 id: sub.id.clone(),
15952 table: sub.table.clone(),
15953 scopes,
15954 params: sub.params.clone().map(RawJson),
15955 cursor: sub.cursor,
15956 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
15957 });
15958 fresh.push((
15959 sub.id.clone(),
15960 sub.cursor < 0 && sub.bootstrap_state.is_none(),
15961 ));
15962 }
15963 let fixed_bytes = match byte_cap {
15966 Some(_) => {
15967 let mut frames = vec![header.clone(), pull.clone()];
15968 frames.extend(sub_frames.iter().cloned());
15969 encode_message(&Message {
15970 wire_version: WIRE_VERSION,
15971 msg_kind: MsgKind::Request,
15972 frames,
15973 })
15974 .len()
15975 }
15976 None => 0,
15977 };
15978 if let Some(cap) = byte_cap {
15979 if fixed_bytes > cap {
15980 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
15981 }
15982 }
15983 let base_bytes = match byte_cap {
15984 Some(_) => encode_message(&Message {
15985 wire_version: WIRE_VERSION,
15986 msg_kind: MsgKind::Request,
15987 frames: vec![header.clone(), pull.clone()],
15988 })
15989 .len(),
15990 None => 0,
15991 };
15992 let total_outbox = if log_epoch.is_some() {
15993 self.outbox.len()
15994 } else {
15995 0
15996 };
15997 let mut push_frames: Vec<Frame> = Vec::new();
15998 let mut pushed_ids = Vec::new();
15999 let mut ops_in_request = 0usize;
16000 let mut bytes_in_request = 0usize;
16001 let mut deferred_commits = 0usize;
16002 #[cfg(feature = "bench-internals")]
16003 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
16004 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
16005 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
16009 deferred_commits = total_outbox - index;
16010 break;
16011 }
16012 if ops_in_request + commit.ops.len() > max_ops {
16013 if push_frames.is_empty() {
16014 return Err(Self::push_capacity_outcome(
16015 "operations",
16016 max_ops,
16017 commit.ops.len(),
16018 Some(commit.client_commit_id.as_str()),
16019 ));
16020 }
16021 deferred_commits = total_outbox - index;
16022 break;
16023 }
16024 let operations = commit
16025 .ops
16026 .iter()
16027 .map(|op| {
16028 let payload = op.values.as_ref().and_then(|values| {
16029 let table = self.schema.table(&op.table)?;
16030 let fallback = self.stored_key_fallback(table, &op.row_id);
16039 encode_sparse_row_json(
16040 table,
16041 &op.row_id,
16042 values,
16043 &self.encryption,
16044 fallback.as_ref(),
16045 )
16046 .ok()
16047 });
16048 ssp2::model::Operation {
16049 table: op.table.clone(),
16050 row_id: op.row_id.clone(),
16051 op: if op.upsert { Op::Upsert } else { Op::Delete },
16052 base_version: op.base_version,
16053 payload,
16054 }
16055 })
16056 .collect();
16057 let frame = Frame::PushCommit {
16058 client_commit_id: commit.client_commit_id.clone(),
16059 operations,
16060 };
16061 let frame_bytes = match byte_cap {
16062 Some(_) => {
16063 encode_message(&Message {
16064 wire_version: WIRE_VERSION,
16065 msg_kind: MsgKind::Request,
16066 frames: vec![header.clone(), frame.clone(), pull.clone()],
16067 })
16068 .len()
16069 - base_bytes
16070 }
16071 None => 0,
16072 };
16073 if let Some(cap) = byte_cap {
16077 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
16078 deferred_commits = total_outbox - index;
16079 break;
16080 }
16081 }
16082 push_frames.push(frame);
16083 pushed_ids.push(commit.client_commit_id.clone());
16084 ops_in_request += commit.ops.len();
16085 bytes_in_request += frame_bytes;
16086 }
16087 #[cfg(feature = "bench-internals")]
16088 drop(outbox_phase);
16089 if let Some(cap) = byte_cap {
16090 if fixed_bytes + bytes_in_request > cap {
16091 let blocked = if push_frames.is_empty() {
16094 None
16095 } else {
16096 self.outbox.first().map(|c| c.client_commit_id.as_str())
16097 };
16098 return Err(Self::push_capacity_outcome(
16099 "bytes",
16100 cap,
16101 fixed_bytes + bytes_in_request,
16102 blocked,
16103 ));
16104 }
16105 }
16106 let mut frames = vec![header];
16107 frames.extend(push_frames);
16108 frames.push(pull);
16109 frames.extend(sub_frames);
16110 let message = Message {
16111 wire_version: WIRE_VERSION,
16112 msg_kind: MsgKind::Request,
16113 frames,
16114 };
16115 Ok((
16116 message,
16117 RequestMeta {
16118 pushed_ids,
16119 fresh,
16120 accept,
16121 deferred_commits,
16122 },
16123 ))
16124 }
16125
16126 fn push_capacity_outcome(
16128 kind: &str,
16129 limit: usize,
16130 size: usize,
16131 client_commit_id: Option<&str>,
16132 ) -> Box<SyncOutcome> {
16133 let mut details = Map::new();
16134 details.insert("kind".to_owned(), Value::from(kind));
16135 details.insert("limit".to_owned(), Value::from(limit as u64));
16136 details.insert("size".to_owned(), Value::from(size as u64));
16137 if let Some(id) = client_commit_id {
16138 details.insert("clientCommitId".to_owned(), Value::from(id));
16139 }
16140 Box::new(SyncOutcome::Failed {
16141 error_code: "client.push_request_too_large".into(),
16142 message: "the push request exceeds a configured capacity limit".into(),
16143 details: Some(Value::Object(details)),
16144 })
16145 }
16146
16147 pub fn progress(&self) -> ProgressObserver {
16151 self.progress.clone()
16152 }
16153
16154 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
16155 let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
16156 Ok(prepared) => prepared,
16157 Err(outcome) => return *outcome,
16158 };
16159 let mut next = prepared;
16164 loop {
16165 match self.apply_sync_round(next.exchange(transport)) {
16166 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
16167 crate::AppliedSyncRound::Complete {
16168 outcome, controls, ..
16169 } => {
16170 if self.transport_enabled {
16171 for text in controls {
16172 let _ = transport.realtime_send(&text);
16173 }
16174 } else {
16175 self.disconnect_realtime(transport);
16176 }
16177 return outcome;
16178 }
16179 }
16180 }
16181 }
16182
16183 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
16184 if let SyncOutcome::Failed {
16185 error_code,
16186 message,
16187 details,
16188 } = &mut outcome
16189 {
16190 if let Some(failure) = self.storage_failure.borrow().as_ref() {
16191 let code = failure
16192 .code
16193 .expect("retained storage failures are classified");
16194 *error_code = code.to_owned();
16195 *message = match code {
16196 "client.storage_busy" => "local SQLite storage is busy",
16197 "client.storage_full" => "local SQLite storage is full",
16198 "client.storage_io" => "local SQLite storage I/O failed",
16199 "client.storage_corrupt" => "local SQLite storage is corrupt",
16200 _ => unreachable!("unrecognized classified storage failure"),
16201 }
16202 .to_owned();
16203 *details = failure.details();
16204 }
16205 }
16206 let retry_delay_ms = self.round_retry_delay_ms;
16207 self.progress.update(|p| match &outcome {
16208 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
16209 if report.failed.is_empty() =>
16210 {
16211 p.state = ProgressState::Complete
16212 }
16213 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
16214 p.state = ProgressState::Failed;
16215 p.error_code = Some("sync.scope_revoked".into());
16216 }
16217 SyncOutcome::Failed { error_code, .. } => {
16218 p.state = ProgressState::Failed;
16219 p.error_code = Some(Self::diagnostic_code(error_code));
16220 p.retry_delay_ms = retry_delay_ms;
16221 }
16222 SyncOutcome::RealtimeUnavailable { .. } => {
16223 p.state = ProgressState::Failed;
16224 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
16225 p.retry_delay_ms = retry_delay_ms;
16226 }
16227 });
16228 let completed_at_ms = self.clock_now_ms();
16229 self.last_round = Some(match &outcome {
16230 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
16231 status: "succeeded".to_owned(),
16232 started_at_ms,
16233 completed_at_ms,
16234 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16235 counters: Some(DiagnosticRoundCounters {
16236 pushed: report.pushed,
16237 applied: report.applied.len(),
16238 rejected: report.rejected.len(),
16239 retryable: report.retryable.len(),
16240 conflicts: report.conflicts,
16241 commits_applied: report.commits_applied,
16242 segment_rows_applied: report.segment_rows_applied,
16243 bootstrapping: report.bootstrapping.len(),
16244 resets: report.resets.len(),
16245 revoked: report.revoked.len(),
16246 failed: report.failed.len(),
16247 deferred_commits: report.deferred_commits,
16248 }),
16249 error_code: None,
16250 },
16251 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
16252 status: "failed".to_owned(),
16253 started_at_ms,
16254 completed_at_ms,
16255 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16256 counters: None,
16257 error_code: Some(Self::diagnostic_code(error_code)),
16258 },
16259 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
16260 status: "failed".to_owned(),
16261 started_at_ms,
16262 completed_at_ms,
16263 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16264 counters: None,
16265 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
16266 },
16267 });
16268 outcome
16269 }
16270
16271 pub fn prepare_sync_round(
16274 &mut self,
16275 url_capable: bool,
16276 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
16277 if !self.transport_enabled {
16278 return Err(Box::new(SyncOutcome::Failed {
16279 error_code: "sync.offline".into(),
16280 message: "transport gate is closed".into(),
16281 details: None,
16282 }));
16283 }
16284 if self.active_round.is_some() {
16285 return Err(Box::new(SyncOutcome::Failed {
16286 error_code: "client.round_in_progress".into(),
16287 message: "a sync round is already in flight".into(),
16288 details: None,
16289 }));
16290 }
16291 self.storage_failure.borrow_mut().take();
16292 self.progress.start();
16293 self.round_retry_delay_ms = None;
16294 let started_at_ms = self.clock_now_ms();
16295 let prepared = (|| {
16296 if self.stopped {
16297 return Err(Box::new(SyncOutcome::Ok(SyncReport {
16298 schema_floor: self.schema_floor.clone(),
16299 ..SyncReport::default()
16300 })));
16301 }
16302 if self.security_preflight {
16303 return Err(Box::new(SyncOutcome::Failed {
16304 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
16305 message: "security preflight must complete before syncing".into(),
16306 details: None,
16307 }));
16308 }
16309 self.set_sync_needed(false, false);
16310 self.drain_pending_evictions()
16311 .and_then(|_| self.drop_incompatible_outbox())
16312 .and_then(|_| self.drop_unencodable_outbox(true, None))
16313 .map_err(|message| {
16314 Box::new(SyncOutcome::Failed {
16315 error_code: "storage.failed".into(),
16316 message,
16317 details: None,
16318 })
16319 })?;
16320 let uploads = if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
16321 self.pending_blob_uploads().map_err(|e| {
16322 Box::new(SyncOutcome::Failed {
16323 error_code: e.code,
16324 message: e.message,
16325 details: e.details,
16326 })
16327 })?
16328 } else {
16329 Vec::new()
16330 };
16331 if self.realtime_state != RealtimeState::Connected
16332 && self.realtime_policy == RealtimePolicy::Required
16333 {
16334 let retry_delay_ms = self.schedule_background_retry();
16335 self.realtime_retry_delay_ms = Some(retry_delay_ms);
16336 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
16337 state: self.realtime_state,
16338 reason_code: self.realtime_reason_code.clone(),
16339 retry_delay_ms,
16340 }));
16341 }
16342 let (message, meta) = self.build_request(url_capable)?;
16343 let id = uuid::Uuid::new_v4();
16344 self.active_round = Some(id);
16345 Ok(crate::PreparedSyncRound {
16346 id,
16347 started_at_ms,
16348 message,
16349 meta,
16350 uploads: uploads.into(),
16351 realtime: self.realtime_state == RealtimeState::Connected,
16352 scopes: self
16353 .subs
16354 .iter()
16355 .map(|s| (s.id.clone(), canonical_scope_json(&s.requested)))
16356 .collect(),
16357 #[cfg(feature = "bench-internals")]
16358 benchmark_phases: self.benchmark_phases.clone(),
16359 progress: self.progress.clone(),
16360 fixed_now: self.now_ms,
16361 round_deadline_at: None,
16362 })
16363 })();
16364 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
16365 }
16366
16367 pub fn cancel_sync_round(&mut self) {
16370 self.active_round = None;
16371 }
16372
16373 pub fn apply_sync_round(
16376 &mut self,
16377 completed: crate::CompletedSyncRound,
16378 ) -> crate::AppliedSyncRound {
16379 let prepared = completed.prepared;
16380 if self.active_round != Some(prepared.id) || self.security_preflight {
16381 return crate::AppliedSyncRound::Complete {
16382 outcome: SyncOutcome::Failed {
16383 error_code: "client.round_cancelled".into(),
16384 message: "sync round belongs to a released client context".into(),
16385 details: None,
16386 },
16387 controls: Vec::new(),
16388 more: false,
16389 bootstrap_advanced: false,
16390 };
16391 }
16392 self.storage_failure.borrow_mut().take();
16393 let (response, mut downloads, transport_failed) = match completed.exchange {
16394 crate::round::ExchangeResult::Upload { id, result } => {
16395 let result = result.and_then(|()| {
16396 self.conn
16397 .execute(
16398 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
16399 rusqlite::params![id],
16400 )
16401 .map(|_| ())
16402 .map_err(|error| {
16403 TransportError::new(
16404 "client.failed",
16405 Self::sqlite_failure(&self.storage_failure, error),
16406 )
16407 })
16408 });
16409 match result {
16410 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
16411 Err(TransportError {
16412 code,
16413 message,
16414 details,
16415 }) => {
16416 self.active_round = None;
16417 if Self::retryable_failure_code(&code) {
16418 self.schedule_background_retry();
16419 }
16420 let outcome = self.finish_sync_round(
16421 prepared.started_at_ms,
16422 SyncOutcome::Failed {
16423 error_code: code,
16424 message,
16425 details,
16426 },
16427 );
16428 return crate::AppliedSyncRound::Complete {
16429 outcome,
16430 controls: Vec::new(),
16431 more: false,
16432 bootstrap_advanced: false,
16433 };
16434 }
16435 }
16436 }
16437 crate::round::ExchangeResult::Reply {
16438 response,
16439 downloads,
16440 transport_failed,
16441 } => (response, downloads, transport_failed),
16442 };
16443 self.active_round = None;
16444 let before = self
16445 .subs
16446 .iter()
16447 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16448 .collect::<Vec<_>>();
16449 let outcome = (|| {
16450 let response = match response {
16451 Ok(response) => response,
16452 Err(TransportError {
16453 code,
16454 message,
16455 details,
16456 }) => {
16457 if transport_failed {
16458 if prepared.realtime {
16459 self.set_realtime_state(RealtimeState::Lost, Some(&code));
16460 }
16461 self.record_lease_error(&code);
16462 }
16463 if Self::retryable_failure_code(&code) {
16464 let delay = self.schedule_background_retry();
16465 if prepared.realtime && transport_failed {
16466 self.realtime_retry_delay_ms = Some(delay);
16467 }
16468 }
16469 return SyncOutcome::Failed {
16470 error_code: code,
16471 message,
16472 details,
16473 };
16474 }
16475 };
16476 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
16477 if let SyncOutcome::Ok(report) = &mut outcome {
16478 report.deferred_commits = prepared.meta.deferred_commits;
16479 }
16480 match &outcome {
16481 SyncOutcome::Ok(_) => self.reset_background_retry(),
16482 SyncOutcome::Failed { error_code, .. }
16483 if Self::retryable_failure_code(error_code) =>
16484 {
16485 self.schedule_background_retry();
16486 }
16487 _ => {}
16488 }
16489 if prepared.meta.deferred_commits > 0 {
16490 self.set_sync_needed(true, true);
16491 }
16492 self.previous_version_lifetime_check();
16493 outcome
16494 })();
16495 let more = self.transport_enabled
16496 && matches!(&outcome, SyncOutcome::Ok(report)
16497 if report.schema_floor.is_none()
16498 && (!report.bootstrapping.is_empty()
16499 || report.commits_applied > 0
16500 || report.segment_rows_applied > 0
16501 || !report.resets.is_empty()
16502 || report.deferred_commits > 0
16503 || self.sync_needed));
16504 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
16505 && before
16506 != self
16507 .subs
16508 .iter()
16509 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16510 .collect::<Vec<_>>();
16511 crate::AppliedSyncRound::Complete {
16512 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
16513 controls: if self.transport_enabled {
16514 downloads.controls
16515 } else {
16516 Vec::new()
16517 },
16518 more,
16519 bootstrap_advanced,
16520 }
16521 }
16522
16523 pub fn sync_until_idle(
16524 &mut self,
16525 transport: &mut dyn Transport,
16526 max_rounds: Option<u32>,
16527 ) -> SyncOutcome {
16528 if max_rounds == Some(0) {
16531 return SyncOutcome::Failed {
16532 details: None,
16533 error_code: "sync.invalid_request".into(),
16534 message: "maxRounds must be a positive integer".into(),
16535 };
16536 }
16537 let rounds = max_rounds.unwrap_or(20);
16538 let mut aggregate = SyncReport::default();
16539 let mut spent = 0;
16540 while spent < rounds {
16541 spent += 1;
16542 let before = self
16543 .subs
16544 .iter()
16545 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16546 .collect::<Vec<_>>();
16547 match self.sync(transport) {
16548 SyncOutcome::Failed {
16549 error_code,
16550 message,
16551 details,
16552 } => {
16553 return SyncOutcome::Failed {
16554 details,
16555 error_code,
16556 message,
16557 };
16558 }
16559 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
16560 SyncOutcome::BudgetExhausted(_) => {
16561 unreachable!("sync() never exhausts a round budget")
16562 }
16563 SyncOutcome::Ok(report) => {
16564 if max_rounds.is_none()
16565 && report.segment_rows_applied > 0
16566 && !report.bootstrapping.is_empty()
16567 && before
16568 != self
16569 .subs
16570 .iter()
16571 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16572 .collect::<Vec<_>>()
16573 {
16574 spent = 0;
16575 }
16576 let stop = !self.transport_enabled() || report.schema_floor.is_some();
16579 aggregate.merge(&report);
16580 if stop {
16581 return SyncOutcome::Ok(aggregate);
16582 }
16583 let more = !report.bootstrapping.is_empty()
16589 || report.commits_applied > 0
16590 || report.segment_rows_applied > 0
16591 || !report.resets.is_empty()
16592 || report.deferred_commits > 0
16593 || self.sync_needed;
16594 if !more {
16595 return SyncOutcome::Ok(aggregate);
16596 }
16597 }
16598 }
16599 }
16600 SyncOutcome::BudgetExhausted(aggregate)
16603 }
16604
16605 fn process_response(
16606 &mut self,
16607 transport: &mut dyn Transport,
16608 response: Message,
16609 meta: &RequestMeta,
16610 ) -> SyncOutcome {
16611 #[cfg(feature = "bench-internals")]
16612 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
16613 let mut report = SyncReport {
16614 pushed: meta.pushed_ids.len() as u32,
16615 ..SyncReport::default()
16616 };
16617 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
16618 HashMap::new();
16619 let pushed_ids = meta
16620 .pushed_ids
16621 .iter()
16622 .map(String::as_str)
16623 .collect::<HashSet<_>>();
16624 let mut last_final_push_result_id: Option<String> = None;
16625 for frame in &response.frames {
16626 match frame {
16627 Frame::PushResultDetails {
16628 client_commit_id,
16629 entries,
16630 } => {
16631 let details = rejection_details_by_commit
16632 .entry(client_commit_id.clone())
16633 .or_default();
16634 for entry in entries {
16635 let parsed = match RejectionDetails::parse(&entry.details.0) {
16636 Ok(value) => value,
16637 Err(message) => {
16638 return SyncOutcome::Failed {
16639 details: None,
16640 error_code: "sync.invalid_request".to_owned(),
16641 message,
16642 };
16643 }
16644 };
16645 details.insert(entry.op_index, parsed);
16646 }
16647 }
16648 Frame::PushResult {
16649 client_commit_id,
16650 status,
16651 results,
16652 ..
16653 } if pushed_ids.contains(client_commit_id.as_str())
16654 && Self::push_result_is_final(*status, results) =>
16655 {
16656 last_final_push_result_id = Some(client_commit_id.clone());
16657 }
16658 _ => {}
16659 }
16660 }
16661 let mut frames = response.frames.into_iter().peekable();
16662 if response.wire_version < 2 {
16663 return SyncOutcome::Failed {
16664 details: None,
16665 error_code: "client.invalid_host_response".to_owned(),
16666 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
16667 };
16668 }
16669 match frames.next() {
16670 Some(Frame::RespHeader {
16671 required_schema_version,
16672 latest_schema_version,
16673 log_epoch,
16674 reset_required,
16675 }) => {
16676 let Some(log_epoch) = log_epoch else {
16677 return SyncOutcome::Failed {
16678 details: None,
16679 error_code: "client.invalid_host_response".to_owned(),
16680 message: "response header omits logEpoch".to_owned(),
16681 };
16682 };
16683 let Some(reset_required) = reset_required else {
16684 return SyncOutcome::Failed {
16685 details: None,
16686 error_code: "client.invalid_host_response".to_owned(),
16687 message: "response header omits resetRequired".to_owned(),
16688 };
16689 };
16690 if let Some(required) = required_schema_version {
16691 let floor = SchemaFloor {
16694 required_schema_version: Some(required),
16695 latest_schema_version,
16696 };
16697 self.set_schema_floor(Some(floor.clone()));
16698 report.schema_floor = Some(floor);
16699 return SyncOutcome::Ok(report);
16700 }
16701 if reset_required {
16702 if frames.next().is_some() {
16703 return SyncOutcome::Failed {
16704 details: None,
16705 error_code: "client.invalid_host_response".to_owned(),
16706 message: "log-epoch reset response contains body frames".to_owned(),
16707 };
16708 }
16709 match self.run_log_epoch_reset(&log_epoch) {
16710 Ok(resets) => {
16711 report.resets = resets;
16712 return SyncOutcome::Ok(report);
16713 }
16714 Err(message) => {
16715 return SyncOutcome::Failed {
16716 details: None,
16717 error_code: "sync.local_corrupt".to_owned(),
16718 message,
16719 };
16720 }
16721 }
16722 }
16723 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
16724 return SyncOutcome::Failed {
16725 details: None,
16726 error_code: "client.invalid_host_response".to_owned(),
16727 message: "server changed logEpoch without requiring a reset".to_owned(),
16728 };
16729 }
16730 }
16731 _ => {
16732 return SyncOutcome::Failed {
16733 details: None,
16734 error_code: "sync.invalid_request".to_owned(),
16735 message: "response does not start with RESP_HEADER".to_owned(),
16736 };
16737 }
16738 }
16739
16740 if let Err(message) = self.begin_coalesced_apply() {
16741 return SyncOutcome::Failed {
16742 details: None,
16743 error_code: "storage.failed".into(),
16744 message,
16745 };
16746 }
16747 let mut failure: Option<(String, String)> = None;
16748 while let Some(frame) = frames.next() {
16749 match frame {
16750 frame @ Frame::PushResult { .. } => {
16751 let successful = matches!(
16752 &frame,
16753 Frame::PushResult {
16754 status: PushStatus::Applied | PushStatus::Cached,
16755 ..
16756 }
16757 );
16758 let mut results = vec![frame];
16759 if successful {
16760 while matches!(
16761 frames.peek(),
16762 Some(Frame::PushResult {
16763 status: PushStatus::Applied | PushStatus::Cached,
16764 ..
16765 })
16766 ) {
16767 if let Some(next) = frames.next() {
16768 results.push(next);
16769 }
16770 }
16771 }
16772 if self
16773 .handle_push_results(
16774 &results,
16775 &pushed_ids,
16776 &rejection_details_by_commit,
16777 &mut report,
16778 last_final_push_result_id.as_deref(),
16779 )
16780 .is_err()
16781 {
16782 failure = Some((
16783 "client.outcome_persistence_failed".to_owned(),
16784 "local commit outcome could not be persisted".to_owned(),
16785 ));
16786 break;
16787 }
16788 }
16789 Frame::PushResultDetails { .. } => {}
16790 Frame::SubStart {
16791 id,
16792 status,
16793 reason_code,
16794 effective_scopes,
16795 bootstrap: _,
16796 } => {
16797 let mut body = Vec::new();
16798 let mut sub_end: Option<(i64, Option<String>)> = None;
16799 for inner in frames.by_ref() {
16800 match inner {
16801 Frame::SubEnd {
16802 next_cursor,
16803 bootstrap_state,
16804 } => {
16805 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
16806 break;
16807 }
16808 Frame::Unknown { .. } => {}
16809 other => body.push(other),
16810 }
16811 }
16812 if let Err(SectionError::Abort(code, message)) = self.process_section(
16813 transport,
16814 &id,
16815 status,
16816 &reason_code,
16817 effective_scopes,
16818 body,
16819 sub_end,
16820 meta,
16821 &mut report,
16822 ) {
16823 failure = Some((code, message));
16824 break;
16825 }
16826 }
16827 Frame::Lease {
16828 lease_id,
16829 expires_at_ms,
16830 } => {
16831 self.set_lease_state(Some(LeaseState {
16834 lease_id: Some(lease_id),
16835 expires_at_ms: Some(expires_at_ms),
16836 error_code: None,
16837 }));
16838 }
16839 Frame::Error { code, message, .. } => {
16840 failure = Some((code, message));
16843 break;
16844 }
16845 Frame::Unknown { .. } => {}
16846 _ => {
16847 failure = Some((
16848 "sync.invalid_request".to_owned(),
16849 "unexpected frame in response".to_owned(),
16850 ));
16851 break;
16852 }
16853 }
16854 }
16855
16856 let applied = match failure {
16857 Some(failure) => Err(failure),
16858 None => self
16863 .rebuild_overlay_if_dirty()
16864 .and_then(|()| self.drain_pending_evictions())
16865 .map_err(|message| ("storage.failed".to_owned(), message)),
16866 };
16867 if let Err(message) = self.finish_coalesced_apply() {
16870 return SyncOutcome::Failed {
16871 details: None,
16872 error_code: "storage.failed".into(),
16873 message,
16874 };
16875 }
16876 if let Err((error_code, message)) = applied {
16877 return SyncOutcome::Failed {
16878 details: None,
16879 error_code,
16880 message,
16881 };
16882 }
16883 self.ack_after_pull(transport);
16884 if self.upgrading && report.bootstrapping.is_empty() {
16887 self.set_upgrading(false);
16888 }
16889 SyncOutcome::Ok(report)
16890 }
16891
16892 fn handle_push_results(
16895 &mut self,
16896 frames: &[Frame],
16897 pushed_ids: &HashSet<&str>,
16898 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
16899 report: &mut SyncReport,
16900 last_final_push_result_id: Option<&str>,
16901 ) -> Result<(), String> {
16902 self.begin_observation("syncular_push_results")?;
16903 let previous_failed = self.failed_commits.clone();
16904 let previous_sync_needed = self.sync_needed;
16905 let conflict_count = self.conflicts.len();
16906 let rejection_count = self.rejections.len();
16907 let applied_count = report.applied.len();
16908 let rejected_count = report.rejected.len();
16909 let retryable_count = report.retryable.len();
16910 let was_dirty = self.overlay_dirty.snapshot();
16911 let mut removed = Vec::new();
16912 let mut batch = ChangeAccumulator::default();
16913 let persisted = (|| {
16914 for frame in frames {
16915 let Frame::PushResult {
16916 client_commit_id,
16917 status,
16918 results,
16919 commit_seq,
16920 } = frame
16921 else {
16922 return Err("unexpected acknowledgement frame".to_owned());
16923 };
16924 if !pushed_ids.contains(client_commit_id.as_str()) {
16925 continue;
16926 }
16927 let Some(index) = self
16928 .outbox
16929 .iter()
16930 .position(|commit| commit.client_commit_id == *client_commit_id)
16931 else {
16932 continue;
16933 };
16934 let status = *status;
16935 let rejection_details = rejection_details_by_commit.get(client_commit_id);
16936 let operations = self.outbox[index].ops.clone();
16937 match status {
16938 PushStatus::Applied | PushStatus::Cached => {
16939 let sequence =
16942 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
16943 let affected: HashSet<_> = operations
16944 .iter()
16945 .map(|op| (op.table.as_str(), op.row_id.as_str()))
16946 .collect();
16947 let prior = self.acknowledged_operations(Some(&affected), None)?;
16948 for (op_index, operation) in operations.iter().enumerate() {
16949 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
16950 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16951 if delivered {
16952 continue;
16953 }
16954 let table = self
16955 .schema
16956 .table(&operation.table)
16957 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16958 let mut intended: Map<String, Value> = self
16959 .stored_row(table, &operation.row_id, true)?
16960 .map(|(row, _)| {
16961 table
16962 .columns
16963 .iter()
16964 .zip(&row)
16965 .map(|(column, value)| {
16966 (column.name.clone(), column_value_to_json(value))
16967 })
16968 .collect()
16969 })
16970 .unwrap_or_default();
16971 let mut scope_image = intended.clone();
16972 for op in prior
16973 .iter()
16974 .chain(
16975 self.failed_commits
16976 .iter()
16977 .flat_map(|(commit, _)| &commit.ops),
16978 )
16979 .chain(
16980 self.outbox
16981 .iter()
16982 .take(index)
16983 .flat_map(|commit| &commit.ops),
16984 )
16985 .chain(operations.iter().take(op_index + 1))
16986 {
16987 if op.table != operation.table || op.row_id != operation.row_id {
16988 continue;
16989 }
16990 if let Some(values) = &op.values {
16991 intended.extend(values.clone());
16992 } else {
16993 scope_image = intended.clone();
16994 intended.clear();
16995 }
16996 }
16997 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
16998 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
16999 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
17000 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
17001 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17002 }
17003 let journal_results = results
17004 .iter()
17005 .map(|result| {
17006 let op_index = match result {
17007 OpResult::Applied { op_index }
17008 | OpResult::Conflict { op_index, .. }
17009 | OpResult::Error { op_index, .. } => *op_index,
17010 };
17011 CommitOperationOutcome::Applied { op_index }
17012 })
17013 .collect::<Vec<_>>();
17014 let outcome_status = if status == PushStatus::Applied {
17015 CommitOutcomeStatus::Applied
17016 } else {
17017 CommitOutcomeStatus::Cached
17018 };
17019 let persisted = self
17020 .persist_commit_outcome(
17021 client_commit_id,
17022 outcome_status,
17023 &journal_results,
17024 None,
17025 )
17026 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
17027 persisted?;
17028 for operation in &operations {
17029 self.overlay_dirty.table(&operation.table);
17030 }
17031 self.sync_needed = true;
17032 batch.status = true;
17033 batch.outcomes = true;
17034 }
17035 PushStatus::Rejected => {
17036 if results.iter().any(|result| {
17037 matches!(
17038 result,
17039 OpResult::Error {
17040 code,
17041 retryable: true,
17042 ..
17043 } if code == "sync.idempotency_cache_miss"
17044 )
17045 }) {
17046 report.retryable.push(client_commit_id.to_owned());
17049 continue;
17050 }
17051
17052 let mut journal_results = Vec::with_capacity(results.len());
17053 let mut conflicts = Vec::new();
17054 let mut rejections = Vec::new();
17055 for result in results {
17056 match result {
17057 OpResult::Applied { op_index } => {
17058 journal_results.push(CommitOperationOutcome::Applied {
17059 op_index: *op_index,
17060 });
17061 }
17062 OpResult::Conflict {
17063 op_index,
17064 code,
17065 message,
17066 server_version,
17067 server_row,
17068 conflict_columns: conflict_columns_bitmap,
17069 } => {
17070 let operation = operations
17071 .get(*op_index as usize)
17072 .map(CommitOperation::from);
17073 let (table, row_id) = operation
17074 .as_ref()
17075 .map(|op| (op.table.clone(), op.row_id.clone()))
17076 .unwrap_or_default();
17077 let server_row_json = self
17078 .schema
17079 .table(&table)
17080 .and_then(|t| {
17081 decode_row_bytes(t, server_row, &self.encryption)
17082 .ok()
17083 .map(|row| (t, row))
17084 })
17085 .map(|(t, row)| {
17086 let mut map = Map::new();
17087 for (i, column) in t.columns.iter().enumerate() {
17088 map.insert(
17089 column.name.clone(),
17090 column_value_to_json(
17091 row.get(i).unwrap_or(&None),
17092 ),
17093 );
17094 }
17095 map
17096 })
17097 .unwrap_or_default();
17098 let conflict_columns = self
17101 .schema
17102 .table(&table)
17103 .map(|t| {
17104 t.columns
17105 .iter()
17106 .enumerate()
17107 .filter(|(index, _)| {
17108 conflict_columns_bitmap
17109 .get(index / 8)
17110 .is_some_and(|byte| {
17111 byte & (1 << (index % 8)) != 0
17112 })
17113 })
17114 .map(|(_, column)| column.name.clone())
17115 .collect::<Vec<_>>()
17116 })
17117 .unwrap_or_default();
17118 let conflict = ConflictRecord {
17119 client_commit_id: client_commit_id.to_owned(),
17120 op_index: *op_index,
17121 table,
17122 row_id,
17123 code: code.clone(),
17124 message: message.clone(),
17125 server_version: *server_version,
17126 server_row: server_row_json,
17127 conflict_columns,
17128 operation,
17129 };
17130 journal_results.push(CommitOperationOutcome::Conflict {
17131 conflict: conflict.clone(),
17132 });
17133 conflicts.push(conflict);
17134 }
17135 OpResult::Error {
17136 op_index,
17137 code,
17138 message,
17139 retryable,
17140 } => {
17141 let rejection = RejectionRecord {
17142 client_commit_id: client_commit_id.to_owned(),
17143 op_index: *op_index,
17144 code: code.clone(),
17145 message: message.clone(),
17146 retryable: *retryable,
17147 details: rejection_details
17148 .and_then(|details| details.get(op_index))
17149 .cloned(),
17150 operation: operations
17151 .get(*op_index as usize)
17152 .map(CommitOperation::from),
17153 };
17154 journal_results.push(CommitOperationOutcome::Error {
17155 rejection: rejection.clone(),
17156 });
17157 rejections.push(rejection);
17158 }
17159 }
17160 }
17161 if self.retain_failed_commits {
17162 self.retain_failed_commit(client_commit_id, &operations)?;
17163 }
17164 let outcome_status = if conflicts.is_empty() {
17165 CommitOutcomeStatus::Rejected
17166 } else {
17167 CommitOutcomeStatus::Conflict
17168 };
17169 let persisted = self
17170 .persist_commit_outcome(
17171 client_commit_id,
17172 outcome_status,
17173 &journal_results,
17174 Some(&operations),
17175 )
17176 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
17177 persisted?;
17178 batch.conflicts = !conflicts.is_empty();
17179 batch.rejections = !rejections.is_empty();
17180 batch.status = true;
17181 batch.outcomes = true;
17182 self.conflicts.extend(conflicts);
17183 self.rejections.extend(rejections);
17184 for operation in &operations {
17185 self.overlay_dirty.table(&operation.table);
17186 }
17187 }
17188 }
17189 removed.push((index, self.outbox.remove(index)));
17190 if batch.status {
17191 for operation in &operations {
17192 if !self.record_row_scopes(
17193 &mut batch,
17194 &operation.table,
17195 &operation.row_id,
17196 false,
17197 ) {
17198 batch.table(&operation.table);
17199 }
17200 }
17201 }
17202 match status {
17203 PushStatus::Applied | PushStatus::Cached => {
17204 report.applied.push(client_commit_id.to_owned())
17205 }
17206 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
17207 }
17208 }
17209 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
17210 self.prune_commit_outcomes()?;
17211 }
17212 self.rebuild_overlay_if_dirty()?;
17213 self.finish_observation("syncular_push_results", batch)
17214 })();
17215 if let Err(error) = persisted {
17216 self.rollback_observation("syncular_push_results");
17217 self.failed_commits = previous_failed;
17218 self.sync_needed = previous_sync_needed;
17219 for (index, commit) in removed.into_iter().rev() {
17220 self.outbox.insert(index, commit);
17221 }
17222 self.conflicts.truncate(conflict_count);
17223 self.rejections.truncate(rejection_count);
17224 self.overlay_dirty.restore(was_dirty.clone());
17225 report.applied.truncate(applied_count);
17226 report.rejected.truncate(rejected_count);
17227 report.retryable.truncate(retryable_count);
17228 return Err(error);
17229 }
17230 if !previous_sync_needed && self.sync_needed {
17231 self.sync_intent_queue.push_back(SyncIntent::Interactive);
17232 }
17233 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
17234 Ok(())
17235 }
17236
17237 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
17238 status != PushStatus::Rejected
17239 || !results.iter().any(|result| {
17240 matches!(
17241 result,
17242 OpResult::Error {
17243 code,
17244 retryable: true,
17245 ..
17246 } if code == "sync.idempotency_cache_miss"
17247 )
17248 })
17249 }
17250
17251 #[allow(clippy::too_many_arguments)]
17254 fn process_section(
17255 &mut self,
17256 transport: &mut dyn Transport,
17257 id: &str,
17258 status: SubStatus,
17259 reason_code: &str,
17260 effective_scopes: Vec<(String, Vec<String>)>,
17261 body: Vec<Frame>,
17262 sub_end: Option<(i64, Option<String>)>,
17263 meta: &RequestMeta,
17264 report: &mut SyncReport,
17265 ) -> Result<(), SectionError> {
17266 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
17267 return Ok(()); };
17269 if status != SubStatus::Active && sub_end.is_none() {
17270 return Err(SectionError::Abort(
17271 "sync.invalid_request".into(),
17272 "subscription section without SUB_END".into(),
17273 ));
17274 }
17275 match status {
17276 SubStatus::Revoked => {
17277 self.begin_observation("syncular_revocation")
17278 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17279 let previous = self.subs[sub_index].clone();
17280 let was_dirty = self.overlay_dirty.snapshot();
17281 let previous_outbox = self.outbox.clone();
17282 let previous_failed = self.failed_commits.clone();
17283 let previous_rejections = self.rejections.clone();
17284 let previous_conflicts = self.conflicts.clone();
17285 let revoked_count = report.revoked.len();
17286 let failed_count = report.failed.len();
17287 let result = (|| {
17288 let mut batch = ChangeAccumulator::default();
17289 let registered = self.window_unit_by_sub_id(id);
17290 let (table, effective) = {
17292 let sub = &self.subs[sub_index];
17293 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17294 };
17295 let scoped_table = self.schema.table(&table).cloned().ok_or_else(|| {
17296 SectionError::Abort(
17297 "sync.unknown_table".to_owned(),
17298 "revoked scope references an unknown table".to_owned(),
17299 )
17300 })?;
17301 let dropped = self
17302 .drop_failed_commits(|name, values| {
17303 name == table
17304 && !effective.is_empty()
17305 && effective.iter().all(|(variable, allowed)| {
17306 scoped_table
17307 .scope_column(variable)
17308 .and_then(|column| values.get(column))
17309 .and_then(Value::as_str)
17310 .is_some_and(|value| {
17311 allowed.iter().any(|allowed| allowed == value)
17312 })
17313 })
17314 })
17315 .map_err(|message| {
17316 SectionError::Abort("storage.failed".to_owned(), message)
17317 })?;
17318 for commit in dropped {
17319 batch.conflicts = true;
17320 batch.rejections = true;
17321 batch.outcomes = true;
17322 for operation in commit.ops {
17323 batch.table(&operation.table);
17324 }
17325 }
17326 let purged = self.purge_scope_rows(&table, &effective);
17327 match purged {
17328 Ok(()) => {
17329 self.record_scope_map(&mut batch, &table, &effective);
17330 let sub = &mut self.subs[sub_index];
17331 sub.state = SubState::Revoked;
17332 sub.reason_code = Some(if reason_code.is_empty() {
17333 "sync.scope_revoked".to_owned()
17334 } else {
17335 reason_code.to_owned()
17336 });
17337 report.revoked.push(id.to_owned());
17338 let doomed_effective = effective;
17339 let sub_table = table;
17340 self.persist_sub(&self.subs[sub_index].clone())
17341 .map_err(|message| {
17342 SectionError::Abort("storage.failed".into(), message)
17343 })?;
17344 let dropped = self
17345 .drop_doomed_outbox(&sub_table, &doomed_effective)
17346 .map_err(|message| {
17347 SectionError::Abort("storage.failed".to_owned(), message)
17348 })?;
17349 if dropped {
17350 batch.status = true;
17351 batch.rejections = true;
17352 batch.outcomes = true;
17353 }
17354 self.delete_unreferenced_cached_blobs().map_err(|message| {
17357 SectionError::Abort("storage.failed".into(), message)
17358 })?;
17359 }
17360 Err(()) => {
17361 let sub = &mut self.subs[sub_index];
17364 sub.state = SubState::Failed;
17365 sub.reason_code = Some("sync.scope_revoked".to_owned());
17366 report.failed.push(id.to_owned());
17367 self.persist_sub(&self.subs[sub_index].clone())
17368 .map_err(|message| {
17369 SectionError::Abort("storage.failed".into(), message)
17370 })?;
17371 }
17372 }
17373 if let Some((base_key, unit)) = registered {
17374 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17375 }
17376 self.rebuild_overlay_if_dirty()
17377 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17378 self.finish_observation("syncular_revocation", batch)
17379 .map_err(|message| {
17380 SectionError::Abort("storage.failed".to_owned(), message)
17381 })?;
17382 Ok(())
17383 })();
17384 if result.is_err() {
17385 self.rollback_observation("syncular_revocation");
17386 self.subs[sub_index] = previous;
17387 self.overlay_dirty.restore(was_dirty.clone());
17388 self.outbox = previous_outbox;
17389 self.failed_commits = previous_failed;
17390 self.rejections = previous_rejections;
17391 self.conflicts = previous_conflicts;
17392 report.revoked.truncate(revoked_count);
17393 report.failed.truncate(failed_count);
17394 }
17395 result
17396 }
17397 SubStatus::Reset => {
17398 self.begin_observation("syncular_reset")
17399 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17400 let previous = self.subs[sub_index].clone();
17401 let was_dirty = self.overlay_dirty.snapshot();
17402 let reset_count = report.resets.len();
17403 let result = (|| {
17404 let mut batch = ChangeAccumulator::default();
17405 let registered = self.window_unit_by_sub_id(id);
17406 let sub = &mut self.subs[sub_index];
17409 sub.cursor = -1;
17410 sub.bootstrap_state = None;
17411 sub.reason_code = Some(if reason_code.is_empty() {
17412 "sync.cursor_expired".to_owned()
17413 } else {
17414 reason_code.to_owned()
17415 });
17416 report.resets.push(id.to_owned());
17417 self.persist_sub(&self.subs[sub_index].clone())
17418 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17419 if let Some((base_key, unit)) = registered {
17420 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17421 }
17422 self.finish_observation("syncular_reset", batch)
17423 .map_err(|message| {
17424 SectionError::Abort("storage.failed".to_owned(), message)
17425 })?;
17426 Ok(())
17427 })();
17428 if result.is_err() {
17429 self.rollback_observation("syncular_reset");
17430 self.subs[sub_index] = previous;
17431 self.overlay_dirty.restore(was_dirty.clone());
17432 report.resets.truncate(reset_count);
17433 }
17434 result
17435 }
17436 SubStatus::Active => {
17437 let fresh = meta
17438 .fresh
17439 .iter()
17440 .find(|(fid, _)| fid == id)
17441 .map(|(_, fresh)| *fresh)
17442 .unwrap_or(false);
17443 let previous = self.subs[sub_index].clone();
17444 self.subs[sub_index].effective = Some(effective_scopes);
17445 let outcome = self
17446 .apply_section_body(transport, sub_index, body, fresh, meta, report)
17447 .and_then(|()| {
17448 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
17449 SectionError::Abort(
17450 "sync.invalid_request".into(),
17451 "subscription section without SUB_END".into(),
17452 )
17453 })?;
17454 self.apply_sub_end(sub_index, cursor, bootstrap_state)
17455 .map_err(|message| {
17456 SectionError::Abort("storage.failed".into(), message)
17457 })
17458 });
17459 match outcome {
17460 Ok(()) => {
17461 if self.subs[sub_index].bootstrap_state.is_some() {
17462 report.bootstrapping.push(id.to_owned());
17463 }
17464 Ok(())
17465 }
17466 Err(SectionError::FailClosed) => {
17467 self.subs[sub_index] = previous.clone();
17468 self.begin_observation("syncular_section_failure")
17469 .map_err(|message| {
17470 SectionError::Abort("storage.failed".into(), message)
17471 })?;
17472 let persisted = (|| {
17473 let mut batch = ChangeAccumulator::default();
17474 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
17475 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17476 }
17477 self.subs[sub_index].state = SubState::Failed;
17478 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
17479 self.persist_sub(&self.subs[sub_index])?;
17480 self.finish_observation("syncular_section_failure", batch)
17481 })();
17482 if let Err(message) = persisted {
17483 self.rollback_observation("syncular_section_failure");
17484 self.subs[sub_index] = previous;
17485 return Err(SectionError::Abort("storage.failed".into(), message));
17486 }
17487 report.failed.push(id.to_owned());
17488 Ok(())
17489 }
17490 Err(error) => {
17491 self.subs[sub_index] = previous;
17494 Err(error)
17495 }
17496 }
17497 }
17498 }
17499 }
17500
17501 fn apply_sub_end(
17502 &mut self,
17503 sub_index: usize,
17504 cursor: i64,
17505 bootstrap_state: Option<String>,
17506 ) -> Result<(), String> {
17507 #[cfg(feature = "bench-internals")]
17508 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
17509 let previous = self.subs[sub_index].clone();
17510 self.begin_observation("syncular_sub_end")?;
17511 let persisted = (|| {
17512 let mut batch = ChangeAccumulator::default();
17513 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
17514 && cursor >= 0
17515 && bootstrap_state.is_none();
17516 if completed {
17517 let table = self
17518 .schema
17519 .table(&previous.table)
17520 .ok_or_else(|| "sync.unknown_table".to_owned())?;
17521 let effective = previous.effective.as_deref().unwrap_or(&[]);
17522 let mut statement = self.conn.prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
17523 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17524 let rows = statement
17525 .query_map(rusqlite::params![previous.table, cursor], |row| {
17526 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
17527 })
17528 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17529 let mut covered = HashSet::new();
17530 for row in rows {
17531 let (id, intent) =
17532 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17533 let values: Map<String, Value> = serde_json::from_str(&intent)
17534 .map_err(|_| "sync.local_corrupt".to_owned())?;
17535 if !effective.is_empty()
17536 && effective.iter().all(|(variable, allowed)| {
17537 table
17538 .scope_column(variable)
17539 .and_then(|column| values.get(column))
17540 .is_some_and(|value| {
17541 allowed.contains(
17542 &value
17543 .as_str()
17544 .map_or_else(|| value.to_string(), str::to_owned),
17545 )
17546 })
17547 })
17548 {
17549 covered.insert(id);
17550 }
17551 }
17552 drop(statement);
17553 for id in covered {
17554 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![previous.table, id, cursor])
17555 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17556 self.overlay_dirty.table(&previous.table);
17557 batch.table(&previous.table);
17558 }
17559 self.rebuild_overlay_if_dirty()?;
17560 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
17561 batch.window(&base_key, &previous.table, &unit);
17562 }
17563 }
17564 let sub = &mut self.subs[sub_index];
17565 sub.cursor = cursor;
17566 sub.bootstrap_state = bootstrap_state;
17567 sub.reason_code = None;
17570 sub.synced_once = true;
17571 self.persist_sub(&self.subs[sub_index])?;
17572 self.finish_observation("syncular_sub_end", batch)
17573 })();
17574 if persisted.is_err() {
17575 self.rollback_observation("syncular_sub_end");
17576 self.subs[sub_index] = previous;
17577 }
17578 persisted
17579 }
17580
17581 fn apply_section_body(
17582 &mut self,
17583 transport: &mut dyn Transport,
17584 sub_index: usize,
17585 body: Vec<Frame>,
17586 fresh: bool,
17587 meta: &RequestMeta,
17588 report: &mut SyncReport,
17589 ) -> Result<(), SectionError> {
17590 let mut saw_segment = false;
17591 for frame in body {
17592 match frame {
17593 Frame::Commit {
17594 tables,
17595 changes,
17596 commit_seq,
17597 ..
17598 } => {
17599 self.apply_commit_frame(&tables, &changes, Some(commit_seq))
17600 .map_err(|(c, m)| SectionError::Abort(c, m))?;
17601 report.commits_applied += 1;
17602 }
17603 Frame::SegmentInline { payload } => {
17604 let segment = decode_rows_segment(&payload)
17605 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
17606 self.progress.update(|p| {
17607 p.phase = ProgressPhase::Import;
17608 p.subscription_id = Some(self.subs[sub_index].id.clone());
17609 p.table = Some(self.subs[sub_index].table.clone());
17610 p.segment_id = None;
17611 p.bytes_received = payload.len() as u64;
17612 p.bytes_total = Some(payload.len() as u64);
17613 p.rows_processed = 0;
17614 p.rows_total =
17615 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
17616 });
17617 let first = !saw_segment;
17618 saw_segment = true;
17619 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17620 report.segment_rows_applied += applied;
17621 }
17622 Frame::SegmentRef {
17623 segment_id,
17624 byte_length,
17625 media_type,
17626 table,
17627 row_count,
17628 as_of_commit_seq,
17629 scope_digest,
17630 row_cursor,
17631 next_row_cursor,
17632 url,
17633 url_expires_at_ms,
17634 ..
17635 } => {
17636 let advertised = match media_type {
17639 MediaType::Rows => {
17640 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
17641 || meta.accept & ACCEPT_INLINE_ROWS != 0
17642 }
17643 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
17644 };
17645 if !advertised {
17646 return Err(SectionError::Abort(
17647 "sync.invalid_request".to_owned(),
17648 format!(
17649 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
17650 media_type.name()
17651 ),
17652 ));
17653 }
17654 self.progress.update(|p| {
17655 p.phase = ProgressPhase::Download;
17656 p.subscription_id = Some(self.subs[sub_index].id.clone());
17657 p.table = Some(table.clone());
17658 p.segment_id = Some(segment_id.clone());
17659 p.bytes_received = 0;
17660 p.bytes_total = u64::try_from(byte_length).ok();
17661 p.rows_processed = 0;
17662 p.rows_total = u64::try_from(row_count).ok();
17663 });
17664 let observer = self.progress.clone();
17665 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
17666 let bytes = if let Some(url) = url {
17667 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
17672 return Err(SectionError::Abort(
17673 "sync.invalid_request".to_owned(),
17674 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
17675 .to_owned(),
17676 ));
17677 }
17678 if url_expires_at_ms.is_some_and(|exp| {
17680 exp <= transport
17681 .segment_fetch_started_at(&url)
17682 .unwrap_or_else(|| self.clock_now_ms())
17683 }) {
17684 return Err(SectionError::Abort(
17685 "sync.segment_expired".to_owned(),
17686 format!(
17687 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
17688 ),
17689 ));
17690 }
17691 transport
17692 .fetch_url(&url, &mut on_progress)
17693 .map_err(|e| SectionError::Abort(e.code, e.message))?
17694 } else {
17695 let requested_scopes_json =
17696 canonical_scope_json(&self.subs[sub_index].requested);
17697 transport
17698 .download_segment(
17699 &SegmentRequest {
17700 segment_id: segment_id.clone(),
17701 table,
17702 requested_scopes_json,
17703 },
17704 &mut on_progress,
17705 )
17706 .map_err(|e| SectionError::Abort(e.code, e.message))?
17707 };
17708 on_progress(bytes.len() as u64);
17710 let digest = Sha256::digest(&bytes);
17711 let expected = segment_id
17712 .strip_prefix("sha256:")
17713 .unwrap_or(segment_id.as_str());
17714 if bytes_to_hex(&digest) != expected {
17715 self.schedule_background_retry();
17717 return Err(SectionError::Abort(
17718 "sync.invalid_request".to_owned(),
17719 "segment bytes do not match the content address (§5.1)".to_owned(),
17720 ));
17721 }
17722 self.progress.update(|p| p.phase = ProgressPhase::Import);
17723 if media_type == MediaType::Sqlite {
17724 if row_cursor.is_some() || next_row_cursor.is_some() {
17727 return Err(SectionError::Abort(
17728 "sync.invalid_request".to_owned(),
17729 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
17730 .to_owned(),
17731 ));
17732 }
17733 let first = !saw_segment;
17734 saw_segment = true;
17735 let applied = self.apply_sqlite_segment(
17736 sub_index,
17737 &bytes,
17738 fresh && first,
17739 row_count,
17740 as_of_commit_seq,
17741 &scope_digest,
17742 )?;
17743 report.segment_rows_applied += applied;
17744 } else {
17745 let segment = decode_rows_segment(&bytes).map_err(|e| {
17746 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
17747 })?;
17748 let first = row_cursor.is_none();
17749 saw_segment = true;
17750 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17751 report.segment_rows_applied += applied;
17752 }
17753 }
17754 Frame::Unknown { .. } => {}
17755 Frame::Error { code, message, .. } => {
17756 return Err(SectionError::Abort(code, message))
17757 }
17758 _ => {
17759 return Err(SectionError::Abort(
17760 "sync.invalid_request".to_owned(),
17761 "unexpected frame inside a subscription section".to_owned(),
17762 ));
17763 }
17764 }
17765 }
17766 Ok(())
17767 }
17768
17769 fn apply_commit_frame(
17771 &mut self,
17772 tables: &[String],
17773 changes: &[ssp2::model::Change],
17774 commit_seq: Option<i64>,
17775 ) -> Result<(), (String, String)> {
17776 #[cfg(feature = "bench-internals")]
17777 let _phase = self.benchmark_phases.start(Phase::CommitApply);
17778 let was_dirty = self.overlay_dirty.snapshot();
17779 self.coalesce_unit(changes.len())
17780 .and_then(|()| self.begin_observation("syncular_commit"))
17781 .map_err(|message| ("storage.failed".into(), message))?;
17782 let applied = (|| {
17783 let mut batch = ChangeAccumulator::default();
17784 self.record_commit_changes(&mut batch, tables, changes);
17785 if let Some(sequence) = commit_seq {
17786 for change in changes {
17787 let table = tables.get(change.table_index as usize).ok_or_else(|| {
17788 (
17789 "sync.invalid_request".to_owned(),
17790 "change tableIndex out of range".to_owned(),
17791 )
17792 })?;
17793 if self
17794 .outbox
17795 .iter()
17796 .flat_map(|commit| &commit.ops)
17797 .any(|op| op.table == *table && op.row_id == change.row_id)
17798 {
17799 self.conn.execute("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)", rusqlite::params![table, change.row_id, sequence])
17800 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
17801 }
17802 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![table, change.row_id, sequence])
17803 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
17804 }
17805 }
17806 self.apply_commit_changes(tables, changes)?;
17807 self.rebuild_overlay_if_dirty()
17808 .map_err(|message| ("storage.failed".into(), message))?;
17809 self.finish_observation("syncular_commit", batch)
17810 .map_err(|message| ("storage.failed".into(), message))
17811 })();
17812 if applied.is_err() {
17813 self.rollback_observation("syncular_commit");
17814 self.overlay_dirty.restore(was_dirty.clone());
17815 }
17816 applied
17817 }
17818
17819 fn apply_commit_changes(
17820 &mut self,
17821 tables: &[String],
17822 changes: &[ssp2::model::Change],
17823 ) -> Result<(), (String, String)> {
17824 let mirror_visible = !self.overlay_dirty.get()
17829 && self.failed_commits.is_empty()
17830 && ((self.outbox.is_empty() && !self.has_acknowledged_rows())
17831 || changes.iter().all(|change| {
17832 tables
17833 .get(change.table_index as usize)
17834 .and_then(|name| self.schema.table(name))
17835 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
17836 }));
17837 for change in changes {
17838 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
17839 (
17840 "sync.invalid_request".to_owned(),
17841 "change tableIndex out of range".to_owned(),
17842 )
17843 })?;
17844 let table = self.schema.table(table_name).ok_or_else(|| {
17845 (
17846 "sync.schema_mismatch".to_owned(),
17847 format!("change targets unknown table {table_name:?}"),
17848 )
17849 })?;
17850 match change.op {
17851 Op::Upsert => {
17852 let payload = change.row.as_ref().ok_or_else(|| {
17853 (
17854 "sync.invalid_request".to_owned(),
17855 "upsert change without row payload".to_owned(),
17856 )
17857 })?;
17858 let row = {
17860 #[cfg(feature = "bench-internals")]
17861 let _phase = self.benchmark_phases.start(Phase::RowDecode);
17862 decode_row_bytes(table, payload, &self.encryption)
17863 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
17864 };
17865 let version = change.row_version.unwrap_or(0);
17866 let table_name = table.name.clone();
17867 self.write_base_row(&table_name, &row, version)
17868 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17869 if mirror_visible {
17870 self.write_row(&visible_table(&table_name), &table_name, &row, version)
17871 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17872 }
17873 }
17874 Op::Delete => {
17875 self.delete_base_row(table_name, &change.row_id)
17876 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17877 if mirror_visible {
17878 let sql = format!(
17879 "DELETE FROM {} WHERE {}",
17880 visible_table(table_name),
17881 row_id_predicate(table)
17882 );
17883 self.conn
17884 .prepare_cached(&sql)
17885 .and_then(|mut statement| {
17886 statement.execute(rusqlite::params![change.row_id])
17887 })
17888 .map_err(|error| {
17889 ("sync.invalid_request".to_owned(), error.to_string())
17890 })?;
17891 }
17892 }
17893 }
17894 }
17895 if mirror_visible {
17896 {
17897 let changed_rows: HashSet<(&str, &str)> = changes
17898 .iter()
17899 .map(|change| {
17900 (
17901 tables[change.table_index as usize].as_str(),
17902 change.row_id.as_str(),
17903 )
17904 })
17905 .collect();
17906 let acknowledged = self
17907 .acknowledged_operations(Some(&changed_rows), None)
17908 .map_err(|error| ("storage.failed".to_owned(), error))?;
17909 self.apply_outbox_ops(&acknowledged)
17910 .map_err(|message| ("storage.failed".into(), message))?;
17911 self.apply_outbox_ops(
17912 self.outbox
17913 .iter()
17914 .flat_map(|commit| &commit.ops)
17915 .filter(|op| {
17916 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
17917 }),
17918 )
17919 .map_err(|message| ("storage.failed".into(), message))?;
17920 }
17921 self.overlay_dirty.set(false);
17922 }
17923 Ok(())
17924 }
17925
17926 fn apply_segment(
17931 &mut self,
17932 sub_index: usize,
17933 segment: &RowsSegment,
17934 first_fresh_page: bool,
17935 ) -> Result<u32, SectionError> {
17936 let (sub_table, effective) = {
17937 let sub = &self.subs[sub_index];
17938 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17939 };
17940 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
17941 SectionError::Abort(
17942 "sync.schema_mismatch".to_owned(),
17943 format!("subscription table {sub_table:?} is not in the client schema"),
17944 )
17945 })?;
17946 let matches = segment.table == table.name
17951 && segment.schema_version == self.schema.version
17952 && segment.columns.len() == table.wire_columns.len()
17953 && segment
17954 .columns
17955 .iter()
17956 .zip(table.wire_columns.iter())
17957 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
17958 if !matches {
17959 return Err(SectionError::Abort(
17960 "sync.schema_mismatch".to_owned(),
17961 "segment column table does not match the generated schema (§5.2)".to_owned(),
17962 ));
17963 }
17964 let mut applied = 0u32;
17965 let empty = Vec::new();
17966 let blocks = if segment.blocks.is_empty() {
17967 std::slice::from_ref(&empty)
17968 } else {
17969 &segment.blocks
17970 };
17971 for (index, block) in blocks.iter().enumerate() {
17972 let clear = first_fresh_page && index == 0;
17973 if !clear && block.is_empty() {
17974 continue;
17975 }
17976 let was_dirty = self.overlay_dirty.snapshot();
17977 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
17978 && self
17979 .outbox
17980 .iter()
17981 .flat_map(|commit| &commit.ops)
17982 .any(|op| op.table == table.name);
17983 let mirror_visible = !reconcile_pending
17984 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
17985 self.coalesce_unit(block.len())
17986 .and_then(|()| self.begin_observation("syncular_segment_block"))
17987 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17988 let outcome = (|| {
17989 self.rebuild_overlay_if_dirty()
17990 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17991 let mut changed_keys = Vec::new();
17992 let mut batch = ChangeAccumulator::default();
17993 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
17994 batch.table(&table.name);
17995 }
17996 if clear {
17997 self.purge_scope_rows(&table.name, &effective)
17998 .map_err(|()| SectionError::FailClosed)?;
17999 }
18000 for (row_index, row) in block.iter().enumerate() {
18001 let decrypted;
18002 let values = if table.has_encrypted_columns() {
18003 let mut values = row.values.clone();
18004 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
18005 .map_err(|message| {
18006 SectionError::Abort("client.decrypt_failed".into(), message)
18007 })?;
18008 decrypted = values;
18009 &decrypted
18010 } else {
18011 &row.values
18012 };
18013 self.write_base_row(&table.name, values, row.server_version)
18014 .map_err(|message| {
18015 SectionError::Abort("sync.invalid_request".into(), message)
18016 })?;
18017 if reconcile_pending {
18018 changed_keys.push(
18019 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
18020 |message| SectionError::Abort("storage.failed".into(), message),
18021 )?,
18022 );
18023 }
18024 if mirror_visible {
18025 self.write_row(
18026 &visible_table(&table.name),
18027 &table.name,
18028 values,
18029 row.server_version,
18030 )
18031 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18032 }
18033 let processed = u64::from(applied) + row_index as u64 + 1;
18034 if processed.is_multiple_of(1024) {
18035 self.progress.update(|p| p.rows_processed = processed);
18036 }
18037 }
18038 if reconcile_pending {
18039 self.reconcile_imported_rows(&table, &mut changed_keys)
18040 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18041 }
18042 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
18043 self.apply_outbox_ops(
18044 self.outbox
18045 .iter()
18046 .flat_map(|commit| &commit.ops)
18047 .filter(|op| op.table == table.name),
18048 )
18049 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18050 self.overlay_dirty.set(false);
18051 } else {
18052 self.rebuild_overlay_if_dirty()
18053 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18054 }
18055 self.finish_observation("syncular_segment_block", batch)
18056 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
18057 })();
18058 if let Err(error) = outcome {
18059 self.rollback_observation("syncular_segment_block");
18060 self.overlay_dirty.restore(was_dirty.clone());
18061 return Err(error);
18062 }
18063 applied += block.len() as u32;
18064 self.progress
18065 .update(|p| p.rows_processed = u64::from(applied));
18066 std::thread::yield_now();
18067 }
18068 self.progress
18069 .update(|p| p.rows_processed = u64::from(applied));
18070 Ok(applied)
18071 }
18072
18073 fn apply_sqlite_segment(
18081 &mut self,
18082 sub_index: usize,
18083 bytes: &[u8],
18084 first_fresh_page: bool,
18085 row_count: i64,
18086 as_of_commit_seq: i64,
18087 scope_digest: &str,
18088 ) -> Result<u32, SectionError> {
18089 let invalid = |detail: &str| {
18090 SectionError::Abort(
18091 "sync.invalid_request".to_owned(),
18092 format!("sqlite segment rejected: {detail} (§5.3)"),
18093 )
18094 };
18095 let (sub_table, effective) = {
18096 let sub = &self.subs[sub_index];
18097 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
18098 };
18099 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
18100 SectionError::Abort(
18101 "sync.schema_mismatch".to_owned(),
18102 format!("subscription table {sub_table:?} is not in the client schema"),
18103 )
18104 })?;
18105
18106 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
18107 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
18108 let img = match rusqlite::Connection::open_with_flags(
18109 &path,
18110 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
18111 ) {
18112 Ok(conn) => conn,
18113 Err(_) => {
18114 let _ = std::fs::remove_file(&path);
18115 return Err(invalid("bytes do not open as a SQLite database"));
18116 }
18117 };
18118 let outcome = self.apply_sqlite_image(
18119 &img,
18120 &table,
18121 first_fresh_page,
18122 &effective,
18123 row_count,
18124 as_of_commit_seq,
18125 scope_digest,
18126 );
18127 drop(img);
18128 let _ = std::fs::remove_file(&path);
18129 outcome
18130 }
18131
18132 #[allow(clippy::too_many_arguments)]
18133 fn apply_sqlite_image(
18134 &mut self,
18135 img: &rusqlite::Connection,
18136 table: &crate::schema::TableSchema,
18137 first_fresh_page: bool,
18138 effective: &[(String, Vec<String>)],
18139 row_count: i64,
18140 as_of_commit_seq: i64,
18141 scope_digest: &str,
18142 ) -> Result<u32, SectionError> {
18143 let invalid = |detail: String| {
18144 SectionError::Abort(
18145 "sync.invalid_request".to_owned(),
18146 format!("sqlite segment rejected: {detail} (§5.3)"),
18147 )
18148 };
18149
18150 type MetaRow = (i64, String, i64, i64, String, i64, i64);
18153 let meta: MetaRow = img
18154 .query_row(
18155 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
18156 \"scopeDigest\", \"rowCount\",
18157 (SELECT count(*) FROM _syncular_segment)
18158 FROM _syncular_segment",
18159 [],
18160 |row| {
18161 Ok((
18162 row.get(0)?,
18163 row.get(1)?,
18164 row.get(2)?,
18165 row.get(3)?,
18166 row.get(4)?,
18167 row.get(5)?,
18168 row.get(6)?,
18169 ))
18170 },
18171 )
18172 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
18173 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
18174 if meta_count != 1 {
18175 return Err(invalid(format!(
18176 "_syncular_segment must contain exactly one row, found {meta_count}"
18177 )));
18178 }
18179 if format != 1 {
18180 return Err(invalid(format!("format {format}")));
18181 }
18182 if meta_table != table.name {
18183 return Err(invalid(format!("image table {meta_table:?}")));
18184 }
18185 if schema_version != i64::from(self.schema.version) {
18186 return Err(invalid(format!("schemaVersion {schema_version}")));
18187 }
18188 if pin != as_of_commit_seq {
18189 return Err(invalid(format!("asOfCommitSeq {pin}")));
18190 }
18191 if digest != scope_digest {
18192 return Err(invalid("scopeDigest mismatch".to_owned()));
18193 }
18194 if meta_rows != row_count {
18195 return Err(invalid(format!("rowCount {meta_rows}")));
18196 }
18197
18198 let mut names: Vec<String> = Vec::new();
18200 let mut primary_keys: Vec<String> = Vec::new();
18201 {
18202 let mut stmt = img
18203 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
18204 .map_err(|_| invalid("image data table missing".to_owned()))?;
18205 let mut rows = stmt
18206 .query([])
18207 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
18208 while let Some(row) = rows
18209 .next()
18210 .map_err(|_| invalid("image data table unreadable".to_owned()))?
18211 {
18212 if row
18213 .get::<_, i64>(5)
18214 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
18215 > 0
18216 {
18217 primary_keys.push(row.get(1).map_err(|error| {
18218 invalid(Self::sqlite_failure(&self.storage_failure, error))
18219 })?);
18220 }
18221 names.push(
18222 row.get::<_, String>(1)
18223 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
18224 );
18225 }
18226 }
18227 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
18228 expected.push("_syncular_version");
18229 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
18230 return Err(SectionError::Abort(
18231 "sync.schema_mismatch".to_owned(),
18232 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
18233 ));
18234 }
18235
18236 if primary_keys != vec![table.primary_key.clone()] {
18237 return Err(invalid(
18238 "image primary key does not match generated schema".into(),
18239 ));
18240 }
18241 let actual_count: i64 = img
18242 .query_row(
18243 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
18244 [],
18245 |row| row.get(0),
18246 )
18247 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18248 if actual_count != row_count {
18249 return Err(invalid("image row count does not match descriptor".into()));
18250 }
18251 let insert = self.insert_row_sql(&base_table(&table.name), table);
18252 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
18253 let column_list = names
18254 .iter()
18255 .map(|n| quote_ident(n))
18256 .collect::<Vec<_>>()
18257 .join(", ");
18258 let mut stmt = img
18259 .prepare(&format!(
18260 "SELECT {column_list} FROM {} ORDER BY {}",
18261 quote_ident(&table.name),
18262 quote_ident(&table.primary_key)
18263 ))
18264 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18265 let mut rows = stmt
18266 .query([])
18267 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18268 let mut applied = 0u32;
18269 loop {
18270 let clear = applied == 0 && first_fresh_page;
18271 let was_dirty = self.overlay_dirty.snapshot();
18272 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
18273 && self
18274 .outbox
18275 .iter()
18276 .flat_map(|commit| &commit.ops)
18277 .any(|op| op.table == table.name);
18278 let mirror_visible = !reconcile_pending
18279 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
18280 self.coalesce_unit(1024)
18282 .and_then(|()| self.begin_observation("syncular_image_chunk"))
18283 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18284 let result = (|| {
18285 self.rebuild_overlay_if_dirty()
18286 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18287 let mut changed_keys = Vec::new();
18288 let mut batch = ChangeAccumulator::default();
18289 if clear {
18290 if self.scoped_rows_exist(&table.name, effective) {
18291 batch.table(&table.name);
18292 }
18293 self.purge_scope_rows(&table.name, effective)
18294 .map_err(|()| SectionError::FailClosed)?;
18295 }
18296 let mut processed = 0u32;
18297 {
18298 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
18299 invalid(Self::sqlite_failure(&self.storage_failure, error))
18300 })?;
18301 let mut mirror = if mirror_visible {
18302 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
18303 invalid(Self::sqlite_failure(&self.storage_failure, error))
18304 })?)
18305 } else {
18306 None
18307 };
18308 for _ in 0..1024 {
18309 let Some(row) = rows.next().map_err(|error| {
18310 invalid(Self::sqlite_failure(&self.storage_failure, error))
18311 })?
18312 else {
18313 break;
18314 };
18315 for (i, column) in table.columns.iter().enumerate() {
18316 let cell = row.get_ref(i).map_err(|error| {
18317 invalid(Self::sqlite_failure(&self.storage_failure, error))
18318 })?;
18319 let param = image_cell_param(column, cell).map_err(invalid)?;
18320 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18321 invalid(Self::sqlite_failure(&self.storage_failure, error))
18322 })?;
18323 if let Some(mirror) = &mut mirror {
18324 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18325 invalid(Self::sqlite_failure(&self.storage_failure, error))
18326 })?;
18327 }
18328 }
18329 let version_index = table.columns.len();
18330 let version: i64 = row.get(version_index).map_err(|error| {
18331 invalid(Self::sqlite_failure(&self.storage_failure, error))
18332 })?;
18333 if version < 1 {
18334 return Err(invalid("image row version must be positive".into()));
18335 }
18336 ins.raw_bind_parameter(version_index + 1, version)
18337 .map_err(|error| {
18338 invalid(Self::sqlite_failure(&self.storage_failure, error))
18339 })?;
18340 ins.raw_execute().map_err(|error| {
18341 invalid(Self::sqlite_failure(&self.storage_failure, error))
18342 })?;
18343 if let Some(mirror) = &mut mirror {
18344 mirror
18345 .raw_bind_parameter(version_index + 1, version)
18346 .map_err(|error| {
18347 invalid(Self::sqlite_failure(&self.storage_failure, error))
18348 })?;
18349 mirror.raw_execute().map_err(|error| {
18350 invalid(Self::sqlite_failure(&self.storage_failure, error))
18351 })?;
18352 }
18353 if reconcile_pending {
18354 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
18355 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
18356 )?);
18357 }
18358 processed += 1;
18359 }
18360 }
18361 if processed > 0 {
18362 batch.table(&table.name);
18363 self.overlay_dirty.table(&table.name);
18364 }
18365 if reconcile_pending {
18366 self.reconcile_imported_rows(table, &mut changed_keys)
18367 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18368 }
18369 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
18370 self.apply_outbox_ops(
18371 self.outbox
18372 .iter()
18373 .flat_map(|commit| &commit.ops)
18374 .filter(|op| op.table == table.name),
18375 )
18376 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18377 self.overlay_dirty.set(false);
18378 } else {
18379 self.rebuild_overlay_if_dirty()
18380 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18381 }
18382 self.finish_observation("syncular_image_chunk", batch)
18383 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18384 Ok(processed)
18385 })();
18386 let processed = match result {
18387 Ok(processed) => processed,
18388 Err(error) => {
18389 self.rollback_observation("syncular_image_chunk");
18390 self.overlay_dirty.restore(was_dirty.clone());
18391 return Err(error);
18392 }
18393 };
18394 applied += processed;
18395 self.progress
18396 .update(|p| p.rows_processed = u64::from(applied));
18397 std::thread::yield_now();
18398 if i64::from(applied) == actual_count {
18399 break;
18400 }
18401 }
18402 Ok(applied)
18403 }
18404
18405 fn reconcile_imported_rows(
18409 &self,
18410 table: &TableSchema,
18411 keys: &mut Vec<SqlValue>,
18412 ) -> Result<(), String> {
18413 let visible = visible_table(&table.name);
18414 let base = base_table(&table.name);
18415 let pk = quote_ident(&table.primary_key);
18416 let predicate = row_id_predicate(table);
18417 let mut lookup = self.conn.prepare_cached(&format!(
18418 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
18419 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18420 let mut deleted = HashSet::new();
18421 for op in self
18422 .outbox
18423 .iter()
18424 .flat_map(|commit| &commit.ops)
18425 .filter(|op| op.table == table.name)
18426 {
18427 if op.upsert {
18428 if let Some(values) = &op.values {
18429 let key = json_to_column_value(
18430 &table.columns[table.pk_index],
18431 values.get(&table.primary_key),
18432 )?;
18433 keys.push(owned_sql_value(RowParam::Cell(&key))?);
18434 }
18435 } else if deleted.insert(&op.row_id) {
18436 keys.extend(
18437 lookup
18438 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
18439 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
18440 .collect::<Result<Vec<_>, _>>()
18441 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18442 );
18443 }
18444 }
18445 for chunk in keys.chunks(400) {
18448 let holes = vec!["?"; chunk.len()].join(",");
18449 self.conn
18450 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
18451 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18452 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18453 }
18454 let updates = table
18455 .columns
18456 .iter()
18457 .map(|column| column.name.as_str())
18458 .chain(std::iter::once("_syncular_version"))
18459 .filter(|name| *name != table.primary_key)
18460 .map(|name| {
18461 let name = quote_ident(name);
18462 format!("{name}=excluded.{name}")
18463 })
18464 .collect::<Vec<_>>()
18465 .join(",");
18466 for chunk in keys.chunks(400) {
18467 let holes = vec!["?"; chunk.len()].join(",");
18468 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
18469 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18470 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18471 }
18472 Ok(())
18473 }
18474
18475 fn compile_local_data_purge(
18478 &self,
18479 input: &LocalDataPurgeInput,
18480 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
18481 let invalid = |message: String| format!("sync.invalid_request: {message}");
18482 if input.purge_id.is_empty()
18483 || input.purge_id.len() > 128
18484 || !is_local_operation_code_like(&input.purge_id)
18485 {
18486 return Err(invalid(
18487 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
18488 ));
18489 }
18490 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
18491 return Err(invalid(format!(
18492 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
18493 )));
18494 }
18495
18496 let mut deduplicated: BTreeMap<
18497 String,
18498 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
18499 > = BTreeMap::new();
18500 for target in &input.targets {
18501 let table = self.schema.table(&target.table).ok_or_else(|| {
18502 invalid(format!(
18503 "local purge names unknown table {:?}",
18504 target.table
18505 ))
18506 })?;
18507 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
18508 return Err(invalid(format!(
18509 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
18510 target.table
18511 )));
18512 }
18513 let mut selectors = Vec::with_capacity(target.selectors.len());
18514 let mut canonical_selectors = BTreeMap::new();
18515 for (column_name, raw_values) in &target.selectors {
18516 let Some((column_index, column)) = table
18517 .columns
18518 .iter()
18519 .enumerate()
18520 .find(|(_, column)| column.name == *column_name)
18521 else {
18522 return Err(invalid(format!(
18523 "local purge target {:?} names unknown column {:?}",
18524 target.table, column_name
18525 )));
18526 };
18527 let encrypted = table
18528 .encrypted_columns
18529 .iter()
18530 .any(|candidate| candidate.index == column_index);
18531 if column.ty != ColumnType::String || encrypted {
18532 return Err(invalid(format!(
18533 "local purge selector {:?}.{:?} must be a plaintext string column",
18534 target.table, column_name
18535 )));
18536 }
18537 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
18538 return Err(invalid(format!(
18539 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
18540 target.table, column_name
18541 )));
18542 }
18543 let mut values = raw_values.clone();
18544 values.sort();
18545 values.dedup();
18546 if values.iter().any(|value| {
18547 value.is_empty()
18548 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
18549 || !is_local_operation_code_like(value)
18550 }) {
18551 return Err(invalid(format!(
18552 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
18553 )));
18554 }
18555 selectors.push((column_name.clone(), values.clone()));
18556 canonical_selectors.insert(column_name.clone(), values);
18557 }
18558 let canonical = LocalDataPurgeTarget {
18559 table: target.table.clone(),
18560 selectors: canonical_selectors,
18561 };
18562 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18563 deduplicated.insert(
18564 key,
18565 (
18566 CompiledLocalDataPurgeTarget {
18567 table: target.table.clone(),
18568 selectors,
18569 },
18570 canonical,
18571 ),
18572 );
18573 }
18574 let targets = deduplicated
18575 .values()
18576 .map(|(compiled, _)| compiled.clone())
18577 .collect::<Vec<_>>();
18578 let canonical = deduplicated
18579 .values()
18580 .map(|(_, target)| target.clone())
18581 .collect::<Vec<_>>();
18582 let canonical_plan =
18583 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18584 Ok((targets, canonical_plan))
18585 }
18586
18587 fn local_purge_base_row_ids(
18588 &self,
18589 targets: &[CompiledLocalDataPurgeTarget],
18590 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
18591 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
18592 for target in targets {
18593 let table = self
18594 .schema
18595 .table(&target.table)
18596 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
18597 let mut clauses = Vec::with_capacity(target.selectors.len());
18598 let mut params = Vec::new();
18599 for (column, values) in &target.selectors {
18600 clauses.push(format!(
18601 "{} IN ({})",
18602 quote_ident(column),
18603 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
18604 ));
18605 params.extend(values.iter().cloned().map(SqlValue::Text));
18606 }
18607 let sql = format!(
18608 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
18609 quote_ident(&table.primary_key),
18610 base_table(&target.table),
18611 clauses.join(" AND ")
18612 );
18613 let mut statement = self
18614 .conn
18615 .prepare(&sql)
18616 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18617 let rows = statement
18618 .query_map(rusqlite::params_from_iter(params), |row| {
18619 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
18620 render_row_id_json(Some(&value))
18621 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
18622 })
18623 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18624 let ids = by_table.entry(target.table.clone()).or_default();
18625 for row in rows {
18626 ids.insert(
18627 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18628 );
18629 }
18630 }
18631 Ok(by_table)
18632 }
18633
18634 fn local_purge_values_match(
18635 target: &CompiledLocalDataPurgeTarget,
18636 values: &Map<String, Value>,
18637 ) -> bool {
18638 target.selectors.iter().all(|(column, allowed)| {
18639 values
18640 .get(column)
18641 .and_then(Value::as_str)
18642 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
18643 })
18644 }
18645
18646 pub fn purge_local_data(
18650 &mut self,
18651 input: &LocalDataPurgeInput,
18652 ) -> Result<LocalDataPurgeResult, String> {
18653 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
18654 let meta_key = format!("localPurge:{}", input.purge_id);
18655 let applied_plan = self
18656 .conn
18657 .query_row(
18658 "SELECT value FROM _syncular_meta WHERE key = ?1",
18659 rusqlite::params![meta_key],
18660 |row| row.get::<_, String>(0),
18661 )
18662 .optional()
18663 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18664 if let Some(applied_plan) = applied_plan {
18665 if applied_plan != canonical_plan {
18666 return Err(format!(
18667 "sync.invalid_request: local purge id {:?} was already used with a different plan",
18668 input.purge_id
18669 ));
18670 }
18671 return Ok(LocalDataPurgeResult {
18672 already_applied: true,
18673 purged_rows: 0,
18674 dropped_commits: 0,
18675 });
18676 }
18677
18678 let _ = self.drop_previous_version();
18682
18683 let prior_outbox = self.outbox.clone();
18684 let prior_failed = self.failed_commits.clone();
18685 let prior_rejections = self.rejections.clone();
18686 let prior_conflicts = self.conflicts.clone();
18687 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18688 self.begin_observation("syncular_local_purge")?;
18689 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
18690 let row_ids = self.local_purge_base_row_ids(&targets)?;
18691 let retained_dropped = self.drop_failed_commits(|table, values| {
18692 targets.iter().any(|target| {
18693 target.table == table && Self::local_purge_values_match(target, values)
18694 })
18695 })?;
18696 let doomed = self
18697 .outbox
18698 .iter()
18699 .filter(|commit| {
18700 commit.ops.iter().any(|operation| {
18701 let matching_targets = targets
18702 .iter()
18703 .filter(|target| target.table == operation.table)
18704 .collect::<Vec<_>>();
18705 if matching_targets.is_empty() {
18706 return false;
18707 }
18708 if row_ids
18709 .get(&operation.table)
18710 .is_some_and(|ids| ids.contains(&operation.row_id))
18711 {
18712 return true;
18713 }
18714 operation.values.as_ref().is_some_and(|values| {
18715 matching_targets
18716 .iter()
18717 .any(|target| Self::local_purge_values_match(target, values))
18718 })
18719 })
18720 })
18721 .cloned()
18722 .collect::<Vec<_>>();
18723 let doomed_ids = doomed
18724 .iter()
18725 .map(|commit| commit.client_commit_id.clone())
18726 .collect::<BTreeSet<_>>();
18727 let mut batch = ChangeAccumulator::default();
18728 for commit in &retained_dropped {
18729 batch.conflicts = true;
18730 batch.rejections = true;
18731 batch.outcomes = true;
18732 for operation in &commit.ops {
18733 batch.table(&operation.table);
18734 }
18735 }
18736 let mut rejections = Vec::new();
18737 for commit in &doomed {
18738 for operation in &commit.ops {
18739 batch.table(&operation.table);
18740 }
18741 let results = commit
18742 .ops
18743 .iter()
18744 .enumerate()
18745 .map(|(op_index, operation)| {
18746 let rejection = RejectionRecord {
18747 client_commit_id: commit.client_commit_id.clone(),
18748 op_index: op_index as i32,
18749 code: "client.local_data_purged".to_owned(),
18750 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
18751 retryable: false,
18752 details: None,
18753 operation: Some(CommitOperation::from(operation)),
18754 };
18755 rejections.push(rejection.clone());
18756 CommitOperationOutcome::Error { rejection }
18757 })
18758 .collect::<Vec<_>>();
18759 self.persist_commit_outcome(
18760 &commit.client_commit_id,
18761 CommitOutcomeStatus::Rejected,
18762 &results,
18763 Some(&commit.ops),
18764 )?;
18765 self.delete_outbox_persisted(&commit.client_commit_id)?;
18766 }
18767 if !doomed.is_empty() {
18768 self.outbox
18769 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
18770 self.rejections.extend(rejections);
18771 self.prune_commit_outcomes()?;
18772 batch.status = true;
18773 batch.rejections = true;
18774 batch.outcomes = true;
18775 }
18776
18777 let mut purged_rows = 0usize;
18778 for (table, ids) in &row_ids {
18779 if ids.is_empty() {
18780 continue;
18781 }
18782 batch.table(table);
18783 for row_id in ids {
18784 self.delete_base_row(table, row_id)?;
18785 purged_rows += 1;
18786 }
18787 }
18788 self.rebuild_overlay_if_dirty()?;
18789 self.delete_unreferenced_cached_blobs()?;
18790 self.conn
18791 .execute(
18792 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
18793 rusqlite::params![meta_key, canonical_plan],
18794 )
18795 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18796 Ok((
18797 batch,
18798 LocalDataPurgeResult {
18799 already_applied: false,
18800 purged_rows,
18801 dropped_commits: doomed.len(),
18802 },
18803 ))
18804 })();
18805 let (batch, result) = match applied {
18806 Ok(value) => value,
18807 Err(error) => {
18808 self.rollback_observation("syncular_local_purge");
18809 self.outbox = prior_outbox;
18810 self.failed_commits = prior_failed;
18811 self.rejections = prior_rejections;
18812 self.conflicts = prior_conflicts;
18813 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18814 return Err(error);
18815 }
18816 };
18817 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
18818 self.rollback_observation("syncular_local_purge");
18819 self.outbox = prior_outbox;
18820 self.failed_commits = prior_failed;
18821 self.rejections = prior_rejections;
18822 self.conflicts = prior_conflicts;
18823 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18824 return Err(error);
18825 }
18826 self.cancel_sync_round();
18827 Ok(result)
18828 }
18829
18830 pub fn rebootstrap_local_data(
18836 &mut self,
18837 input: &LocalDataRebootstrapInput,
18838 ) -> Result<LocalDataRebootstrapResult, String> {
18839 if input.rebootstrap_id.is_empty()
18840 || input.rebootstrap_id.len() > 128
18841 || !is_local_operation_code_like(&input.rebootstrap_id)
18842 {
18843 return Err(
18844 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
18845 .to_owned(),
18846 );
18847 }
18848 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
18849 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
18850 let (retained_commits, reset_subscriptions) =
18851 decode_local_rebootstrap_receipt(&persisted)?;
18852 return Ok(LocalDataRebootstrapResult {
18853 already_applied: true,
18854 retained_commits,
18855 reset_subscriptions,
18856 });
18857 }
18858 if self.stopped || self.schema_floor.is_some() {
18859 return Err(
18860 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
18861 .to_owned(),
18862 );
18863 }
18864
18865 let retained_commits = self.outbox.len();
18866 let reset_subscriptions = self.subs.len();
18867 let prior_subs = self.subs.clone();
18868 let prior_upgrading = self.upgrading;
18869 let prior_stopped = self.stopped;
18870 let prior_schema_floor = self.schema_floor.clone();
18871 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18872 let prior_sync_needed = self.sync_needed;
18873 let prior_sync_intents = self.sync_intent_queue.clone();
18874 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
18875
18876 let prior_active_round = self.active_round;
18877 self.begin_observation("syncular_local_rebootstrap")?;
18878 let mut batch = ChangeAccumulator::default();
18879 let applied = (|| -> Result<(), String> {
18880 self.run_schema_reset_observed(&mut batch, false)?;
18881 self.conn
18882 .execute(
18883 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
18884 rusqlite::params![meta_key, receipt],
18885 )
18886 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18887 self.sync_needed = true;
18888 self.sync_intent_queue.push_back(SyncIntent::Interactive);
18889 batch.status = true;
18890 Ok(())
18891 })();
18892 if let Err(error) = applied {
18893 self.rollback_observation("syncular_local_rebootstrap");
18894 self.active_round = prior_active_round;
18895 self.subs = prior_subs;
18896 self.upgrading = prior_upgrading;
18897 self.stopped = prior_stopped;
18898 self.schema_floor = prior_schema_floor;
18899 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18900 self.sync_needed = prior_sync_needed;
18901 self.sync_intent_queue = prior_sync_intents;
18902 return Err(error);
18903 }
18904 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
18905 self.rollback_observation("syncular_local_rebootstrap");
18906 self.active_round = prior_active_round;
18907 self.subs = prior_subs;
18908 self.upgrading = prior_upgrading;
18909 self.stopped = prior_stopped;
18910 self.schema_floor = prior_schema_floor;
18911 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18912 self.sync_needed = prior_sync_needed;
18913 self.sync_intent_queue = prior_sync_intents;
18914 return Err(error);
18915 }
18916
18917 Ok(LocalDataRebootstrapResult {
18918 already_applied: false,
18919 retained_commits,
18920 reset_subscriptions,
18921 })
18922 }
18923
18924 fn purge_scope_rows(
18929 &mut self,
18930 table_name: &str,
18931 effective: &[(String, Vec<String>)],
18932 ) -> Result<(), ()> {
18933 if effective.is_empty() {
18934 return Ok(());
18935 }
18936 let table = self.schema.table(table_name).ok_or(())?.clone();
18937 let mut clauses = Vec::new();
18938 let mut params: Vec<SqlValue> = Vec::new();
18939 for (variable, values) in effective {
18940 let column = table.scope_column(variable).ok_or(())?;
18941 let placeholders: Vec<String> = values
18942 .iter()
18943 .map(|v| {
18944 params.push(SqlValue::Text(v.clone()));
18945 "?".to_owned()
18946 })
18947 .collect();
18948 clauses.push(format!(
18949 "{} IN ({})",
18950 quote_ident(column),
18951 placeholders.join(", ")
18952 ));
18953 }
18954 let sql = format!(
18955 "DELETE FROM {} WHERE {}",
18956 base_table(table_name),
18957 clauses.join(" AND ")
18958 );
18959 self.overlay_dirty.table(table_name);
18960 self.conn
18961 .execute(&sql, rusqlite::params_from_iter(¶ms))
18962 .map_err(|_| ())?;
18963 self.conn
18964 .execute(
18965 &format!(
18966 "DELETE FROM {} WHERE {}",
18967 visible_table(table_name),
18968 clauses.join(" AND ")
18969 ),
18970 rusqlite::params_from_iter(¶ms),
18971 )
18972 .map_err(|_| ())?;
18973 Ok(())
18974 }
18975
18976 fn drop_doomed_outbox(
18979 &mut self,
18980 table_name: &str,
18981 effective: &[(String, Vec<String>)],
18982 ) -> Result<bool, String> {
18983 if effective.is_empty() {
18984 return Ok(false);
18985 }
18986 let Some(table) = self.schema.table(table_name).cloned() else {
18987 return Ok(false);
18988 };
18989 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
18990 for (variable, values) in effective {
18991 match table.scope_column(variable) {
18992 Some(column) => mappings.push((column, values)),
18993 None => return Ok(false), }
18995 }
18996 let doomed: Vec<OutboxCommit> = self
18997 .outbox
18998 .iter()
18999 .filter(|commit| {
19000 commit.ops.iter().any(|op| {
19001 op.upsert
19002 && op.table == table_name
19003 && op.values.as_ref().is_some_and(|values| {
19004 mappings
19005 .iter()
19006 .all(|(column, allowed)| match values.get(*column) {
19007 Some(Value::String(s)) => allowed.contains(s),
19008 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
19009 Some(Value::Bool(b)) => allowed.contains(&b.to_string()),
19010 _ => false,
19011 })
19012 })
19013 })
19014 })
19015 .cloned()
19016 .collect();
19017 if doomed.is_empty() {
19018 return Ok(false);
19019 }
19020 let mut rejections = Vec::new();
19021 for commit in &doomed {
19022 let results = commit
19023 .ops
19024 .iter()
19025 .enumerate()
19026 .map(|(op_index, operation)| {
19027 let rejection = RejectionRecord {
19028 client_commit_id: commit.client_commit_id.clone(),
19029 op_index: op_index as i32,
19030 code: "sync.scope_revoked".to_owned(),
19031 message: "the commit was dropped because its effective scope was revoked"
19032 .to_owned(),
19033 retryable: false,
19034 details: None,
19035 operation: Some(CommitOperation::from(operation)),
19036 };
19037 rejections.push(rejection.clone());
19038 CommitOperationOutcome::Error { rejection }
19039 })
19040 .collect::<Vec<_>>();
19041 self.persist_commit_outcome(
19042 &commit.client_commit_id,
19043 CommitOutcomeStatus::Rejected,
19044 &results,
19045 Some(&commit.ops),
19046 )?;
19047 self.delete_outbox_persisted(&commit.client_commit_id)?;
19048 }
19049 self.prune_commit_outcomes()?;
19050 let doomed_ids = doomed
19051 .iter()
19052 .map(|commit| commit.client_commit_id.as_str())
19053 .collect::<BTreeSet<_>>();
19054 self.outbox
19055 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
19056 self.rejections.extend(rejections);
19057 Ok(true)
19058 }
19059
19060 pub fn upload_blob(
19067 &mut self,
19068 bytes: &[u8],
19069 media_type: Option<String>,
19070 name: Option<String>,
19071 ) -> Result<Value, String> {
19072 let blob_id = blob_id_for(bytes);
19073 let now = self.clock_now_ms();
19074 let tx = self
19075 .conn
19076 .transaction()
19077 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19078 tx.execute(
19079 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
19080 ON CONFLICT(blob_id) DO NOTHING",
19081 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
19082 )
19083 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19084 tx.execute(
19085 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
19086 rusqlite::params![blob_id, media_type, now],
19087 )
19088 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19089 tx.commit()
19090 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19091 self.enforce_blob_cache_cap();
19092 let mut obj = Map::new();
19093 obj.insert("blobId".to_owned(), Value::from(blob_id));
19094 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
19095 if let Some(mt) = media_type {
19096 obj.insert("mediaType".to_owned(), Value::from(mt));
19097 }
19098 if let Some(n) = name {
19099 obj.insert("name".to_owned(), Value::from(n));
19100 }
19101 Ok(Value::Object(obj))
19102 }
19103
19104 pub fn fetch_blob_bytes(
19108 &mut self,
19109 transport: &mut dyn Transport,
19110 blob_id_or_ref: &str,
19111 ) -> Result<FetchedBlob, (String, String)> {
19112 let simple = |m: String| ("client.failed".to_owned(), m);
19113 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
19114 blob_id_or_ref.to_owned()
19115 } else {
19116 let value: Value = serde_json::from_str(blob_id_or_ref)
19117 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
19118 value
19119 .get("blobId")
19120 .and_then(Value::as_str)
19121 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
19122 .to_owned()
19123 };
19124 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
19125 return Ok(cached);
19126 }
19127 if !self.transport_enabled {
19128 return Err(("sync.offline".into(), "transport gate is closed".into()));
19129 }
19130 #[cfg(feature = "bench-internals")]
19136 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
19137 let bytes = match transport
19138 .blob_download(&blob_id)
19139 .map_err(|e| (e.code, e.message))?
19140 {
19141 BlobDownload::Bytes(bytes) => bytes,
19142 BlobDownload::Url {
19143 url,
19144 url_expires_at_ms,
19145 } => {
19146 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
19148 return Err((
19149 "sync.segment_expired".to_owned(),
19150 format!(
19151 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
19152 ),
19153 ));
19154 }
19155 transport
19156 .fetch_blob_url(&url)
19157 .map_err(|e| (e.code, e.message))?
19158 }
19159 };
19160 #[cfg(feature = "bench-internals")]
19161 drop(download_phase);
19162 #[cfg(feature = "bench-internals")]
19163 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
19164 if blob_id_for(&bytes) != blob_id {
19166 return Err(simple(format!(
19167 "blob content address mismatch for {blob_id}"
19168 )));
19169 }
19170 #[cfg(feature = "bench-internals")]
19171 drop(validate_phase);
19172 #[cfg(feature = "bench-internals")]
19173 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
19174 let now = self.clock_now_ms();
19175 self.conn
19176 .execute(
19177 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
19178 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
19179 )
19180 .map_err(|e| simple(e.to_string()))?;
19181 self.conn
19182 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
19183 .map_err(|e| simple(e.to_string()))?;
19184 self.enforce_blob_cache_cap();
19185 #[cfg(feature = "bench-internals")]
19186 drop(insert_phase);
19187 self.get_cached_blob_bytes(&blob_id)
19188 .map_err(simple)?
19189 .ok_or_else(|| simple("blob cache write failed".to_owned()))
19190 }
19191
19192 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
19193 #[cfg(feature = "bench-internals")]
19194 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
19195 let mut stmt = self
19196 .conn
19197 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
19198 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19199 let mut rows = stmt
19200 .query(rusqlite::params![blob_id])
19201 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19202 if let Some(row) = rows
19203 .next()
19204 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
19205 {
19206 let bytes: Vec<u8> = row
19207 .get(0)
19208 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19209 let byte_length: i64 = row
19210 .get(1)
19211 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19212 let media_type: Option<String> = row
19213 .get(2)
19214 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19215 return Ok(Some(FetchedBlob {
19216 blob_id: blob_id.to_owned(),
19217 byte_length,
19218 bytes,
19219 media_type,
19220 }));
19221 }
19222 Ok(None)
19223 }
19224
19225 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
19227 let pending: Vec<(String, Option<String>)> = {
19228 let mut stmt = self
19229 .conn
19230 .prepare(
19231 "SELECT blob_id, media_type FROM (
19232 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
19233 UNION ALL
19234 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
19235 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
19236 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
19237 ) ORDER BY created_at_ms, blob_id",
19238 )
19239 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19240 let rows = stmt
19241 .query_map([], |row| {
19242 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
19243 })
19244 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19245 rows.collect::<Result<_, _>>()
19246 .map_err(|error| match error {
19247 rusqlite::Error::InvalidColumnType(..)
19248 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
19249 "sync.local_corrupt",
19250 "Pending blob upload metadata is invalid",
19251 ),
19252 _ => TransportError::new("client.failed", error.to_string()),
19253 })?
19254 };
19255 let mut uploads = Vec::new();
19256 for (blob_id, media_type) in pending {
19257 let bytes = (|| {
19258 let stored: Option<(SqlValue, SqlValue)> = self
19259 .conn
19260 .query_row(
19261 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
19262 rusqlite::params![blob_id],
19263 |row| Ok((row.get(0)?, row.get(1)?)),
19264 )
19265 .optional()
19266 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19267 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
19268 return Err(TransportError::new(
19269 "sync.local_corrupt",
19270 "Pending blob upload body is missing or corrupt",
19271 ));
19272 };
19273 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
19274 return Err(TransportError::new(
19275 "sync.local_corrupt",
19276 "Pending blob upload body is missing or corrupt",
19277 ));
19278 }
19279 Ok(bytes)
19280 })();
19281 let failed = bytes.is_err();
19282 uploads.push(crate::round::PendingUpload {
19283 id: blob_id,
19284 bytes,
19285 media_type,
19286 });
19287 if failed {
19288 break;
19289 }
19290 }
19291 Ok(uploads)
19292 }
19293
19294 #[cfg(test)]
19295 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
19296 for upload in self.pending_blob_uploads()? {
19297 self.upload_one(
19298 transport,
19299 &upload.id,
19300 &upload.bytes?,
19301 upload.media_type.as_deref(),
19302 )?;
19303 self.conn
19304 .execute(
19305 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
19306 rusqlite::params![upload.id],
19307 )
19308 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19309 }
19310 Ok(())
19311 }
19312
19313 #[cfg(test)]
19320 fn upload_one(
19321 &self,
19322 transport: &mut dyn Transport,
19323 blob_id: &str,
19324 bytes: &[u8],
19325 media_type: Option<&str>,
19326 ) -> Result<(), TransportError> {
19327 crate::round::upload_one(transport, blob_id, bytes, media_type, self.now_ms)
19328 }
19329
19330 fn visible_blob_ids_sql(&self) -> String {
19331 let mut selects = Vec::new();
19332 for table in &self.schema.tables {
19333 for column in table
19334 .columns
19335 .iter()
19336 .filter(|column| column.ty == ColumnType::BlobRef)
19337 {
19338 let identifier = quote_ident(&column.name);
19339 selects.push(format!(
19340 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
19341 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
19342 AND json_type({identifier}, '$.blobId') = 'text'",
19343 quote_ident(&table.name),
19344 ));
19345 }
19346 }
19347 if selects.is_empty() {
19348 "SELECT NULL AS blob_id WHERE 0".to_owned()
19349 } else {
19350 selects.join(" UNION ")
19351 }
19352 }
19353
19354 fn enforce_blob_cache_cap(&self) {
19357 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
19358 return;
19359 };
19360 let mut total: i64 = self
19361 .conn
19362 .query_row(
19363 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
19364 [],
19365 |row| row.get(0),
19366 )
19367 .unwrap_or(0);
19368 if total <= max_bytes {
19369 return;
19370 }
19371 let query = format!(
19372 "SELECT blob_id, byte_length FROM _syncular_blobs
19373 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19374 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19375 AND blob_id NOT IN ({})
19376 ORDER BY created_at_ms ASC, blob_id ASC",
19377 self.visible_blob_ids_sql(),
19378 );
19379 let candidates: Vec<(String, i64)> = {
19380 let Ok(mut stmt) = self.conn.prepare(&query) else {
19381 return;
19382 };
19383 let Ok(rows) = stmt.query_map([], |row| {
19384 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
19385 }) else {
19386 return;
19387 };
19388 rows.filter_map(Result::ok).collect()
19389 };
19390 for (blob_id, byte_length) in candidates {
19391 if total <= max_bytes {
19392 break;
19393 }
19394 let _ = self.conn.execute(
19395 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
19396 rusqlite::params![blob_id],
19397 );
19398 total -= byte_length;
19399 }
19400 }
19401
19402 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
19403 #[cfg(feature = "bench-internals")]
19404 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
19405 if !self.schema_has_blobs() {
19406 return Ok(());
19407 }
19408 self.rebuild_overlay_if_dirty()?;
19409 self.conn
19410 .execute(
19411 &format!(
19412 "DELETE FROM _syncular_blobs
19413 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19414 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19415 AND blob_id NOT IN ({})",
19416 self.visible_blob_ids_sql(),
19417 ),
19418 [],
19419 )
19420 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19421 Ok(())
19422 }
19423
19424 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
19429 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
19430 return sql.clone();
19431 }
19432 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
19433 columns.push(quote_ident("_syncular_version"));
19434 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
19435 let primary_key = quote_ident(&table.primary_key);
19436 let updates = columns
19437 .iter()
19438 .filter(|column| **column != primary_key)
19439 .map(|column| format!("{column}=excluded.{column}"))
19440 .collect::<Vec<_>>()
19441 .join(", ");
19442 let sql = format!(
19443 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
19444 columns.join(", "),
19445 placeholders.join(", ")
19446 );
19447 self.insert_sql
19448 .borrow_mut()
19449 .insert(full_table.to_owned(), sql.clone());
19450 sql
19451 }
19452
19453 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
19454 self.overlay_dirty.table(table_name);
19455 self.write_row(&base_table(table_name), table_name, row, version)
19456 }
19457
19458 fn write_row(
19459 &self,
19460 full_table: &str,
19461 table_name: &str,
19462 row: &Row,
19463 version: i64,
19464 ) -> Result<(), String> {
19465 #[cfg(feature = "bench-internals")]
19466 let _phase = self.benchmark_phases.start(Phase::RowWrite);
19467 let table = self
19468 .schema
19469 .table(table_name)
19470 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19471 let sql = self.insert_row_sql(full_table, table);
19472 let mut stmt = self
19473 .conn
19474 .prepare_cached(&sql)
19475 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19476 let params = row
19477 .iter()
19478 .map(RowParam::Cell)
19479 .chain(std::iter::once(RowParam::Version(version)));
19480 stmt.execute(rusqlite::params_from_iter(params))
19481 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19482 Ok(())
19483 }
19484
19485 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
19486 let table = self
19487 .schema
19488 .table(table_name)
19489 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19490 self.overlay_dirty.table(table_name);
19491 let sql = format!(
19492 "DELETE FROM {} WHERE {}",
19493 base_table(table_name),
19494 row_id_predicate(table)
19495 );
19496 let mut stmt = self
19497 .conn
19498 .prepare_cached(&sql)
19499 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19500 stmt.execute(rusqlite::params![row_id])
19501 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19502 Ok(())
19503 }
19504
19505 fn has_acknowledged_rows(&self) -> bool {
19506 self.conn
19507 .query_row(
19508 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
19509 [],
19510 |row| row.get(0),
19511 )
19512 .expect("read acknowledged intent")
19513 }
19514
19515 fn acknowledged_operations(
19516 &self,
19517 rows: Option<&HashSet<(&str, &str)>>,
19518 tables: Option<&BTreeSet<&str>>,
19519 ) -> Result<Vec<OutboxOp>, String> {
19520 let keys = rows.map(|rows| {
19521 serde_json::to_string(
19522 &rows
19523 .iter()
19524 .map(|(table, id)| serde_json::to_string(&(table, id)).expect("row key"))
19525 .collect::<Vec<_>>(),
19526 )
19527 .expect("row keys")
19528 });
19529 let mut statement = self
19530 .conn
19531 .prepare("SELECT op_json,intent_json FROM _syncular_acked_rows WHERE (?1 IS NULL OR json_array(tbl,id) IN(SELECT value FROM json_each(?1))) AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx")
19532 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19533 let rows = statement
19534 .query_map(
19535 rusqlite::params![
19536 keys,
19537 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
19538 ],
19539 |row| {
19540 #[cfg(test)]
19541 self.acknowledged_replay_count
19542 .set(self.acknowledged_replay_count.get() + 1);
19543 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
19544 },
19545 )
19546 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19547 rows.map(|row| {
19548 let (op, intent) =
19549 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19550 let op: CommitOperation =
19551 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
19552 let table = self
19553 .schema
19554 .table(&op.table)
19555 .ok_or_else(|| "sync.unknown_table".to_owned())?;
19556 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
19557 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
19558 } else {
19559 op.values
19560 };
19561 Ok(OutboxOp {
19562 table: op.table,
19563 row_id: op.row_id,
19564 upsert: op.op == "upsert",
19565 base_version: op.base_version,
19566 values,
19567 })
19568 })
19569 .collect()
19570 }
19571
19572 fn drop_acknowledged_rows(
19573 &self,
19574 matches: impl Fn(&str, &Map<String, Value>) -> bool,
19575 ) -> Result<Vec<OutboxCommit>, String> {
19576 let mut statement = self
19577 .conn
19578 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
19579 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19580 let rows = statement
19581 .query_map([], |row| {
19582 Ok((
19583 row.get::<_, String>(0)?,
19584 row.get::<_, String>(1)?,
19585 row.get::<_, String>(2)?,
19586 row.get::<_, String>(3)?,
19587 ))
19588 })
19589 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19590 let mut doomed = HashSet::new();
19591 for row in rows {
19592 let (id, table, intent, row_id) =
19593 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19594 let values =
19595 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
19596 let base_matches = self
19597 .schema
19598 .table(&table)
19599 .map(|schema| self.stored_row(schema, &row_id, true))
19600 .transpose()?
19601 .flatten()
19602 .is_some_and(|(base, _)| {
19603 let schema = self.schema.table(&table).expect("acknowledged table");
19604 let values = schema
19605 .columns
19606 .iter()
19607 .zip(base.iter())
19608 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
19609 .collect();
19610 matches(&table, &values)
19611 });
19612 if matches(&table, &values) || base_matches {
19613 doomed.insert(id);
19614 }
19615 }
19616 let mut dropped = Vec::new();
19617 for id in doomed {
19618 let mut statement = self
19619 .conn
19620 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
19621 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19622 let operations = statement
19623 .query_map([&id], |row| row.get::<_, String>(0))
19624 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19625 let mut ops = Vec::new();
19626 for operation in operations {
19627 let json = operation
19628 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19629 let op: CommitOperation =
19630 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
19631 self.overlay_dirty.table(&op.table);
19632 ops.push(OutboxOp {
19633 table: op.table,
19634 row_id: op.row_id,
19635 upsert: op.op == "upsert",
19636 base_version: op.base_version,
19637 values: op.values,
19638 });
19639 }
19640 dropped.push(OutboxCommit {
19641 client_commit_id: id.clone(),
19642 ops,
19643 });
19644 self.conn
19645 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
19646 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19647 }
19648 Ok(dropped)
19649 }
19650
19651 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
19654 if self.overlay_dirty.get() {
19655 self.rebuild_overlay()?;
19656 }
19657 Ok(())
19658 }
19659
19660 fn rebuild_overlay(&mut self) -> Result<(), String> {
19664 #[cfg(feature = "bench-internals")]
19665 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
19666 #[cfg(test)]
19667 self.overlay_rebuild_count
19668 .set(self.overlay_rebuild_count.get() + 1);
19669 let dirty = self.overlay_dirty.snapshot();
19670 let tables: Vec<_> = self
19671 .schema
19672 .tables
19673 .iter()
19674 .filter(|table| {
19675 dirty
19676 .as_ref()
19677 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
19678 })
19679 .cloned()
19680 .collect();
19681 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
19682 self.begin_observation("syncular_overlay")?;
19683 let rebuilt = (|| {
19684 for table in &tables {
19685 for index in &table.fts_indexes {
19686 self.drop_fts_triggers(index)?;
19687 }
19688 let visible = visible_table(&table.name);
19689 let base = base_table(&table.name);
19690 self.conn
19691 .execute_batch(&format!(
19692 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
19693 ))
19694 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19695 }
19696 let acknowledged = self.acknowledged_operations(None, Some(&names))?;
19697 self.apply_outbox_ops(
19698 acknowledged
19699 .iter()
19700 .filter(|op| names.contains(op.table.as_str())),
19701 )?;
19702 for (commit, initial) in &self.failed_commits {
19703 for operation in &commit.ops {
19704 if !names.contains(operation.table.as_str()) || !operation.upsert {
19705 continue;
19706 }
19707 if let Some(table) = self.schema.table(&operation.table) {
19708 if self.visible_row(table, &operation.row_id)?.is_none()
19709 && operation.values.as_ref().is_some_and(|values| {
19710 table
19711 .columns
19712 .iter()
19713 .all(|column| values.contains_key(&column.name))
19714 })
19715 {
19716 let key = serde_json::to_string(&(
19717 operation.table.clone(),
19718 operation.row_id.clone(),
19719 ))
19720 .expect("row key serialization");
19721 if let Some(Value::Object(values)) = initial.get(&key) {
19722 self.apply_outbox_ops(&[OutboxOp {
19723 values: Some(values.clone()),
19724 ..operation.clone()
19725 }])?;
19726 }
19727 }
19728 }
19729 }
19730 self.apply_outbox_ops(
19731 commit
19732 .ops
19733 .iter()
19734 .filter(|op| names.contains(op.table.as_str())),
19735 )?;
19736 }
19737 for commit in &self.outbox {
19738 self.apply_outbox_ops(
19739 commit
19740 .ops
19741 .iter()
19742 .filter(|op| names.contains(op.table.as_str())),
19743 )?;
19744 }
19745 for table in &tables {
19746 for index in &table.fts_indexes {
19747 self.rebuild_fts_projection(table, index)?;
19748 self.create_fts_triggers(table, index)?;
19749 }
19750 }
19751 self.conn
19752 .execute_batch("RELEASE syncular_overlay")
19753 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19754 Ok(())
19755 })();
19756 if rebuilt.is_err() {
19757 self.conn
19758 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
19759 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19760 } else {
19761 self.overlay_dirty.set(false);
19762 }
19763 rebuilt
19764 }
19765
19766 fn apply_outbox_ops<'a>(
19771 &self,
19772 ops: impl IntoIterator<Item = &'a OutboxOp>,
19773 ) -> Result<(), String> {
19774 #[cfg(feature = "bench-internals")]
19775 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
19776 for op in ops {
19777 #[cfg(feature = "bench-internals")]
19778 if let Some(phase) = &mut phase {
19779 phase.unit();
19780 }
19781 if op.upsert {
19782 if let (Some(table), Some(values)) = (
19783 self.schema
19784 .table(&op.table)
19785 .filter(|table| table.indexes.iter().any(|index| index.unique)),
19786 op.values.as_ref(),
19787 ) {
19788 let mut intended: Map<String, Value> = self
19789 .visible_row(table, &op.row_id)?
19790 .map(|(row, _)| {
19791 table
19792 .columns
19793 .iter()
19794 .zip(&row)
19795 .map(|(column, value)| {
19796 (column.name.clone(), column_value_to_json(value))
19797 })
19798 .collect()
19799 })
19800 .unwrap_or_default();
19801 intended.extend(values.clone());
19802 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
19803 continue;
19804 }
19805 }
19806 }
19807 self.apply_outbox_op(op)?;
19808 }
19809 Ok(())
19810 }
19811
19812 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
19813 let Some(table) = self.schema.table(&op.table) else {
19814 return Ok(());
19815 };
19816 let visible = visible_table(&table.name);
19817 if !op.upsert {
19818 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
19819 self.conn
19820 .prepare_cached(&sql)
19821 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
19822 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19823 return Ok(());
19824 }
19825 let Some(values) = op.values.as_ref() else {
19826 return Ok(());
19827 };
19828 let full = table
19833 .columns
19834 .iter()
19835 .all(|column| values.contains_key(&column.name));
19836 let local = self.visible_row(table, &op.row_id)?;
19837 if local.is_none() && !full {
19838 return Ok(());
19839 }
19840 let mut row: Row = Vec::with_capacity(table.columns.len());
19841 for (index, column) in table.columns.iter().enumerate() {
19842 let incoming = match values.get(&column.name) {
19843 Some(raw) => json_to_column_value(
19844 if raw.get("$bytes").is_some()
19845 && table
19846 .encrypted_columns
19847 .iter()
19848 .any(|encrypted| encrypted.index == index)
19849 {
19850 &table.wire_columns[index]
19851 } else {
19852 column
19853 },
19854 Some(raw),
19855 ),
19856 None => Ok(local
19857 .as_ref()
19858 .and_then(|(values, _)| values.get(index).cloned())
19859 .flatten()),
19860 };
19861 row.push(incoming.map_err(|_| {
19862 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
19863 })?);
19864 }
19865 let version = local.as_ref().map_or(-1, |(_, version)| *version);
19866 self.write_row(&visible, &table.name, &row, version)
19867 .map_err(|error| {
19868 if error.starts_with("UNIQUE constraint failed:") {
19869 "sync.constraint_violation: local write violates a unique constraint".to_owned()
19870 } else {
19871 error
19872 }
19873 })
19874 }
19875
19876 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
19879 self.stored_row(table, row_id, false)
19880 }
19881
19882 fn stored_row(
19883 &self,
19884 table: &TableSchema,
19885 row_id: &str,
19886 base: bool,
19887 ) -> Result<Option<(Row, i64)>, String> {
19888 let sql = format!(
19889 "SELECT * FROM {} WHERE {}",
19890 if base {
19891 base_table(&table.name)
19892 } else {
19893 visible_table(&table.name)
19894 },
19895 row_id_predicate(table)
19896 );
19897 let mut stmt = self
19898 .conn
19899 .prepare_cached(&sql)
19900 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19901 let mut rows = stmt
19902 .query(rusqlite::params![row_id])
19903 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19904 let Some(row) = rows
19905 .next()
19906 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
19907 else {
19908 return Ok(None);
19909 };
19910 let mut values = Vec::with_capacity(table.columns.len());
19911 for (index, column) in table.columns.iter().enumerate() {
19912 let raw = row
19913 .get_ref(index)
19914 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19915 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
19916 if table
19917 .encrypted_columns
19918 .iter()
19919 .any(|encrypted| encrypted.index == index)
19920 {
19921 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
19922 continue;
19923 }
19924 }
19925 let json = sql_ref_to_json(column, raw);
19926 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
19927 "sync.local_corrupt: persisted local row value is invalid".to_owned()
19928 })?);
19929 }
19930 let version: i64 = row
19931 .get(table.columns.len())
19932 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19933 Ok(Some((values, version)))
19934 }
19935
19936 #[must_use]
19940 pub fn realtime_policy(&self) -> RealtimePolicy {
19941 self.realtime_policy
19942 }
19943
19944 #[must_use]
19946 pub fn realtime_state(&self) -> RealtimeState {
19947 self.realtime_state
19948 }
19949
19950 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
19953 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
19954 return Err(
19955 "sync.invalid_request: disconnect realtime before changing the realtime policy"
19956 .to_owned(),
19957 );
19958 }
19959 self.realtime_policy = policy;
19960 match policy {
19961 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
19962 RealtimePolicy::Required | RealtimePolicy::Optional
19963 if self.realtime_state == RealtimeState::Disabled =>
19964 {
19965 self.set_realtime_state(RealtimeState::Disconnected, None);
19966 }
19967 RealtimePolicy::Required | RealtimePolicy::Optional => {}
19968 }
19969 Ok(())
19970 }
19971
19972 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
19975 self.realtime_state = state;
19976 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
19977 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
19978 self.realtime_reason_code = None;
19979 self.realtime_retry_delay_ms = None;
19980 }
19981 }
19982
19983 #[must_use]
19984 fn realtime_connected(&self) -> bool {
19985 self.realtime_state == RealtimeState::Connected
19986 }
19987
19988 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
19989 if !self.transport_enabled {
19990 return Err("sync.offline: transport gate is closed".into());
19991 }
19992 if self.realtime_policy == RealtimePolicy::Off {
19993 return Err(
19994 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
19995 );
19996 }
19997 if self.realtime_connected() {
19998 return Ok(());
19999 }
20000 self.realtime_state = RealtimeState::Connecting;
20001 match transport.realtime_connect_for_client(&self.client_id) {
20002 Ok(()) => {
20003 self.set_realtime_state(RealtimeState::Connected, None);
20004 Ok(())
20005 }
20006 Err(error) => {
20007 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
20008 Err(format!("{}: {}", error.code, error.message))
20009 }
20010 }
20011 }
20012
20013 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
20014 self.cancel_sync_round();
20015 if self.realtime_state == RealtimeState::Disconnected
20018 || self.realtime_state == RealtimeState::Disabled
20019 {
20020 return;
20021 }
20022 let _ = transport.realtime_close();
20023 self.set_realtime_state(
20024 if self.realtime_policy == RealtimePolicy::Off {
20025 RealtimeState::Disabled
20026 } else {
20027 RealtimeState::Disconnected
20028 },
20029 None,
20030 );
20031 self.presence.clear(); }
20033
20034 pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
20040 if self.realtime_state != RealtimeState::Connected {
20041 return;
20042 }
20043 self.cancel_sync_round();
20044 let _ = transport.realtime_close();
20045 self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
20046 self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
20047 self.presence.clear();
20048 }
20049
20050 pub fn set_presence(
20056 &mut self,
20057 transport: &mut dyn Transport,
20058 scope_key: &str,
20059 doc: Option<&Value>,
20060 ) -> Result<(), String> {
20061 if !self.transport_enabled {
20062 return Err("sync.offline: transport gate is closed".into());
20063 }
20064 if !self.realtime_connected() {
20065 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
20066 }
20067 let text = encode_presence_publish(scope_key, doc);
20068 transport
20069 .realtime_send(&text)
20070 .map_err(|e| format!("{}: {}", e.code, e.message))
20071 }
20072
20073 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
20075 self.presence
20076 .get(scope_key)
20077 .map(|peers| peers.values().cloned().collect())
20078 .unwrap_or_default()
20079 }
20080
20081 fn apply_presence(
20083 &mut self,
20084 scope_key: String,
20085 kind: Option<PresenceKind>,
20086 actor_id: Option<String>,
20087 client_id: Option<String>,
20088 doc: Option<Value>,
20089 error: Option<String>,
20090 ) {
20091 if error.is_some() {
20094 return;
20095 }
20096 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
20097 return;
20098 };
20099 let peer_key = format!("{actor_id} {client_id}");
20100 match kind {
20101 PresenceKind::Leave => {
20102 if let Some(peers) = self.presence.get_mut(&scope_key) {
20103 peers.remove(&peer_key);
20104 if peers.is_empty() {
20105 self.presence.remove(&scope_key);
20106 }
20107 }
20108 }
20109 _ => {
20110 let doc = match doc {
20111 Some(Value::Object(_)) => doc.unwrap(),
20112 _ => return,
20113 };
20114 self.presence.entry(scope_key).or_default().insert(
20115 peer_key,
20116 PresencePeer {
20117 actor_id,
20118 client_id,
20119 doc,
20120 },
20121 );
20122 }
20123 }
20124 }
20125
20126 pub fn on_realtime_text(&mut self, text: &str) {
20128 if self.stopped || !self.transport_enabled || self.security_preflight {
20129 return;
20130 }
20131 match parse_control(text) {
20132 Ok(ControlMessage::Hello { requires_sync, .. }) => {
20133 if requires_sync {
20134 self.set_sync_needed(true, true);
20136 }
20137 }
20138 Ok(ControlMessage::Presence {
20139 scope_key,
20140 kind,
20141 actor_id,
20142 client_id,
20143 doc,
20144 error,
20145 ..
20146 }) => {
20147 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
20148 }
20149 Ok(ControlMessage::Wake { .. }) => {
20150 self.set_sync_needed(true, true);
20152 }
20153 _ => {}
20154 }
20155 }
20156
20157 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
20161 let mut transport = crate::round::DownloadResults::default();
20162 self.on_realtime_binary(&mut transport, bytes);
20163 transport.controls
20164 }
20165
20166 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
20167 if self.stopped || !self.transport_enabled || self.security_preflight {
20168 return;
20169 }
20170 #[cfg(feature = "bench-internals")]
20171 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
20172 let message = match decode_message(bytes) {
20173 Ok(m) if m.msg_kind == MsgKind::Response => m,
20174 _ => {
20175 self.set_sync_needed(true, true);
20176 return;
20177 }
20178 };
20179 #[cfg(feature = "bench-internals")]
20180 drop(decode_phase);
20181 let mut frames = message.frames.into_iter();
20182 let mut applied_cursor: Option<i64> = None;
20183 let mut any_covered = false;
20184 let mut dropped = self.begin_coalesced_apply().is_err();
20185 while let Some(frame) = frames.next() {
20186 let Frame::SubStart {
20187 id,
20188 status,
20189 effective_scopes,
20190 ..
20191 } = frame
20192 else {
20193 continue;
20194 };
20195 let mut body = Vec::new();
20196 let mut next_cursor: Option<i64> = None;
20197 for inner in frames.by_ref() {
20198 match inner {
20199 Frame::SubEnd {
20200 next_cursor: nc, ..
20201 } => {
20202 next_cursor = Some(nc);
20203 break;
20204 }
20205 Frame::Unknown { .. } => {}
20206 other => body.push(other),
20207 }
20208 }
20209 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
20210 dropped = true;
20211 continue;
20212 };
20213 let sub = &self.subs[sub_index];
20214 if status != SubStatus::Active
20217 || sub.state != SubState::Active
20218 || sub.bootstrap_state.is_some()
20219 || !sub.synced_once
20220 {
20221 dropped = true;
20222 continue;
20223 }
20224 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
20225 any_covered = true;
20227 continue;
20228 }
20229 let previous_effective = self.subs[sub_index].effective.clone();
20230 let previous_cursor = self.subs[sub_index].cursor;
20231 self.subs[sub_index].effective = Some(effective_scopes);
20232 let mut failed = false;
20233 for inner in body {
20234 match inner {
20235 Frame::Commit {
20236 tables,
20237 changes,
20238 commit_seq,
20239 ..
20240 } => {
20241 if self
20242 .apply_commit_frame(&tables, &changes, Some(commit_seq))
20243 .is_err()
20244 {
20245 failed = true;
20246 break;
20247 }
20248 }
20249 _ => {
20250 failed = true;
20251 break;
20252 }
20253 }
20254 }
20255 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
20256 self.apply_sub_end(sub_index, cursor, None)
20257 .ok()
20258 .map(|()| cursor)
20259 });
20260 let Some(next_cursor) = committed else {
20261 self.subs[sub_index].effective = previous_effective;
20262 self.subs[sub_index].cursor = previous_cursor;
20263 dropped = true;
20264 continue;
20265 };
20266 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
20267 }
20268 if self.finish_coalesced_apply().is_err() {
20271 applied_cursor = None;
20272 dropped = true;
20273 }
20274 if let Some(cursor) = applied_cursor {
20275 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
20277 let _ = transport.realtime_send(&ack);
20278 } else if !any_covered || dropped {
20279 self.set_sync_needed(true, true);
20281 }
20282 }
20283
20284 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
20288 if !self.realtime_connected() {
20289 return;
20290 }
20291 let floor = self
20292 .subs
20293 .iter()
20294 .filter(|s| {
20295 s.state == SubState::Active
20296 && s.bootstrap_state.is_none()
20297 && s.synced_once
20298 && s.cursor >= 0
20299 })
20300 .map(|s| s.cursor)
20301 .min();
20302 if let Some(cursor) = floor {
20303 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
20304 let _ = transport.realtime_send(&ack);
20305 }
20306 }
20307}
20308
20309#[cfg(test)]
20310mod previous_version_wiring_tests {
20311 use super::*;
20312 use serde_json::{json, Value};
20313
20314 fn previous_version_schema(version: i32) -> Value {
20315 json!({
20316 "version": version,
20317 "tables": [{
20318 "name": "tasks",
20319 "primaryKey": "id",
20320 "columns": [
20321 { "name": "id", "type": "string", "nullable": false },
20322 { "name": "note", "type": "string", "nullable": true }
20323 ],
20324 "scopes": []
20325 }]
20326 })
20327 }
20328
20329 fn previous_version_limits() -> crate::api::ClientLimits {
20330 crate::api::ClientLimits {
20331 previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
20332 enabled: true,
20333 ..Default::default()
20334 }),
20335 ..Default::default()
20336 }
20337 }
20338
20339 fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
20341 previous_version_limits()
20342 .previous_version_context
20343 .expect("configured")
20344 }
20345
20346 fn recapture_previous_version(client: &SyncClient) {
20349 let replica_path = client
20350 .previous_version_replica_path()
20351 .expect("file-backed replica");
20352 let outcome = crate::previous_version::capture_previous_version_from_replica(
20353 &client.conn,
20354 &replica_path,
20355 2,
20356 2,
20357 &previous_version_config(),
20358 1,
20359 )
20360 .expect("capture");
20361 assert!(matches!(
20362 outcome,
20363 crate::previous_version::CaptureOutcome::Captured(_)
20364 ));
20365 }
20366
20367 fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
20368 Subscription {
20369 id: "wiring-sub".to_owned(),
20370 table: "tasks".to_owned(),
20371 requested: Vec::new(),
20372 params: None,
20373 cursor,
20374 bootstrap_state: bootstrap_state.map(str::to_owned),
20375 state,
20376 reason_code: None,
20377 effective: None,
20378 synced_once: false,
20379 }
20380 }
20381
20382 #[test]
20383 fn schema_downgrade_refuses_all_opens_and_recreate_without_mutation() {
20384 let path =
20385 std::env::temp_dir().join(format!("syncular-downgrade-{}.db", uuid::Uuid::new_v4()));
20386 let path_str = path.to_str().expect("temp path");
20387 let schema = previous_version_schema(3);
20388 let mut older = previous_version_schema(2);
20389 older["tables"][0]["columns"] =
20390 json!([{ "name": "id", "type": "string", "nullable": false }]);
20391 let mut client = SyncClient::open_path(
20392 "downgrade".into(),
20393 &previous_version_schema(1),
20394 previous_version_limits(),
20395 path_str,
20396 )
20397 .expect("fresh");
20398 client
20399 .mutate(vec![Mutation::Upsert {
20400 table: "tasks".into(),
20401 values: Map::from_iter([
20402 ("id".into(), json!("accepted")),
20403 ("note".into(), json!("old")),
20404 ]),
20405 base_version: None,
20406 }])
20407 .expect("v1 write");
20408 client
20409 .recreate_with_schema(&schema)
20410 .expect("upgrade captures context");
20411 let commit = client
20412 .mutate(vec![Mutation::Upsert {
20413 table: "tasks".into(),
20414 values: Map::from_iter([
20415 ("id".into(), json!("queued")),
20416 ("note".into(), json!("v3-only")),
20417 ]),
20418 base_version: None,
20419 }])
20420 .expect("queued v3 write");
20421 let changes = client.conn.total_changes();
20422 let outbox = client.pending_commit_ids();
20423 assert!(outbox.contains(&commit));
20424 let descriptor = client.get_meta("localSchemaDescriptor");
20425 let container = format!(
20426 "{path_str}{}",
20427 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
20428 );
20429 let context = std::fs::read(&container).expect("captured context");
20430 assert!(client
20431 .recreate_with_schema(&older)
20432 .expect_err("refuse recreate")
20433 .starts_with("client.schema_downgrade:"));
20434 assert_eq!(client.schema.version, 3);
20435 assert_eq!(client.conn.total_changes(), changes);
20436 assert_eq!(client.pending_commit_ids(), outbox);
20437 assert_eq!(client.get_meta("localSchemaDescriptor"), descriptor);
20438 assert_eq!(
20439 client
20440 .conn
20441 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20442 row.get::<_, String>(0)
20443 })
20444 .expect("queued row"),
20445 "v3-only"
20446 );
20447 assert_eq!(std::fs::read(&container).expect("context remains"), context);
20448 drop(client);
20449 let before = std::fs::read(&path).expect("replica bytes");
20450 for entry in 0..3 {
20453 let error = match entry {
20454 0 => SyncClient::open_path(
20455 "downgrade".into(),
20456 &older,
20457 previous_version_limits(),
20458 path_str,
20459 ),
20460 1 => SyncClient::open_path_with_identity(
20461 None,
20462 &older,
20463 previous_version_limits(),
20464 path_str,
20465 ),
20466 _ => SyncClient::with_connection(
20467 "downgrade".into(),
20468 &older,
20469 previous_version_limits(),
20470 Connection::open(&path).expect("connection"),
20471 ),
20472 }
20473 .err()
20474 .expect("refuse open");
20475 assert!(error.starts_with("client.schema_downgrade:"));
20476 let conn = Connection::open(&path).expect("inspect");
20477 assert_eq!(
20478 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20479 .expect("journal mode"),
20480 "delete"
20481 );
20482 drop(conn);
20483 assert_eq!(std::fs::read(&path).expect("replica unchanged"), before);
20484 assert_eq!(
20485 std::fs::read(&container).expect("context unchanged"),
20486 context
20487 );
20488 }
20489 let reopened = SyncClient::open_path(
20490 "downgrade".into(),
20491 &schema,
20492 previous_version_limits(),
20493 path_str,
20494 )
20495 .expect("compatible reopen");
20496 assert_eq!(reopened.pending_commit_ids(), outbox);
20497 assert_eq!(
20498 reopened
20499 .conn
20500 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20501 row.get::<_, String>(0)
20502 })
20503 .expect("queued row remains"),
20504 "v3-only"
20505 );
20506 reopened.delete_meta(LOCAL_SCHEMA_VERSION_KEY);
20507 reopened.delete_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20508 drop(reopened);
20509 let legacy = SyncClient::open_path(
20510 "downgrade".into(),
20511 &schema,
20512 previous_version_limits(),
20513 path_str,
20514 )
20515 .expect("legacy no-marker reopen");
20516 assert_eq!(legacy.pending_commit_ids(), outbox);
20517 assert!(!legacy.upgrading());
20518 assert_eq!(
20519 legacy
20520 .conn
20521 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20522 row.get::<_, String>(0)
20523 })
20524 .expect("legacy row remains"),
20525 "v3-only"
20526 );
20527 drop(legacy);
20528 std::fs::remove_file(path).expect("remove replica");
20529 let _ = std::fs::remove_file(container);
20530 }
20531
20532 #[test]
20533 fn schema_marker_errors_refuse_open_and_recreate_without_mutation() {
20534 for marker in [
20535 "''",
20536 "'bad'",
20537 "'0'",
20538 "'-1'",
20539 "'2.5'",
20540 "'03'",
20541 "'3\n'",
20542 "' 3'",
20543 "'+3'",
20544 "'3e0'",
20545 "'2147483648'",
20546 "NULL",
20547 "X'33'",
20548 ] {
20549 let conn = Connection::open_in_memory().expect("connection");
20550 conn.execute_batch(&format!("CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value); INSERT INTO _syncular_meta VALUES ('localSchemaVersion', {marker});")).expect("invalid marker fixture");
20551 let error = SyncClient::with_connection(
20552 "marker".into(),
20553 &previous_version_schema(3),
20554 ClientLimits::default(),
20555 conn,
20556 )
20557 .err()
20558 .expect("refuse marker");
20559 assert!(
20560 error.starts_with("sync.local_corrupt:"),
20561 "{marker}: {error}"
20562 );
20563 }
20564 for fault in [
20565 "UPDATE _syncular_meta SET value = 'bad' WHERE key = 'localSchemaVersion'",
20566 "ALTER TABLE _syncular_meta RENAME COLUMN value TO unreadable_value",
20567 ] {
20568 let mut client = SyncClient::new(
20569 "marker".into(),
20570 &previous_version_schema(3),
20571 ClientLimits::default(),
20572 )
20573 .expect("fresh");
20574 let commit = client
20575 .mutate(vec![Mutation::Upsert {
20576 table: "tasks".into(),
20577 values: Map::from_iter([("id".into(), json!("queued"))]),
20578 base_version: None,
20579 }])
20580 .expect("queued write");
20581 client.conn.execute_batch(fault).expect("marker fault");
20582 let changes = client.conn.total_changes();
20583 let error = client
20584 .recreate_with_schema(&previous_version_schema(4))
20585 .expect_err("refuse corrupt marker");
20586 assert!(error.starts_with("sync.local_corrupt:"));
20587 assert_eq!(client.conn.total_changes(), changes);
20588 assert_eq!(client.schema.version, 3);
20589 assert_eq!(client.pending_commit_ids(), vec![commit]);
20590 }
20591 for setup in [
20592 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY)",
20593 "CREATE VIEW _syncular_meta AS SELECT 'localSchemaVersion' AS key, '3' AS value",
20594 ] {
20595 let path =
20596 std::env::temp_dir().join(format!("syncular-marker-{}.db", uuid::Uuid::new_v4()));
20597 let conn = Connection::open(&path).expect("connection");
20598 conn.execute_batch(setup)
20599 .expect("unreadable metadata fixture");
20600 drop(conn);
20601 let before = std::fs::read(&path).expect("before");
20602 let error = SyncClient::open_path_with_identity(
20603 None,
20604 &previous_version_schema(3),
20605 ClientLimits::default(),
20606 path.to_str().expect("path"),
20607 )
20608 .err()
20609 .expect("refuse unreadable marker");
20610 assert!(error.starts_with("sync.local_corrupt:"));
20611 assert_eq!(std::fs::read(&path).expect("after"), before);
20612 std::fs::remove_file(path).expect("remove fixture");
20613 }
20614 }
20615
20616 std::thread_local! {
20617 static CONCURRENT_UPGRADE: RefCell<Option<Connection>> =
20620 const { RefCell::new(None) };
20621 }
20622
20623 fn commit_concurrent_upgrade(attempts: i32) -> bool {
20624 CONCURRENT_UPGRADE.with(|slot| {
20625 if let Some(banner) = slot.borrow_mut().take() {
20626 banner
20627 .execute_batch(
20628 "UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion'; COMMIT",
20629 )
20630 .expect("commit the concurrent upgrade");
20631 }
20632 });
20633 attempts < 10
20634 }
20635
20636 fn journal_mode(conn: &Connection) -> String {
20637 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20638 .expect("journal mode")
20639 }
20640
20641 fn temp_replica(label: &str) -> std::path::PathBuf {
20642 std::env::temp_dir().join(format!("syncular-{label}-{}.db", uuid::Uuid::new_v4()))
20643 }
20644
20645 #[test]
20646 fn file_identity_constructor_requires_the_returned_wal_mode() {
20647 for path in [":memory:", ""] {
20648 let error = SyncClient::open_path_with_identity(
20649 None,
20650 &previous_version_schema(1),
20651 ClientLimits::default(),
20652 path,
20653 )
20654 .err()
20655 .expect("temporary SQLite databases cannot enter WAL mode");
20656 assert_eq!(
20657 error,
20658 "sync.invalid_request: file-backed client requires WAL journal mode"
20659 );
20660 }
20661 let path = temp_replica("confirmed-wal");
20662 let client = SyncClient::open_path_with_identity(
20663 None,
20664 &previous_version_schema(1),
20665 ClientLimits::default(),
20666 path.to_str().unwrap(),
20667 )
20668 .unwrap();
20669 assert_eq!(journal_mode(&client.conn), "wal");
20670 drop(client);
20671 std::fs::remove_file(path).unwrap();
20672 }
20673
20674 #[test]
20675 fn non_positive_generated_schema_versions_are_refused_before_storage() {
20676 for version in [json!(0), json!(-1), json!(i32::MIN), json!(2147483648i64)] {
20677 let mut schema = previous_version_schema(1);
20678 schema["version"] = version.clone();
20679 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20680 .err()
20681 .expect("refuse an out-of-range generated version");
20682 assert_eq!(
20683 error,
20684 "sync.invalid_request: generated schema version must be an integer in 1..=2147483647"
20685 );
20686 let path = temp_replica("bad-version");
20688 let path_str = path.to_str().expect("path");
20689 for identity in [false, true] {
20690 let error = if identity {
20691 SyncClient::open_path_with_identity(
20692 None,
20693 &schema,
20694 ClientLimits::default(),
20695 path_str,
20696 )
20697 } else {
20698 SyncClient::open_path(
20699 "bad-version".into(),
20700 &schema,
20701 ClientLimits::default(),
20702 path_str,
20703 )
20704 }
20705 .err()
20706 .expect("refuse before storage");
20707 assert!(
20708 error.starts_with("sync.invalid_request:"),
20709 "{version}: {error}"
20710 );
20711 assert!(
20712 !path.exists(),
20713 "{version}: a refused constructor creates no replica file"
20714 );
20715 }
20716 }
20717 let mut schema = previous_version_schema(1);
20719 schema
20720 .as_object_mut()
20721 .expect("schema object")
20722 .remove("version");
20723 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20724 .err()
20725 .expect("refuse a missing version");
20726 assert!(error.starts_with("sync.invalid_request:"), "{error}");
20727 }
20728
20729 #[test]
20730 fn a_duplicated_schema_marker_is_refused_without_a_reset() {
20731 let path = temp_replica("duplicate-marker");
20732 let path_str = path.to_str().expect("path");
20733 {
20734 let conn = Connection::open(path_str).expect("fixture");
20737 conn.execute_batch(
20738 "CREATE TABLE _syncular_meta(key TEXT, value);
20739 INSERT INTO _syncular_meta VALUES
20740 ('localSchemaVersion', '1'),
20741 ('localSchemaVersion', '2'),
20742 ('clientId', 'duplicate');",
20743 )
20744 .expect("duplicate marker fixture");
20745 }
20746 let error = SyncClient::open_path_with_identity(
20747 None,
20748 &previous_version_schema(3),
20749 ClientLimits::default(),
20750 path_str,
20751 )
20752 .err()
20753 .expect("refuse a duplicated marker");
20754 assert!(error.starts_with("sync.local_corrupt:"), "{error}");
20755 let probe = Connection::open(path_str).expect("probe");
20756 assert_eq!(
20757 probe
20758 .query_row(
20759 "SELECT COUNT(*) FROM _syncular_meta WHERE key = 'localSchemaVersion'",
20760 [],
20761 |row| row.get::<_, i64>(0),
20762 )
20763 .expect("marker rows"),
20764 2,
20765 "the duplicated rows are left for the operator, not collapsed"
20766 );
20767 assert_eq!(
20768 probe
20769 .query_row(
20770 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_syncular_%' AND name NOT LIKE 'sqlite_%'",
20771 [],
20772 |row| row.get::<_, i64>(0),
20773 )
20774 .expect("synced tables"),
20775 0,
20776 "a duplicated marker never drives the reset that recreates v3 tables"
20777 );
20778 assert_eq!(
20779 journal_mode(&probe),
20780 "delete",
20781 "the refusal leaves the journal mode alone"
20782 );
20783 drop(probe);
20784 std::fs::remove_file(path).expect("remove replica");
20785 }
20786
20787 #[test]
20788 fn a_refused_open_does_not_persist_wal_or_mutate_the_replica() {
20789 let path = temp_replica("refused-wal");
20790 let path_str = path.to_str().expect("path");
20791 drop(
20794 SyncClient::open_path(
20795 "owner".into(),
20796 &previous_version_schema(2),
20797 ClientLimits::default(),
20798 path_str,
20799 )
20800 .expect("v2 replica"),
20801 );
20802 let before = std::fs::read(&path).expect("before");
20803 let error = SyncClient::open_path_with_identity(
20804 Some("intruder".into()),
20805 &previous_version_schema(2),
20806 ClientLimits::default(),
20807 path_str,
20808 )
20809 .err()
20810 .expect("refuse the identity mismatch");
20811 assert!(error.starts_with("client.identity_mismatch:"), "{error}");
20812 let probe = Connection::open(path_str).expect("probe");
20813 assert_eq!(
20814 journal_mode(&probe),
20815 "delete",
20816 "a refused open must not persist WAL"
20817 );
20818 assert_eq!(std::fs::read(&path).expect("after"), before);
20819 assert_eq!(
20820 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20821 Some("2")
20822 );
20823 drop(probe);
20824 std::fs::remove_file(path).expect("remove replica");
20825 }
20826
20827 #[test]
20828 fn a_concurrent_marker_upgrade_refuses_a_stale_open() {
20829 let path = temp_replica("marker-race");
20830 let path_str = path.to_str().expect("path");
20831 drop(
20832 SyncClient::open_path_with_identity(
20833 Some("stale".into()),
20834 &previous_version_schema(1),
20835 ClientLimits::default(),
20836 path_str,
20837 )
20838 .expect("v1 replica"),
20839 );
20840 let banner = Connection::open(path_str).expect("banner");
20844 banner
20845 .execute_batch(
20846 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
20847 )
20848 .expect("pending upgrade");
20849 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
20850 let conn = Connection::open(path_str).expect("stale connection");
20851 conn.busy_handler(Some(commit_concurrent_upgrade))
20852 .expect("busy handler");
20853 let error = SyncClient::with_connection(
20854 "stale".into(),
20855 &previous_version_schema(1),
20856 ClientLimits::default(),
20857 conn,
20858 )
20859 .err()
20860 .expect("a concurrent upgrade must refuse the stale open");
20861 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
20862 let probe = Connection::open(path_str).expect("probe");
20863 assert_eq!(
20864 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20865 Some("2"),
20866 "the concurrent upgrade survives"
20867 );
20868 drop(probe);
20869 std::fs::remove_file(path).expect("remove replica");
20870 }
20871
20872 #[test]
20873 fn a_concurrent_marker_upgrade_refuses_a_stale_recreate() {
20874 let path = temp_replica("recreate-race");
20875 let path_str = path.to_str().expect("path");
20876 let mut client = SyncClient::open_path_with_identity(
20877 Some("stale".into()),
20878 &previous_version_schema(1),
20879 ClientLimits::default(),
20880 path_str,
20881 )
20882 .expect("v1 replica");
20883 client
20884 .mutate(vec![Mutation::Upsert {
20885 table: "tasks".into(),
20886 values: Map::from_iter([("id".into(), json!("queued"))]),
20887 base_version: None,
20888 }])
20889 .expect("queued write");
20890 let commit = client.pending_commit_ids();
20891 let banner = Connection::open(path_str).expect("banner");
20892 banner
20893 .execute_batch(
20894 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
20895 )
20896 .expect("pending upgrade");
20897 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
20898 client
20899 .conn
20900 .busy_handler(Some(commit_concurrent_upgrade))
20901 .expect("busy handler");
20902 let error = client
20903 .recreate_with_schema(&previous_version_schema(1))
20904 .expect_err("a concurrent upgrade must refuse the stale recreate");
20905 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
20906 assert_eq!(client.schema.version, 1, "the compiled schema is restored");
20907 assert_eq!(
20908 client.pending_commit_ids(),
20909 commit,
20910 "the outbox is preserved"
20911 );
20912 let probe = Connection::open(path_str).expect("probe");
20913 assert_eq!(
20914 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20915 Some("2")
20916 );
20917 drop(probe);
20918 drop(client);
20919 std::fs::remove_file(path).expect("remove replica");
20920 }
20921
20922 #[test]
20923 fn failed_schema_record_writes_roll_back_recreation() {
20924 for key in [
20925 LOCAL_SCHEMA_VERSION_KEY,
20926 crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY,
20927 ] {
20928 let mut client = SyncClient::new(
20929 "schema-record".into(),
20930 &previous_version_schema(1),
20931 ClientLimits::default(),
20932 )
20933 .expect("v1");
20934 let marker = client.get_meta(LOCAL_SCHEMA_VERSION_KEY);
20935 let descriptor = client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20936 client.conn.execute_batch(&format!("CREATE TRIGGER refuse_schema_record BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{key}' BEGIN SELECT RAISE(ABORT, 'refuse schema record'); END")).expect("fault");
20937 client
20938 .recreate_with_schema(&previous_version_schema(2))
20939 .expect_err("both record writes must propagate failure");
20940 assert_eq!(client.schema.version, 1);
20941 assert_eq!(client.get_meta(LOCAL_SCHEMA_VERSION_KEY), marker);
20942 assert_eq!(
20943 client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY),
20944 descriptor
20945 );
20946 client
20947 .conn
20948 .execute_batch("DROP TRIGGER refuse_schema_record")
20949 .expect("remove fault");
20950 client
20951 .recreate_with_schema(&previous_version_schema(2))
20952 .expect("retry");
20953 assert_eq!(client.schema.version, 2);
20954 }
20955 }
20956
20957 #[test]
20958 fn a_failed_outer_commit_restores_every_reset_side_effect() {
20959 let path = temp_replica("outer-commit");
20960 let path_str = path.to_str().expect("path");
20961 let conn = Connection::open(path_str).expect("replica connection");
20964 let mut client = SyncClient::with_connection(
20965 "outer-commit".into(),
20966 &previous_version_schema(1),
20967 previous_version_limits(),
20968 conn,
20969 )
20970 .expect("v1 replica");
20971 assert_eq!(journal_mode(&client.conn), "delete");
20972 let commit = client
20973 .mutate(vec![Mutation::Upsert {
20974 table: "tasks".into(),
20975 values: Map::from_iter([
20976 ("id".into(), json!("queued")),
20977 ("note".into(), json!("dropped-by-v2")),
20978 ]),
20979 base_version: None,
20980 }])
20981 .expect("queued write");
20982 client.drain_change_batches();
20983 let mut dropped = previous_version_schema(2);
20985 dropped["tables"][0]["columns"] =
20986 json!([{ "name": "id", "type": "string", "nullable": false }]);
20987 let marker = meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY);
20990 let revision = client.local_revision();
20991 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
20992 let columns: i64 = client
20993 .conn
20994 .query_row(
20995 "SELECT count(*) FROM pragma_table_info('tasks')",
20996 [],
20997 |row| row.get(0),
20998 )
20999 .expect("column count");
21000 let rebuilds = client.overlay_rebuild_count.get();
21001 let prior_subs = client.subs.len();
21002 let prior_changes = client.change_queue.len();
21003 let prior_intents = client.sync_intent_queue.len();
21004 let prior_last_change = client.last_change.clone();
21005 let prior_overlay_dirty = client.overlay_dirty.get();
21006 let prior_sync_needed = client.sync_needed;
21007 let prior_upgrading = client.upgrading;
21008 let prior_stopped = client.stopped;
21009 let prior_schema_floor = client.schema_floor.clone();
21010 let prior_active_round = client.active_round;
21011 let reader = Connection::open(path_str).expect("reader");
21014 reader
21015 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
21016 .expect("reader snapshot");
21017 client
21018 .conn
21019 .busy_timeout(std::time::Duration::ZERO)
21020 .expect("busy timeout");
21021 let error = client
21022 .recreate_with_schema(&dropped)
21023 .expect_err("a blocked COMMIT must refuse the recreation");
21024 assert!(error.starts_with("client.storage_busy:"), "{error}");
21025 assert!(
21026 client.overlay_rebuild_count.get() > rebuilds,
21027 "the reset ran to its overlay rebuild inside the transaction"
21028 );
21029 assert!(client.conn.is_autocommit(), "the transaction is released");
21030 assert_eq!(meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY), marker);
21032 assert_eq!(client.local_revision(), revision);
21033 assert_eq!(
21034 client.query("SELECT * FROM tasks", &[]).expect("rows"),
21035 rows
21036 );
21037 assert_eq!(
21038 client
21039 .conn
21040 .query_row(
21041 "SELECT count(*) FROM pragma_table_info('tasks')",
21042 [],
21043 |row| { row.get::<_, i64>(0) }
21044 )
21045 .expect("column count"),
21046 columns,
21047 "the previous table layout survives the rollback"
21048 );
21049 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
21050 assert_eq!(client.schema.version, 1, "compiled schema");
21053 assert_eq!(client.subs.len(), prior_subs);
21054 assert_eq!(client.change_queue.len(), prior_changes);
21055 assert_eq!(client.sync_intent_queue.len(), prior_intents);
21056 assert_eq!(client.last_change, prior_last_change);
21057 assert_eq!(client.overlay_dirty.get(), prior_overlay_dirty);
21058 assert_eq!(client.sync_needed, prior_sync_needed);
21059 assert_eq!(client.upgrading, prior_upgrading);
21060 assert_eq!(client.stopped, prior_stopped);
21061 assert_eq!(client.schema_floor, prior_schema_floor);
21062 assert_eq!(client.active_round, prior_active_round);
21063 assert!(
21064 client.rejections.is_empty(),
21065 "the dropped-column rejection is rolled back too"
21066 );
21067 let container = crate::previous_version::previous_version_container_path(path_str);
21070 assert!(std::path::Path::new(&container).exists());
21071 drop(reader);
21074 let snapshot = client
21075 .previous_version_snapshot(&PreviousVersionReadSpec {
21076 table: "tasks".into(),
21077 ..Default::default()
21078 })
21079 .expect("refuse stale capture");
21080 assert!(!snapshot.available);
21081 assert_eq!(snapshot.current_version, 1);
21082 assert!(!std::path::Path::new(&container).exists());
21083 client
21088 .conn
21089 .busy_timeout(std::time::Duration::from_secs(1))
21090 .expect("busy timeout");
21091 client
21092 .recreate_with_schema(&dropped)
21093 .expect("the recreation succeeds once the reader releases");
21094 assert_eq!(client.schema.version, 2);
21095 assert_eq!(
21096 client.pending_commit_ids(),
21097 vec![commit.clone()],
21098 "the reset kept the incompatible commit pending"
21099 );
21100 assert!(
21101 client.rejections.is_empty(),
21102 "no rejection exists before the send-time prepass"
21103 );
21104 client
21106 .prepare_sync_round(false)
21107 .expect("the send prepass runs during round preparation");
21108 assert!(client.pending_commit_ids().is_empty());
21109 assert!(
21110 client
21111 .rejections
21112 .iter()
21113 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE),
21114 "the dropped-column commit surfaced as a rejection at send time"
21115 );
21116 client.previous_version_discard();
21117 drop(client);
21118 std::fs::remove_file(path).expect("remove replica");
21119 }
21120
21121 #[test]
21122 fn the_send_time_incompatible_drop_is_atomic_and_publishes_once() {
21123 let v1 = previous_version_schema(1);
21124 let mut v2 = previous_version_schema(2);
21125 v2["tables"][0]["columns"] = json!([{ "name": "id", "type": "string", "nullable": false }]);
21127 let mut client = SyncClient::new("drop".to_string(), &v1, ClientLimits::default()).unwrap();
21128 let dropped = client
21129 .mutate(vec![Mutation::Upsert {
21130 table: "tasks".into(),
21131 values: Map::from_iter([
21132 ("id".into(), json!("dropped")),
21133 ("note".into(), json!("gone")),
21134 ]),
21135 base_version: None,
21136 }])
21137 .unwrap();
21138 let survivor = client
21142 .mutate(vec![Mutation::Delete {
21143 table: "tasks".into(),
21144 row_id: "gone".into(),
21145 base_version: None,
21146 }])
21147 .unwrap();
21148 client
21149 .recreate_with_schema(&v2)
21150 .expect("the reset preserves the outbox");
21151 assert_eq!(
21152 client.pending_commit_ids(),
21153 vec![dropped.clone(), survivor.clone()],
21154 "the reset preserves both commits"
21155 );
21156 assert!(client.rejections.is_empty(), "the reset classifies nothing");
21157 client.drain_change_batches();
21158 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21159 assert_eq!(rows.len(), 1, "only the dropped upsert has a visible row");
21160
21161 for fault in [
21167 "outcome-insert",
21168 "outbox-delete",
21169 "overlay-rebuild",
21170 "publish-revision",
21171 ] {
21172 client.set_sync_needed(false, false);
21173 client.drain_change_batches();
21174 let revision = client.local_revision();
21175 let trigger = match fault {
21176 "outcome-insert" => {
21177 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21178 }
21179 "outbox-delete" => {
21180 "CREATE TRIGGER fail_drop BEFORE DELETE ON _syncular_outbox BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21181 }
21182 "publish-revision" => {
21187 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21188 }
21189 _ => {
21190 "CREATE TRIGGER fail_drop BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21191 }
21192 };
21193 client.conn.execute_batch(trigger).unwrap();
21194 let outcome = client
21195 .prepare_sync_round(false)
21196 .err()
21197 .unwrap_or_else(|| panic!("{fault}: a fault in the drop must fail the round"));
21198 assert!(
21199 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "storage.failed"),
21200 "{fault}: {outcome:?}"
21201 );
21202 client.conn.execute_batch("DROP TRIGGER fail_drop").unwrap();
21203 assert!(client.conn.is_autocommit(), "{fault}: no leaked savepoint");
21204 assert_eq!(
21205 client.pending_commit_ids(),
21206 vec![dropped.clone(), survivor.clone()],
21207 "{fault}: the outbox is restored"
21208 );
21209 assert!(
21210 client.rejections.is_empty(),
21211 "{fault}: no in-memory rejection survives"
21212 );
21213 assert!(
21214 client.commit_outcome(&dropped).unwrap().is_none(),
21215 "{fault}: the durable outcome rolled back"
21216 );
21217 assert_eq!(client.local_revision(), revision, "{fault}: revision");
21218 assert_eq!(
21219 client.query("SELECT * FROM tasks", &[]).unwrap(),
21220 rows,
21221 "{fault}: the visible projection is unchanged"
21222 );
21223 assert!(
21224 client.drain_change_batches().is_empty(),
21225 "{fault}: nothing is published"
21226 );
21227 }
21228
21229 client
21232 .conn
21233 .execute_batch(
21234 "BEGIN; INSERT INTO _syncular_meta(key,value) VALUES ('outerProbe','kept');
21235 CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes
21236 BEGIN SELECT RAISE(ABORT,'nested drop fault'); END;",
21237 )
21238 .unwrap();
21239 assert!(client.drop_incompatible_outbox().is_err());
21240 assert!(!client.conn.is_autocommit());
21241 assert_eq!(client.get_meta("outerProbe").as_deref(), Some("kept"));
21242 assert_eq!(
21243 client.pending_commit_ids(),
21244 vec![dropped.clone(), survivor.clone()]
21245 );
21246 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21247 client
21248 .conn
21249 .execute_batch("DROP TRIGGER fail_drop; COMMIT")
21250 .unwrap();
21251
21252 client.set_meta(LOG_EPOCH_KEY, "epoch");
21257 client.set_sync_needed(false, false);
21258 client.drain_change_batches();
21259 let revision = client.local_revision();
21260 let prepared = client.prepare_sync_round(false).expect("the drop succeeds");
21261 assert_eq!(client.pending_commit_ids(), vec![survivor.clone()]);
21262 assert_eq!(
21263 prepared.meta.pushed_ids,
21264 vec![survivor.clone()],
21265 "the incompatible commit is classified before any operation is sent"
21266 );
21267 let rejection = client
21268 .rejections
21269 .iter()
21270 .find(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE)
21271 .expect("a sync.outbox_incompatible rejection");
21272 assert_eq!(rejection.client_commit_id, dropped);
21273 assert!(
21274 !rejection.retryable,
21275 "a schema-incompatible commit is final"
21276 );
21277 assert_eq!(
21278 client
21279 .commit_outcome(&dropped)
21280 .unwrap()
21281 .expect("a durable rejected outcome")
21282 .status,
21283 CommitOutcomeStatus::Rejected
21284 );
21285 assert_eq!(client.local_revision(), revision + 1, "one revision");
21286 let batches = client.drain_change_batches();
21287 assert_eq!(batches.len(), 1, "one change batch");
21288 assert!(
21289 batches[0].rejections_changed
21290 && batches[0].outcomes_changed
21291 && batches[0].status.is_some(),
21292 "{:?}",
21293 batches[0]
21294 );
21295 assert!(
21296 batches[0].tables.iter().any(|entry| entry.table == "tasks"),
21297 "{:?}",
21298 batches[0]
21299 );
21300 let visible = client.query("SELECT * FROM tasks", &[]).unwrap();
21301 assert!(
21302 visible.is_empty(),
21303 "the dropped projection is undone: {visible:?}"
21304 );
21305
21306 let path = temp_replica("outbox-drop-commit");
21310 let path_str = path.to_str().expect("path");
21311 let conn = Connection::open(path_str).expect("replica connection");
21312 let mut client =
21313 SyncClient::with_connection("drop-commit".into(), &v1, ClientLimits::default(), conn)
21314 .expect("v1 replica");
21315 let dropped = client
21316 .mutate(vec![Mutation::Upsert {
21317 table: "tasks".into(),
21318 values: Map::from_iter([
21319 ("id".into(), json!("dropped")),
21320 ("note".into(), json!("gone")),
21321 ]),
21322 base_version: None,
21323 }])
21324 .unwrap();
21325 client
21326 .recreate_with_schema(&v2)
21327 .expect("the reset preserves the outbox");
21328 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21329 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21330 assert_eq!(rows.len(), 1, "the optimistic row is visible");
21331 client.set_sync_needed(false, false);
21332 client.drain_change_batches();
21333 let revision = client.local_revision();
21334 let reader = Connection::open(path_str).expect("reader");
21335 reader
21336 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
21337 .expect("reader snapshot");
21338 client
21339 .conn
21340 .busy_timeout(std::time::Duration::ZERO)
21341 .expect("busy timeout");
21342 let outcome = client
21343 .prepare_sync_round(false)
21344 .err()
21345 .expect("a blocked COMMIT must fail the round");
21346 assert!(
21347 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "client.storage_busy"),
21348 "{outcome:?}"
21349 );
21350 assert!(client.conn.is_autocommit(), "the savepoint is released");
21351 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21352 assert!(client.rejections.is_empty());
21353 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21354 assert_eq!(client.local_revision(), revision);
21355 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
21356 assert!(client.drain_change_batches().is_empty());
21357 drop(reader);
21359 client
21360 .conn
21361 .busy_timeout(std::time::Duration::from_secs(1))
21362 .expect("busy timeout");
21363 client
21364 .prepare_sync_round(false)
21365 .expect("the drop succeeds after the reader releases");
21366 assert!(client.pending_commit_ids().is_empty());
21367 assert!(client
21368 .rejections
21369 .iter()
21370 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE));
21371 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
21372 drop(client);
21373 std::fs::remove_file(path).expect("remove replica");
21374 }
21375
21376 #[test]
21377 fn previous_version_read_reasons_come_from_live_client_state() {
21378 use crate::previous_version::{
21379 PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
21380 };
21381 let path = std::env::temp_dir()
21382 .join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
21383 .to_string_lossy()
21384 .into_owned();
21385 let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
21386 {
21387 let mut setup = SyncClient::open_path(
21388 "wiring".to_owned(),
21389 &previous_version_schema(1),
21390 previous_version_limits(),
21391 &path,
21392 )
21393 .expect("v1 install");
21394 setup
21395 .mutate(vec![Mutation::Upsert {
21396 table: "tasks".to_owned(),
21397 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21398 base_version: None,
21399 }])
21400 .expect("seed a row");
21401 }
21402 let mut client = SyncClient::open_path(
21403 "wiring".to_owned(),
21404 &previous_version_schema(2),
21405 previous_version_limits(),
21406 &path,
21407 )
21408 .expect("v2 bump captures");
21409 assert!(std::path::Path::new(&container).exists());
21410 let spec = PreviousVersionReadSpec {
21411 table: "tasks".to_owned(),
21412 ..Default::default()
21413 };
21414
21415 recapture_previous_version(&client);
21417 client.lease_state = Some(LeaseState {
21418 lease_id: None,
21419 expires_at_ms: None,
21420 error_code: Some("lease.expired".to_owned()),
21421 });
21422 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21423 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21424 assert!(!snapshot.available);
21425 assert!(!std::path::Path::new(&container).exists());
21426
21427 recapture_previous_version(&client);
21429 client.set_now_ms(1_000);
21430 client.lease_state = Some(LeaseState {
21431 lease_id: Some("lease".to_owned()),
21432 expires_at_ms: Some(1_000),
21433 error_code: None,
21434 });
21435 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21436 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21437 assert!(!std::path::Path::new(&container).exists());
21438 client.lease_state = None;
21439
21440 recapture_previous_version(&client);
21442 client.subs.push(subscription(SubState::Revoked, 0, None));
21443 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21444 assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
21445 assert!(!std::path::Path::new(&container).exists());
21446 client.subs.clear();
21447
21448 recapture_previous_version(&client);
21451 client
21452 .subs
21453 .push(subscription(SubState::Active, 0, Some("resume")));
21454 assert!(!client.previous_version_coverage_complete());
21455 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21456 assert!(snapshot.available);
21457 assert_eq!(snapshot.previous_version, Some(2));
21458 assert_eq!(snapshot.rows.len(), 1);
21459
21460 recapture_previous_version(&client);
21461 client.subs[0].bootstrap_state = None;
21462 assert!(client.previous_version_coverage_complete());
21463 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21464 assert_eq!(
21465 snapshot.reason,
21466 Some(PreviousVersionReason::CoverageComplete)
21467 );
21468 assert!(!std::path::Path::new(&container).exists());
21469 client.subs.clear();
21470
21471 assert!(!client.previous_version_coverage_complete());
21473
21474 drop(client);
21475 let _ = std::fs::remove_file(&path);
21476 }
21477
21478 #[test]
21485 fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
21486 let path = std::env::temp_dir()
21487 .join(format!(
21488 "syncular-prev-lifetime-{}.db",
21489 uuid::Uuid::new_v4()
21490 ))
21491 .to_string_lossy()
21492 .into_owned();
21493 let container = format!(
21494 "{path}{}",
21495 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
21496 );
21497 let exists = || std::path::Path::new(&container).exists();
21498 {
21499 let mut setup = SyncClient::open_path(
21500 "lifetime".to_owned(),
21501 &previous_version_schema(1),
21502 previous_version_limits(),
21503 &path,
21504 )
21505 .expect("v1 install");
21506 setup
21507 .mutate(vec![Mutation::Upsert {
21508 table: "tasks".to_owned(),
21509 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21510 base_version: None,
21511 }])
21512 .expect("seed a row");
21513 }
21514 let mut client = SyncClient::open_path(
21515 "lifetime".to_owned(),
21516 &previous_version_schema(2),
21517 previous_version_limits(),
21518 &path,
21519 )
21520 .expect("v2 bump captures");
21521 client.set_now_ms(1_000);
21524 assert!(exists());
21525
21526 client.lease_state = Some(LeaseState {
21528 lease_id: None,
21529 expires_at_ms: None,
21530 error_code: Some("lease.expired".to_owned()),
21531 });
21532 client.previous_version_lifetime_check();
21533 assert!(!exists());
21534 client.lease_state = None;
21535
21536 recapture_previous_version(&client);
21539 client.subs.push(subscription(SubState::Active, 0, None));
21540 client.previous_version_lifetime_check();
21541 assert!(!exists());
21542 client.subs.clear();
21543
21544 recapture_previous_version(&client);
21546 client.subs.push(subscription(SubState::Revoked, 0, None));
21547 client.previous_version_lifetime_check();
21548 assert!(!exists());
21549 client.subs.clear();
21550
21551 recapture_previous_version(&client);
21554 client
21555 .subs
21556 .push(subscription(SubState::Active, 0, Some("resume")));
21557 client.previous_version_lifetime_check();
21558 assert!(exists());
21559 client.subs.clear();
21560
21561 drop(client);
21562 let _ = std::fs::remove_file(&path);
21563 let _ = std::fs::remove_file(&container);
21564 let _ = std::fs::remove_file(format!("{container}-wal"));
21565 let _ = std::fs::remove_file(format!("{container}-shm"));
21566 }
21567}