1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12#[cfg(test)]
13use std::cell::Cell;
14use std::cell::RefCell;
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
16
17use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
18use rusqlite::{Connection, OpenFlags, OptionalExtension};
19use serde::{Deserialize, Serialize};
20use serde_json::{json, Map, Value};
21use sha2::{Digest, Sha256};
22use ssp2::decode::WIRE_VERSION;
23use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
24use ssp2::primitives::RawJson;
25use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
26use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
27
28use crate::api::{
29 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
30 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
31 ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects, CommitDelivery,
32 CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
33 CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
34 DiagnosticLastChange, DiagnosticLastRound, DiagnosticQueryFailure, DiagnosticRoundCounters,
35 DiagnosticSubscription, FetchedBlob, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
36 LocalDataPurgeTarget, LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation,
37 PresencePeer, PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot,
38 QueryValue, RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
39 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SubscriptionCatchup,
40 SubscriptionRequest, SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport,
41 SyncStatusSnapshot, TableChange, WindowBase, WindowChange, WindowCoverage, WindowState,
42 WindowUnitRef, CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
43 MAX_DIAGNOSTIC_QUERY_FAILURES,
44};
45use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
46#[cfg(feature = "bench-internals")]
47use crate::bench::{Phase, Recorder};
48use crate::previous_version::{
49 capture_previous_version_from_replica, first_incompatibility,
50 reconcile_previous_version_at_boot, set_local_schema_descriptor,
51 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
52 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
53};
54use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
55use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
56use crate::values::{
57 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
58 encode_sparse_row_json, full_row_values, is_canonical_bytes_envelope, json_to_column_value,
59 json_to_scope_map, normalize_values_casing, render_row_id_json, scope_map_to_json,
60 sort_scope_map,
61};
62
63const DEFAULT_ACCEPT: u8 = 0b0111;
67const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
68const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
69const ACCEPT_SQLITE: u8 = 1 << 2;
70const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
71const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
72
73pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
75const LOCAL_REVISION_KEY: &str = "localRevision";
76const CLIENT_ID_KEY: &str = "clientId";
77const LEASE_STATE_KEY: &str = "leaseState";
78const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
79const LOG_EPOCH_KEY: &str = "logEpoch";
80const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
85const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
86const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
87const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
90const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
93const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
94const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
96 "the persisted commit carries values the current codec refuses";
97const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
99
100#[derive(Debug, Serialize, Deserialize)]
101#[serde(rename_all = "camelCase", deny_unknown_fields)]
102struct PersistedLocalDataRebootstrapReceipt {
103 version: u8,
104 retained_commits: u64,
105 reset_subscriptions: u64,
106}
107
108fn invalid_local_rebootstrap_receipt() -> String {
109 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
110}
111
112fn encode_local_rebootstrap_receipt(
113 retained_commits: usize,
114 reset_subscriptions: usize,
115) -> Result<String, String> {
116 let retained_commits =
117 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
118 let reset_subscriptions =
119 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
120 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
121 return Err(invalid_local_rebootstrap_receipt());
122 }
123 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
124 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
125 retained_commits,
126 reset_subscriptions,
127 })
128 .map_err(|_| invalid_local_rebootstrap_receipt())
129}
130
131fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
132 if value == "v1" {
134 return Ok((0, 0));
135 }
136 let receipt: PersistedLocalDataRebootstrapReceipt =
137 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
138 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
139 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
140 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
141 {
142 return Err(invalid_local_rebootstrap_receipt());
143 }
144 Ok((
145 usize::try_from(receipt.retained_commits)
146 .map_err(|_| invalid_local_rebootstrap_receipt())?,
147 usize::try_from(receipt.reset_subscriptions)
148 .map_err(|_| invalid_local_rebootstrap_receipt())?,
149 ))
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153enum SubState {
154 Active,
155 Revoked,
156 Failed,
157}
158
159#[cfg(test)]
160mod observation_tests {
161 use super::*;
162 use crate::native_transport::HostTransport;
163 use crate::values::encode_row_json;
164 use serde_json::json;
165
166 #[test]
167 fn authority_policy_refuses_encrypted_and_undeclared_columns_before_open() {
168 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
169 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "secret", "type": "bytes", "encrypted": true, "declaredType": "string", "nullable": true}
170 ]}]});
171 for name in [
172 "secret",
173 "missing",
174 "*",
175 "id FROM authority; DELETE FROM authority",
176 "_syncular_version",
177 ] {
178 let limits = ClientLimits {
179 authority_reads: vec![crate::api::AuthorityReadDeclaration {
180 table: "authority".into(),
181 columns: vec!["id".into(), "actor_id".into(), name.into()],
182 scopes: [("actor_id".into(), vec!["a".into()])].into(),
183 }],
184 ..Default::default()
185 };
186 let error = SyncClient::new("authority".into(), &schema, limits)
187 .err()
188 .expect("forbidden policy");
189 assert!(
190 error.starts_with("client.authority_read_forbidden:"),
191 "{error}"
192 );
193 }
194 }
195
196 #[test]
197 fn authority_snapshot_observes_one_file_transaction_during_concurrent_writes() {
198 let path = std::env::temp_dir().join(format!(
199 "syncular-authority-snapshot-{}.db",
200 uuid::Uuid::new_v4()
201 ));
202 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
203 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "revision", "type": "integer", "nullable": false}
204 ]}]});
205 let limits = ClientLimits {
206 authority_reads: vec![crate::api::AuthorityReadDeclaration {
207 table: "authority".into(),
208 columns: vec!["id".into(), "actor_id".into(), "revision".into()],
209 scopes: [("actor_id".into(), vec!["a".into()])].into(),
210 }],
211 ..Default::default()
212 };
213 let mut client =
214 SyncClient::open_path("authority".into(), &schema, limits, path.to_str().unwrap())
215 .unwrap();
216 client.begin_security_preflight();
217 let update = |conn: &Connection, revision: i64| {
218 conn.execute_batch("BEGIN IMMEDIATE").unwrap();
219 conn.execute(
220 "INSERT OR REPLACE INTO _syncular_base_authority VALUES('one','a',?1,1)",
221 [revision],
222 )
223 .unwrap();
224 conn.execute(
225 "UPDATE _syncular_meta SET value=?1 WHERE key='localRevision'",
226 [revision.to_string()],
227 )
228 .unwrap();
229 conn.execute("INSERT OR REPLACE INTO _syncular_subscriptions VALUES('s','authority',?1)", [json!({"requested": {"actor_id": ["a"]}, "effectiveScopes": {"actor_id": ["a"]}, "status": "active", "cursor": revision}).to_string()]).unwrap();
230 conn.execute_batch("COMMIT").unwrap();
231 };
232 update(&client.conn, 1);
233 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
234 let writer_barrier = barrier.clone();
235 let writer_path = path.clone();
236 let writer = std::thread::spawn(move || {
237 let conn = Connection::open(writer_path).unwrap();
238 conn.busy_timeout(std::time::Duration::from_secs(5))
239 .unwrap();
240 writer_barrier.wait();
241 for revision in 2..=100 {
242 update(&conn, revision);
243 }
244 });
245 barrier.wait();
246 for _ in 0..100 {
247 let snapshot = client.authority_snapshot().unwrap();
248 let revision = snapshot["revision"]
249 .as_str()
250 .unwrap()
251 .parse::<i64>()
252 .unwrap();
253 assert_eq!(
254 snapshot["tables"][0]["rows"][0]["values"]["revision"],
255 revision
256 );
257 assert_eq!(snapshot["tables"][0]["persisted"][0]["cursor"], revision);
258 assert_eq!(snapshot["complete"], true);
259 }
260 writer.join().unwrap();
261 assert_eq!(client.authority_snapshot().unwrap()["revision"], "100");
262 drop(client);
263 let _ = std::fs::remove_file(path);
264 }
265
266 fn client() -> SyncClient {
267 SyncClient::new(
268 "retry-test".to_owned(),
269 &json!({
270 "version": 1,
271 "tables": [{
272 "name": "tasks",
273 "primaryKey": "id",
274 "columns": [
275 { "name": "id", "type": "string", "nullable": false },
276 { "name": "project_id", "type": "string", "nullable": false }
277 ],
278 "scopes": [{ "pattern": "project:{project_id}" }]
279 }]
280 }),
281 ClientLimits::default(),
282 )
283 .expect("test client")
284 }
285
286 #[test]
287 fn subscribe_many_registers_all_or_nothing() {
288 let request = |id: &str, project: &str| SubscriptionRequest {
289 id: id.to_owned(),
290 table: "tasks".to_owned(),
291 scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
292 params: None,
293 };
294 let mut client = client();
295 client
296 .subscribe_many(vec![request("a", "p1"), request("b", "p2")])
297 .unwrap();
298 let persisted = |client: &SyncClient| -> i64 {
299 client
300 .conn
301 .query_row("SELECT count(*) FROM _syncular_subscriptions", [], |row| {
302 row.get(0)
303 })
304 .unwrap()
305 };
306 assert_eq!(persisted(&client), 2);
307 let error = client
309 .subscribe_many(vec![request("c", "p3"), request("b", "p9")])
310 .unwrap_err();
311 assert!(error.starts_with("client.subscription_intent_mismatch:"));
312 assert!(client.subscription_state("c").is_none());
313 assert_eq!(persisted(&client), 2);
314 assert!(client.conn.is_autocommit());
315 }
316
317 #[cfg(feature = "e2ee")]
318 #[test]
319 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
320 let schema = json!({
321 "version": 1,
322 "tables": [{
323 "name": "secrets",
324 "primaryKey": "id",
325 "columns": [
326 { "name": "id", "type": "string", "nullable": false },
327 { "name": "encryption_key_id", "type": "string", "nullable": true },
328 { "name": "note", "type": "bytes", "nullable": true,
329 "encrypted": true, "declaredType": "string" }
330 ],
331 "scopes": []
332 }]
333 });
334 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
335 .expect("test client");
336 let mut config = crate::values::EncryptionConfig::default();
337 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
338 config
339 .key_id_columns
340 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
341 client.set_encryption(config);
342 client
345 .write_base_row(
346 "secrets",
347 &vec![Some(ColumnValue::String("ghost".into())), None, None],
348 1,
349 )
350 .unwrap();
351 client.rebuild_overlay().expect("rebuild overlay");
352 let commit_id = client
353 .patch(
354 "secrets",
355 "ghost",
356 Map::from_iter([("note".to_owned(), json!("x"))]),
357 None,
358 )
359 .expect("patch records without an author-time encrypt failure");
360 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
361 assert!(client.drop_unencodable_outbox(true, None).unwrap());
363 assert!(client.pending_commit_ids().is_empty());
364 let rejections = client.rejections();
365 assert_eq!(rejections.len(), 1);
366 assert_eq!(rejections[0].code, "client.encrypt_failed");
367 assert_eq!(rejections[0].client_commit_id, commit_id);
368 assert_eq!(
369 client
370 .commit_outcome(&commit_id)
371 .unwrap()
372 .map(|outcome| outcome.status),
373 Some(CommitOutcomeStatus::Rejected)
374 );
375 }
376
377 #[test]
378 fn patch_scope_read_preserves_sqlite_failure() {
379 let mut client = client();
380 client
381 .conn
382 .execute_batch(
383 "DROP TABLE tasks;
384 CREATE VIEW tasks AS SELECT 't1' AS id, abs(-9223372036854775808) AS project_id;",
385 )
386 .unwrap();
387 let error = client
388 .patch(
389 "tasks",
390 "t1",
391 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
392 None,
393 )
394 .expect_err("the scope read must fail");
395 assert_ne!(error.code, "sync.invalid_request");
396 assert_eq!(error.code, "client.failed");
397 assert!(error.details.as_ref().unwrap()["legacyCause"]
398 .as_str()
399 .unwrap()
400 .contains("integer overflow"));
401 assert!(client.pending_commit_ids().is_empty());
402 assert!(client.conn.is_autocommit());
403 }
404
405 #[test]
406 fn patch_with_a_scope_column_matches_the_stored_local_row() {
407 let mut client = client();
408 client.create_synced_tables().unwrap();
409 client
410 .conn
411 .execute(
412 "INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
413 [],
414 )
415 .unwrap();
416
417 client
420 .patch(
421 "tasks",
422 "t1",
423 Map::from_iter([
424 ("project_id".to_owned(), json!("p1")),
425 ("id".to_owned(), json!("t1")),
426 ]),
427 None,
428 )
429 .expect("an equal scope column is a no-op");
430 let values = client.outbox[0].ops[0]
431 .values
432 .clone()
433 .expect("the patch records values");
434 assert_eq!(values.get("id"), Some(&json!("t1")));
435 assert!(!values.contains_key("project_id"));
436
437 let differing = client.patch(
439 "tasks",
440 "t1",
441 Map::from_iter([("project_id".to_owned(), json!("p2"))]),
442 None,
443 );
444 let differing = differing.expect_err("a differing scope column is rejected");
445 assert_eq!(differing.code, "sync.invalid_request");
446 assert_eq!(differing.message, "the authoring request is invalid");
447 assert!(
448 differing
449 .details
450 .as_ref()
451 .and_then(|details| details["legacyCause"].as_str())
452 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
453 "{differing:?}"
454 );
455
456 let absent = client.patch(
459 "tasks",
460 "ghost",
461 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
462 None,
463 );
464 let absent = absent.expect_err("an absent local row is rejected");
465 assert_eq!(absent.code, "sync.row_missing");
466 assert!(absent.message.contains("requires a local row"));
467 assert_eq!(client.pending_commit_ids().len(), 1);
468 }
469
470 #[test]
471 fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
472 let schema = json!({
473 "version": 1,
474 "tables": [
475 {
476 "name": "tenants",
477 "primaryKey": "tenant_id",
478 "columns": [
479 { "name": "tenant_id", "type": "string", "nullable": false },
480 { "name": "body", "type": "string", "nullable": false }
481 ],
482 "scopes": [{ "pattern": "tenant:{tenant_id}" }]
483 },
484 {
485 "name": "buckets",
486 "primaryKey": "id",
487 "columns": [
488 { "name": "id", "type": "string", "nullable": false },
489 { "name": "bucket", "type": "float", "nullable": false },
490 { "name": "body", "type": "string", "nullable": false }
491 ],
492 "scopes": [{ "pattern": "bucket:{bucket}" }]
493 }
494 ]
495 });
496 let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
497 .expect("test client");
498 client.create_synced_tables().unwrap();
499
500 client
502 .conn
503 .execute(
504 "INSERT INTO tenants (tenant_id, body, _syncular_version) VALUES ('t1', 'seed', 1)",
505 [],
506 )
507 .unwrap();
508 client
509 .patch(
510 "tenants",
511 "t1",
512 Map::from_iter([
513 ("tenant_id".to_owned(), json!("t1")),
514 ("body".to_owned(), json!("created")),
515 ]),
516 None,
517 )
518 .expect("the primary-key scope column is proven equal by construction");
519
520 client
522 .conn
523 .execute(
524 "INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
525 [],
526 )
527 .unwrap();
528 client
529 .patch(
530 "buckets",
531 "b1",
532 Map::from_iter([
533 ("bucket".to_owned(), json!(2)),
534 ("body".to_owned(), json!("coerced")),
535 ]),
536 None,
537 )
538 .expect("the stored REAL value coerces from a supplied integer");
539 let values = client.outbox[1].ops[0]
540 .values
541 .clone()
542 .expect("the patch records values");
543 assert!(!values.contains_key("bucket"));
544
545 let differing = client.patch(
547 "buckets",
548 "b1",
549 Map::from_iter([("bucket".to_owned(), json!(3))]),
550 None,
551 );
552 let differing = differing.expect_err("a differing scope value is rejected");
553 assert_eq!(differing.code, "sync.invalid_request");
554 assert_eq!(differing.message, "the authoring request is invalid");
555 assert!(
556 differing
557 .details
558 .as_ref()
559 .and_then(|details| details["legacyCause"].as_str())
560 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
561 "{differing:?}"
562 );
563 }
564
565 #[cfg(feature = "e2ee")]
566 #[test]
567 fn stored_key_fallback_resolves_an_integer_primary_key() {
568 let schema = json!({
569 "version": 1,
570 "tables": [{
571 "name": "patients",
572 "primaryKey": "id",
573 "columns": [
574 { "name": "id", "type": "integer", "nullable": false },
575 { "name": "encryption_key_id", "type": "string", "nullable": true },
576 { "name": "note", "type": "bytes", "nullable": true,
577 "encrypted": true, "declaredType": "string" }
578 ],
579 "scopes": []
580 }]
581 });
582 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
583 .expect("test client");
584 client.create_synced_tables().unwrap();
585 client
586 .conn
587 .execute(
588 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
589 VALUES (5, 'k1', NULL, 1)",
590 [],
591 )
592 .unwrap();
593 let mut config = crate::values::EncryptionConfig::default();
594 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
595 config
596 .key_id_columns
597 .insert("patients".to_owned(), "encryption_key_id".to_owned());
598 client.set_encryption(config);
599 let table = client.schema.table("patients").unwrap();
600 let fallback = client
601 .stored_key_fallback(table, "5")
602 .expect("the stored integer key resolves the fallback");
603 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
604 }
605
606 #[test]
607 fn patch_types_an_integer_primary_key_from_its_canonical_row_id() {
608 let schema = json!({
609 "version": 1,
610 "tables": [{
611 "name": "counters",
612 "primaryKey": "id",
613 "columns": [
614 { "name": "id", "type": "integer", "nullable": false },
615 { "name": "title", "type": "string", "nullable": false }
616 ],
617 "scopes": []
618 }]
619 });
620 let mut client = SyncClient::new("int-patch".into(), &schema, ClientLimits::default())
621 .expect("test client");
622 client
623 .write_base_row(
624 "counters",
625 &vec![
626 Some(ColumnValue::Integer(42)),
627 Some(ColumnValue::String("before".into())),
628 ],
629 1,
630 )
631 .unwrap();
632 client.rebuild_overlay().expect("rebuild overlay");
633 client
634 .patch(
635 "counters",
636 "42",
637 Map::from_iter([
638 ("id".to_owned(), json!(42)),
639 ("title".to_owned(), json!("after")),
640 ]),
641 None,
642 )
643 .expect("an integer key that renders as the rowId is not a change");
644 let rows = client.query("SELECT id, title FROM counters", &[]).unwrap();
645 assert_eq!(Value::from(rows), json!([{"id": 42, "title": "after"}]));
646 for row_id in ["042", "+42", "42.0"] {
647 let error = client
648 .patch(
649 "counters",
650 row_id,
651 Map::from_iter([("title".to_owned(), json!("x"))]),
652 None,
653 )
654 .expect_err("a non-canonical integer rowId names no row");
655 assert_eq!(error.code, "sync.row_missing", "{row_id}");
656 }
657 }
658
659 #[test]
660 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
661 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
662 {"name":"id","type":"string","nullable":false},
663 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
664 for state in ["absent", "cached", "pinned"] {
665 let path = std::env::temp_dir().join(format!(
666 "syncular-blob-stage-{}.sqlite",
667 uuid::Uuid::new_v4()
668 ));
669 let mut client = SyncClient::open_path(
670 "stage-test".into(),
671 &schema,
672 Default::default(),
673 path.to_str().unwrap(),
674 )
675 .unwrap();
676 let bytes = b"atomic stage";
677 if state != "absent" {
678 client
679 .upload_blob(bytes, Some("text/plain".into()), None)
680 .unwrap();
681 if state == "cached" {
682 client
683 .conn
684 .execute("DELETE FROM _syncular_blob_uploads", [])
685 .unwrap();
686 }
687 }
688 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
689 client
690 .conn
691 .execute_batch(
692 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
693 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
694 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
695 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
696 )
697 .unwrap();
698 let error = client
699 .upload_blob(bytes, Some("application/octet-stream".into()), None)
700 .unwrap_err();
701 assert!(error.contains("injected stage failure"), "{state}: {error}");
702 assert!(client.conn.is_autocommit());
703 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
704 drop(client);
705 let mut reopened = SyncClient::open_path(
706 "stage-test".into(),
707 &schema,
708 Default::default(),
709 path.to_str().unwrap(),
710 )
711 .unwrap();
712 let durable = reopened
713 .query("SELECT * FROM _syncular_blobs", &[])
714 .unwrap();
715 reopened
716 .conn
717 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
718 .unwrap();
719 let retried = reopened.upload_blob(bytes, None, None).unwrap();
720 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
721 assert_eq!(
722 reopened
723 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
724 .unwrap(),
725 vec![Map::from_iter([(
726 "blob_id".into(),
727 Value::from(blob_id_for(bytes)),
728 )])]
729 );
730 drop(reopened);
731 std::fs::remove_file(&path).unwrap();
732 assert_eq!(durable, bodies, "{state}: reopened body state");
733 assert_eq!(immediate, bodies, "{state}: immediate body state");
734 }
735 }
736
737 #[test]
738 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
739 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
740 {"name":"id","type":"string","nullable":false},
741 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
742 for fault in [
743 "missing",
744 "body-type",
745 "length",
746 "hash",
747 "pin-metadata",
748 "read",
749 "pin-delete",
750 ] {
751 let path = std::env::temp_dir().join(format!(
752 "syncular-blob-upload-{}.sqlite",
753 uuid::Uuid::new_v4()
754 ));
755 let mut client = SyncClient::open_path(
756 "upload-test".into(),
757 &schema,
758 Default::default(),
759 path.to_str().unwrap(),
760 )
761 .unwrap();
762 let bytes = b"durable upload";
763 let ref_value = client
764 .upload_blob(bytes, Some("text/plain".into()), None)
765 .unwrap();
766 let commit = client
767 .mutate(vec![Mutation::Upsert {
768 table: "attachments".into(),
769 values: Map::from_iter([
770 ("id".into(), Value::from("pending")),
771 (
772 "file".into(),
773 Value::from(format!(
774 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
775 blob_id_for(bytes),
776 bytes.len()
777 )),
778 ),
779 ]),
780 base_version: None,
781 }])
782 .unwrap();
783 client
784 .conn
785 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
786 .unwrap();
787 client.conn.execute_batch(match fault {
788 "missing" => "DELETE FROM _syncular_blobs",
789 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
790 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
791 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
792 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
793 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
794 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
795 }).unwrap();
796 let state = if fault == "read" {
797 Vec::new()
798 } else {
799 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
800 };
801 let mut transport = CountingRealtimeTransport::default();
802 let error = client.flush_blob_uploads(&mut transport).unwrap_err();
803 assert_eq!(
804 error.code,
805 if matches!(fault, "read" | "pin-delete") {
806 "client.failed"
807 } else {
808 "sync.local_corrupt"
809 },
810 "{fault}"
811 );
812 assert_eq!(
813 transport.blob_uploads.len(),
814 usize::from(fault == "pin-delete"),
815 "{fault}: corrupt bytes must never be uploaded"
816 );
817 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
818 if fault != "read" {
819 assert_eq!(
820 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
821 state
822 );
823 } else {
824 client
825 .conn
826 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
827 .unwrap();
828 }
829 drop(client);
830 let mut reopened = SyncClient::open_path(
831 "upload-test".into(),
832 &schema,
833 Default::default(),
834 path.to_str().unwrap(),
835 )
836 .unwrap();
837 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
838 if fault != "read" {
839 assert_eq!(
840 reopened
841 .query("SELECT * FROM _syncular_blobs", &[])
842 .unwrap(),
843 state
844 );
845 }
846 if fault == "pin-delete" {
847 reopened
848 .conn
849 .execute_batch("DROP TRIGGER fail_pin_delete")
850 .unwrap();
851 }
852 if fault == "pin-metadata" {
853 reopened
854 .conn
855 .execute(
856 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
857 [],
858 )
859 .unwrap();
860 }
861 reopened
862 .conn
863 .execute_batch(
864 "DELETE FROM _syncular_blobs;
865 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
866 )
867 .unwrap();
868 transport.blob_uploads.clear();
869 reopened.flush_blob_uploads(&mut transport).unwrap();
870 assert_eq!(
871 transport.blob_uploads,
872 [(blob_id_for(bytes), bytes.to_vec())]
873 );
874 assert_eq!(
875 reopened
876 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
877 .unwrap(),
878 Vec::<Map<String, Value>>::new()
879 );
880 assert_eq!(reopened.pending_commit_ids(), [commit]);
881 assert_eq!(
882 reopened
883 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
884 .unwrap()
885 .unwrap()
886 .bytes,
887 bytes
888 );
889 drop(reopened);
890 std::fs::remove_file(path).unwrap();
891 }
892 }
893
894 fn coalesced_fixture(sections: usize, commits: usize, rows: usize) -> (SyncClient, Message) {
898 let mut client = client();
899 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
900 let mut response = Message {
901 wire_version: WIRE_VERSION,
902 msg_kind: MsgKind::Response,
903 frames: vec![Frame::RespHeader {
904 required_schema_version: None,
905 latest_schema_version: None,
906 log_epoch: Some("epoch-1".into()),
907 reset_required: Some(false),
908 }],
909 };
910 let mut seq = 0;
911 for section in 0..sections {
912 let project = format!("p{section}");
913 let scopes = vec![("project_id".to_owned(), vec![project.clone()])];
914 client
915 .subscribe(format!("s{section}"), "tasks".into(), scopes.clone(), None)
916 .unwrap();
917 let sub = client.subs.last_mut().unwrap();
918 sub.cursor = 0;
919 sub.synced_once = true;
920 sub.effective = Some(scopes.clone());
921 let sub = sub.clone();
922 client.persist_sub(&sub).unwrap();
923 response.frames.push(Frame::SubStart {
924 id: format!("s{section}"),
925 status: SubStatus::Active,
926 reason_code: "".into(),
927 effective_scopes: scopes,
928 bootstrap: false,
929 });
930 for _ in 0..commits {
931 seq += 1;
932 let changes = (0..rows)
933 .map(|row| {
934 let id = format!("{project}-{seq}-{row}");
935 let values = Map::from_iter([
936 ("id".into(), Value::from(id.clone())),
937 ("project_id".into(), Value::from(project.clone())),
938 ]);
939 ssp2::model::Change {
940 table_index: 0,
941 row_id: id.clone(),
942 op: Op::Upsert,
943 row_version: Some(seq),
944 scopes: vec![("project_id".into(), project.clone())],
945 row: Some(
946 encode_row_json(
947 client.schema.table("tasks").unwrap(),
948 &id,
949 &values,
950 &client.encryption,
951 )
952 .unwrap(),
953 ),
954 }
955 })
956 .collect();
957 response.frames.push(Frame::Commit {
958 commit_seq: seq,
959 actor_id: "writer".into(),
960 created_at_ms: 1,
961 tables: vec!["tasks".into()],
962 changes,
963 });
964 }
965 response.frames.push(Frame::SubEnd {
966 next_cursor: seq,
967 bootstrap_state: None,
968 });
969 }
970 client.drain_change_batches();
971 (client, response)
972 }
973
974 fn count_commits(client: &SyncClient) -> std::sync::Arc<std::sync::atomic::AtomicUsize> {
975 let commits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
976 let counter = commits.clone();
977 client
978 .conn
979 .commit_hook(Some(move || {
980 counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
981 false
982 }))
983 .expect("commit hook");
984 commits
985 }
986
987 #[test]
988 fn a_response_of_many_small_sections_commits_once() {
989 for realtime in [false, true] {
990 let (mut client, response) = coalesced_fixture(70, 1, 1);
991 client.realtime_state = RealtimeState::Connected;
992 client.realtime_reason_code = None;
993 let mut transport = CountingRealtimeTransport::default();
994 let (_, meta) = client.build_request(false).unwrap();
995 let commits = count_commits(&client);
996 if realtime {
997 client.on_realtime_binary(&mut transport, &encode_message(&response));
998 } else {
999 assert!(matches!(
1000 client.process_response(&mut transport, response, &meta),
1001 SyncOutcome::Ok(_)
1002 ));
1003 }
1004 assert_eq!(
1005 commits.load(std::sync::atomic::Ordering::SeqCst),
1006 1,
1007 "{realtime}"
1008 );
1009 assert!(client.conn.is_autocommit());
1010 assert!(client.subs.iter().all(|sub| sub.cursor > 0));
1011 let revisions = client
1013 .drain_change_batches()
1014 .into_iter()
1015 .filter(|batch| !batch.tables.is_empty())
1016 .map(|batch| batch.revision)
1017 .collect::<Vec<_>>();
1018 assert_eq!(
1019 revisions,
1020 (1..=70).map(|r: u64| r.to_string()).collect::<Vec<_>>()
1021 );
1022 assert_eq!(client.query("SELECT id FROM tasks", &[]).unwrap().len(), 70);
1023 assert_eq!(
1025 transport
1026 .messages
1027 .iter()
1028 .filter(|m| m.contains("\"ack\""))
1029 .count(),
1030 1,
1031 "{realtime}"
1032 );
1033 }
1034 }
1035
1036 #[test]
1037 fn a_response_past_the_row_bound_commits_per_bound() {
1038 let (mut client, response) = coalesced_fixture(1, 3, 600);
1040 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1041 let (_, meta) = client.build_request(false).unwrap();
1042 let commits = count_commits(&client);
1043 assert!(matches!(
1044 client.process_response(&mut transport, response, &meta),
1045 SyncOutcome::Ok(_)
1046 ));
1047 assert_eq!(commits.load(std::sync::atomic::Ordering::SeqCst), 3);
1048 assert_eq!(
1049 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1050 1800
1051 );
1052 assert_eq!(client.subs[0].cursor, 3);
1053 }
1054
1055 #[test]
1056 fn a_failed_coalesced_commit_restores_memory_and_publishes_nothing() {
1057 let (mut client, response) = coalesced_fixture(2, 2, 600);
1061 client
1062 .conn
1063 .execute_batch(
1064 "PRAGMA foreign_keys = ON; CREATE TABLE fail_parent(id INTEGER PRIMARY KEY); CREATE TABLE fail_child(id INTEGER REFERENCES fail_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_commit AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'p0-1-0' BEGIN INSERT INTO fail_child VALUES(1); END",
1065 )
1066 .unwrap();
1067 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1068 let (_, meta) = client.build_request(false).unwrap();
1069 let subs = client.subs.clone();
1070 let revision = client.local_revision();
1071 let outcome = client.process_response(&mut transport, response.clone(), &meta);
1072 assert!(
1073 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == "storage.failed"),
1074 "{outcome:?}"
1075 );
1076 assert!(client.conn.is_autocommit());
1077 assert!(client.coalesced.is_none() && client.coalesced_failure.is_none());
1078 assert_eq!(client.local_revision(), revision);
1079 assert!(client
1080 .drain_change_batches()
1081 .iter()
1082 .all(|batch| batch.tables.is_empty()));
1083 assert!(client
1084 .query("SELECT id FROM tasks", &[])
1085 .unwrap()
1086 .is_empty());
1087 assert_eq!(
1088 client.subs.iter().map(|s| s.cursor).collect::<Vec<_>>(),
1089 subs.iter().map(|s| s.cursor).collect::<Vec<_>>()
1090 );
1091 client
1093 .conn
1094 .execute_batch("DROP TRIGGER fail_commit")
1095 .unwrap();
1096 let (_, meta) = client.build_request(false).unwrap();
1097 assert!(matches!(
1098 client.process_response(&mut transport, response, &meta),
1099 SyncOutcome::Ok(_)
1100 ));
1101 assert_eq!(
1102 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1103 2400
1104 );
1105 }
1106
1107 #[test]
1108 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
1109 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1110 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
1111 "scopes":[{"pattern":"project:{project_id}"}]}]});
1112 for realtime in [false, true] {
1113 for fault in [
1114 "none",
1115 "row",
1116 "revision",
1117 "commit",
1118 "cursor",
1119 "error",
1120 "missing-end",
1121 ] {
1122 if realtime && fault == "missing-end" {
1123 continue;
1124 }
1125 let path = std::env::temp_dir()
1126 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
1127 let mut client = SyncClient::open_path(
1128 "frame-test".into(),
1129 &schema,
1130 Default::default(),
1131 path.to_str().unwrap(),
1132 )
1133 .unwrap();
1134 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1135 client
1136 .subscribe(
1137 "tasks".into(),
1138 "tasks".into(),
1139 vec![("project_id".into(), vec!["p1".into()])],
1140 None,
1141 )
1142 .unwrap();
1143 client.subs[0].cursor = 0;
1144 client.subs[0].synced_once = true;
1145 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1146 client.persist_sub(&client.subs[0]).unwrap();
1147 client.realtime_state = RealtimeState::Connected;
1148 client.realtime_reason_code = None;
1149 client.drain_change_batches();
1150 let mut transport = CountingRealtimeTransport::default();
1151 let (_, meta) = client.build_request(false).unwrap();
1152 let mut response = Message {
1153 wire_version: WIRE_VERSION,
1154 msg_kind: MsgKind::Response,
1155 frames: vec![
1156 Frame::RespHeader {
1157 required_schema_version: None,
1158 latest_schema_version: None,
1159 log_epoch: Some("epoch-1".into()),
1160 reset_required: Some(false),
1161 },
1162 Frame::SubStart {
1163 id: "tasks".into(),
1164 status: SubStatus::Active,
1165 reason_code: "".into(),
1166 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1167 bootstrap: false,
1168 },
1169 ],
1170 };
1171 for (index, id) in ["first", "second"].iter().enumerate() {
1172 let values = Map::from_iter([
1173 ("id".into(), Value::from(*id)),
1174 ("project_id".into(), Value::from("p1")),
1175 ]);
1176 response.frames.push(Frame::Commit {
1177 commit_seq: index as i64 + 1,
1178 actor_id: "writer".into(),
1179 created_at_ms: 1,
1180 tables: vec!["tasks".into()],
1181 changes: vec![ssp2::model::Change {
1182 table_index: 0,
1183 row_id: (*id).into(),
1184 op: Op::Upsert,
1185 row_version: Some(index as i64 + 1),
1186 scopes: vec![("project_id".into(), "p1".into())],
1187 row: Some(
1188 encode_row_json(
1189 client.schema.table("tasks").unwrap(),
1190 id,
1191 &values,
1192 &client.encryption,
1193 )
1194 .unwrap(),
1195 ),
1196 }],
1197 });
1198 }
1199 response.frames.push(Frame::SubEnd {
1200 next_cursor: 2,
1201 bootstrap_state: None,
1202 });
1203 let valid = response.clone();
1204 match fault {
1205 "row" => {
1206 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
1207 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
1208 },
1209 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
1210 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
1211 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
1212 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
1213 "missing-end" => {response.frames.truncate(3);},
1214 _ => {}
1215 }
1216 if realtime {
1217 client.on_realtime_binary(&mut transport, &encode_message(&response));
1218 } else {
1219 assert_eq!(
1220 matches!(
1221 client.process_response(&mut transport, response, &meta),
1222 SyncOutcome::Ok(_)
1223 ),
1224 fault == "none",
1225 "{fault}"
1226 );
1227 }
1228 assert!(
1229 client.conn.is_autocommit(),
1230 "no open transaction after {realtime}/{fault}"
1231 );
1232 let count = match fault {
1236 "none" | "cursor" => 2,
1237 "commit" => 0,
1238 _ => 1,
1239 };
1240 let expected_revision = count + u64::from(realtime && fault != "none");
1241 assert_eq!(
1242 client.local_revision(),
1243 expected_revision,
1244 "{realtime}/{fault}"
1245 );
1246 let batches = client.drain_change_batches();
1247 assert_eq!(
1248 batches.len(),
1249 expected_revision as usize,
1250 "{realtime}/{fault}"
1251 );
1252 let row_batches: Vec<_> = batches
1253 .iter()
1254 .filter(|batch| !batch.tables.is_empty())
1255 .collect();
1256 assert_eq!(row_batches.len(), count as usize);
1257 assert_eq!(
1258 row_batches
1259 .iter()
1260 .map(|b| b.revision.clone())
1261 .collect::<Vec<_>>(),
1262 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
1263 );
1264 for table in ["tasks", "_syncular_base_tasks"] {
1265 assert_eq!(
1266 client
1267 .conn
1268 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
1269 .get::<_, u64>(0))
1270 .unwrap(),
1271 count,
1272 "{realtime}/{fault}/{table}"
1273 );
1274 }
1275 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1276 assert_eq!(
1277 transport
1278 .messages
1279 .iter()
1280 .filter(|m| m.contains("\"ack\""))
1281 .count(),
1282 usize::from(fault == "none")
1283 );
1284 if matches!(fault, "revision" | "commit" | "cursor") {
1285 client
1286 .conn
1287 .execute_batch("DROP TRIGGER fail_frame")
1288 .unwrap();
1289 }
1290 drop(client);
1291 let mut reopened = SyncClient::open_path(
1292 "frame-test".into(),
1293 &schema,
1294 Default::default(),
1295 path.to_str().unwrap(),
1296 )
1297 .unwrap();
1298 assert_eq!(reopened.local_revision(), expected_revision);
1299 assert_eq!(
1300 reopened
1301 .query("SELECT id FROM tasks ORDER BY id", &[])
1302 .unwrap()
1303 .len(),
1304 count as usize
1305 );
1306 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1307 let (_, meta) = reopened.build_request(false).unwrap();
1308 assert!(matches!(
1309 reopened.process_response(&mut transport, valid, &meta),
1310 SyncOutcome::Ok(_)
1311 ));
1312 assert_eq!(
1313 reopened
1314 .query("SELECT id FROM tasks ORDER BY id", &[])
1315 .unwrap()
1316 .len(),
1317 2
1318 );
1319 assert_eq!(reopened.subs[0].cursor, 2);
1320 drop(reopened);
1321 std::fs::remove_file(path).unwrap();
1322 }
1323 }
1324 }
1325
1326 #[test]
1327 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
1328 for status in [SubStatus::Reset, SubStatus::Revoked] {
1329 for fault in ["subscription", "revision"] {
1330 let mut client = client();
1331 client
1332 .subscribe(
1333 "tasks".into(),
1334 "tasks".into(),
1335 vec![("project_id".into(), vec!["p1".into()])],
1336 None,
1337 )
1338 .unwrap();
1339 client.subs[0].cursor = 41;
1340 client.subs[0].synced_once = true;
1341 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1342 client.persist_sub(&client.subs[0]).unwrap();
1343 client
1344 .mutate(vec![Mutation::Upsert {
1345 table: "tasks".into(),
1346 values: Map::from_iter([
1347 ("id".into(), Value::from("pending")),
1348 ("project_id".into(), Value::from("p1")),
1349 ]),
1350 base_version: None,
1351 }])
1352 .unwrap();
1353 let ids = client.pending_commit_ids();
1354 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
1355 let revision = client.local_revision();
1356 client.drain_change_batches();
1357 let (_, meta) = client.build_request(false).unwrap();
1358 client.conn.execute_batch(if fault == "subscription" {
1359 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
1360 } else {
1361 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
1362 }).unwrap();
1363 let mut transport = CountingRealtimeTransport::default();
1364 let mut report = SyncReport::default();
1365 if status == SubStatus::Reset {
1367 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
1368 }
1369 let result = client.process_section(
1370 &mut transport,
1371 "tasks",
1372 status,
1373 "sync.scope_revoked",
1374 vec![],
1375 vec![],
1376 Some((42, None)),
1377 &meta,
1378 &mut report,
1379 );
1380 assert!(result.is_err(), "{status:?}/{fault}");
1381 assert!(client.conn.is_autocommit());
1382 assert_eq!(client.subs[0].cursor, 41);
1383 assert_eq!(client.subs[0].state, SubState::Active);
1384 assert_eq!(client.pending_commit_ids(), ids);
1385 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
1386 assert!(client.rejections.is_empty());
1387 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
1388 assert_eq!(client.local_revision(), revision);
1389 assert!(client.drain_change_batches().is_empty());
1390 assert!(report.revoked.is_empty() && report.resets.is_empty());
1391 }
1392 }
1393 }
1394
1395 #[test]
1396 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
1397 let mut client = client();
1398 client
1399 .subscribe(
1400 "tasks".into(),
1401 "tasks".into(),
1402 vec![("project_id".into(), vec!["p1".into()])],
1403 None,
1404 )
1405 .unwrap();
1406 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1407 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
1408 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
1409 let row = |id: &str| ssp2::segment::SegmentRow {
1410 server_version: 1,
1411 values: vec![
1412 Some(ColumnValue::String(id.into())),
1413 Some(ColumnValue::String("p1".into())),
1414 ],
1415 };
1416 let segment = RowsSegment {
1417 table: "tasks".into(),
1418 schema_version: 1,
1419 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
1420 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
1421 };
1422 assert!(client.apply_segment(0, &segment, true).is_err());
1423 assert!(client.conn.is_autocommit());
1424 assert_eq!(
1425 client
1426 .query("SELECT id FROM tasks ORDER BY id", &[])
1427 .unwrap(),
1428 vec![Map::from_iter([("id".into(), Value::from("first"))])]
1429 );
1430 assert_eq!(client.local_revision(), 1);
1431 assert_eq!(client.drain_change_batches().len(), 1);
1432 assert_eq!(client.subs[0].cursor, -1);
1433 client
1434 .conn
1435 .execute_batch("DROP TRIGGER fail_block")
1436 .unwrap();
1437 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
1438 assert_eq!(client.local_revision(), 3);
1439 assert_eq!(client.drain_change_batches().len(), 2);
1440 }
1441
1442 struct SegmentBatchTransport {
1446 response: Message,
1447 bodies: BTreeMap<String, Vec<u8>>,
1448 batches: Vec<Vec<String>>,
1449 single_fetches: usize,
1450 }
1451
1452 impl Transport for SegmentBatchTransport {
1453 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1454 Ok(encode_message(&self.response))
1455 }
1456
1457 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1458 self.sync(request)
1459 }
1460
1461 fn download_segment(
1462 &mut self,
1463 _request: &SegmentRequest,
1464 _on_progress: &mut dyn FnMut(u64),
1465 ) -> Result<Vec<u8>, TransportError> {
1466 self.single_fetches += 1;
1467 Err(TransportError::new(
1468 "sync.transport_failed",
1469 "per-descriptor fetch",
1470 ))
1471 }
1472
1473 fn download_segments(
1474 &mut self,
1475 fetches: &[crate::SegmentFetch],
1476 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1477 ) -> Vec<Result<Vec<u8>, TransportError>> {
1478 self.batches.push(
1479 fetches
1480 .iter()
1481 .map(|fetch| match fetch {
1482 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
1483 crate::SegmentFetch::Url(url) => url.clone(),
1484 })
1485 .collect(),
1486 );
1487 fetches
1488 .iter()
1489 .enumerate()
1490 .map(|(index, fetch)| {
1491 let crate::SegmentFetch::Direct(request) = fetch else {
1492 panic!("the descriptors are direct-endpoint requests")
1493 };
1494 let bytes = self.bodies[&request.segment_id].clone();
1495 on_progress(index, bytes.len() as u64);
1496 Ok(bytes)
1497 })
1498 .collect()
1499 }
1500
1501 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1502 Ok(())
1503 }
1504
1505 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1506 Ok(())
1507 }
1508
1509 fn realtime_close(&mut self) -> Result<(), TransportError> {
1510 Ok(())
1511 }
1512 }
1513
1514 #[test]
1515 fn a_response_fetches_all_its_segment_pages_in_one_ordered_batch() {
1516 let mut client = client();
1517 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1518 client
1519 .subscribe(
1520 "tasks".into(),
1521 "tasks".into(),
1522 vec![("project_id".into(), vec!["p1".into()])],
1523 None,
1524 )
1525 .unwrap();
1526 client.subs[0].cursor = 0;
1527 client.subs[0].synced_once = true;
1528 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1529 client.persist_sub(&client.subs[0]).unwrap();
1530 client.drain_change_batches();
1531 let columns = client.schema.table("tasks").unwrap().wire_columns.clone();
1532
1533 let pages = ["one", "two", "three", "four"].map(|id| {
1537 let bytes = ssp2::segment::encode_rows_segment(&RowsSegment {
1538 table: "tasks".into(),
1539 schema_version: 1,
1540 columns: columns.clone(),
1541 blocks: vec![vec![ssp2::segment::SegmentRow {
1542 server_version: 1,
1543 values: vec![
1544 Some(ColumnValue::String(id.into())),
1545 Some(ColumnValue::String("p1".into())),
1546 ],
1547 }]],
1548 });
1549 (
1550 format!("sha256:{}", bytes_to_hex(&Sha256::digest(&bytes))),
1551 bytes,
1552 )
1553 });
1554
1555 let mut frames = vec![
1556 Frame::RespHeader {
1557 required_schema_version: None,
1558 latest_schema_version: None,
1559 log_epoch: Some("epoch-1".into()),
1560 reset_required: Some(false),
1561 },
1562 Frame::SubStart {
1563 id: "tasks".into(),
1564 status: SubStatus::Active,
1565 reason_code: String::new(),
1566 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1567 bootstrap: false,
1568 },
1569 ];
1570 for (segment_id, bytes) in &pages {
1571 frames.push(Frame::SegmentRef {
1572 segment_id: segment_id.clone(),
1573 media_type: MediaType::Rows,
1574 table: "tasks".into(),
1575 byte_length: bytes.len() as i64,
1576 row_count: 1,
1577 as_of_commit_seq: 1,
1578 scope_digest: "digest".into(),
1579 row_cursor: None,
1580 next_row_cursor: None,
1581 url: None,
1582 url_expires_at_ms: None,
1583 });
1584 }
1585 frames.push(Frame::SubEnd {
1586 next_cursor: 1,
1587 bootstrap_state: None,
1588 });
1589 let mut transport = SegmentBatchTransport {
1590 response: Message {
1591 wire_version: WIRE_VERSION,
1592 msg_kind: MsgKind::Response,
1593 frames,
1594 },
1595 bodies: pages
1596 .iter()
1597 .map(|(segment_id, bytes)| (segment_id.clone(), bytes.clone()))
1598 .collect(),
1599 batches: Vec::new(),
1600 single_fetches: 0,
1601 };
1602
1603 let outcome = client.sync(&mut transport);
1604 assert!(matches!(outcome, SyncOutcome::Ok(_)), "{outcome:?}");
1605 assert_eq!(
1606 transport.batches,
1607 vec![pages
1608 .iter()
1609 .map(|(segment_id, _)| segment_id.clone())
1610 .collect::<Vec<_>>()],
1611 "one batch per response, in wire order"
1612 );
1613 assert_eq!(
1614 transport.single_fetches, 0,
1615 "the batch seam replaces the per-descriptor calls"
1616 );
1617 assert_eq!(
1618 client
1619 .query("SELECT id FROM tasks ORDER BY id", &[])
1620 .unwrap(),
1621 ["four", "one", "three", "two"]
1622 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
1623 .to_vec()
1624 );
1625 assert_eq!(client.subs[0].cursor, 1);
1626 }
1627
1628 #[test]
1629 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
1630 assert_eq!(
1631 client().diagnostics_storage().blob_cache_bytes_approx,
1632 Some(0)
1633 );
1634 let client = SyncClient::new(
1635 "diagnostic-storage".to_owned(),
1636 &json!({
1637 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
1638 "columns": [{"name": "id", "type": "string", "nullable": false},
1639 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
1640 }),
1641 ClientLimits {
1642 blob_cache_max_bytes: Some(5),
1643 ..Default::default()
1644 },
1645 )
1646 .unwrap();
1647 let empty = client.diagnostics_storage();
1648 assert_eq!(empty.status, "healthy");
1649 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
1650 assert_eq!(empty.retained_outcome_entries, Some(0));
1651 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
1652 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
1653 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
1654 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
1655 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
1656 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
1657 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
1658 let written = client.diagnostics_storage();
1659 assert_eq!(written.status, "pressure");
1660 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
1661 assert_eq!(written.retained_outcome_entries, Some(2));
1662 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
1663 assert_eq!(written.blob_cache_bytes_approx, Some(8));
1664 assert_eq!(
1665 written.pressure_reason_code.as_deref(),
1666 Some("client.blob_cache_over_limit")
1667 );
1668 client
1669 .conn
1670 .execute_batch(
1671 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
1672 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
1673 )
1674 .unwrap();
1675 let staged = client.diagnostics_storage();
1676 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
1677 assert_eq!(staged.retained_outcome_entries, Some(0));
1678 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
1679 assert_eq!(staged.status, "healthy");
1680 client.conn.execute_batch("ROLLBACK").unwrap();
1681 assert_eq!(client.diagnostics_storage(), written);
1682 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
1683 let unreadable = client.diagnostics_storage();
1684 assert_eq!(unreadable.status, "unreadable");
1685 assert!(unreadable.database_bytes_approx.is_none());
1686 assert!(unreadable.pending_outbox_bytes_approx.is_none());
1687 assert!(unreadable.retained_outcome_entries.is_none());
1688 assert!(unreadable.retained_outcome_bytes_approx.is_none());
1689 assert!(unreadable.blob_cache_bytes_approx.is_none());
1690 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
1691 let restored = client.diagnostics_storage();
1692 assert_eq!(restored.status, written.status);
1693 assert_eq!(
1694 restored.retained_outcome_bytes_approx,
1695 written.retained_outcome_bytes_approx
1696 );
1697 client.conn.set_prepared_statement_cache_capacity(0);
1698 assert_eq!(client.diagnostics_storage(), restored);
1699 }
1700
1701 struct ServerErrorTransport {
1703 code: &'static str,
1704 retryable: bool,
1705 }
1706
1707 impl Transport for ServerErrorTransport {
1708 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1709 Ok(encode_message(&Message {
1710 wire_version: WIRE_VERSION,
1711 msg_kind: MsgKind::Response,
1712 frames: vec![
1713 Frame::RespHeader {
1714 required_schema_version: None,
1715 latest_schema_version: None,
1716 log_epoch: Some("epoch-1".to_owned()),
1717 reset_required: Some(false),
1718 },
1719 Frame::Error {
1720 code: self.code.to_owned(),
1721 message: "server error".to_owned(),
1722 category: "internal".to_owned(),
1723 retryable: self.retryable,
1724 recommended_action: "retryLater".to_owned(),
1725 details: None,
1726 },
1727 ],
1728 }))
1729 }
1730
1731 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1732 self.sync(request)
1733 }
1734
1735 fn download_segment(
1736 &mut self,
1737 _request: &SegmentRequest,
1738 _on_progress: &mut dyn FnMut(u64),
1739 ) -> Result<Vec<u8>, TransportError> {
1740 Err(TransportError::new("sync.transport_failed", "offline"))
1741 }
1742
1743 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1744 Ok(())
1745 }
1746
1747 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1748 Ok(())
1749 }
1750
1751 fn realtime_close(&mut self) -> Result<(), TransportError> {
1752 Ok(())
1753 }
1754 }
1755
1756 #[test]
1757 fn retryable_server_error_schedules_a_background_retry() {
1758 let mut client = client();
1759 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1760 let mut transport = ServerErrorTransport {
1761 code: "sync.internal_error",
1762 retryable: true,
1763 };
1764 assert!(matches!(
1765 client.sync(&mut transport),
1766 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
1767 ));
1768 assert!(matches!(
1769 client.drain_sync_intents().as_slice(),
1770 [SyncIntent::Background { delay_ms: 250 }]
1771 ));
1772 assert_eq!(
1773 client.progress().snapshot().unwrap().retry_delay_ms,
1774 Some(250)
1775 );
1776 let mut transport = ServerErrorTransport {
1777 code: "sync.invalid_request",
1778 retryable: false,
1779 };
1780 assert!(matches!(
1781 client.sync(&mut transport),
1782 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
1783 ));
1784 assert!(client.drain_sync_intents().is_empty());
1785 let progress = client.progress().snapshot().unwrap();
1786 assert_eq!(progress.state, ProgressState::Failed);
1787 assert_eq!(progress.retry_delay_ms, None);
1788 }
1789
1790 #[test]
1791 fn sync_until_idle_reports_budget_exhaustion_or_failure() {
1792 let exhausted = SyncOutcome::BudgetExhausted(SyncReport::default()).to_json();
1795 assert_eq!(exhausted["ok"], json!(true));
1796 assert_eq!(exhausted["budgetExhausted"], json!(true));
1797 assert!(exhausted["report"].is_object());
1798
1799 let mut client = client();
1800 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1801 let mut idle_transport = HostTransport::new_from_config(&json!({})).unwrap();
1803 let zero = client.sync_until_idle(&mut idle_transport, Some(0));
1804 assert!(
1805 matches!(zero, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
1806 "{zero:?}"
1807 );
1808 let mut erroring = ServerErrorTransport {
1810 code: "sync.internal_error",
1811 retryable: false,
1812 };
1813 let failed = client.sync_until_idle(&mut erroring, Some(3));
1814 assert!(
1815 matches!(failed, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1816 "{failed:?}"
1817 );
1818 }
1819
1820 struct ScriptedSyncTransport {
1823 responses: std::collections::VecDeque<Message>,
1824 calls: usize,
1825 }
1826
1827 impl Transport for ScriptedSyncTransport {
1828 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1829 self.calls += 1;
1830 let response = self.responses.pop_front().expect("scripted response");
1831 Ok(encode_message(&response))
1832 }
1833
1834 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1835 self.sync(request)
1836 }
1837
1838 fn download_segment(
1839 &mut self,
1840 _request: &SegmentRequest,
1841 _on_progress: &mut dyn FnMut(u64),
1842 ) -> Result<Vec<u8>, TransportError> {
1843 Err(TransportError::new("sync.transport_failed", "offline"))
1844 }
1845
1846 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1847 Ok(())
1848 }
1849
1850 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1851 Ok(())
1852 }
1853
1854 fn realtime_close(&mut self) -> Result<(), TransportError> {
1855 Ok(())
1856 }
1857 }
1858
1859 #[test]
1860 fn sync_until_idle_success_then_failure_keeps_earlier_work() {
1861 let path = std::env::temp_dir().join(format!(
1862 "syncular-budget-script-{}.sqlite",
1863 uuid::Uuid::new_v4()
1864 ));
1865 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1866 {"name":"id","type":"string","nullable":false},
1867 {"name":"project_id","type":"string","nullable":false}],
1868 "scopes":[{"pattern":"project:{project_id}"}]}]});
1869 let mut client = SyncClient::open_path(
1870 "budget-script".into(),
1871 &schema,
1872 ClientLimits::default(),
1873 path.to_str().unwrap(),
1874 )
1875 .expect("open client");
1876 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1877 client
1878 .subscribe(
1879 "tasks".into(),
1880 "tasks".into(),
1881 vec![("project_id".into(), vec!["p1".into()])],
1882 None,
1883 )
1884 .unwrap();
1885 client.subs[0].cursor = 0;
1886 client.subs[0].synced_once = true;
1887 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1888 client.persist_sub(&client.subs[0]).unwrap();
1889 client.drain_change_batches();
1890
1891 let row = encode_row_json(
1892 client.schema.table("tasks").unwrap(),
1893 "kept",
1894 &Map::from_iter([
1895 ("id".into(), json!("kept")),
1896 ("project_id".into(), json!("p1")),
1897 ]),
1898 &client.encryption,
1899 )
1900 .unwrap();
1901 let success = Message {
1902 wire_version: WIRE_VERSION,
1903 msg_kind: MsgKind::Response,
1904 frames: vec![
1905 Frame::RespHeader {
1906 required_schema_version: None,
1907 latest_schema_version: None,
1908 log_epoch: Some("epoch-1".into()),
1909 reset_required: Some(false),
1910 },
1911 Frame::SubStart {
1912 id: "tasks".into(),
1913 status: SubStatus::Active,
1914 reason_code: String::new(),
1915 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1916 bootstrap: false,
1917 },
1918 Frame::Commit {
1919 commit_seq: 1,
1920 created_at_ms: 1,
1921 actor_id: "writer".into(),
1922 tables: vec!["tasks".into()],
1923 changes: vec![ssp2::model::Change {
1924 table_index: 0,
1925 row_id: "kept".into(),
1926 op: Op::Upsert,
1927 row_version: Some(1),
1928 scopes: vec![("project_id".into(), "p1".into())],
1929 row: Some(row),
1930 }],
1931 },
1932 Frame::SubEnd {
1933 next_cursor: 1,
1934 bootstrap_state: None,
1935 },
1936 ],
1937 };
1938 let failure = Message {
1939 wire_version: WIRE_VERSION,
1940 msg_kind: MsgKind::Response,
1941 frames: vec![
1942 Frame::RespHeader {
1943 required_schema_version: None,
1944 latest_schema_version: None,
1945 log_epoch: Some("epoch-1".into()),
1946 reset_required: Some(false),
1947 },
1948 Frame::Error {
1949 code: "sync.internal_error".into(),
1950 message: "second round failed".into(),
1951 category: "internal".into(),
1952 retryable: false,
1953 recommended_action: "retryLater".into(),
1954 details: None,
1955 },
1956 ],
1957 };
1958 let mut transport = ScriptedSyncTransport {
1959 responses: std::collections::VecDeque::from([success, failure]),
1960 calls: 0,
1961 };
1962 let outcome = client.sync_until_idle(&mut transport, Some(2));
1963 assert!(
1964 matches!(outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1965 "{outcome:?}"
1966 );
1967 assert_eq!(
1968 transport.calls, 2,
1969 "one successful round then the failing round"
1970 );
1971 assert_eq!(
1972 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
1973 json!("kept"),
1974 "the earlier round's durable work survives the later failure"
1975 );
1976 drop(client);
1977 let _ = std::fs::remove_file(&path);
1978 }
1979
1980 #[test]
1981 fn sync_until_idle_stops_on_a_schema_floor() {
1982 let mut client = client();
1983 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1984 let floor = Message {
1985 wire_version: WIRE_VERSION,
1986 msg_kind: MsgKind::Response,
1987 frames: vec![Frame::RespHeader {
1988 required_schema_version: Some(2),
1989 latest_schema_version: Some(2),
1990 log_epoch: Some("epoch-1".into()),
1991 reset_required: Some(false),
1992 }],
1993 };
1994 let mut transport = ScriptedSyncTransport {
1995 responses: std::collections::VecDeque::from([floor.clone(), floor]),
1996 calls: 0,
1997 };
1998 let outcome = client.sync_until_idle(&mut transport, Some(3));
1999 match outcome {
2000 SyncOutcome::Ok(report) => assert_eq!(
2001 report
2002 .schema_floor
2003 .as_ref()
2004 .and_then(|floor| floor.required_schema_version),
2005 Some(2),
2006 "the stop retains the latest schema floor"
2007 ),
2008 other => panic!("expected a schema-floor stop, got {other:?}"),
2009 }
2010 assert_eq!(
2011 transport.calls, 1,
2012 "a schema floor stops the budget loop after one round"
2013 );
2014 }
2015
2016 #[test]
2017 fn sync_report_merge_schema_floor_is_latest_state() {
2018 let mut aggregate = SyncReport {
2019 schema_floor: Some(SchemaFloor {
2020 required_schema_version: Some(2),
2021 latest_schema_version: Some(2),
2022 }),
2023 ..SyncReport::default()
2024 };
2025 aggregate.merge(&SyncReport::default());
2026 assert!(
2027 aggregate.schema_floor.is_none(),
2028 "a later round without a floor clears it"
2029 );
2030 }
2031
2032 #[test]
2033 fn push_capacity_limits_are_hard_and_keep_intent() {
2034 let mut c = client();
2036 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2037 c.limits.max_push_operations_per_request = Some(1);
2038 let id = c
2039 .mutate(vec![
2040 Mutation::Upsert {
2041 table: "tasks".into(),
2042 values: Map::from_iter([
2043 ("id".into(), json!("a")),
2044 ("project_id".into(), json!("p1")),
2045 ]),
2046 base_version: None,
2047 },
2048 Mutation::Upsert {
2049 table: "tasks".into(),
2050 values: Map::from_iter([
2051 ("id".into(), json!("b")),
2052 ("project_id".into(), json!("p1")),
2053 ]),
2054 base_version: None,
2055 },
2056 ])
2057 .unwrap();
2058 let error = c.build_request(false).unwrap_err();
2059 let SyncOutcome::Failed {
2060 error_code,
2061 details,
2062 ..
2063 } = *error
2064 else {
2065 panic!("expected a capacity failure");
2066 };
2067 assert_eq!(error_code, "client.push_request_too_large");
2068 let details = details.expect("capacity details");
2069 assert_eq!(details["kind"], json!("operations"));
2070 assert_eq!(details["limit"], json!(1));
2071 assert_eq!(details["size"], json!(2));
2072 assert_eq!(details["clientCommitId"], json!(id));
2073 assert_eq!(c.pending_commit_ids(), vec![id]);
2074
2075 let mut c = client();
2077 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2078 c.limits.max_push_commits_per_request = Some(1);
2079 let first = c
2080 .mutate(vec![Mutation::Upsert {
2081 table: "tasks".into(),
2082 values: Map::from_iter([
2083 ("id".into(), json!("one")),
2084 ("project_id".into(), json!("p1")),
2085 ]),
2086 base_version: None,
2087 }])
2088 .unwrap();
2089 let second = c
2090 .mutate(vec![Mutation::Upsert {
2091 table: "tasks".into(),
2092 values: Map::from_iter([
2093 ("id".into(), json!("two")),
2094 ("project_id".into(), json!("p1")),
2095 ]),
2096 base_version: None,
2097 }])
2098 .unwrap();
2099 let (_, meta) = c.build_request(false).unwrap();
2100 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2101 assert_eq!(meta.deferred_commits, 1);
2102 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2103 }
2104
2105 #[test]
2106 fn push_request_byte_cap_defers_the_prefix_boundary_and_errors_the_head() {
2107 let mut c = client();
2110 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2111 let first = c
2112 .mutate(vec![Mutation::Upsert {
2113 table: "tasks".into(),
2114 values: Map::from_iter([
2115 ("id".into(), json!("one")),
2116 ("project_id".into(), json!("p1")),
2117 ]),
2118 base_version: None,
2119 }])
2120 .unwrap();
2121 let second = c
2122 .mutate(vec![Mutation::Upsert {
2123 table: "tasks".into(),
2124 values: Map::from_iter([
2125 ("id".into(), json!("two")),
2126 ("project_id".into(), json!("p1")),
2127 ]),
2128 base_version: None,
2129 }])
2130 .unwrap();
2131 let (message, meta) = c.build_request(false).unwrap();
2132 assert_eq!(meta.pushed_ids.len(), 2);
2133 let full = encode_message(&message).len();
2134 c.limits.max_push_request_bytes = Some(full - 1);
2135 let (_, meta) = c.build_request(false).unwrap();
2136 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2137 assert_eq!(meta.deferred_commits, 1);
2138 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2139
2140 let mut c = client();
2143 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2144 let huge = c
2145 .mutate(vec![Mutation::Upsert {
2146 table: "tasks".into(),
2147 values: Map::from_iter([
2148 ("id".into(), json!("x".repeat(20_000))),
2149 ("project_id".into(), json!("p1")),
2150 ]),
2151 base_version: None,
2152 }])
2153 .unwrap();
2154 let (message, meta) = c.build_request(false).unwrap();
2155 assert_eq!(meta.pushed_ids, vec![huge.clone()]);
2156 let with_huge = encode_message(&message).len();
2157 c.limits.max_push_request_bytes = Some(with_huge - 1);
2158 let error = c.build_request(false).unwrap_err();
2159 let SyncOutcome::Failed {
2160 error_code,
2161 details,
2162 ..
2163 } = *error
2164 else {
2165 panic!("expected a capacity failure");
2166 };
2167 assert_eq!(error_code, "client.push_request_too_large");
2168 let details = details.expect("capacity details");
2169 assert_eq!(details["kind"], json!("bytes"));
2170 assert_eq!(details["limit"], json!((with_huge - 1) as u64));
2171 assert_eq!(details["size"], json!(with_huge as u64));
2172 assert_eq!(details["clientCommitId"], json!(huge));
2173 assert_eq!(c.pending_commit_ids(), vec![huge]);
2174
2175 let mut c = client();
2177 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2178 c.limits.max_push_request_bytes = Some(1);
2179 let error = c.build_request(false).unwrap_err();
2180 let SyncOutcome::Failed { details, .. } = *error else {
2181 panic!("expected a capacity failure");
2182 };
2183 let details = details.expect("capacity details");
2184 assert_eq!(details["kind"], json!("bytes"));
2185 assert!(details.get("clientCommitId").is_none(), "{details}");
2186 }
2187
2188 #[test]
2189 fn push_limits_validate_at_construction() {
2190 for limits in [
2191 ClientLimits {
2192 max_push_commits_per_request: Some(0),
2193 ..Default::default()
2194 },
2195 ClientLimits {
2196 max_push_operations_per_request: Some(0),
2197 ..Default::default()
2198 },
2199 ClientLimits {
2200 max_push_request_bytes: Some(0),
2201 ..Default::default()
2202 },
2203 ClientLimits {
2204 max_push_request_bytes: Some(u32::MAX as usize + 1),
2205 ..Default::default()
2206 },
2207 ] {
2208 let schema = json!({"version":1,"tables":[]});
2209 let error = SyncClient::with_connection(
2210 "bad-limits".into(),
2211 &schema,
2212 limits,
2213 Connection::open_in_memory().unwrap(),
2214 )
2215 .err();
2216 assert_eq!(
2217 error.as_deref(),
2218 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2219 );
2220 }
2221 }
2222
2223 #[test]
2224 fn open_path_rejects_invalid_push_limits_before_storage() {
2225 for identity in [false, true] {
2226 let path = std::env::temp_dir().join(format!(
2227 "syncular-bad-limits-{}.sqlite",
2228 uuid::Uuid::new_v4()
2229 ));
2230 let schema = json!({"version":1,"tables":[]});
2231 let limits = ClientLimits {
2232 max_push_request_bytes: Some(0),
2233 ..Default::default()
2234 };
2235 let error = if identity {
2236 SyncClient::open_path_with_identity(None, &schema, limits, path.to_str().unwrap())
2237 } else {
2238 SyncClient::open_path("bad-limits".into(), &schema, limits, path.to_str().unwrap())
2239 }
2240 .err();
2241 assert_eq!(
2242 error.as_deref(),
2243 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2244 );
2245 assert!(
2246 !path.exists(),
2247 "an invalid limit is rejected before the database file is created"
2248 );
2249 }
2250 }
2251
2252 #[test]
2253 fn failed_progress_retry_delay_doubles_to_the_cap() {
2254 let mut client = client();
2255 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2256 let mut transport = ServerErrorTransport {
2257 code: "sync.internal_error",
2258 retryable: true,
2259 };
2260 let delays: Vec<Option<u64>> = (0..9)
2261 .map(|_| {
2262 client.sync(&mut transport);
2263 client.progress().snapshot().unwrap().retry_delay_ms
2264 })
2265 .collect();
2266 assert_eq!(
2267 delays,
2268 [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
2269 );
2270 }
2271
2272 #[test]
2273 fn background_retry_deadlines_back_off_and_reset() {
2274 let mut client = client();
2275 client.schedule_background_retry();
2276 assert!(matches!(
2277 client.drain_sync_intents().as_slice(),
2278 [SyncIntent::Background { delay_ms: 250 }]
2279 ));
2280 client.schedule_background_retry();
2281 assert!(matches!(
2282 client.drain_sync_intents().as_slice(),
2283 [SyncIntent::Background { delay_ms: 500 }]
2284 ));
2285 client.reset_background_retry();
2286 client.schedule_background_retry();
2287 assert!(matches!(
2288 client.drain_sync_intents().as_slice(),
2289 [SyncIntent::Background { delay_ms: 250 }]
2290 ));
2291 }
2292
2293 #[test]
2294 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
2295 let mut client = client();
2296 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2297 client
2298 .subscribe(
2299 "epoch-tasks".to_owned(),
2300 "tasks".to_owned(),
2301 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2302 None,
2303 )
2304 .expect("subscribe");
2305 client.drain_sync_intents();
2306
2307 client
2308 .run_log_epoch_reset("epoch-2")
2309 .expect("reset partition log epoch");
2310
2311 assert!(matches!(
2312 client.drain_sync_intents().as_slice(),
2313 [SyncIntent::Interactive]
2314 ));
2315 }
2316
2317 #[derive(Default)]
2318 struct CountingRealtimeTransport {
2319 connects: usize,
2320 closes: usize,
2321 messages: Vec<String>,
2322 blob_uploads: Vec<(String, Vec<u8>)>,
2323 blob_download: Option<Vec<u8>>,
2324 sync_calls: usize,
2325 realtime_sync_calls: usize,
2326 refuse_connect: bool,
2327 }
2328
2329 impl Transport for CountingRealtimeTransport {
2330 fn blob_upload(
2331 &mut self,
2332 blob_id: &str,
2333 bytes: &[u8],
2334 _media_type: Option<&str>,
2335 ) -> Result<(), TransportError> {
2336 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
2337 Ok(())
2338 }
2339
2340 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
2341 self.blob_download
2342 .clone()
2343 .map(BlobDownload::Bytes)
2344 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
2345 }
2346
2347 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2348 self.sync_calls += 1;
2349 Err(TransportError::new("sync.transport_failed", "offline"))
2350 }
2351
2352 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2353 self.realtime_sync_calls += 1;
2354 Err(TransportError::new("sync.transport_failed", "offline"))
2355 }
2356
2357 fn download_segment(
2358 &mut self,
2359 _request: &SegmentRequest,
2360 _on_progress: &mut dyn FnMut(u64),
2361 ) -> Result<Vec<u8>, TransportError> {
2362 Err(TransportError::new("sync.transport_failed", "offline"))
2363 }
2364
2365 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2366 self.connects += 1;
2367 if self.refuse_connect {
2368 return Err(TransportError::new("transport.failed", "refused"));
2369 }
2370 Ok(())
2371 }
2372
2373 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2374 self.messages.push(text.to_owned());
2375 Ok(())
2376 }
2377
2378 fn realtime_close(&mut self) -> Result<(), TransportError> {
2379 self.closes += 1;
2380 Ok(())
2381 }
2382 }
2383
2384 #[test]
2385 fn blob_results_own_downloaded_bytes() {
2386 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
2387 {"name":"id","type":"string","nullable":false},
2388 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
2389 let bytes = b"owned typed result".to_vec();
2390 let blob_id = blob_id_for(&bytes);
2391 let mut transport = CountingRealtimeTransport {
2392 blob_download: Some(bytes.clone()),
2393 ..Default::default()
2394 };
2395 let result = {
2396 let mut client =
2397 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
2398 .unwrap();
2399 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
2400 assert_eq!(result.blob_id, blob_id);
2401 assert_eq!(result.byte_length, bytes.len() as i64);
2402 assert_eq!(result.bytes, bytes);
2403 assert_eq!(result.media_type, None);
2404 assert_eq!(
2405 client
2406 .conn
2407 .query_row(
2408 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
2409 rusqlite::params![blob_id],
2410 |row| row.get::<_, i64>(0),
2411 )
2412 .unwrap(),
2413 0
2414 );
2415 result
2416 };
2417 assert_eq!(result.bytes, bytes);
2418
2419 let mut client =
2420 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
2421 assert_eq!(
2422 client
2423 .fetch_blob_bytes(&mut transport, "not json")
2424 .unwrap_err(),
2425 (
2426 "client.failed".to_owned(),
2427 "blob ref is not JSON".to_owned()
2428 )
2429 );
2430 }
2431
2432 #[test]
2433 fn visible_blob_references_survive_cache_trimming() {
2434 let schema = json!({"version":1,"tables":[
2435 {"name":"attachments","primaryKey":"id","columns":[
2436 {"name":"id","type":"string","nullable":false},
2437 {"name":"body","type":"blob_ref","nullable":true},
2438 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
2439 {"name":"avatars","primaryKey":"id","columns":[
2440 {"name":"id","type":"string","nullable":false},
2441 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
2442 ]});
2443 let bytes = b"referenced above cap".to_vec();
2444 let blob_id = blob_id_for(&bytes);
2445 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
2446 let mut client = SyncClient::new(
2447 "visible-blob-refs".to_owned(),
2448 &schema,
2449 ClientLimits {
2450 blob_cache_max_bytes: Some(1),
2451 ..Default::default()
2452 },
2453 )
2454 .unwrap();
2455 client.conn.execute(
2456 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
2457 rusqlite::params![reference, reference, reference, reference],
2458 ).unwrap();
2459 client
2460 .conn
2461 .execute(
2462 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
2463 rusqlite::params![reference],
2464 )
2465 .unwrap();
2466 client.conn.execute(
2467 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
2468 [],
2469 ).unwrap();
2470 let mut transport = CountingRealtimeTransport {
2471 blob_download: Some(bytes.clone()),
2472 ..Default::default()
2473 };
2474
2475 assert_eq!(
2476 client
2477 .fetch_blob_bytes(&mut transport, &blob_id)
2478 .unwrap()
2479 .bytes,
2480 bytes
2481 );
2482 let rows = client
2483 .conn
2484 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
2485 .unwrap()
2486 .query_map([], |row| row.get::<_, String>(0))
2487 .unwrap()
2488 .collect::<Result<Vec<_>, _>>()
2489 .unwrap();
2490 assert_eq!(rows, vec![blob_id]);
2491 }
2492
2493 #[test]
2494 fn closed_transport_never_invokes_network_and_reopen_is_host_owned() {
2495 let mut transport = CountingRealtimeTransport::default();
2496 let path = std::env::temp_dir().join(format!(
2497 "syncular-transport-gate-{}.db",
2498 uuid::Uuid::new_v4()
2499 ));
2500 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
2501 {"name":"id","type":"string","nullable":false},
2502 {"name":"project_id","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
2503 let mut client = SyncClient::open_path(
2504 "gate".into(),
2505 &schema,
2506 ClientLimits::default(),
2507 path.to_str().unwrap(),
2508 )
2509 .unwrap();
2510 client.connect_realtime(&mut transport).unwrap();
2511 client.set_transport_enabled(&mut transport, false);
2512 assert_eq!(transport.closes, 1);
2513 client.begin_security_preflight();
2514 client.activate_security(Default::default()).unwrap();
2515 client
2516 .subscribe(
2517 "tasks".into(),
2518 "tasks".into(),
2519 vec![("project_id".into(), vec!["p1".into()])],
2520 None,
2521 )
2522 .unwrap();
2523 for id in ["one", "two"] {
2524 client
2525 .mutate(vec![Mutation::Upsert {
2526 table: "tasks".into(),
2527 values: Map::from_iter([
2528 ("id".into(), json!(id)),
2529 ("project_id".into(), json!("p1")),
2530 ]),
2531 base_version: None,
2532 }])
2533 .unwrap();
2534 }
2535 assert_eq!(
2536 client
2537 .query("SELECT id FROM tasks ORDER BY id", &[])
2538 .unwrap()
2539 .len(),
2540 2
2541 );
2542 assert!(
2543 matches!(client.sync(&mut transport), SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.offline")
2544 );
2545 assert!(client
2546 .connect_realtime(&mut transport)
2547 .unwrap_err()
2548 .starts_with("sync.offline:"));
2549 assert!(client
2550 .set_presence(&mut transport, "project:p1", None)
2551 .unwrap_err()
2552 .starts_with("sync.offline:"));
2553 assert_eq!(transport.sync_calls, 0);
2554 assert_eq!(transport.realtime_sync_calls, 0);
2555 assert_eq!(transport.connects, 1);
2556 assert!(transport.messages.is_empty());
2557 assert!(matches!(
2558 client.drain_sync_intents().as_slice(),
2559 [SyncIntent::None]
2560 ));
2561 let pending = client.pending_commit_ids();
2562 drop(client);
2563 let mut reopened = SyncClient::open_path(
2564 "gate".into(),
2565 &schema,
2566 ClientLimits::default(),
2567 path.to_str().unwrap(),
2568 )
2569 .unwrap();
2570 assert!(reopened.transport_enabled());
2571 reopened.set_transport_enabled(&mut transport, false);
2572 assert_eq!(reopened.pending_commit_ids(), pending);
2573 assert_eq!(
2574 reopened.query("SELECT id FROM tasks", &[]).unwrap().len(),
2575 2
2576 );
2577 reopened.set_transport_enabled(&mut transport, true);
2578 assert!(matches!(
2579 reopened.drain_sync_intents().as_slice(),
2580 [SyncIntent::Interactive]
2581 ));
2582 reopened.set_transport_enabled(&mut transport, true);
2583 assert!(reopened.drain_sync_intents().is_empty());
2584 drop(reopened);
2585 std::fs::remove_file(path).unwrap();
2586 }
2587
2588 #[test]
2589 fn realtime_connection_ownership_is_idempotent() {
2590 let mut client = client();
2591 let mut transport = CountingRealtimeTransport {
2592 connects: 0,
2593 closes: 0,
2594 ..Default::default()
2595 };
2596 client
2597 .connect_realtime(&mut transport)
2598 .expect("first connect");
2599 client
2600 .connect_realtime(&mut transport)
2601 .expect("idempotent connect");
2602 assert_eq!(transport.connects, 1);
2603 client.disconnect_realtime(&mut transport);
2604 client.disconnect_realtime(&mut transport);
2605 assert_eq!(transport.closes, 1);
2606 client
2607 .connect_realtime(&mut transport)
2608 .expect("deliberate reconnect");
2609 assert_eq!(transport.connects, 2);
2610 }
2611
2612 #[test]
2613 fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
2614 let mut client = client();
2615 client
2616 .set_realtime_policy(RealtimePolicy::Required)
2617 .expect("set policy");
2618 let mut transport = CountingRealtimeTransport::default();
2619 match client.sync(&mut transport) {
2620 SyncOutcome::RealtimeUnavailable {
2621 state,
2622 reason_code,
2623 retry_delay_ms,
2624 } => {
2625 assert_eq!(state, RealtimeState::Disconnected);
2626 assert_eq!(reason_code, None);
2627 assert_eq!(retry_delay_ms, 250);
2628 }
2629 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2630 }
2631 assert_eq!(transport.sync_calls, 0);
2632 assert!(matches!(
2633 client.drain_sync_intents().as_slice(),
2634 [SyncIntent::Background { delay_ms: 250 }]
2635 ));
2636 let snapshot = client
2637 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2638 .expect("diagnostics");
2639 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2640 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
2641 assert_eq!(snapshot.host.realtime_reason_code, None);
2642 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2643 assert_eq!(
2644 snapshot
2645 .last_round
2646 .as_ref()
2647 .and_then(|round| round.error_code.clone()),
2648 Some(REALTIME_UNAVAILABLE_CODE.to_owned())
2649 );
2650 }
2651
2652 #[test]
2653 fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
2654 let mut client = client();
2655 client
2656 .set_realtime_policy(RealtimePolicy::Required)
2657 .expect("set policy");
2658 let mut transport = CountingRealtimeTransport {
2659 refuse_connect: true,
2660 ..Default::default()
2661 };
2662 assert!(client.connect_realtime(&mut transport).is_err());
2663 assert_eq!(client.realtime_state(), RealtimeState::Refused);
2664 let snapshot = client
2665 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2666 .expect("diagnostics");
2667 assert_eq!(
2668 snapshot.host.realtime_reason_code.as_deref(),
2669 Some("transport.failed")
2670 );
2671 match client.sync(&mut transport) {
2672 SyncOutcome::RealtimeUnavailable {
2673 state, reason_code, ..
2674 } => {
2675 assert_eq!(state, RealtimeState::Refused);
2676 assert_eq!(reason_code.as_deref(), Some("transport.failed"));
2677 }
2678 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2679 }
2680 assert_eq!(transport.sync_calls, 0);
2681 assert_eq!(transport.realtime_sync_calls, 0);
2682 }
2683
2684 #[test]
2685 fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
2686 let mut client = client();
2687 client
2688 .set_realtime_policy(RealtimePolicy::Required)
2689 .expect("set policy");
2690 let mut transport = CountingRealtimeTransport::default();
2691 client.connect_realtime(&mut transport).expect("connect");
2692 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2693 assert!(
2694 client.set_realtime_policy(RealtimePolicy::Off).is_err(),
2695 "a policy change cannot silently re-route a connected socket"
2696 );
2697 assert!(matches!(
2698 client.sync(&mut transport),
2699 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
2700 ));
2701 assert_eq!(client.realtime_state(), RealtimeState::Lost);
2702 assert_eq!(transport.realtime_sync_calls, 1);
2703 assert_eq!(transport.sync_calls, 0);
2704 let snapshot = client
2705 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2706 .expect("diagnostics");
2707 assert_eq!(
2708 snapshot.host.realtime_reason_code.as_deref(),
2709 Some("sync.transport_failed")
2710 );
2711 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
2712 match client.sync(&mut transport) {
2713 SyncOutcome::RealtimeUnavailable {
2714 state, reason_code, ..
2715 } => {
2716 assert_eq!(state, RealtimeState::Lost);
2717 assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
2718 }
2719 other => panic!("expected RealtimeUnavailable, got {other:?}"),
2720 }
2721 assert_eq!(transport.sync_calls, 0);
2722 client.disconnect_realtime(&mut transport);
2723 assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
2724 assert_eq!(transport.closes, 1);
2725 client.connect_realtime(&mut transport).expect("reconnect");
2726 assert_eq!(client.realtime_state(), RealtimeState::Connected);
2727 assert_eq!(transport.connects, 2);
2728 }
2729
2730 #[test]
2731 fn off_policy_uses_http_and_refuses_an_explicit_connect() {
2732 let mut client = client();
2733 client
2734 .set_realtime_policy(RealtimePolicy::Off)
2735 .expect("set policy");
2736 let mut transport = CountingRealtimeTransport::default();
2737 assert!(client.connect_realtime(&mut transport).is_err());
2738 assert_eq!(client.realtime_state(), RealtimeState::Disabled);
2739 assert!(matches!(
2740 client.sync(&mut transport),
2741 SyncOutcome::Failed { .. }
2742 ));
2743 assert_eq!(transport.sync_calls, 1);
2744 assert_eq!(transport.realtime_sync_calls, 0);
2745 let snapshot = client
2746 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2747 .expect("diagnostics");
2748 assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
2749 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
2750 }
2751
2752 #[test]
2753 fn optional_policy_keeps_the_http_round() {
2754 let mut client = client();
2755 let mut transport = CountingRealtimeTransport::default();
2756 assert!(matches!(
2757 client.sync(&mut transport),
2758 SyncOutcome::Failed { .. }
2759 ));
2760 assert_eq!(transport.sync_calls, 1);
2761 assert_eq!(transport.realtime_sync_calls, 0);
2762 let snapshot = client
2763 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2764 .expect("diagnostics");
2765 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
2766 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
2767 }
2768
2769 #[test]
2770 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
2771 let mut client = client();
2772 client
2773 .subscribe(
2774 "repair-tasks".to_owned(),
2775 "tasks".to_owned(),
2776 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2777 None,
2778 )
2779 .expect("subscribe");
2780 {
2781 let sub = client
2782 .subs
2783 .iter_mut()
2784 .find(|sub| sub.id == "repair-tasks")
2785 .expect("subscription");
2786 sub.cursor = 42;
2787 sub.synced_once = true;
2788 let persisted = sub.clone();
2789 client.persist_sub(&persisted).unwrap();
2790 }
2791 for table in ["_syncular_base_tasks", "tasks"] {
2792 client
2793 .conn
2794 .execute(
2795 &format!(
2796 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
2797 ),
2798 rusqlite::params!["server-row", "p1"],
2799 )
2800 .expect("seed server row");
2801 }
2802 let pending = client
2803 .mutate(vec![Mutation::Upsert {
2804 table: "tasks".to_owned(),
2805 values: Map::from_iter([
2806 ("id".to_owned(), Value::from("offline-row")),
2807 ("project_id".to_owned(), Value::from("p1")),
2808 ]),
2809 base_version: None,
2810 }])
2811 .expect("queue offline work");
2812 client.drain_change_batches();
2813 client.drain_sync_intents();
2814
2815 assert_eq!(
2816 client
2817 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2818 rebootstrap_id: "support-case-001".to_owned(),
2819 })
2820 .expect("rebootstrap"),
2821 LocalDataRebootstrapResult {
2822 already_applied: false,
2823 retained_commits: 1,
2824 reset_subscriptions: 1,
2825 }
2826 );
2827 let visible_ids = client
2828 .conn
2829 .prepare("SELECT id FROM tasks ORDER BY id")
2830 .expect("prepare visible ids")
2831 .query_map([], |row| row.get::<_, String>(0))
2832 .expect("query visible ids")
2833 .collect::<Result<Vec<_>, _>>()
2834 .expect("collect visible ids");
2835 assert_eq!(visible_ids, vec!["offline-row"]);
2836 assert_eq!(client.pending_commit_ids(), vec![pending]);
2837 assert_eq!(
2838 client
2839 .subscription_state("repair-tasks")
2840 .expect("subscription")
2841 .cursor,
2842 -1
2843 );
2844 assert!(client.upgrading());
2845 assert!(client.sync_needed());
2846 assert!(matches!(
2847 client.drain_sync_intents().as_slice(),
2848 [SyncIntent::Interactive]
2849 ));
2850 assert_eq!(client.drain_change_batches().len(), 1);
2851
2852 assert_eq!(
2853 client
2854 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2855 rebootstrap_id: "support-case-001".to_owned(),
2856 })
2857 .expect("idempotent retry"),
2858 LocalDataRebootstrapResult {
2859 already_applied: true,
2860 retained_commits: 1,
2861 reset_subscriptions: 1,
2862 }
2863 );
2864 assert!(client.drain_change_batches().is_empty());
2865 }
2866
2867 #[test]
2868 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
2869 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
2870 assert_eq!(
2871 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
2872 (3, 4)
2873 );
2874 assert_eq!(
2875 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
2876 (0, 0)
2877 );
2878 for malformed in [
2879 "",
2880 "{}",
2881 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
2882 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
2883 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
2884 ] {
2885 assert_eq!(
2886 decode_local_rebootstrap_receipt(malformed)
2887 .expect_err("malformed receipt must fail"),
2888 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
2889 );
2890 }
2891 }
2892
2893 #[test]
2894 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
2895 let path = std::env::temp_dir().join(format!(
2896 "syncular-rebootstrap-receipt-{}.db",
2897 uuid::Uuid::new_v4()
2898 ));
2899 let schema = json!({
2900 "version": 1,
2901 "tables": [{
2902 "name": "tasks",
2903 "primaryKey": "id",
2904 "columns": [
2905 { "name": "id", "type": "string", "nullable": false },
2906 { "name": "project_id", "type": "string", "nullable": false }
2907 ],
2908 "scopes": [{ "pattern": "project:{project_id}" }]
2909 }]
2910 });
2911 let path_string = path.to_str().expect("UTF-8 temp path");
2912
2913 {
2914 let mut first = SyncClient::open_path(
2915 "repair-restart-client".to_owned(),
2916 &schema,
2917 ClientLimits::default(),
2918 path_string,
2919 )
2920 .expect("first open");
2921 first
2922 .subscribe(
2923 "repair-tasks".to_owned(),
2924 "tasks".to_owned(),
2925 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2926 None,
2927 )
2928 .expect("subscribe");
2929 first
2930 .mutate(vec![Mutation::Upsert {
2931 table: "tasks".to_owned(),
2932 values: Map::from_iter([
2933 ("id".to_owned(), Value::from("offline-row")),
2934 ("project_id".to_owned(), Value::from("p1")),
2935 ]),
2936 base_version: None,
2937 }])
2938 .expect("queue offline work");
2939 assert_eq!(
2940 first
2941 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2942 rebootstrap_id: "restart-receipt".to_owned(),
2943 })
2944 .expect("first rebootstrap"),
2945 LocalDataRebootstrapResult {
2946 already_applied: false,
2947 retained_commits: 1,
2948 reset_subscriptions: 1,
2949 }
2950 );
2951 }
2952
2953 let mut reopened = SyncClient::open_path(
2954 "repair-restart-client".to_owned(),
2955 &schema,
2956 ClientLimits::default(),
2957 path_string,
2958 )
2959 .expect("reopen");
2960 reopened.drain_change_batches();
2961 reopened.drain_sync_intents();
2962 assert_eq!(
2963 reopened
2964 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2965 rebootstrap_id: "restart-receipt".to_owned(),
2966 })
2967 .expect("receipt replay"),
2968 LocalDataRebootstrapResult {
2969 already_applied: true,
2970 retained_commits: 1,
2971 reset_subscriptions: 1,
2972 }
2973 );
2974 assert!(reopened.drain_change_batches().is_empty());
2975 assert!(reopened.drain_sync_intents().is_empty());
2976 drop(reopened);
2977 std::fs::remove_file(path).expect("remove temp database");
2978 }
2979
2980 #[test]
2981 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
2982 let mut malformed = client();
2983 malformed
2984 .subscribe(
2985 "repair-tasks".to_owned(),
2986 "tasks".to_owned(),
2987 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2988 None,
2989 )
2990 .expect("subscribe");
2991 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
2992 malformed.drain_change_batches();
2993 malformed.drain_sync_intents();
2994 let malformed_error = malformed
2995 .rebootstrap_local_data(&LocalDataRebootstrapInput {
2996 rebootstrap_id: "malformed".to_owned(),
2997 })
2998 .expect_err("malformed receipt must fail closed");
2999 assert_eq!(
3000 malformed_error,
3001 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3002 );
3003 assert!(!malformed.upgrading());
3004 assert_eq!(
3005 malformed
3006 .subscription_state("repair-tasks")
3007 .expect("unchanged subscription")
3008 .cursor,
3009 -1
3010 );
3011 assert!(malformed.drain_change_batches().is_empty());
3012 assert!(malformed.drain_sync_intents().is_empty());
3013
3014 let mut unreadable = client();
3015 unreadable
3016 .conn
3017 .execute(
3018 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
3019 rusqlite::params!["server-row", "p1"],
3020 )
3021 .expect("seed visible row");
3022 unreadable
3023 .conn
3024 .execute("DROP TABLE _syncular_meta", [])
3025 .expect("break marker storage");
3026 let unreadable_error = unreadable
3027 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3028 rebootstrap_id: "unreadable".to_owned(),
3029 })
3030 .expect_err("unreadable marker storage must fail closed");
3031 assert_eq!(
3032 unreadable_error,
3033 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
3034 );
3035 let visible_rows = unreadable
3036 .conn
3037 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
3038 .expect("visible projection remains");
3039 assert_eq!(visible_rows, 1);
3040 assert!(!unreadable.upgrading());
3041 assert!(unreadable.drain_change_batches().is_empty());
3042 assert!(unreadable.drain_sync_intents().is_empty());
3043 }
3044
3045 #[test]
3046 fn local_rebootstrap_cannot_bypass_schema_floor() {
3047 let mut client = client();
3048 client.set_schema_floor(Some(SchemaFloor {
3049 required_schema_version: Some(2),
3050 latest_schema_version: Some(2),
3051 }));
3052 let error = client
3053 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3054 rebootstrap_id: "blocked-floor".to_owned(),
3055 })
3056 .expect_err("schema floor must block repair");
3057 assert!(error.contains("cannot bypass an active schema-floor stop"));
3058 }
3059
3060 #[test]
3061 fn diego_applied_sparse_moves_stay_visible_until_pull_image_arrives() {
3062 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3063 "columns":[{"name":"id","type":"string","nullable":false},
3064 {"name":"project_id","type":"string","nullable":false},
3065 {"name":"starts_at_ms","type":"integer","nullable":false},
3066 {"name":"version","type":"integer","nullable":false}],
3067 "scopes":[{"pattern":"project:{project_id}"}]}]});
3068 let mut client =
3069 SyncClient::new("diego-ack-gap".into(), &schema, ClientLimits::default()).unwrap();
3070 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3071 client
3072 .write_base_row(
3073 "tasks",
3074 &vec![
3075 Some(ColumnValue::String("t1".into())),
3076 Some(ColumnValue::String("p1".into())),
3077 Some(ColumnValue::Integer(100)),
3078 Some(ColumnValue::Integer(1)),
3079 ],
3080 1,
3081 )
3082 .unwrap();
3083 client.rebuild_overlay().expect("rebuild overlay");
3084 let first = client
3085 .patch(
3086 "tasks",
3087 "t1",
3088 Map::from_iter([
3089 ("starts_at_ms".into(), json!(200)),
3090 ("version".into(), json!(2)),
3091 ]),
3092 None,
3093 )
3094 .unwrap();
3095 let (_, first_request) = client.build_request(false).unwrap();
3096 let second = client
3098 .patch(
3099 "tasks",
3100 "t1",
3101 Map::from_iter([
3102 ("starts_at_ms".into(), json!(300)),
3103 ("version".into(), json!(3)),
3104 ]),
3105 None,
3106 )
3107 .unwrap();
3108 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3109 for (id, seq, meta) in [
3110 (first, 2, first_request),
3111 (second, 3, client.build_request(false).unwrap().1),
3112 ] {
3113 let response = Message {
3114 wire_version: WIRE_VERSION,
3115 msg_kind: MsgKind::Response,
3116 frames: vec![
3117 Frame::RespHeader {
3118 required_schema_version: None,
3119 latest_schema_version: None,
3120 log_epoch: Some("epoch-1".into()),
3121 reset_required: Some(false),
3122 },
3123 Frame::PushResult {
3124 client_commit_id: id,
3125 status: PushStatus::Applied,
3126 commit_seq: Some(seq),
3127 results: vec![OpResult::Applied { op_index: 0 }],
3128 },
3129 ],
3130 };
3131 assert!(matches!(
3132 client.process_response(&mut transport, response, &meta),
3133 SyncOutcome::Ok(_)
3134 ));
3135 let row = &client
3136 .query("SELECT starts_at_ms, version FROM tasks WHERE id='t1'", &[])
3137 .unwrap()[0];
3138 assert_eq!(
3139 row["starts_at_ms"],
3140 json!(300),
3141 "an applied ACK without a pull image must not expose the old base"
3142 );
3143 assert_eq!(
3144 row["version"],
3145 json!(3),
3146 "the queued domain version must remain monotonic"
3147 );
3148 }
3149 }
3150
3151 #[test]
3152 fn first_epoch_acquisition_preserves_tables_subscriptions_and_local_intent() {
3153 for stored in [None, Some("epoch-old")] {
3154 let mut client = client();
3155 client
3156 .subscribe(
3157 "tasks".into(),
3158 "tasks".into(),
3159 vec![("project_id".into(), vec!["p1".into()])],
3160 None,
3161 )
3162 .unwrap();
3163 client
3164 .conn
3165 .execute_batch(
3166 "CREATE TRIGGER epoch_table_identity AFTER INSERT ON tasks BEGIN SELECT 1; END",
3167 )
3168 .unwrap();
3169 let id = client
3170 .mutate(vec![Mutation::Upsert {
3171 table: "tasks".into(),
3172 values: Map::from_iter([
3173 ("id".into(), json!("offline")),
3174 ("project_id".into(), json!("p1")),
3175 ]),
3176 base_version: None,
3177 }])
3178 .unwrap();
3179 if let Some(epoch) = stored {
3180 client.set_meta(LOG_EPOCH_KEY, epoch);
3181 }
3182 let before = serde_json::to_value(client.subscription_state("tasks")).unwrap();
3183 let (_, meta) = client.build_request(false).unwrap();
3184 client.drain_change_batches();
3185 client.drain_sync_intents();
3186 let response = Message {
3187 wire_version: WIRE_VERSION,
3188 msg_kind: MsgKind::Response,
3189 frames: vec![Frame::RespHeader {
3190 required_schema_version: None,
3191 latest_schema_version: None,
3192 log_epoch: Some("epoch-new".into()),
3193 reset_required: Some(true),
3194 }],
3195 };
3196 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3197 let outcome = client.process_response(&mut transport, response, &meta);
3198 let SyncOutcome::Ok(report) = outcome else {
3199 panic!("epoch round failed: {outcome:?}");
3200 };
3201 assert_eq!(
3202 report.resets,
3203 if stored.is_none() {
3204 Vec::<String>::new()
3205 } else {
3206 vec!["tasks".into()]
3207 }
3208 );
3209 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3210 assert_eq!(client.pending_commit_ids(), vec![id]);
3211 assert_eq!(
3212 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3213 json!("offline")
3214 );
3215 let trigger: i64 = client
3216 .conn
3217 .query_row(
3218 "SELECT count(*) FROM sqlite_master WHERE name='epoch_table_identity'",
3219 [],
3220 |row| row.get(0),
3221 )
3222 .unwrap();
3223 if stored.is_none() {
3224 assert!(!client.upgrading);
3225 assert!(client
3226 .drain_change_batches()
3227 .iter()
3228 .all(|batch| batch.status.as_ref().is_none_or(|status| !status.upgrading)));
3229 assert_eq!(
3230 serde_json::to_value(client.subscription_state("tasks")).unwrap(),
3231 before
3232 );
3233 assert_eq!(trigger, 1, "acquisition must not drop the synced table");
3234 } else {
3235 assert!(client.upgrading);
3236 assert_eq!(trigger, 0, "an epoch change must drop the old table");
3237 }
3238 assert!(matches!(
3239 client.drain_sync_intents().as_slice(),
3240 [SyncIntent::Interactive]
3241 ));
3242 }
3243 }
3244
3245 #[test]
3246 fn epoch_acquisition_storage_failures_preserve_ready_state_and_intent() {
3247 for key in ["logEpoch", "localRevision"] {
3248 let mut client = client();
3249 let id = client
3250 .mutate(vec![Mutation::Upsert {
3251 table: "tasks".into(),
3252 values: Map::from_iter([
3253 ("id".into(), json!("offline")),
3254 ("project_id".into(), json!("p1")),
3255 ]),
3256 base_version: None,
3257 }])
3258 .unwrap();
3259 let revision = client.local_revision();
3260 let needed = client.sync_needed;
3261 client.drain_change_batches();
3262 client.drain_sync_intents();
3263 client.conn.execute_batch(&format!("CREATE TRIGGER fail_acquisition BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected acquisition failure'); END")).unwrap();
3264 assert!(client.run_log_epoch_reset("epoch-new").is_err());
3265 assert!(client.get_meta(LOG_EPOCH_KEY).is_none());
3266 assert!(!client.upgrading);
3267 assert_eq!(client.sync_needed, needed);
3268 assert_eq!(client.local_revision(), revision);
3269 assert_eq!(client.pending_commit_ids(), [id]);
3270 assert!(client.drain_change_batches().is_empty());
3271 assert!(client.drain_sync_intents().is_empty());
3272 }
3273 }
3274
3275 #[test]
3276 fn failed_log_epoch_resets_restore_memory_and_durable_state() {
3277 for key in [
3278 "localSchemaVersion",
3279 "localSchemaDescriptor",
3280 "logEpoch",
3281 "localRevision",
3282 ] {
3283 let mut client = client();
3284 client.set_meta(LOG_EPOCH_KEY, "epoch-old");
3285 let commit = client
3286 .mutate(vec![Mutation::Upsert {
3287 table: "tasks".into(),
3288 values: Map::from_iter([
3289 ("id".into(), json!("offline")),
3290 ("project_id".into(), json!("p1")),
3291 ]),
3292 base_version: None,
3293 }])
3294 .unwrap();
3295 client.active_round = Some(uuid::Uuid::new_v4());
3296 client.overlay_dirty.set(false);
3297 let round = client.active_round;
3298 let needed = client.sync_needed;
3299 let revision = client.local_revision();
3300 let rows = serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap();
3301 client.drain_change_batches();
3302 client.drain_sync_intents();
3303 client.conn.execute_batch(&format!("CREATE TRIGGER fail_reset BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected reset failure'); END")).unwrap();
3304 assert!(client.run_log_epoch_reset("epoch-new").is_err(), "{key}");
3305 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-old"));
3306 assert!(!client.upgrading);
3307 assert_eq!(client.active_round, round);
3308 assert!(!client.overlay_dirty.get());
3309 assert_eq!(client.sync_needed, needed);
3310 assert_eq!(client.local_revision(), revision);
3311 assert_eq!(
3312 client.pending_commit_ids().as_slice(),
3313 std::slice::from_ref(&commit)
3314 );
3315 assert_eq!(
3316 serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap(),
3317 rows
3318 );
3319 assert!(client.drain_change_batches().is_empty());
3320 assert!(client.drain_sync_intents().is_empty());
3321 assert!(client.conn.is_autocommit());
3322 client
3323 .conn
3324 .execute_batch("DROP TRIGGER fail_reset")
3325 .unwrap();
3326 client.run_log_epoch_reset("epoch-new").unwrap();
3327 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3328 assert_eq!(client.pending_commit_ids(), [commit]);
3329 }
3330 }
3331
3332 #[test]
3333 fn acknowledged_restore_and_replay_ignore_unrelated_subscription_count() {
3334 for subscriptions in [0, 2, 6, 12] {
3335 let mut client = SyncClient::new("ack-scope".into(), &json!({ "version": 1, "tables": [
3336 { "name": "tasks", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] },
3337 { "name": "docs", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] }
3338 ] }), ClientLimits::default()).unwrap();
3339 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3340 client.conn.execute_batch("CREATE TABLE ack_writes(kind TEXT); CREATE TRIGGER count_ack_restore AFTER DELETE ON tasks BEGIN INSERT INTO ack_writes VALUES('restore'); END; CREATE TRIGGER count_ack_replay AFTER INSERT ON tasks BEGIN INSERT INTO ack_writes VALUES('replay'); END;").unwrap();
3341 for index in 0..32 {
3342 client
3343 .mutate(vec![Mutation::Upsert {
3344 table: "tasks".into(),
3345 values: Map::from_iter([
3346 ("id".into(), json!(format!("task-{index}"))),
3347 ("project_id".into(), json!("p1")),
3348 ]),
3349 base_version: None,
3350 }])
3351 .unwrap();
3352 }
3353 let (_, meta) = client.build_request(false).unwrap();
3354 let mut frames = vec![Frame::RespHeader {
3355 required_schema_version: None,
3356 latest_schema_version: None,
3357 log_epoch: Some("epoch-1".into()),
3358 reset_required: Some(false),
3359 }];
3360 frames.extend(meta.pushed_ids.iter().enumerate().map(|(index, id)| {
3361 Frame::PushResult {
3362 client_commit_id: id.clone(),
3363 status: PushStatus::Applied,
3364 commit_seq: Some(index as i64 + 1),
3365 results: vec![OpResult::Applied { op_index: 0 }],
3366 }
3367 }));
3368 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3369 client.acknowledged_replay_count.set(0);
3370 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3371 assert!(matches!(
3372 client.process_response(
3373 &mut transport,
3374 Message {
3375 wire_version: WIRE_VERSION,
3376 msg_kind: MsgKind::Response,
3377 frames
3378 },
3379 &meta
3380 ),
3381 SyncOutcome::Ok(_)
3382 ));
3383 assert_eq!(client.acknowledged_replay_count.get(), 32);
3384 assert_eq!(
3385 client
3386 .query(
3387 "SELECT count(*) AS n FROM ack_writes WHERE kind='restore'",
3388 &[]
3389 )
3390 .unwrap()[0]["n"],
3391 json!(32)
3392 );
3393 assert_eq!(
3394 client
3395 .query(
3396 "SELECT count(*) AS n FROM ack_writes WHERE kind='replay'",
3397 &[]
3398 )
3399 .unwrap()[0]["n"],
3400 json!(32)
3401 );
3402 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
3403 client.acknowledged_replay_count.set(0);
3404 for index in 0..subscriptions {
3405 client
3406 .subscribe(
3407 format!("docs-{index}"),
3408 "docs".into(),
3409 vec![("project_id".into(), vec![format!("scope-{index}")])],
3410 None,
3411 )
3412 .unwrap();
3413 let sub = client.subs.len() - 1;
3414 client.subs[sub].effective =
3415 Some(vec![("project_id".into(), vec![format!("scope-{index}")])]);
3416 let segment = RowsSegment {
3417 table: "docs".into(),
3418 schema_version: 1,
3419 columns: client.schema.table("docs").unwrap().wire_columns.clone(),
3420 blocks: if index % 2 == 0 {
3421 vec![vec![ssp2::segment::SegmentRow {
3422 values: vec![
3423 Some(ColumnValue::String(format!("doc-{index}"))),
3424 Some(ColumnValue::String(format!("scope-{index}"))),
3425 ],
3426 server_version: 1,
3427 }]]
3428 } else {
3429 vec![]
3430 },
3431 };
3432 assert!(client.apply_segment(sub, &segment, true).is_ok());
3433 }
3434 assert_eq!(
3435 client
3436 .query("SELECT count(*) AS n FROM ack_writes", &[])
3437 .unwrap()[0]["n"],
3438 json!(0),
3439 "unrelated bootstrap must not restore/replay tasks"
3440 );
3441 assert_eq!(
3442 client.acknowledged_replay_count.get(),
3443 0,
3444 "unrelated bootstrap must not decode task ACKs"
3445 );
3446 assert_eq!(
3447 client
3448 .query("SELECT count(*) AS n FROM tasks", &[])
3449 .unwrap()[0]["n"],
3450 json!(32)
3451 );
3452 assert_eq!(
3453 client
3454 .query("SELECT count(*) AS n FROM _syncular_acked_rows", &[])
3455 .unwrap()[0]["n"],
3456 json!(32)
3457 );
3458 }
3459 }
3460
3461 #[test]
3462 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
3463 let mut client = client();
3464 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3465 const COMMIT_COUNT: usize = 32;
3466
3467 for index in 0..COMMIT_COUNT {
3468 client
3469 .mutate(vec![Mutation::Upsert {
3470 table: "tasks".to_owned(),
3471 values: Map::from_iter([
3472 ("id".to_owned(), Value::from(format!("task-{index}"))),
3473 ("project_id".to_owned(), Value::from("project-1")),
3474 ]),
3475 base_version: None,
3476 }])
3477 .expect("queue commit");
3478 }
3479
3480 let (_, request_meta) = client.build_request(false).unwrap();
3481 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
3482 let mut frames = vec![Frame::RespHeader {
3483 required_schema_version: None,
3484 latest_schema_version: None,
3485 log_epoch: Some("epoch-1".to_owned()),
3486 reset_required: Some(false),
3487 }];
3488 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
3489 |(index, client_commit_id)| Frame::PushResult {
3490 client_commit_id: client_commit_id.clone(),
3491 status: PushStatus::Applied,
3492 commit_seq: Some(index as i64 + 1),
3493 results: vec![OpResult::Applied { op_index: 0 }],
3494 },
3495 ));
3496 let response = Message {
3497 wire_version: WIRE_VERSION,
3498 msg_kind: MsgKind::Response,
3499 frames,
3500 };
3501 let mut transport =
3502 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
3503
3504 client.overlay_rebuild_count.set(0);
3505 client.outcome_prune_count.set(0);
3506 let outcome = client.process_response(&mut transport, response, &request_meta);
3507 assert!(matches!(outcome, SyncOutcome::Ok(_)));
3508 assert!(client.pending_commit_ids().is_empty());
3509 assert_eq!(
3510 client.overlay_rebuild_count.get(),
3511 1,
3512 "one response must reconcile its acknowledged commits with one overlay rebuild"
3513 );
3514 assert_eq!(
3515 client.outcome_prune_count.get(),
3516 1,
3517 "one response must enforce outcome retention once"
3518 );
3519 }
3520
3521 #[test]
3522 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
3523 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
3524 "columns": [
3525 {"name":"id", "type":"string", "nullable":false},
3526 {"name":"project_id", "type":"string", "nullable":false},
3527 {"name":"title", "type":"string", "nullable":false}
3528 ], "scopes": [{"pattern":"project:{project_id}"}],
3529 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
3530 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
3531 }] });
3532 let mut incremental =
3533 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3534 let mut reference =
3535 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
3536 let tables = vec!["tasks".to_owned()];
3537 for index in 0..32 {
3538 let id = format!("task-{}", index % 8);
3539 let values = Map::from_iter([
3540 ("id".to_owned(), Value::from(id.clone())),
3541 (
3542 "project_id".to_owned(),
3543 Value::from(format!("p{}", index % 2)),
3544 ),
3545 ("title".to_owned(), Value::from(format!("needle{index}"))),
3546 ]);
3547 let payload = encode_row_json(
3548 incremental.schema.table("tasks").unwrap(),
3549 &id,
3550 &values,
3551 &incremental.encryption,
3552 )
3553 .unwrap();
3554 let change = ssp2::model::Change {
3555 table_index: 0,
3556 row_id: id,
3557 op: if index % 5 == 0 {
3558 Op::Delete
3559 } else {
3560 Op::Upsert
3561 },
3562 row_version: Some(index + 1),
3563 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
3564 row: Some(payload),
3565 };
3566 reference.overlay_dirty.set(true);
3567 for instance in [&mut incremental, &mut reference] {
3568 instance.begin_observation("mirror_test").unwrap();
3569 let mut batch = ChangeAccumulator::default();
3570 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
3571 instance
3572 .apply_commit_changes(&tables, std::slice::from_ref(&change))
3573 .unwrap();
3574 instance
3575 .rebuild_overlay_if_dirty()
3576 .expect("rebuild overlay");
3577 instance.finish_observation("mirror_test", batch).unwrap();
3578 }
3579 for sql in [
3580 "SELECT * FROM tasks ORDER BY id",
3581 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
3582 ] {
3583 assert_eq!(
3584 incremental.query(sql, &[]).unwrap(),
3585 reference.query(sql, &[]).unwrap()
3586 );
3587 }
3588 assert_eq!(incremental.local_revision(), reference.local_revision());
3589 assert_eq!(
3590 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3591 serde_json::to_value(reference.drain_change_batches()).unwrap()
3592 );
3593 }
3594 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
3595 assert_eq!(reference.overlay_rebuild_count.get(), 32);
3596 }
3597
3598 #[test]
3599 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
3600 let cases = [
3601 (
3602 "string",
3603 vec![
3604 json!("1"),
3605 json!("01"),
3606 json!(""),
3607 json!("nul\u{0}id"),
3608 json!("é"),
3609 ],
3610 ),
3611 (
3612 "integer",
3613 vec![
3614 json!(i64::MIN),
3615 json!(-1),
3616 json!(0),
3617 json!(1),
3618 json!(i64::MAX),
3619 ],
3620 ),
3621 ("boolean", vec![json!(false), json!(true)]),
3622 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
3623 ];
3624 for (kind, ids) in cases {
3625 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3626 "columns":[{"name":"id","type":kind,"nullable":false},
3627 {"name":"project_id","type":"string","nullable":false}],
3628 "scopes":[{"pattern":"project:{project_id}"}]}]});
3629 let mut client =
3630 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
3631 for (index, id) in ids.iter().enumerate() {
3632 let table = client.schema.table("tasks").unwrap();
3633 client
3634 .write_base_row(
3635 "tasks",
3636 &vec![
3637 if kind == "integer" {
3638 Some(ColumnValue::Integer(id.as_i64().unwrap()))
3641 } else {
3642 json_to_column_value(&table.columns[0], Some(id)).unwrap()
3643 },
3644 Some(ColumnValue::String(format!("p{index}"))),
3645 ],
3646 1,
3647 )
3648 .unwrap();
3649 }
3650 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3651 for base in [true, false] {
3652 let full = if base {
3653 base_table("tasks")
3654 } else {
3655 visible_table("tasks")
3656 };
3657 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
3658 for row_id in [
3659 "1",
3660 "01",
3661 "1.0",
3662 "1.0000000000000002",
3663 "+1",
3664 "1e0",
3665 " 1",
3666 "-1",
3667 "0",
3668 "9223372036854775807",
3669 "-9223372036854775808",
3670 "true",
3671 "false",
3672 "",
3673 "nul\u{0}id",
3674 "é",
3675 "null",
3676 "{\"a\":1}",
3677 ] {
3678 let old_sql =
3679 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
3680 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
3681 let old = client
3682 .conn
3683 .prepare(&old_sql)
3684 .unwrap()
3685 .query_map([row_id], |row| row.get::<_, String>(0))
3686 .unwrap()
3687 .collect::<Result<Vec<_>, _>>()
3688 .unwrap();
3689 let new = client
3690 .conn
3691 .prepare_cached(&new_sql)
3692 .unwrap()
3693 .query_map([row_id], |row| row.get::<_, String>(0))
3694 .unwrap()
3695 .collect::<Result<Vec<_>, _>>()
3696 .unwrap();
3697 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
3698 let mut batch = ChangeAccumulator::default();
3699 assert_eq!(
3700 client.record_row_scopes(&mut batch, "tasks", row_id, base),
3701 !old.is_empty()
3702 );
3703 let expected = old.first().map(|scope| {
3704 BTreeMap::from([(
3705 "tasks".to_owned(),
3706 Some(BTreeSet::from([format!("project:{scope}")])),
3707 )])
3708 });
3709 assert_eq!(batch.tables, expected.unwrap_or_default());
3710 client
3711 .conn
3712 .execute_batch("SAVEPOINT lookup_delete")
3713 .unwrap();
3714 if base {
3715 client.delete_base_row("tasks", row_id).unwrap();
3716 } else {
3717 client
3718 .apply_outbox_ops(&[OutboxOp {
3719 upsert: false,
3720 table: "tasks".into(),
3721 row_id: row_id.into(),
3722 base_version: None,
3723 values: None,
3724 }])
3725 .expect("replay delete");
3726 }
3727 let remaining: i64 = client
3728 .conn
3729 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
3730 row.get(0)
3731 })
3732 .unwrap();
3733 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
3734 client
3735 .conn
3736 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
3737 .unwrap();
3738 client.overlay_dirty.set(false);
3739 }
3740 for (predicate, expected) in [
3741 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
3742 (predicate, "SEARCH"),
3743 ] {
3744 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
3745 let details = client
3746 .conn
3747 .prepare(&sql)
3748 .unwrap()
3749 .query_map(["1"], |row| row.get::<_, String>(3))
3750 .unwrap()
3751 .collect::<Result<Vec<_>, _>>()
3752 .unwrap();
3753 assert!(
3754 details.iter().any(|detail| detail.contains(expected)),
3755 "{kind}: {details:?}"
3756 );
3757 }
3758 }
3759 }
3760 }
3761
3762 #[test]
3763 fn primary_keys_reject_every_ineligible_column_type() {
3764 for ty in ["float", "bytes", "crdt", "blob_ref"] {
3768 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3769 "columns":[{"name":"id","type":ty,"nullable":false},
3770 {"name":"project_id","type":"string","nullable":false}],
3771 "scopes":[{"pattern":"project:{project_id}"}]}]});
3772 let error =
3773 match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
3774 Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
3775 Err(error) => error,
3776 };
3777 assert!(
3778 error.contains("has an ineligible column type (§2.4)"),
3779 "{ty}: {error}"
3780 );
3781 }
3782 for ty in ["string", "integer", "boolean", "json"] {
3783 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3784 "columns":[{"name":"id","type":ty,"nullable":false},
3785 {"name":"project_id","type":"string","nullable":false}],
3786 "scopes":[{"pattern":"project:{project_id}"}]}]});
3787 SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
3788 .unwrap_or_else(|error| panic!("{ty}: {error}"));
3789 }
3790 }
3791
3792 #[test]
3793 fn pending_row_reconciliation_matches_full_fifo_replay() {
3794 for unique in [false, true] {
3795 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
3796 "name": name, "primaryKey": "id", "columns": [
3797 {"name":"id", "type":"string", "nullable":false},
3798 {"name":"project_id", "type":"string", "nullable":false},
3799 {"name":"title", "type":"string", "nullable":false}],
3800 "scopes": [{"pattern":"project:{project_id}"}],
3801 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
3802 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
3803 })))});
3804 let mut incremental =
3805 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
3806 let mut reference =
3807 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
3808 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
3809 for instance in [&mut incremental, &mut reference] {
3810 for table in &tables {
3811 for index in 0..12 {
3812 instance
3813 .write_base_row(
3814 table,
3815 &vec![
3816 Some(ColumnValue::String(format!("row{index}"))),
3817 Some(ColumnValue::String("p0".into())),
3818 Some(ColumnValue::String(format!("base{index}"))),
3819 ],
3820 1,
3821 )
3822 .unwrap();
3823 }
3824 }
3825 instance
3826 .rebuild_overlay_if_dirty()
3827 .expect("rebuild overlay");
3828 }
3829 for index in 0..80 {
3830 let mutations = (0..3)
3831 .map(|offset| {
3832 let table = tables[(index + offset) % 2].clone();
3833 let id = format!("row{}", (index + offset) % 16);
3834 if (index + offset) % 7 == 0 {
3835 Mutation::Delete {
3836 table,
3837 row_id: id,
3838 base_version: None,
3839 }
3840 } else {
3841 Mutation::Upsert {
3842 table,
3843 values: Map::from_iter([
3844 ("id".into(), Value::from(id)),
3845 ("project_id".into(), Value::from(format!("p{}", index % 3))),
3846 (
3847 "title".into(),
3848 Value::from(format!("needle{}", (index + offset) % 16)),
3849 ),
3850 ]),
3851 base_version: None,
3852 }
3853 }
3854 })
3855 .collect::<Vec<_>>();
3856 for instance in [&mut incremental, &mut reference] {
3857 instance.mutate(mutations.clone()).unwrap();
3858 }
3859 let changes = (0..3)
3862 .map(|offset| {
3863 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
3864 let id = format!("row{}", (index + offset) % 16);
3865 let values = Map::from_iter([
3866 ("id".into(), Value::from(id.clone())),
3867 (
3868 "project_id".into(),
3869 Value::from(format!("p{}", (index + 1) % 3)),
3870 ),
3871 (
3872 "title".into(),
3873 Value::from(format!("remote{index}_{offset}")),
3874 ),
3875 ]);
3876 ssp2::model::Change {
3877 table_index: table_index as u16,
3878 row_id: id.clone(),
3879 op: if (index + offset) % 5 == 0 {
3880 Op::Delete
3881 } else {
3882 Op::Upsert
3883 },
3884 row_version: Some(index as i64 + 2),
3885 scopes: vec![],
3886 row: Some(
3887 encode_row_json(
3888 incremental.schema.table(&tables[table_index]).unwrap(),
3889 &id,
3890 &values,
3891 &incremental.encryption,
3892 )
3893 .unwrap(),
3894 ),
3895 }
3896 })
3897 .collect::<Vec<_>>();
3898 reference.overlay_dirty.set(true);
3899 incremental
3900 .apply_commit_frame(&tables, &changes, None)
3901 .unwrap();
3902 reference
3903 .apply_commit_frame(&tables, &changes, None)
3904 .unwrap();
3905 for table in &tables {
3906 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
3907 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
3908 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
3909 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
3910 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
3911 }
3912 }
3913 assert_eq!(incremental.local_revision(), reference.local_revision());
3914 assert_eq!(
3915 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
3916 serde_json::to_value(reference.drain_change_batches()).unwrap()
3917 );
3918 }
3919 assert_eq!(
3920 incremental.overlay_rebuild_count.get(),
3921 if unique { 81 } else { 21 }
3922 );
3923 assert_eq!(reference.overlay_rebuild_count.get(), 81);
3924 }
3925 }
3926
3927 #[test]
3928 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
3929 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
3930 "columns":[{"name":"id", "type":"string", "nullable":false},
3931 {"name":"title", "type":"string", "nullable":false}],
3932 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
3933 let mut client =
3934 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
3935 for (id, title) in [("a", "original"), ("b", "occupied")] {
3936 client
3937 .write_base_row(
3938 "tasks",
3939 &vec![
3940 Some(ColumnValue::String(id.into())),
3941 Some(ColumnValue::String(title.into())),
3942 ],
3943 1,
3944 )
3945 .unwrap();
3946 }
3947 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
3948 let pending = client
3949 .mutate(vec![Mutation::Upsert {
3950 table: "tasks".into(),
3951 values: Map::from_iter([
3952 ("id".into(), Value::from("a")),
3953 ("title".into(), Value::from("available")),
3954 ]),
3955 base_version: None,
3956 }])
3957 .unwrap();
3958 assert_eq!(
3959 client
3960 .conn
3961 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
3962 .get::<_, String>(
3963 0
3964 ))
3965 .unwrap(),
3966 "available"
3967 );
3968 for (title, expected) in [("available", "original"), ("occupied", "available")] {
3969 let change = ssp2::model::Change {
3970 table_index: 0,
3971 row_id: "b".into(),
3972 op: Op::Upsert,
3973 row_version: Some(2),
3974 scopes: vec![],
3975 row: Some(
3976 encode_row_json(
3977 client.schema.table("tasks").unwrap(),
3978 "b",
3979 &Map::from_iter([
3980 ("id".into(), Value::from("b")),
3981 ("title".into(), Value::from(title)),
3982 ]),
3983 &client.encryption,
3984 )
3985 .unwrap(),
3986 ),
3987 };
3988 client
3989 .apply_commit_frame(&["tasks".into()], &[change], None)
3990 .unwrap();
3991 assert_eq!(
3992 client
3993 .conn
3994 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
3995 .get::<_, String>(
3996 0
3997 ))
3998 .unwrap(),
3999 expected
4000 );
4001 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
4002 }
4003 assert_eq!(client.overlay_rebuild_count.get(), 3);
4004 }
4005
4006 fn measure_scoped_overlay(unchanged_rows: i64, rounds: i64) -> Vec<u128> {
4007 let schema = json!({ "version": 1, "tables": [
4008 { "name":"tasks", "primaryKey":"id", "columns":[
4009 {"name":"id","type":"string","nullable":false},
4010 {"name":"project_id","type":"string","nullable":false}],
4011 "scopes":[{"pattern":"project:{project_id}"}] },
4012 { "name":"catalogue", "primaryKey":"id", "columns":[
4013 {"name":"id","type":"string","nullable":false},
4014 {"name":"project_id","type":"string","nullable":false},
4015 {"name":"title","type":"string","nullable":false}],
4016 "scopes":[{"pattern":"project:{project_id}"}],
4017 "ftsIndexes":[{"name":"catalogue_fts","columns":["title"],"tokenize":"unicode61"}] }
4018 ] });
4019 let mut client =
4020 SyncClient::new("overlay-scope".into(), &schema, ClientLimits::default()).unwrap();
4021 if unchanged_rows > 0 {
4022 client.conn.execute_batch(&format!("WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i<{unchanged_rows}) INSERT INTO _syncular_base_catalogue SELECT 'c'||i,'p1','needle '||i,1 FROM n")).unwrap();
4023 }
4024 client.rebuild_overlay().expect("rebuild overlay");
4025 client.conn.execute_batch("CREATE TABLE audit(tbl TEXT); CREATE TRIGGER catalogue_changed AFTER DELETE ON catalogue BEGIN INSERT INTO audit VALUES ('catalogue'); END;").unwrap();
4026 let mut elapsed = Vec::new();
4027 for round in 1..=rounds {
4028 let pending = client
4029 .mutate(vec![Mutation::Upsert {
4030 table: "tasks".into(),
4031 values: Map::from_iter([
4032 ("id".into(), json!("t1")),
4033 ("project_id".into(), json!("p1")),
4034 ]),
4035 base_version: None,
4036 }])
4037 .unwrap();
4038 let started = std::time::Instant::now();
4039 client
4040 .write_base_row(
4041 "tasks",
4042 &vec![
4043 Some(ColumnValue::String("t1".into())),
4044 Some(ColumnValue::String("p1".into())),
4045 ],
4046 round,
4047 )
4048 .unwrap();
4049 let mut report = SyncReport::default();
4050 client
4051 .handle_push_results(
4052 &[Frame::PushResult {
4053 client_commit_id: pending.clone(),
4054 status: PushStatus::Applied,
4055 commit_seq: Some(round),
4056 results: vec![OpResult::Applied { op_index: 0 }],
4057 }],
4058 &HashSet::from([pending.as_str()]),
4059 &HashMap::new(),
4060 &mut report,
4061 Some(&pending),
4062 )
4063 .unwrap();
4064 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4065 elapsed.push(started.elapsed().as_nanos());
4066 }
4067 assert_eq!(
4068 client
4069 .conn
4070 .query_row("SELECT count(*) FROM audit", [], |r| r.get::<_, i64>(0))
4071 .unwrap(),
4072 0,
4073 "unchanged catalogue must never be copied"
4074 );
4075 assert_eq!(
4076 client
4077 .conn
4078 .query_row(
4079 "SELECT count(*) FROM catalogue_fts WHERE catalogue_fts MATCH 'needle'",
4080 [],
4081 |r| r.get::<_, i64>(0)
4082 )
4083 .unwrap(),
4084 unchanged_rows
4085 );
4086 assert_eq!(
4087 client.query("SELECT * FROM tasks", &[]).unwrap()[0]["_syncular_version"],
4088 json!(rounds)
4089 );
4090 elapsed
4091 }
4092
4093 #[test]
4094 fn changed_overlay_leaves_large_unchanged_table_and_fts_untouched() {
4095 let elapsed = measure_scoped_overlay(100_000, 1);
4096 eprintln!(
4097 "table-scoped overlay with 100k unchanged rows: {} ns",
4098 elapsed[0]
4099 );
4100 }
4101
4102 #[test]
4103 #[ignore = "isolated performance measurement; run with the shared heavy-check lock"]
4104 fn benchmark_scoped_overlay() {
4105 for rows in [0, 100_000, 200_000] {
4106 let mut elapsed = measure_scoped_overlay(rows, 30);
4107 elapsed.sort_unstable();
4108 eprintln!(
4109 "{}",
4110 json!({"unchangedRows": rows, "rounds": elapsed.len(),
4111 "medianNs": elapsed[elapsed.len()/2], "minNs": elapsed[0],
4112 "maxNs": elapsed[elapsed.len()-1], "unchangedDeletes": 0})
4113 );
4114 }
4115 }
4116
4117 #[test]
4118 fn released_context_discards_in_flight_ack_and_server_rows() {
4119 for barrier in [
4120 "preflight",
4121 "purge",
4122 "rebootstrap",
4123 "subscription",
4124 "failed-purge-replay",
4125 "failed-purge-publish",
4126 "failed-rebootstrap",
4127 ] {
4128 let mut client = client();
4129 client.set_meta(LOG_EPOCH_KEY, "epoch");
4130 client
4131 .subscribe(
4132 "tasks".into(),
4133 "tasks".into(),
4134 vec![("project_id".into(), vec!["p1".into()])],
4135 None,
4136 )
4137 .unwrap();
4138 let commit = client
4139 .mutate(vec![Mutation::Upsert {
4140 table: "tasks".into(),
4141 values: Map::from_iter([
4142 ("id".into(), json!("local")),
4143 ("project_id".into(), json!("p1")),
4144 ]),
4145 base_version: None,
4146 }])
4147 .unwrap();
4148 let prepared = client.prepare_sync_round(false).unwrap();
4149 client.set_transport_enabled(&mut CountingRealtimeTransport::default(), false);
4150 let payload = encode_row_json(
4151 &client.schema.tables[0],
4152 "server",
4153 &Map::from_iter([
4154 ("id".into(), json!("server")),
4155 ("project_id".into(), json!("p1")),
4156 ]),
4157 &client.encryption,
4158 )
4159 .unwrap();
4160 let completed = crate::CompletedSyncRound {
4161 prepared,
4162 exchange: crate::round::ExchangeResult::Reply {
4163 transport_failed: false,
4164 downloads: Default::default(),
4165 response: Ok(Message {
4166 wire_version: WIRE_VERSION,
4167 msg_kind: MsgKind::Response,
4168 frames: vec![
4169 Frame::RespHeader {
4170 required_schema_version: None,
4171 latest_schema_version: None,
4172 log_epoch: Some("epoch".into()),
4173 reset_required: Some(false),
4174 },
4175 Frame::PushResult {
4176 client_commit_id: commit.clone(),
4177 status: PushStatus::Applied,
4178 commit_seq: Some(1),
4179 results: vec![OpResult::Applied { op_index: 0 }],
4180 },
4181 Frame::SubStart {
4182 id: "tasks".into(),
4183 status: SubStatus::Active,
4184 reason_code: String::new(),
4185 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
4186 bootstrap: true,
4187 },
4188 Frame::Commit {
4189 commit_seq: 1,
4190 created_at_ms: 0,
4191 actor_id: "actor".into(),
4192 tables: vec!["tasks".into()],
4193 changes: vec![ssp2::model::Change {
4194 table_index: 0,
4195 row_id: "server".into(),
4196 op: Op::Upsert,
4197 row_version: Some(1),
4198 scopes: vec![("project_id".into(), "p1".into())],
4199 row: Some(payload),
4200 }],
4201 },
4202 Frame::SubEnd {
4203 next_cursor: 1,
4204 bootstrap_state: None,
4205 },
4206 ],
4207 }),
4208 },
4209 };
4210 match barrier {
4211 "preflight" => client.begin_security_preflight(),
4212 "purge" => {
4213 client
4214 .purge_local_data(&LocalDataPurgeInput {
4215 purge_id: "pending-round".into(),
4216 targets: vec![LocalDataPurgeTarget {
4217 table: "tasks".into(),
4218 selectors: BTreeMap::from([(
4219 "project_id".into(),
4220 vec!["p1".into()],
4221 )]),
4222 }],
4223 })
4224 .unwrap();
4225 }
4226 "rebootstrap" => {
4227 client
4228 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4229 rebootstrap_id: "pending-round".into(),
4230 })
4231 .unwrap();
4232 }
4233 "failed-purge-replay" | "failed-purge-publish" | "failed-rebootstrap" => {
4234 let trigger = if barrier == "failed-purge-replay" {
4235 "CREATE TRIGGER fail_local BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4236 } else {
4237 "CREATE TRIGGER fail_local BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4238 };
4239 client.conn.execute_batch(trigger).unwrap();
4240 let revision = client.local_revision();
4241 if barrier == "failed-rebootstrap" {
4242 assert!(client
4243 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4244 rebootstrap_id: "failed-round".into(),
4245 })
4246 .is_err());
4247 } else {
4248 assert!(client
4249 .purge_local_data(&LocalDataPurgeInput {
4250 purge_id: "failed-round".into(),
4251 targets: vec![LocalDataPurgeTarget {
4252 table: "tasks".into(),
4253 selectors: BTreeMap::from([(
4254 "project_id".into(),
4255 vec!["p1".into()]
4256 )]),
4257 }],
4258 })
4259 .is_err());
4260 }
4261 assert_eq!(client.local_revision(), revision);
4262 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
4263 client
4264 .conn
4265 .execute_batch("DROP TRIGGER fail_local")
4266 .unwrap();
4267 let applied = client.apply_sync_round(completed);
4268 assert!(
4269 matches!(
4270 applied,
4271 crate::AppliedSyncRound::Complete {
4272 outcome: SyncOutcome::Ok(_),
4273 ..
4274 }
4275 ),
4276 "{barrier}"
4277 );
4278 assert!(client.pending_commit_ids().is_empty());
4279 assert_eq!(
4280 client
4281 .query("SELECT id FROM tasks WHERE id = 'server'", &[])
4282 .unwrap()
4283 .len(),
4284 1
4285 );
4286 continue;
4287 }
4288 "subscription" => {
4289 client.unsubscribe("tasks");
4290 client
4291 .subscribe(
4292 "tasks".into(),
4293 "tasks".into(),
4294 vec![("project_id".into(), vec!["p2".into()])],
4295 None,
4296 )
4297 .unwrap();
4298 }
4299 _ => unreachable!(),
4300 }
4301 let pending_before_reply = client.pending_commit_ids();
4302 let revision = client.local_revision();
4303 let applied = client.apply_sync_round(completed);
4304 assert!(
4305 matches!(applied, crate::AppliedSyncRound::Complete { outcome: SyncOutcome::Failed { ref error_code, .. }, ref controls, .. } if error_code == "client.round_cancelled" && controls.is_empty())
4306 );
4307 assert_eq!(client.local_revision(), revision);
4308 assert!(client
4309 .conn
4310 .query_row(
4311 "SELECT NOT EXISTS(SELECT 1 FROM tasks WHERE id='server')",
4312 [],
4313 |r| r.get::<_, bool>(0)
4314 )
4315 .unwrap());
4316 assert_eq!(client.pending_commit_ids(), pending_before_reply);
4317 }
4318 }
4319
4320 #[test]
4321 fn sync_round_deadline_is_anchored_once_per_round_and_carried_across_continues() {
4322 struct DeadlineRecording {
4323 budget: std::time::Duration,
4324 seen: Vec<Option<std::time::Instant>>,
4325 }
4326 impl Transport for DeadlineRecording {
4327 fn round_deadline(&self) -> Option<std::time::Duration> {
4328 Some(self.budget)
4329 }
4330 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4331 self.seen.push(deadline);
4332 }
4333 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4334 Err(TransportError::new("transport.failed", "injected"))
4335 }
4336 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4337 self.sync(request)
4338 }
4339 fn download_segment(
4340 &mut self,
4341 _: &SegmentRequest,
4342 _: &mut dyn FnMut(u64),
4343 ) -> Result<Vec<u8>, TransportError> {
4344 Err(TransportError::new("transport.failed", "injected"))
4345 }
4346 fn blob_upload_grant(
4347 &mut self,
4348 _: &str,
4349 _: u64,
4350 _: Option<&str>,
4351 ) -> Result<crate::BlobUploadGrant, TransportError> {
4352 Ok(crate::BlobUploadGrant::Present)
4353 }
4354 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4355 Ok(())
4356 }
4357 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4358 Ok(())
4359 }
4360 fn realtime_close(&mut self) -> Result<(), TransportError> {
4361 Ok(())
4362 }
4363 }
4364
4365 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4366 {"name":"id","type":"string","nullable":false},
4367 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4368 let mut client =
4369 SyncClient::new("deadline-round".into(), &schema, Default::default()).unwrap();
4370 client.set_meta(LOG_EPOCH_KEY, "epoch");
4371 client
4372 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4373 .unwrap();
4374
4375 let mut transport = DeadlineRecording {
4376 budget: std::time::Duration::from_secs(30),
4377 seen: Vec::new(),
4378 };
4379 let prepared = client.prepare_sync_round(false).unwrap();
4380 assert_eq!(prepared.uploads.len(), 1, "one staged upload");
4381 let prepared = match client.apply_sync_round(prepared.exchange(&mut transport)) {
4383 crate::AppliedSyncRound::Continue(prepared) => prepared,
4384 crate::AppliedSyncRound::Complete { .. } => panic!("the upload must continue"),
4385 };
4386 let completed = prepared.exchange(&mut transport);
4388 let anchor = completed
4389 .prepared
4390 .round_deadline_at
4391 .expect("the upload round anchors a deadline");
4392 assert!(matches!(
4393 client.apply_sync_round(completed),
4394 crate::AppliedSyncRound::Complete { .. }
4395 ));
4396 assert_eq!(transport.seen.len(), 4, "one scope per exchange");
4397 assert_eq!(
4398 transport.seen[0],
4399 Some(anchor),
4400 "the upload anchors the round"
4401 );
4402 assert_eq!(
4403 transport.seen[1], None,
4404 "the upload exchange clears its scope"
4405 );
4406 assert_eq!(
4407 transport.seen[2],
4408 Some(anchor),
4409 "a continuation reuses the carried anchor instead of refreshing it"
4410 );
4411 assert_eq!(transport.seen[3], None, "the continuation clears its scope");
4412
4413 transport.budget = std::time::Duration::from_secs(60);
4416 let prepared = client.prepare_sync_round(false).unwrap();
4417 let completed = prepared.exchange(&mut transport);
4418 let fresh = completed
4419 .prepared
4420 .round_deadline_at
4421 .expect("the next round anchors a deadline");
4422 client.apply_sync_round(completed);
4423 assert!(
4424 fresh >= anchor + std::time::Duration::from_secs(30),
4425 "the next round anchors a fresh deadline from the new budget"
4426 );
4427 assert_eq!(transport.seen[4], Some(fresh));
4428 assert_eq!(
4429 transport.seen[5], None,
4430 "the new round clears its scope too"
4431 );
4432 }
4433
4434 #[derive(Default)]
4437 struct RoundDeadlineProbe {
4438 budget: Option<std::time::Duration>,
4439 budget_queries: std::cell::Cell<usize>,
4440 scoped: Vec<Option<std::time::Instant>>,
4441 }
4442
4443 impl Transport for RoundDeadlineProbe {
4444 fn round_deadline(&self) -> Option<std::time::Duration> {
4445 self.budget_queries.set(self.budget_queries.get() + 1);
4446 self.budget
4447 }
4448 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
4449 self.scoped.push(deadline);
4450 }
4451 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4452 Err(TransportError::new("transport.failed", "injected"))
4453 }
4454 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4455 self.sync(request)
4456 }
4457 fn download_segment(
4458 &mut self,
4459 _: &SegmentRequest,
4460 _: &mut dyn FnMut(u64),
4461 ) -> Result<Vec<u8>, TransportError> {
4462 Err(TransportError::new("transport.failed", "injected"))
4463 }
4464 fn blob_upload_grant(
4465 &mut self,
4466 _: &str,
4467 _: u64,
4468 _: Option<&str>,
4469 ) -> Result<crate::BlobUploadGrant, TransportError> {
4470 Ok(crate::BlobUploadGrant::Present)
4471 }
4472 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4473 Ok(())
4474 }
4475 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4476 Ok(())
4477 }
4478 fn realtime_close(&mut self) -> Result<(), TransportError> {
4479 Ok(())
4480 }
4481 }
4482
4483 #[test]
4484 fn an_unusable_upload_body_does_not_anchor_the_round_deadline() {
4485 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
4486 {"name":"id","type":"string","nullable":false},
4487 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
4488 let mut client =
4489 SyncClient::new("deadline-unusable".into(), &schema, ClientLimits::default()).unwrap();
4490 client.set_meta(LOG_EPOCH_KEY, "epoch");
4491 client
4492 .upload_blob(b"pending upload", Some("text/plain".into()), None)
4493 .unwrap();
4494 client
4495 .conn
4496 .execute_batch("DELETE FROM _syncular_blobs")
4497 .unwrap();
4498 let prepared = client.prepare_sync_round(false).unwrap();
4499 assert!(
4500 prepared.uploads.front().is_some_and(|u| u.bytes.is_err()),
4501 "the staged body must fail locally"
4502 );
4503 let mut transport = RoundDeadlineProbe {
4504 budget: Some(std::time::Duration::from_secs(30)),
4505 ..Default::default()
4506 };
4507 let completed = prepared.exchange(&mut transport);
4508 assert!(matches!(
4509 completed.exchange,
4510 crate::round::ExchangeResult::Upload { result: Err(_), .. }
4511 ));
4512 assert_eq!(
4513 transport.budget_queries.get(),
4514 0,
4515 "a round with no network work never asks for a budget"
4516 );
4517 assert_eq!(transport.scoped, vec![None]);
4518 }
4519
4520 #[test]
4521 fn a_transport_budget_the_clock_cannot_represent_fails_before_network() {
4522 struct HugeBudget;
4523 impl Transport for HugeBudget {
4524 fn round_deadline(&self) -> Option<std::time::Duration> {
4525 Some(std::time::Duration::MAX)
4526 }
4527 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
4528 panic!("no network for an invalid budget")
4529 }
4530 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4531 self.sync(request)
4532 }
4533 fn download_segment(
4534 &mut self,
4535 _: &SegmentRequest,
4536 _: &mut dyn FnMut(u64),
4537 ) -> Result<Vec<u8>, TransportError> {
4538 panic!("no network for an invalid budget")
4539 }
4540 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4541 Ok(())
4542 }
4543 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
4544 Ok(())
4545 }
4546 fn realtime_close(&mut self) -> Result<(), TransportError> {
4547 Ok(())
4548 }
4549 }
4550
4551 let mut client = client();
4552 client.set_meta(LOG_EPOCH_KEY, "epoch");
4553 let prepared = client.prepare_sync_round(false).unwrap();
4554 let outcome = match client.apply_sync_round(prepared.exchange(&mut HugeBudget)) {
4555 crate::AppliedSyncRound::Complete { outcome, .. } => outcome,
4556 crate::AppliedSyncRound::Continue(_) => {
4557 panic!("an unrepresentable budget must not continue")
4558 }
4559 };
4560 match outcome {
4561 SyncOutcome::Failed {
4562 error_code,
4563 message,
4564 ..
4565 } => {
4566 assert_eq!(error_code, "sync.invalid_request");
4567 assert_eq!(message, "transport round deadline is out of range");
4568 }
4569 _ => panic!("an unrepresentable budget must fail synchronously"),
4570 }
4571 }
4572
4573 #[test]
4574 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
4575 for failure in ["visible", "revision", "commit"] {
4576 let mut client = client();
4577 let id = client
4578 .mutate(vec![Mutation::Upsert {
4579 table: "tasks".into(),
4580 values: Map::from_iter([
4581 ("id".into(), Value::from("t1")),
4582 ("project_id".into(), Value::from("pending")),
4583 ]),
4584 base_version: None,
4585 }])
4586 .unwrap();
4587 client.drain_change_batches();
4588 let revision = client.local_revision();
4589 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
4590 let trigger = match failure {
4591 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
4592 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
4593 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
4594 };
4595 client.conn.execute_batch(&trigger).unwrap();
4596 let change = ssp2::model::Change {
4597 table_index: 0,
4598 row_id: "t1".into(),
4599 op: Op::Upsert,
4600 row_version: Some(2),
4601 scopes: vec![],
4602 row: Some(
4603 encode_row_json(
4604 client.schema.table("tasks").unwrap(),
4605 "t1",
4606 &Map::from_iter([
4607 ("id".into(), Value::from("t1")),
4608 ("project_id".into(), Value::from("remote")),
4609 ]),
4610 &client.encryption,
4611 )
4612 .unwrap(),
4613 ),
4614 };
4615 assert!(client
4616 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), None)
4617 .is_err());
4618 assert!(client.conn.is_autocommit());
4619 assert!(!client.overlay_dirty.get());
4620 assert_eq!(client.local_revision(), revision);
4621 assert!(client.drain_change_batches().is_empty());
4622 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
4623 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
4624 assert!(client
4625 .query("SELECT * FROM _syncular_base_tasks", &[])
4626 .unwrap()
4627 .is_empty());
4628 client
4629 .conn
4630 .execute_batch("DROP TRIGGER fail_pending")
4631 .unwrap();
4632 client
4633 .apply_commit_frame(&["tasks".into()], &[change], None)
4634 .unwrap();
4635 assert_eq!(client.local_revision(), revision + 1);
4636 assert_eq!(client.pending_commit_ids(), vec![id]);
4637 let mut expected = before[0].clone();
4640 expected.insert("_syncular_version".to_owned(), Value::from(2));
4641 assert_eq!(
4642 client.query("SELECT * FROM tasks", &[]).unwrap(),
4643 vec![expected]
4644 );
4645 assert_eq!(
4646 client
4647 .conn
4648 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
4649 .get::<_, String>(0))
4650 .unwrap(),
4651 "remote"
4652 );
4653 }
4654 }
4655
4656 #[test]
4657 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
4658 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
4659 "columns": [
4660 {"name":"id", "type":"string", "nullable":false},
4661 {"name":"project_id", "type":"string", "nullable":false},
4662 {"name":"title", "type":"string", "nullable":false}],
4663 "scopes": [{"pattern":"project:{project_id}"}],
4664 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
4665 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
4666 let mut incremental =
4667 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4668 let mut reference =
4669 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
4670 for instance in [&mut incremental, &mut reference] {
4671 for index in 0..16 {
4672 instance
4673 .write_base_row(
4674 "tasks",
4675 &vec![
4676 Some(ColumnValue::String(format!("task{index}"))),
4677 Some(ColumnValue::String("p0".to_owned())),
4678 Some(ColumnValue::String(format!("base{index}"))),
4679 ],
4680 1,
4681 )
4682 .unwrap();
4683 }
4684 instance
4685 .rebuild_overlay_if_dirty()
4686 .expect("rebuild overlay");
4687 instance.overlay_rebuild_count.set(0);
4688 }
4689 for index in 0..100 {
4690 let mutations = (0..1 + index % 4)
4691 .map(|offset| {
4692 let id = format!("task{}", (index + offset) % 24);
4693 if (index + offset) % 7 == 0 {
4694 Mutation::Delete {
4695 table: "tasks".to_owned(),
4696 row_id: id,
4697 base_version: None,
4698 }
4699 } else {
4700 Mutation::Upsert {
4701 table: "tasks".to_owned(),
4702 values: Map::from_iter([
4703 ("id".to_owned(), Value::from(id)),
4704 (
4705 "project_id".to_owned(),
4706 Value::from(format!("p{}", index % 2)),
4707 ),
4708 (
4709 "title".to_owned(),
4710 Value::from(format!("needle{}", (index + offset) % 24)),
4711 ),
4712 ]),
4713 base_version: None,
4714 }
4715 }
4716 })
4717 .collect::<Vec<_>>();
4718 reference.overlay_dirty.set(true);
4719 incremental.mutate(mutations.clone()).unwrap();
4720 reference.mutate(mutations).unwrap();
4721 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
4722 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
4723 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
4724 }
4725 assert_eq!(incremental.local_revision(), reference.local_revision());
4726 assert_eq!(
4727 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4728 serde_json::to_value(reference.drain_change_batches()).unwrap()
4729 );
4730 }
4731 assert_eq!(incremental.outbox.len(), 100);
4732 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
4733 assert_eq!(reference.overlay_rebuild_count.get(), 100);
4734 }
4735
4736 #[test]
4737 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
4738 for fail_at in ["outbox", "revision"] {
4739 let mut client = client();
4740 let values = Map::from_iter([
4741 ("id".to_owned(), Value::from("t1")),
4742 ("project_id".to_owned(), Value::from("p1")),
4743 ]);
4744 let mutation = Mutation::Upsert {
4745 table: "tasks".to_owned(),
4746 values,
4747 base_version: None,
4748 };
4749 let trigger = if fail_at == "outbox" {
4750 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
4751 } else {
4752 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
4753 };
4754 client.conn.execute_batch(&trigger).unwrap();
4755 let revision = client.local_revision();
4756 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
4757 assert_eq!(client.local_revision(), revision);
4758 assert!(client.outbox.is_empty());
4759 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
4760 assert_eq!(
4761 client
4762 .conn
4763 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
4764 .get::<_, i64>(0))
4765 .unwrap(),
4766 0
4767 );
4768 assert!(client.drain_change_batches().is_empty());
4769 assert!(!client.overlay_dirty.get());
4770 client
4771 .conn
4772 .execute_batch("DROP TRIGGER fail_append")
4773 .unwrap();
4774 client.mutate(vec![mutation]).unwrap();
4775 assert_eq!(client.outbox.len(), 1);
4776 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
4777 }
4778 }
4779
4780 #[test]
4781 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
4782 let mut client = client();
4783 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
4784 let values = Map::from_iter([
4785 ("id".to_owned(), Value::from("t1")),
4786 ("project_id".to_owned(), Value::from("p1")),
4787 ]);
4788 let payload = encode_row_json(
4789 client.schema.table("tasks").unwrap(),
4790 "t1",
4791 &values,
4792 &client.encryption,
4793 )
4794 .unwrap();
4795 let first = ssp2::model::Change {
4796 table_index: 0,
4797 row_id: "t1".to_owned(),
4798 op: Op::Upsert,
4799 row_version: Some(1),
4800 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
4801 row: Some(payload),
4802 };
4803 let mut invalid = first.clone();
4804 invalid.table_index = 1;
4805 let revision = client.local_revision();
4806 client.begin_observation("failed_mirror").unwrap();
4807 assert!(client
4808 .apply_commit_changes(&tables, &[first, invalid])
4809 .is_err());
4810 client.rollback_observation("failed_mirror");
4811 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4812 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
4813 assert_eq!(
4814 client
4815 .conn
4816 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
4817 .get::<_, i64>(
4818 0
4819 ))
4820 .unwrap(),
4821 0
4822 );
4823 assert_eq!(client.local_revision(), revision);
4824 assert!(client.drain_change_batches().is_empty());
4825 }
4826
4827 #[test]
4828 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
4829 for failure in ["journal", "revision", "commit", "later-frame"] {
4830 let mut client = client();
4831 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4832 for id in ["first", "second"] {
4833 client
4834 .mutate(vec![Mutation::Upsert {
4835 table: "tasks".to_owned(),
4836 values: Map::from_iter([
4837 ("id".to_owned(), Value::from(id)),
4838 ("project_id".to_owned(), Value::from("p1")),
4839 ]),
4840 base_version: None,
4841 }])
4842 .unwrap();
4843 }
4844 let (_, meta) = client.build_request(false).unwrap();
4845 client
4847 .mutate(vec![Mutation::Upsert {
4848 table: "tasks".to_owned(),
4849 values: Map::from_iter([
4850 ("id".to_owned(), Value::from("later")),
4851 ("project_id".to_owned(), Value::from("p1")),
4852 ]),
4853 base_version: None,
4854 }])
4855 .unwrap();
4856 let ids = client.pending_commit_ids();
4857 let rows = client
4858 .query("SELECT * FROM tasks ORDER BY id", &[])
4859 .unwrap();
4860 let revision = client.local_revision();
4861 client.drain_change_batches();
4862 if failure != "later-frame" {
4863 client.conn.execute_batch(match failure {
4864 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
4865 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
4866 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
4867 }).unwrap();
4868 }
4869 let mut response = Message {
4870 wire_version: WIRE_VERSION,
4871 msg_kind: MsgKind::Response,
4872 frames: vec![Frame::RespHeader {
4873 required_schema_version: None,
4874 latest_schema_version: None,
4875 log_epoch: Some("epoch-1".to_owned()),
4876 reset_required: Some(false),
4877 }],
4878 };
4879 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
4880 response.frames.push(Frame::PushResult {
4881 client_commit_id: id.to_owned(),
4882 status: PushStatus::Applied,
4883 commit_seq: Some(1),
4884 results: vec![OpResult::Applied { op_index: 0 }],
4885 });
4886 }
4887 if failure == "later-frame" {
4888 response.frames.push(Frame::Error {
4889 code: "sync.invalid_request".to_owned(),
4890 message: "later error".to_owned(),
4891 category: "protocol".to_owned(),
4892 retryable: false,
4893 recommended_action: "retry".to_owned(),
4894 details: None,
4895 });
4896 }
4897 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
4898 let outcome = client.process_response(&mut transport, response.clone(), &meta);
4899 let expected = match failure {
4902 "later-frame" => "sync.invalid_request",
4903 "commit" => "storage.failed",
4904 _ => "client.outcome_persistence_failed",
4905 };
4906 assert!(
4907 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
4908 "{failure}: {outcome:?}"
4909 );
4910 if failure == "later-frame" {
4911 assert_eq!(client.pending_commit_ids(), ids[2..]);
4912 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
4913 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
4914 let batches = client.drain_change_batches();
4915 assert_eq!(batches.len(), 1);
4916 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
4917 assert_eq!(client.local_revision(), revision + 1);
4918 response.frames.pop();
4919 } else {
4920 assert_eq!(client.pending_commit_ids(), ids);
4921 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
4922 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
4923 assert_eq!(client.local_revision(), revision);
4924 assert!(client.drain_change_batches().is_empty());
4925 assert_eq!(
4926 client
4927 .query("SELECT * FROM tasks ORDER BY id", &[])
4928 .unwrap(),
4929 rows
4930 );
4931 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
4932 }
4933 for frame in &mut response.frames {
4934 if let Frame::PushResult { status, .. } = frame {
4935 *status = PushStatus::Cached;
4936 }
4937 }
4938 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
4939 else {
4940 panic!("retry failed");
4941 };
4942 assert_eq!(
4943 report.applied,
4944 if failure == "later-frame" {
4945 vec![]
4946 } else {
4947 ids[..2].to_vec()
4948 }
4949 );
4950 assert_eq!(client.pending_commit_ids(), ids[2..]);
4951 let batches = client.drain_change_batches();
4952 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
4953 if let Some(batch) = batches.first() {
4954 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
4955 }
4956 assert_eq!(client.local_revision(), revision + 1);
4957 }
4958 }
4959
4960 #[test]
4961 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
4962 for failure in ["revision", "journal", "commit"] {
4963 for status in ["applied", "cached", "rejected", "conflict"] {
4964 let mut client = client();
4965 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4966 for id in ["first", "later"] {
4967 client
4968 .mutate(vec![Mutation::Upsert {
4969 table: "tasks".to_owned(),
4970 values: Map::from_iter([
4971 ("id".to_owned(), Value::from(id)),
4972 ("project_id".to_owned(), Value::from("p1")),
4973 ]),
4974 base_version: None,
4975 }])
4976 .unwrap();
4977 }
4978 let (_, meta) = client.build_request(false).unwrap();
4979 let ids = client.pending_commit_ids();
4980 let rows = client
4981 .query("SELECT * FROM tasks ORDER BY id", &[])
4982 .unwrap();
4983 let revision = client.local_revision();
4984 client.drain_change_batches();
4985 let (push_status, results) = match status {
4986 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
4987 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
4988 "conflict" => (
4989 PushStatus::Rejected,
4990 vec![OpResult::Conflict {
4991 op_index: 0,
4992 code: "sync.version_conflict".to_owned(),
4993 message: "conflict".to_owned(),
4994 server_version: 1,
4995 server_row: encode_row_json(
4996 client.schema.table("tasks").unwrap(),
4997 "first",
4998 &Map::from_iter([
4999 ("id".to_owned(), Value::from("first")),
5000 ("project_id".to_owned(), Value::from("p1")),
5001 ]),
5002 &client.encryption,
5003 )
5004 .unwrap(),
5005 conflict_columns: Vec::new(),
5006 }],
5007 ),
5008 _ => (
5009 PushStatus::Rejected,
5010 vec![OpResult::Error {
5011 op_index: 0,
5012 code: "sync.validation_failed".to_owned(),
5013 message: "rejected".to_owned(),
5014 retryable: false,
5015 }],
5016 ),
5017 };
5018 let response = Message {
5019 wire_version: WIRE_VERSION,
5020 msg_kind: MsgKind::Response,
5021 frames: vec![
5022 Frame::RespHeader {
5023 required_schema_version: None,
5024 latest_schema_version: None,
5025 log_epoch: Some("epoch-1".to_owned()),
5026 reset_required: Some(false),
5027 },
5028 Frame::PushResult {
5029 client_commit_id: ids[0].clone(),
5030 status: push_status,
5031 commit_seq: Some(1),
5032 results,
5033 },
5034 ],
5035 };
5036 client.conn.execute_batch(match failure {
5037 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
5038 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
5039 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
5040 }).unwrap();
5041 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5042 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5043 assert_eq!(
5044 client.pending_commit_ids(),
5045 ids,
5046 "{failure}/{status}: pending identity"
5047 );
5048 assert_eq!(
5049 client.local_revision(),
5050 revision,
5051 "{failure}/{status}: revision"
5052 );
5053 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5054 assert!(client.conflicts().is_empty());
5055 assert!(client.rejections().is_empty());
5056 assert!(client.drain_change_batches().is_empty());
5057 assert_eq!(
5058 client
5059 .query("SELECT * FROM tasks ORDER BY id", &[])
5060 .unwrap(),
5061 rows
5062 );
5063 let expected = if failure == "commit" {
5066 "storage.failed"
5067 } else {
5068 "client.outcome_persistence_failed"
5069 };
5070 assert!(
5071 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5072 "{failure}/{status}: failed round"
5073 );
5074 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
5075 assert!(matches!(
5076 client.process_response(&mut transport, response, &meta),
5077 SyncOutcome::Ok(_)
5078 ));
5079 assert_eq!(client.pending_commit_ids(), ids[1..]);
5080 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5081 assert_eq!(client.drain_change_batches().len(), 1);
5082 }
5083 }
5084 }
5085
5086 #[test]
5087 fn mixed_push_results_reconcile_each_boundary_and_prune_once_per_response() {
5088 let mut client = client();
5089 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5090 for index in 0..4 {
5091 client
5092 .mutate(vec![Mutation::Upsert {
5093 table: "tasks".to_owned(),
5094 values: Map::from_iter([
5095 ("id".to_owned(), Value::from(format!("task-{index}"))),
5096 ("project_id".to_owned(), Value::from("project-1")),
5097 ]),
5098 base_version: None,
5099 }])
5100 .expect("queue commit");
5101 }
5102
5103 let (_, request_meta) = client.build_request(false).unwrap();
5104 let ids = &request_meta.pushed_ids;
5105 assert_eq!(ids.len(), 4);
5106 let response = Message {
5107 wire_version: WIRE_VERSION,
5108 msg_kind: MsgKind::Response,
5109 frames: vec![
5110 Frame::RespHeader {
5111 required_schema_version: None,
5112 latest_schema_version: None,
5113 log_epoch: Some("epoch-1".to_owned()),
5114 reset_required: Some(false),
5115 },
5116 Frame::PushResult {
5117 client_commit_id: ids[0].clone(),
5118 status: PushStatus::Applied,
5119 commit_seq: Some(1),
5120 results: vec![OpResult::Applied { op_index: 0 }],
5121 },
5122 Frame::PushResult {
5123 client_commit_id: ids[1].clone(),
5124 status: PushStatus::Cached,
5125 commit_seq: Some(2),
5126 results: vec![OpResult::Applied { op_index: 0 }],
5127 },
5128 Frame::PushResult {
5129 client_commit_id: ids[2].clone(),
5130 status: PushStatus::Rejected,
5131 commit_seq: None,
5132 results: vec![OpResult::Error {
5133 op_index: 0,
5134 code: "sync.validation_failed".to_owned(),
5135 message: "rejected".to_owned(),
5136 retryable: false,
5137 }],
5138 },
5139 Frame::PushResult {
5140 client_commit_id: ids[3].clone(),
5141 status: PushStatus::Rejected,
5142 commit_seq: None,
5143 results: vec![OpResult::Error {
5144 op_index: 0,
5145 code: "sync.idempotency_cache_miss".to_owned(),
5146 message: "retry".to_owned(),
5147 retryable: true,
5148 }],
5149 },
5150 ],
5151 };
5152 let mut transport =
5153 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
5154
5155 client.overlay_rebuild_count.set(0);
5156 client.outcome_prune_count.set(0);
5157 let outcome = client.process_response(&mut transport, response, &request_meta);
5158 let SyncOutcome::Ok(report) = outcome else {
5159 panic!("mixed push-result response failed");
5160 };
5161
5162 assert_eq!(report.applied, ids[..2]);
5163 assert_eq!(report.rejected, ids[2..3]);
5164 assert_eq!(report.retryable, ids[3..4]);
5165 assert_eq!(client.pending_commit_ids(), ids[3..4]);
5166 assert_eq!(
5167 client
5168 .query("SELECT id FROM tasks ORDER BY id", &[])
5169 .expect("query visible overlay"),
5170 ["task-0", "task-1", "task-3"]
5171 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
5172 .to_vec()
5173 );
5174 assert_eq!(client.overlay_rebuild_count.get(), 2);
5176 assert_eq!(client.outcome_prune_count.get(), 1);
5177 }
5178
5179 fn replay_schema(fts: bool) -> Value {
5180 json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
5181 {"name":"id","type":"string","nullable":false},
5182 {"name":"project_id","type":"string","nullable":false},
5183 {"name":"title","type":"string","nullable":false},
5184 {"name":"done","type":"boolean","nullable":false}
5185 ],"scopes":[{"pattern":"project:{project_id}"}], "ftsIndexes": if fts { json!([{"name":"tasks_fts","columns":["title"],"tokenize":"unicode61"}]) } else { json!([]) }}]})
5186 }
5187
5188 #[test]
5189 fn aborted_schema_and_epoch_resets_restore_memory_and_durable_intent() {
5190 for schema_reset in [false, true] {
5191 let mut schema = replay_schema(false);
5192 let mut client =
5193 SyncClient::new("reset-failure".into(), &schema, ClientLimits::default()).unwrap();
5194 let id = client
5195 .mutate(vec![Mutation::Upsert {
5196 table: "tasks".into(),
5197 values: Map::from_iter([
5198 ("id".into(), json!("queued")),
5199 ("project_id".into(), json!("p1")),
5200 ("title".into(), json!("old")),
5201 ("done".into(), json!(false)),
5202 ]),
5203 base_version: None,
5204 }])
5205 .unwrap();
5206 client.set_meta(LOG_EPOCH_KEY, "old");
5207 let revision = client.local_revision();
5208 let needed = client.sync_needed;
5209 client.drain_change_batches();
5210 client.drain_sync_intents();
5211 client.conn.execute_batch("CREATE TRIGGER refuse_revision BEFORE INSERT ON _syncular_meta WHEN NEW.key='localRevision' BEGIN SELECT RAISE(ABORT,'reset completion failed'); END;").unwrap();
5212 if schema_reset {
5213 schema["version"] = json!(2);
5214 schema["tables"][0]["columns"]
5215 .as_array_mut()
5216 .unwrap()
5217 .retain(|column| column["name"] != "title");
5218 client.schema = parse_schema_json(&schema).unwrap();
5219 assert!(client.run_schema_reset().is_err());
5220 } else {
5221 assert!(client.run_log_epoch_reset("new").is_err());
5222 }
5223 assert!(!client.upgrading);
5224 assert_eq!(client.sync_needed, needed);
5225 assert_eq!(client.local_revision(), revision);
5226 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5227 assert!(client.rejections.is_empty());
5228 assert!(client.commit_outcome(&id).unwrap().is_none());
5229 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("old"));
5230 assert_eq!(
5231 client
5232 .query("SELECT title FROM tasks WHERE id='queued'", &[])
5233 .unwrap()[0]["title"],
5234 "old"
5235 );
5236 assert_eq!(
5237 client
5238 .conn
5239 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5240 .get::<_, i64>(0))
5241 .unwrap(),
5242 1
5243 );
5244 assert!(client.drain_change_batches().is_empty());
5245 assert!(client.drain_sync_intents().is_empty());
5246 assert!(client.conn.is_autocommit());
5247 }
5248 }
5249
5250 #[test]
5251 fn cached_blob_cleanup_propagates_delete_failures() {
5252 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5253 {"name":"id","type":"string","nullable":false},
5254 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5255 let mut client =
5256 SyncClient::new("blob-cleanup".into(), &schema, ClientLimits::default()).unwrap();
5257 client.conn.execute_batch("CREATE TRIGGER refuse_blob_cleanup BEFORE DELETE ON _syncular_blobs BEGIN SELECT RAISE(ABORT,'blob cleanup failed'); END;").unwrap();
5258 client.conn.execute("INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,created_at_ms) VALUES ('test',X'01',1,0)", []).unwrap();
5259 client
5260 .subscribe("attachments".into(), "attachments".into(), vec![], None)
5261 .unwrap();
5262 client.subs[0].effective = Some(vec![]);
5263 client.persist_sub(&client.subs[0]).unwrap();
5264 let revision = client.local_revision();
5265 let (_, meta) = client.build_request(false).unwrap();
5266 let mut transport = CountingRealtimeTransport::default();
5267 let mut report = SyncReport::default();
5268 assert!(client
5269 .process_section(
5270 &mut transport,
5271 "attachments",
5272 SubStatus::Revoked,
5273 "sync.scope_revoked",
5274 vec![],
5275 vec![],
5276 Some((0, None)),
5277 &meta,
5278 &mut report
5279 )
5280 .is_err());
5281 assert_eq!(client.subs[0].state, SubState::Active);
5282 assert_eq!(client.local_revision(), revision);
5283 assert!(report.revoked.is_empty());
5284 assert!(client.conn.is_autocommit());
5285 client
5286 .conn
5287 .execute_batch("DROP TRIGGER refuse_blob_cleanup")
5288 .unwrap();
5289 assert!(client
5290 .process_section(
5291 &mut transport,
5292 "attachments",
5293 SubStatus::Revoked,
5294 "sync.scope_revoked",
5295 vec![],
5296 vec![],
5297 Some((0, None)),
5298 &meta,
5299 &mut report
5300 )
5301 .is_ok());
5302 assert_eq!(client.subs[0].state, SubState::Revoked);
5303 assert_eq!(
5304 client
5305 .conn
5306 .query_row("SELECT count(*) FROM _syncular_blobs", [], |row| row
5307 .get::<_, i64>(0))
5308 .unwrap(),
5309 0
5310 );
5311 }
5312
5313 #[test]
5314 fn overlay_replay_trigger_refuses_reopen_and_preserves_durable_intent() {
5315 let path =
5316 std::env::temp_dir().join(format!("syncular-replay-{}.db", uuid::Uuid::new_v4()));
5317 let schema = replay_schema(false);
5318 let mut client = SyncClient::open_path(
5319 "replay".into(),
5320 &schema,
5321 ClientLimits::default(),
5322 path.to_str().unwrap(),
5323 )
5324 .unwrap();
5325 let ids = ["early", "bad"].map(|id| {
5326 client
5327 .mutate(vec![Mutation::Upsert {
5328 table: "tasks".into(),
5329 values: Map::from_iter([
5330 ("id".into(), json!(id)),
5331 ("project_id".into(), json!("p1")),
5332 ("title".into(), json!(id)),
5333 ("done".into(), json!(false)),
5334 ]),
5335 base_version: None,
5336 }])
5337 .unwrap()
5338 });
5339 client.conn.execute_batch("DELETE FROM tasks; CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.id='bad' BEGIN SELECT RAISE(ABORT,'replay failed'); END;").unwrap();
5340 let revision = client.local_revision();
5341 drop(client);
5342 let error = SyncClient::open_path(
5343 "replay".into(),
5344 &schema,
5345 ClientLimits::default(),
5346 path.to_str().unwrap(),
5347 )
5348 .err()
5349 .expect("reopen must fail");
5350 assert!(error.contains("replay failed"));
5351 let conn = Connection::open(&path).unwrap();
5352 assert_eq!(
5353 conn.query_row("SELECT count(*) FROM tasks", [], |row| row.get::<_, i64>(0))
5354 .unwrap(),
5355 0
5356 );
5357 assert_eq!(
5358 conn.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5359 .get::<_, i64>(0))
5360 .unwrap(),
5361 2
5362 );
5363 assert_eq!(
5364 meta_get(&conn, LOCAL_REVISION_KEY),
5365 Some(revision.to_string())
5366 );
5367 conn.execute_batch("DROP TRIGGER fail_replay").unwrap();
5368 drop(conn);
5369 let reopened = SyncClient::open_path(
5370 "replay".into(),
5371 &schema,
5372 ClientLimits::default(),
5373 path.to_str().unwrap(),
5374 )
5375 .unwrap();
5376 assert_eq!(reopened.pending_commit_ids(), ids);
5377 assert_eq!(
5378 reopened
5379 .query("SELECT id FROM tasks ORDER BY id", &[])
5380 .unwrap()
5381 .len(),
5382 2
5383 );
5384 drop(reopened);
5385 std::fs::remove_file(path).unwrap();
5386 }
5387
5388 #[test]
5389 fn overlay_replay_failure_rolls_back_push_and_pull_apply_boundaries() {
5390 let schema = replay_schema(false);
5391 let mut client =
5392 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5393 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5394 client
5395 .write_base_row(
5396 "tasks",
5397 &vec![
5398 Some(ColumnValue::String("queued".into())),
5399 Some(ColumnValue::String("p1".into())),
5400 Some(ColumnValue::String("server".into())),
5401 Some(ColumnValue::Boolean(false)),
5402 ],
5403 1,
5404 )
5405 .unwrap();
5406 client.rebuild_overlay().unwrap();
5407 let id = client
5408 .patch(
5409 "tasks",
5410 "queued",
5411 Map::from_iter([("title".into(), json!("local"))]),
5412 Some(1),
5413 )
5414 .unwrap();
5415 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
5416 let base = client
5417 .query("SELECT * FROM _syncular_base_tasks", &[])
5418 .unwrap();
5419 let revision = client.local_revision();
5420 client.drain_change_batches();
5421 client.conn.execute_batch("CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.title='local' BEGIN SELECT RAISE(ABORT,'replay failed'); END").unwrap();
5422 let remote = vec![
5423 Some(ColumnValue::String("queued".into())),
5424 Some(ColumnValue::String("p1".into())),
5425 Some(ColumnValue::String("remote".into())),
5426 Some(ColumnValue::Boolean(false)),
5427 ];
5428 let bytes = crate::values::encode_row_json(
5429 client.schema.table("tasks").unwrap(),
5430 "queued",
5431 &Map::from_iter(
5432 client
5433 .schema
5434 .table("tasks")
5435 .unwrap()
5436 .columns
5437 .iter()
5438 .zip(&remote)
5439 .map(|(column, value)| (column.name.clone(), column_value_to_json(value))),
5440 ),
5441 &client.encryption,
5442 )
5443 .unwrap();
5444 let change = ssp2::model::Change {
5445 table_index: 0,
5446 op: Op::Upsert,
5447 row_id: "queued".into(),
5448 row_version: Some(2),
5449 scopes: vec![("project_id".into(), "p1".into())],
5450 row: Some(bytes),
5451 };
5452 assert!(client
5453 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), Some(2))
5454 .is_err());
5455 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5456 assert_eq!(
5457 client
5458 .query("SELECT * FROM _syncular_base_tasks", &[])
5459 .unwrap(),
5460 base
5461 );
5462 assert_eq!(client.local_revision(), revision);
5463 assert!(client.drain_change_batches().is_empty());
5464 let response = Message {
5465 wire_version: 2,
5466 msg_kind: MsgKind::Response,
5467 frames: vec![
5468 Frame::RespHeader {
5469 required_schema_version: None,
5470 latest_schema_version: None,
5471 log_epoch: Some("epoch-1".into()),
5472 reset_required: Some(false),
5473 },
5474 Frame::PushResult {
5475 client_commit_id: id.clone(),
5476 status: PushStatus::Applied,
5477 commit_seq: Some(2),
5478 results: vec![OpResult::Applied { op_index: 0 }],
5479 },
5480 ],
5481 };
5482 let (_, meta) = client.build_request(false).unwrap();
5483 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5484 assert!(
5485 matches!(client.process_response(&mut transport,response.clone(),&meta),SyncOutcome::Failed { error_code,.. } if error_code=="client.outcome_persistence_failed")
5486 );
5487 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5488 assert!(client.commit_outcome(&id).unwrap().is_none());
5489 assert_eq!(
5490 client
5491 .conn
5492 .query_row("SELECT count(*) FROM _syncular_acked_rows", [], |row| row
5493 .get::<_, i64>(
5494 0
5495 ))
5496 .unwrap(),
5497 0
5498 );
5499 assert_eq!(
5500 client
5501 .conn
5502 .query_row("SELECT count(*) FROM _syncular_row_deliveries", [], |row| {
5503 row.get::<_, i64>(0)
5504 })
5505 .unwrap(),
5506 0
5507 );
5508 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
5509 assert_eq!(client.local_revision(), revision);
5510 assert!(client.drain_change_batches().is_empty());
5511 client
5512 .conn
5513 .execute_batch("DROP TRIGGER fail_replay")
5514 .unwrap();
5515 assert!(matches!(
5516 client.process_response(&mut transport, response, &meta),
5517 SyncOutcome::Ok(_)
5518 ));
5519 client
5520 .apply_commit_frame(&["tasks".into()], &[change], Some(2))
5521 .unwrap();
5522 assert!(client.pending_commit_ids().is_empty());
5523 assert_eq!(
5524 client.query("SELECT title FROM tasks", &[]).unwrap()[0]["title"],
5525 json!("remote")
5526 );
5527 }
5528
5529 #[test]
5530 fn overlay_replay_distinguishes_absence_from_sql_and_value_decode_errors() {
5531 let client = SyncClient::new(
5532 "replay".into(),
5533 &replay_schema(false),
5534 ClientLimits::default(),
5535 )
5536 .unwrap();
5537 let table = client.schema.table("tasks").unwrap();
5538 let patch = OutboxOp {
5539 table: "tasks".into(),
5540 row_id: "absent".into(),
5541 upsert: true,
5542 base_version: Some(1),
5543 values: Some(Map::from_iter([
5544 ("id".into(), json!("absent")),
5545 ("title".into(), json!("patch")),
5546 ])),
5547 };
5548 assert!(client.visible_row(table, "absent").unwrap().is_none());
5549 client
5550 .apply_outbox_ops([&patch])
5551 .expect("sparse replay over truly absent row");
5552 client
5553 .conn
5554 .execute_batch("INSERT INTO tasks VALUES ('bad','p1','title','invalid',1)")
5555 .unwrap();
5556 assert!(client
5557 .visible_row(table, "bad")
5558 .unwrap_err()
5559 .starts_with("sync.local_corrupt:"));
5560 let mut invalid = patch.clone();
5561 invalid.values = Some(Map::from_iter([
5562 ("id".into(), json!("bad-value")),
5563 ("project_id".into(), json!("p1")),
5564 ("title".into(), json!("title")),
5565 ("done".into(), json!({"bad":true})),
5566 ]));
5567 assert!(client
5568 .apply_outbox_ops([&invalid])
5569 .unwrap_err()
5570 .starts_with("sync.local_corrupt:"));
5571 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5572 assert!(
5573 client.apply_outbox_ops([&patch]).is_err(),
5574 "SQL failure must not count as absence"
5575 );
5576 }
5577
5578 #[test]
5579 fn overlay_replay_keeps_secondary_unique_deferral_and_lazy_base_precedence() {
5580 let mut schema = replay_schema(false);
5581 schema["tables"][0]["indexes"] =
5582 json!([{"name":"tasks_project_title", "columns":["project_id","title"],"unique":true}]);
5583 let mut client =
5584 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
5585 let id = client
5586 .mutate(vec![Mutation::Upsert {
5587 table: "tasks".into(),
5588 values: Map::from_iter([
5589 ("id".into(), json!("queued")),
5590 ("project_id".into(), json!("p1")),
5591 ("title".into(), json!("occupied")),
5592 ("done".into(), json!(false)),
5593 ]),
5594 base_version: None,
5595 }])
5596 .unwrap();
5597 client
5598 .write_base_row(
5599 "tasks",
5600 &vec![
5601 Some(ColumnValue::String("server".into())),
5602 Some(ColumnValue::String("p1".into())),
5603 Some(ColumnValue::String("occupied".into())),
5604 Some(ColumnValue::Boolean(false)),
5605 ],
5606 1,
5607 )
5608 .unwrap();
5609 client.rebuild_overlay_if_dirty().unwrap();
5610 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5611 assert!(client
5612 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5613 .unwrap()
5614 .is_none());
5615 assert!(client.rejections().is_empty());
5616 client.delete_base_row("tasks", "server").unwrap();
5617 client.rebuild_overlay_if_dirty().unwrap();
5618 assert!(client
5619 .visible_row(client.schema.table("tasks").unwrap(), "queued")
5620 .unwrap()
5621 .is_some());
5622 client
5623 .write_base_row(
5624 "tasks",
5625 &vec![
5626 Some(ColumnValue::String("base".into())),
5627 Some(ColumnValue::String("p1".into())),
5628 Some(ColumnValue::String("base-title".into())),
5629 Some(ColumnValue::Boolean(false)),
5630 ],
5631 1,
5632 )
5633 .unwrap();
5634 client.conn.execute_batch("DROP TABLE tasks").unwrap();
5635 client
5636 .retain_failed_commit(
5637 "retained",
5638 &[OutboxOp {
5639 table: "tasks".into(),
5640 row_id: "base".into(),
5641 upsert: true,
5642 base_version: Some(1),
5643 values: Some(Map::from_iter([
5644 ("id".into(), json!("base")),
5645 ("title".into(), json!("patch")),
5646 ])),
5647 }],
5648 )
5649 .expect("an existing base must not read the unavailable visible table");
5650 }
5651
5652 #[test]
5653 fn overlay_replay_fts_and_savepoint_release_failures_roll_back_all_visible_changes() {
5654 for failure in ["fts", "release"] {
5655 let mut client = SyncClient::new(
5656 "replay".into(),
5657 &replay_schema(true),
5658 ClientLimits::default(),
5659 )
5660 .unwrap();
5661 client
5662 .mutate(vec![Mutation::Upsert {
5663 table: "tasks".into(),
5664 values: Map::from_iter([
5665 ("id".into(), json!("queued")),
5666 ("project_id".into(), json!("p1")),
5667 ("title".into(), json!("needle")),
5668 ("done".into(), json!(false)),
5669 ]),
5670 base_version: None,
5671 }])
5672 .unwrap();
5673 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
5674 let fts_before = client
5675 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5676 .unwrap();
5677 let triggers_before: Vec<String> = client
5678 .conn
5679 .prepare("SELECT sql FROM sqlite_master WHERE type='trigger' ORDER BY name")
5680 .unwrap()
5681 .query_map([], |row| row.get(0))
5682 .unwrap()
5683 .collect::<Result<_, _>>()
5684 .unwrap();
5685 if failure == "fts" {
5686 client.conn.execute_batch("CREATE TRIGGER fail_fts BEFORE INSERT ON _syncular_fts_tasks_fts BEGIN SELECT RAISE(ABORT,'fts replay failed'); END;").unwrap();
5687 } else {
5688 client.conn.execute_batch("PRAGMA foreign_keys=ON; CREATE TABLE parent(id TEXT PRIMARY KEY); CREATE TABLE deferred(id TEXT REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_release AFTER INSERT ON tasks BEGIN INSERT INTO deferred VALUES ('missing'); END;").unwrap();
5689 }
5690 client.overlay_dirty.set(true);
5691 assert!(client.rebuild_overlay().is_err());
5692 assert!(client.overlay_dirty.get());
5693 assert!(client.conn.is_autocommit());
5694 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
5695 assert_eq!(
5696 client
5697 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
5698 .unwrap(),
5699 fts_before
5700 );
5701 let original_triggers: Vec<String> = client.conn.prepare("SELECT sql FROM sqlite_master WHERE type='trigger' AND name NOT LIKE 'fail_%' ORDER BY name").unwrap().query_map([], |row| row.get(0)).unwrap().collect::<Result<_,_>>().unwrap();
5702 assert_eq!(original_triggers, triggers_before);
5703 assert_eq!(client.pending_commit_ids().len(), 1);
5704 client
5705 .conn
5706 .execute_batch(if failure == "fts" {
5707 "DROP TRIGGER fail_fts"
5708 } else {
5709 "DROP TRIGGER fail_release"
5710 })
5711 .unwrap();
5712 client
5713 .rebuild_overlay()
5714 .expect("retry after removing the fault");
5715 }
5716 }
5717
5718 #[test]
5719 fn secondary_unique_collision_preserves_existing_synced_row() {
5720 let client = SyncClient::new(
5721 "unique-upsert-test".to_owned(),
5722 &json!({
5723 "version": 1,
5724 "tables": [{
5725 "name": "tasks",
5726 "primaryKey": "id",
5727 "columns": [
5728 { "name": "id", "type": "string", "nullable": false },
5729 { "name": "project_id", "type": "string", "nullable": false },
5730 { "name": "title", "type": "string", "nullable": false }
5731 ],
5732 "scopes": [{ "pattern": "project:{project_id}" }],
5733 "indexes": [{
5734 "name": "tasks_by_project_title",
5735 "columns": ["project_id", "title"],
5736 "unique": true
5737 }]
5738 }]
5739 }),
5740 ClientLimits::default(),
5741 )
5742 .expect("test client");
5743 let table = client.schema.table("tasks").expect("tasks table");
5744 let sql = client.insert_row_sql(&base_table("tasks"), table);
5745
5746 client
5747 .conn
5748 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
5749 .expect("insert first row");
5750 client
5751 .conn
5752 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
5753 .expect("update same primary key");
5754 client
5755 .conn
5756 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
5757 .expect("insert second row");
5758 assert!(client
5759 .conn
5760 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
5761 .is_err());
5762
5763 let rows = client
5764 .conn
5765 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
5766 .expect("prepare rows")
5767 .query_map([], |row| {
5768 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
5769 })
5770 .expect("query rows")
5771 .collect::<Result<Vec<_>, _>>()
5772 .expect("collect rows");
5773 assert_eq!(
5774 rows,
5775 vec![
5776 ("t1".to_owned(), "updated".to_owned()),
5777 ("t2".to_owned(), "original".to_owned())
5778 ]
5779 );
5780 }
5781
5782 #[test]
5783 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
5784 let path = std::env::temp_dir().join(format!(
5785 "syncular-startup-intent-{}.db",
5786 uuid::Uuid::new_v4()
5787 ));
5788 let schema = json!({
5789 "version": 1,
5790 "tables": [{
5791 "name": "tasks",
5792 "primaryKey": "id",
5793 "columns": [
5794 { "name": "id", "type": "string", "nullable": false },
5795 { "name": "project_id", "type": "string", "nullable": false }
5796 ],
5797 "scopes": [{ "pattern": "project:{project_id}" }]
5798 }]
5799 });
5800
5801 {
5802 let mut first = SyncClient::open_path_with_identity(
5803 None,
5804 &schema,
5805 ClientLimits::default(),
5806 path.to_str().expect("UTF-8 temp path"),
5807 )
5808 .expect("first open");
5809 first
5810 .set_window(
5811 &WindowBase {
5812 table: "tasks".to_owned(),
5813 variable: "project_id".to_owned(),
5814 fixed_scopes: Vec::new(),
5815 params: None,
5816 },
5817 &["persisted".to_owned()],
5818 )
5819 .expect("persist window");
5820 }
5821
5822 let mut reopened = SyncClient::open_path_with_identity(
5823 None,
5824 &schema,
5825 ClientLimits::default(),
5826 path.to_str().expect("UTF-8 temp path"),
5827 )
5828 .expect("reopen");
5829 assert!(reopened.sync_needed());
5830 assert!(matches!(
5831 reopened.drain_sync_intents().as_slice(),
5832 [SyncIntent::Interactive]
5833 ));
5834 drop(reopened);
5835 std::fs::remove_file(path).expect("remove temp database");
5836 }
5837
5838 #[test]
5839 fn reopening_preserves_immutable_subscription_identity_and_progress() {
5840 let path = std::env::temp_dir().join(format!(
5841 "syncular-subscription-identity-{}.db",
5842 uuid::Uuid::new_v4()
5843 ));
5844 let schema = json!({
5845 "version": 1,
5846 "tables": [
5847 {
5848 "name": "tasks",
5849 "primaryKey": "id",
5850 "columns": [
5851 { "name": "id", "type": "string", "nullable": false },
5852 { "name": "project_id", "type": "string", "nullable": false }
5853 ],
5854 "scopes": [{ "pattern": "project:{project_id}" }]
5855 },
5856 {
5857 "name": "docs",
5858 "primaryKey": "id",
5859 "columns": [
5860 { "name": "id", "type": "string", "nullable": false },
5861 { "name": "org_id", "type": "string", "nullable": false },
5862 { "name": "project_id", "type": "string", "nullable": false }
5863 ],
5864 "scopes": [
5865 { "pattern": "org:{org_id}" },
5866 { "pattern": "project:{projectId}", "column": "project_id" }
5867 ]
5868 }
5869 ]
5870 });
5871
5872 {
5873 let mut first = SyncClient::open_path_with_identity(
5874 None,
5875 &schema,
5876 ClientLimits::default(),
5877 path.to_str().expect("UTF-8 temp path"),
5878 )
5879 .expect("first open");
5880 first
5881 .subscribe(
5882 "stable-subscription".to_owned(),
5883 "tasks".to_owned(),
5884 vec![(
5885 "project_id".to_owned(),
5886 vec!["p2".to_owned(), "p1".to_owned()],
5887 )],
5888 Some(r#"{"view":"v1"}"#.to_owned()),
5889 )
5890 .expect("persist subscription");
5891 let persisted = {
5892 let subscription = first
5893 .subs
5894 .iter_mut()
5895 .find(|subscription| subscription.id == "stable-subscription")
5896 .expect("subscription");
5897 subscription.cursor = 41;
5898 subscription.bootstrap_state = Some("resume-token".to_owned());
5899 subscription.effective = Some(vec![(
5900 "project_id".to_owned(),
5901 vec!["p1".to_owned(), "p2".to_owned()],
5902 )]);
5903 subscription.synced_once = true;
5904 subscription.clone()
5905 };
5906 first.persist_sub(&persisted).unwrap();
5907 }
5908
5909 let mut reopened = SyncClient::open_path_with_identity(
5910 None,
5911 &schema,
5912 ClientLimits::default(),
5913 path.to_str().expect("UTF-8 temp path"),
5914 )
5915 .expect("reopen");
5916 let progress = reopened
5917 .subscription_state("stable-subscription")
5918 .expect("persisted state");
5919 assert_eq!(progress.cursor, 41);
5920 assert!(progress.has_resume_token);
5921
5922 reopened
5923 .subscribe(
5924 "stable-subscription".to_owned(),
5925 "tasks".to_owned(),
5926 vec![(
5927 "project_id".to_owned(),
5928 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
5929 )],
5930 Some(r#"{"view":"v1"}"#.to_owned()),
5931 )
5932 .expect("canonical intent is idempotent");
5933 assert_eq!(
5934 reopened
5935 .subscription_state("stable-subscription")
5936 .expect("unchanged state")
5937 .cursor,
5938 progress.cursor
5939 );
5940
5941 for (table, scopes, params) in [
5942 (
5943 "tasks",
5944 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
5945 Some(r#"{"view":"v1"}"#.to_owned()),
5946 ),
5947 (
5948 "tasks",
5949 vec![(
5950 "project_id".to_owned(),
5951 vec!["p2".to_owned(), "p1".to_owned()],
5952 )],
5953 Some(r#"{"view":"v2"}"#.to_owned()),
5954 ),
5955 (
5956 "docs",
5957 vec![
5958 ("org_id".to_owned(), vec!["o1".to_owned()]),
5959 ("projectId".to_owned(), vec!["p1".to_owned()]),
5960 ],
5961 Some(r#"{"view":"v1"}"#.to_owned()),
5962 ),
5963 ] {
5964 let error = reopened
5965 .subscribe(
5966 "stable-subscription".to_owned(),
5967 table.to_owned(),
5968 scopes,
5969 params,
5970 )
5971 .expect_err("identity rebind must fail");
5972 assert!(error.starts_with("client.subscription_intent_mismatch:"));
5973 assert_eq!(
5974 reopened
5975 .subscription_state("stable-subscription")
5976 .expect("unchanged state")
5977 .cursor,
5978 progress.cursor
5979 );
5980 }
5981
5982 drop(reopened);
5983 std::fs::remove_file(path).expect("remove temp database");
5984 }
5985
5986 #[test]
5987 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
5988 let path = std::env::temp_dir().join(format!(
5989 "syncular-satisfied-schema-floor-{}.db",
5990 uuid::Uuid::new_v4()
5991 ));
5992 let schema = json!({
5993 "version": 23,
5994 "tables": [{
5995 "name": "tasks",
5996 "primaryKey": "id",
5997 "columns": [
5998 { "name": "id", "type": "string", "nullable": false },
5999 { "name": "project_id", "type": "string", "nullable": false }
6000 ],
6001 "scopes": [{ "pattern": "project:{project_id}" }]
6002 }]
6003 });
6004
6005 {
6006 let mut first = SyncClient::open_path_with_identity(
6007 None,
6008 &schema,
6009 ClientLimits::default(),
6010 path.to_str().expect("UTF-8 temp path"),
6011 )
6012 .expect("first open");
6013 first
6014 .subscribe(
6015 "tasks".to_owned(),
6016 "tasks".to_owned(),
6017 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6018 None,
6019 )
6020 .expect("persist subscription");
6021 first.set_schema_floor(Some(SchemaFloor {
6022 required_schema_version: Some(22),
6023 latest_schema_version: Some(22),
6024 }));
6025 }
6026
6027 let mut reopened = SyncClient::open_path_with_identity(
6028 None,
6029 &schema,
6030 ClientLimits::default(),
6031 path.to_str().expect("UTF-8 temp path"),
6032 )
6033 .expect("reopen");
6034 assert!(reopened.schema_floor().is_none());
6035 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
6036 assert!(reopened.sync_needed());
6037 assert!(matches!(
6038 reopened.drain_sync_intents().as_slice(),
6039 [SyncIntent::Interactive]
6040 ));
6041 drop(reopened);
6042 std::fs::remove_file(path).expect("remove temp database");
6043 }
6044
6045 #[test]
6046 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
6047 let path = std::env::temp_dir().join(format!(
6048 "syncular-unsatisfied-schema-floor-{}.db",
6049 uuid::Uuid::new_v4()
6050 ));
6051 let schema = json!({
6052 "version": 1,
6053 "tables": [{
6054 "name": "tasks",
6055 "primaryKey": "id",
6056 "columns": [
6057 { "name": "id", "type": "string", "nullable": false },
6058 { "name": "project_id", "type": "string", "nullable": false }
6059 ],
6060 "scopes": [{ "pattern": "project:{project_id}" }]
6061 }]
6062 });
6063
6064 {
6065 let mut first = SyncClient::open_path_with_identity(
6066 None,
6067 &schema,
6068 ClientLimits::default(),
6069 path.to_str().expect("UTF-8 temp path"),
6070 )
6071 .expect("first open");
6072 first
6073 .subscribe(
6074 "tasks".to_owned(),
6075 "tasks".to_owned(),
6076 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6077 None,
6078 )
6079 .expect("persist subscription");
6080 first.set_schema_floor(Some(SchemaFloor {
6081 required_schema_version: Some(2),
6082 latest_schema_version: Some(2),
6083 }));
6084 }
6085
6086 let reopened = SyncClient::open_path_with_identity(
6087 None,
6088 &schema,
6089 ClientLimits::default(),
6090 path.to_str().expect("UTF-8 temp path"),
6091 )
6092 .expect("reopen");
6093 assert_eq!(
6094 reopened.schema_floor(),
6095 Some(&SchemaFloor {
6096 required_schema_version: Some(2),
6097 latest_schema_version: Some(2),
6098 })
6099 );
6100 assert!(!reopened.sync_needed());
6101 drop(reopened);
6102 std::fs::remove_file(path).expect("remove temp database");
6103 }
6104
6105 #[test]
6106 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
6107 let path = std::env::temp_dir().join(format!(
6108 "syncular-indexed-column-bump-{}.db",
6109 uuid::Uuid::new_v4()
6110 ));
6111 let old_schema = json!({
6112 "version": 1,
6113 "tables": [
6114 {
6115 "name": "tasks",
6116 "primaryKey": "id",
6117 "columns": [
6118 { "name": "id", "type": "string", "nullable": false },
6119 { "name": "project_id", "type": "string", "nullable": false }
6120 ],
6121 "scopes": [{ "pattern": "project:{project_id}" }]
6122 },
6123 {
6124 "name": "legacy",
6125 "primaryKey": "id",
6126 "columns": [
6127 { "name": "id", "type": "string", "nullable": false },
6128 { "name": "project_id", "type": "string", "nullable": false }
6129 ],
6130 "scopes": [{ "pattern": "project:{project_id}" }]
6131 }
6132 ]
6133 });
6134 {
6135 let mut first = SyncClient::open_path_with_identity(
6136 None,
6137 &old_schema,
6138 ClientLimits::default(),
6139 path.to_str().expect("UTF-8 temp path"),
6140 )
6141 .expect("open old schema");
6142 first
6143 .subscribe(
6144 "legacy-sub".to_owned(),
6145 "legacy".to_owned(),
6146 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6147 None,
6148 )
6149 .expect("persist legacy subscription");
6150 }
6151
6152 let new_schema = json!({
6153 "version": 2,
6154 "tables": [{
6155 "name": "tasks",
6156 "primaryKey": "id",
6157 "columns": [
6158 { "name": "id", "type": "string", "nullable": false },
6159 { "name": "project_id", "type": "string", "nullable": false },
6160 { "name": "facility_membership_id", "type": "string", "nullable": true }
6161 ],
6162 "scopes": [{ "pattern": "project:{project_id}" }],
6163 "indexes": [{
6164 "name": "tasks_by_membership",
6165 "columns": ["project_id", "facility_membership_id"],
6166 "unique": false
6167 }]
6168 }]
6169 });
6170 let upgraded = SyncClient::open_path_with_identity(
6171 None,
6172 &new_schema,
6173 ClientLimits::default(),
6174 path.to_str().expect("UTF-8 temp path"),
6175 )
6176 .expect("open upgraded schema");
6177 let columns = upgraded
6178 .conn
6179 .prepare("PRAGMA table_info(tasks)")
6180 .expect("prepare columns")
6181 .query_map([], |row| row.get::<_, String>(1))
6182 .expect("query columns")
6183 .collect::<Result<Vec<_>, _>>()
6184 .expect("collect columns");
6185 assert!(columns.contains(&"facility_membership_id".to_owned()));
6186 let index_count: i64 = upgraded
6187 .conn
6188 .query_row(
6189 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
6190 [],
6191 |row| row.get(0),
6192 )
6193 .expect("query index");
6194 assert_eq!(index_count, 1);
6195 assert!(upgraded.subscription_state("legacy-sub").is_none());
6196 drop(upgraded);
6197 std::fs::remove_file(path).expect("remove temp database");
6198 }
6199
6200 #[test]
6201 fn migrates_pre_envelope_outcome_journal_additively() {
6202 let path = std::env::temp_dir().join(format!(
6203 "syncular-outcome-migration-{}.db",
6204 uuid::Uuid::new_v4()
6205 ));
6206 let conn = Connection::open(&path).expect("open legacy database");
6207 conn.execute_batch(
6208 "CREATE TABLE _syncular_commit_outcomes (
6209 seq INTEGER PRIMARY KEY AUTOINCREMENT,
6210 client_commit_id TEXT NOT NULL UNIQUE,
6211 status TEXT NOT NULL,
6212 recorded_at_ms INTEGER NOT NULL,
6213 results_json TEXT NOT NULL,
6214 resolution TEXT NOT NULL DEFAULT 'active',
6215 resolved_at_ms INTEGER,
6216 replacement_client_commit_id TEXT);",
6217 )
6218 .expect("create legacy outcome journal");
6219 drop(conn);
6220 let schema = json!({
6221 "version": 1,
6222 "tables": [{
6223 "name": "tasks",
6224 "primaryKey": "id",
6225 "columns": [
6226 { "name": "id", "type": "string", "nullable": false },
6227 { "name": "project_id", "type": "string", "nullable": false }
6228 ],
6229 "scopes": [{ "pattern": "project:{project_id}" }]
6230 }]
6231 });
6232 let client = SyncClient::open_path(
6233 "migration-native".to_owned(),
6234 &schema,
6235 ClientLimits::default(),
6236 path.to_str().expect("UTF-8 temp path"),
6237 )
6238 .expect("migrate database");
6239 let has_operations = client
6240 .conn
6241 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
6242 .expect("prepare table info")
6243 .query_map([], |row| row.get::<_, String>(1))
6244 .expect("query table info")
6245 .filter_map(Result::ok)
6246 .any(|column| column == "operations_json");
6247 assert!(has_operations);
6248 drop(client);
6249 std::fs::remove_file(path).expect("remove temp database");
6250 }
6251
6252 #[test]
6253 fn retained_rejection_survives_file_reopen_and_security_purge() {
6254 let path =
6255 std::env::temp_dir().join(format!("syncular-retained-{}.db", uuid::Uuid::new_v4()));
6256 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id", "columns": [
6257 { "name": "id", "type": "string", "nullable": false },
6258 { "name": "project_id", "type": "string", "nullable": false },
6259 { "name": "title", "type": "string", "nullable": false }
6260 ], "scopes": [{ "pattern": "project:{project_id}" }] }] });
6261 {
6262 let mut first = SyncClient::open_path(
6263 "retained-native".to_owned(),
6264 &schema,
6265 ClientLimits::default(),
6266 path.to_str().unwrap(),
6267 )
6268 .unwrap();
6269 first
6270 .write_base_row(
6271 "tasks",
6272 &vec![
6273 Some(ColumnValue::String("t1".into())),
6274 Some(ColumnValue::String("p1".into())),
6275 Some(ColumnValue::String("server".into())),
6276 ],
6277 1,
6278 )
6279 .unwrap();
6280 first.rebuild_overlay().expect("rebuild overlay");
6281 let operation = OutboxOp {
6282 table: "tasks".into(),
6283 row_id: "t1".into(),
6284 upsert: true,
6285 base_version: Some(1),
6286 values: Some(Map::from_iter([
6287 ("id".into(), json!("t1")),
6288 ("title".into(), json!("mine")),
6289 ])),
6290 };
6291 first
6292 .retain_failed_commit("retained", std::slice::from_ref(&operation))
6293 .unwrap();
6294 first
6295 .persist_commit_outcome(
6296 "retained",
6297 CommitOutcomeStatus::Rejected,
6298 &[CommitOperationOutcome::Error {
6299 rejection: RejectionRecord {
6300 client_commit_id: "retained".into(),
6301 op_index: 0,
6302 code: "validation.denied".into(),
6303 message: "denied".into(),
6304 retryable: false,
6305 details: None,
6306 operation: None,
6307 },
6308 }],
6309 Some(&[operation]),
6310 )
6311 .unwrap();
6312 first.rebuild_overlay().expect("rebuild overlay");
6313 }
6314 {
6315 let mut reopened = SyncClient::open_path(
6316 "retained-native".to_owned(),
6317 &schema,
6318 ClientLimits::default(),
6319 path.to_str().unwrap(),
6320 )
6321 .unwrap();
6322 let row = reopened
6323 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6324 .expect("read visible row")
6325 .unwrap();
6326 assert_eq!(row.0[2], Some(ColumnValue::String("mine".into())));
6327 assert_eq!(
6328 reopened
6329 .commit_outcome("retained")
6330 .unwrap()
6331 .unwrap()
6332 .retained_rows
6333 .unwrap()[0]
6334 .server_row
6335 .as_ref()
6336 .unwrap()["title"],
6337 json!("server")
6338 );
6339 reopened
6340 .conn
6341 .execute("DELETE FROM _syncular_base_tasks WHERE id = 't1'", [])
6342 .unwrap();
6343 reopened.rebuild_overlay().expect("rebuild overlay");
6344 assert!(reopened
6345 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6346 .expect("read visible row")
6347 .is_none());
6348 let retained = reopened.commit_outcome("retained").unwrap().unwrap();
6349 assert!(retained.retained_rows.is_some());
6350 assert_eq!(
6351 retained.operations.unwrap()[0].values.as_ref().unwrap()["title"],
6352 json!("mine")
6353 );
6354 reopened
6355 .purge_local_data(&LocalDataPurgeInput {
6356 purge_id: "protected-purge".into(),
6357 targets: vec![LocalDataPurgeTarget {
6358 table: "tasks".into(),
6359 selectors: BTreeMap::from_iter([("project_id".into(), vec!["p1".into()])]),
6360 }],
6361 })
6362 .unwrap();
6363 assert!(reopened
6364 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
6365 .expect("read visible row")
6366 .is_none());
6367 assert!(reopened
6368 .commit_outcome("retained")
6369 .unwrap()
6370 .unwrap()
6371 .retained_rows
6372 .is_none());
6373 }
6374 std::fs::remove_file(path).unwrap();
6375 }
6376
6377 #[test]
6378 fn durable_conflict_outcome_and_resolution_survive_reopen() {
6379 let path = std::env::temp_dir().join(format!(
6380 "syncular-durable-outcome-{}.db",
6381 uuid::Uuid::new_v4()
6382 ));
6383 let schema = json!({
6384 "version": 1,
6385 "tables": [{
6386 "name": "tasks",
6387 "primaryKey": "id",
6388 "columns": [
6389 { "name": "id", "type": "string", "nullable": false },
6390 { "name": "project_id", "type": "string", "nullable": false }
6391 ],
6392 "scopes": [{ "pattern": "project:{project_id}" }]
6393 }]
6394 });
6395 let conflict = ConflictRecord {
6396 client_commit_id: "losing-commit".to_owned(),
6397 op_index: 0,
6398 table: "tasks".to_owned(),
6399 row_id: "t1".to_owned(),
6400 code: "sync.version_conflict".to_owned(),
6401 message: "stale base version".to_owned(),
6402 server_version: 2,
6403 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6404 conflict_columns: vec!["project_id".to_owned()],
6405 operation: Some(CommitOperation {
6406 table: "tasks".to_owned(),
6407 row_id: "t1".to_owned(),
6408 op: "upsert".to_owned(),
6409 base_version: Some(1),
6410 values: Some(Map::from_iter([
6411 ("id".into(), json!("t1")),
6412 ("project_id".into(), json!("p1")),
6413 ])),
6414 }),
6415 };
6416 let failed_operations = vec![
6417 OutboxOp {
6418 upsert: true,
6419 table: "tasks".to_owned(),
6420 row_id: "t1".to_owned(),
6421 base_version: Some(1),
6422 values: Some(Map::from_iter([
6423 ("id".into(), json!("t1")),
6424 ("project_id".into(), json!("p1")),
6425 ])),
6426 },
6427 OutboxOp {
6428 upsert: true,
6429 table: "tasks".to_owned(),
6430 row_id: "status-event-1".to_owned(),
6431 base_version: Some(0),
6432 values: Some(Map::from_iter([
6433 ("id".into(), json!("status-event-1")),
6434 ("project_id".into(), json!("p1")),
6435 ])),
6436 },
6437 ];
6438
6439 {
6440 let mut first = SyncClient::open_path(
6441 "durable-native".to_owned(),
6442 &schema,
6443 ClientLimits::default(),
6444 path.to_str().expect("UTF-8 temp path"),
6445 )
6446 .expect("first open");
6447 first
6448 .begin_observation("test_outcome")
6449 .expect("begin outcome");
6450 first
6451 .persist_commit_outcome(
6452 "losing-commit",
6453 CommitOutcomeStatus::Conflict,
6454 &[CommitOperationOutcome::Conflict {
6455 conflict: conflict.clone(),
6456 }],
6457 Some(&failed_operations),
6458 )
6459 .expect("persist outcome");
6460 first.conflicts.push(conflict);
6461 first
6462 .finish_observation(
6463 "test_outcome",
6464 ChangeAccumulator {
6465 conflicts: true,
6466 outcomes: true,
6467 ..ChangeAccumulator::default()
6468 },
6469 )
6470 .expect("commit outcome");
6471 }
6472
6473 {
6474 let mut reopened = SyncClient::open_path(
6475 "durable-native".to_owned(),
6476 &schema,
6477 ClientLimits::default(),
6478 path.to_str().expect("UTF-8 temp path"),
6479 )
6480 .expect("reopen");
6481 assert_eq!(reopened.conflicts().len(), 1);
6482 assert_eq!(
6483 reopened.conflicts()[0].conflict_columns,
6484 vec!["project_id".to_owned()],
6485 "conflictColumns survives the journal round trip"
6486 );
6487 let outcome = reopened
6488 .commit_outcome("losing-commit")
6489 .expect("read outcome")
6490 .expect("outcome");
6491 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6492 let operations = outcome.operations.expect("aggregate envelope");
6493 assert_eq!(operations.len(), 2);
6494 assert_eq!(operations[1].row_id, "status-event-1");
6495 let resolved = reopened
6496 .resolve_commit_outcome(ResolveCommitOutcomeInput {
6497 client_commit_id: "losing-commit".to_owned(),
6498 resolution: CommitOutcomeResolution::ResolvedKeepServer,
6499 replacement_client_commit_id: None,
6500 })
6501 .expect("resolve");
6502 assert_eq!(
6503 resolved.resolution,
6504 CommitOutcomeResolution::ResolvedKeepServer
6505 );
6506 assert!(reopened.conflicts().is_empty());
6507 }
6508
6509 let reopened = SyncClient::open_path(
6510 "durable-native".to_owned(),
6511 &schema,
6512 ClientLimits::default(),
6513 path.to_str().expect("UTF-8 temp path"),
6514 )
6515 .expect("second reopen");
6516 assert!(reopened.conflicts().is_empty());
6517 assert_eq!(
6518 reopened
6519 .commit_outcome("losing-commit")
6520 .expect("read outcome")
6521 .expect("outcome")
6522 .resolution,
6523 CommitOutcomeResolution::ResolvedKeepServer
6524 );
6525 drop(reopened);
6526 std::fs::remove_file(path).expect("remove temp database");
6527 }
6528
6529 #[test]
6530 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
6531 let path =
6532 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6533 let schema = json!({
6534 "version": 1,
6535 "tables": [{
6536 "name": "tasks",
6537 "primaryKey": "id",
6538 "columns": [
6539 { "name": "id", "type": "string", "nullable": false },
6540 { "name": "project_id", "type": "string", "nullable": false }
6541 ],
6542 "scopes": [{ "pattern": "project:{project_id}" }]
6543 }]
6544 });
6545 let mut client = SyncClient::open_path_with_identity(
6546 Some("sidecar-client".to_owned()),
6547 &schema,
6548 ClientLimits::default(),
6549 path.to_str().expect("UTF-8 temp path"),
6550 )
6551 .expect("open owner");
6552 let base = WindowBase {
6553 table: "tasks".to_owned(),
6554 variable: "project_id".to_owned(),
6555 fixed_scopes: Vec::new(),
6556 params: None,
6557 };
6558 client
6559 .set_window(&base, &["one".to_owned()])
6560 .expect("set window");
6561 client
6562 .mutate(vec![Mutation::Upsert {
6563 table: "tasks".to_owned(),
6564 values: Map::from_iter([
6565 ("id".to_owned(), Value::from("t1")),
6566 ("project_id".to_owned(), Value::from("one")),
6567 ]),
6568 base_version: None,
6569 }])
6570 .expect("local mutate");
6571
6572 let coverage = [WindowCoverage {
6573 base,
6574 units: vec!["one".to_owned(), "missing".to_owned()],
6575 }];
6576 let owner = client
6577 .query_snapshot(
6578 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6579 &[],
6580 &coverage,
6581 None,
6582 )
6583 .expect("owner snapshot");
6584 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6585 let sidecar = reader
6586 .query_snapshot(
6587 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
6588 &[],
6589 &coverage,
6590 )
6591 .expect("sidecar snapshot");
6592
6593 assert_eq!(sidecar.revision, owner.revision);
6594 assert_eq!(sidecar.rows, owner.rows);
6595 assert_eq!(
6596 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
6597 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
6598 );
6599 assert_eq!(sidecar.revision, "2");
6600 assert_eq!(sidecar.rows[0]["id"], "t1");
6601 assert_eq!(sidecar.rows[0]["server_version"], -1);
6602 assert!(!sidecar.coverage.complete);
6603 assert_eq!(sidecar.coverage.pending.len(), 1);
6604 assert_eq!(sidecar.coverage.missing.len(), 1);
6605
6606 drop(reader);
6607 drop(client);
6608 std::fs::remove_file(path).expect("remove temp database");
6609 }
6610
6611 #[test]
6612 fn file_snapshot_reader_reports_catchup_delivery_and_multi_statement() {
6613 let path =
6614 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6615 let schema = json!({
6616 "version": 1,
6617 "tables": [{
6618 "name": "tasks",
6619 "primaryKey": "id",
6620 "columns": [
6621 { "name": "id", "type": "string", "nullable": false },
6622 { "name": "project_id", "type": "string", "nullable": false }
6623 ],
6624 "scopes": [{ "pattern": "project:{project_id}" }]
6625 }]
6626 });
6627 let mut client = SyncClient::open_path_with_identity(
6628 Some("sidecar-catchup".to_owned()),
6629 &schema,
6630 ClientLimits::default(),
6631 path.to_str().expect("UTF-8 temp path"),
6632 )
6633 .expect("open owner");
6634 client
6635 .subscribe(
6636 "s1".into(),
6637 "tasks".into(),
6638 vec![("project_id".into(), vec!["one".into()])],
6639 None,
6640 )
6641 .expect("subscribe");
6642 let commit_id = client
6643 .mutate(vec![Mutation::Upsert {
6644 table: "tasks".to_owned(),
6645 values: Map::from_iter([
6646 ("id".to_owned(), Value::from("t1")),
6647 ("project_id".to_owned(), Value::from("one")),
6648 ]),
6649 base_version: None,
6650 }])
6651 .expect("mutate");
6652 let revision = client.local_revision().to_string();
6653
6654 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6655 assert_eq!(reader.local_revision().expect("revision"), revision);
6656
6657 let read = reader
6658 .snapshot_read(&SnapshotReadRequest {
6659 statements: vec![
6660 SnapshotStatement {
6661 sql: "SELECT 1 AS n",
6662 params: &[],
6663 },
6664 SnapshotStatement {
6665 sql: "SELECT id FROM tasks ORDER BY id",
6666 params: &[],
6667 },
6668 ],
6669 coverage: &[],
6670 subscriptions: vec!["s1".to_owned(), "absent".to_owned()],
6671 commit_ids: vec![commit_id.clone(), "never-known".to_owned()],
6672 })
6673 .expect("snapshot read");
6674 assert_eq!(read.revision, revision);
6675 assert_eq!(read.queries.len(), 2);
6676 assert_eq!(read.queries[0][0]["n"], 1);
6677 assert_eq!(read.queries[1][0]["id"], "t1");
6678 match &read.subscriptions[0] {
6679 SubscriptionCatchup::Known {
6680 status,
6681 cursor,
6682 bootstrap_complete,
6683 known_pending_pages,
6684 ..
6685 } => {
6686 assert_eq!(status, "active");
6687 assert_eq!(*cursor, -1);
6688 assert!(!bootstrap_complete);
6689 assert!(known_pending_pages);
6690 }
6691 other => panic!("expected a known subscription: {other:?}"),
6692 }
6693 assert!(matches!(
6694 &read.subscriptions[1],
6695 SubscriptionCatchup::Unknown { id } if id.as_str() == "absent"
6696 ));
6697 assert!(matches!(
6698 &read.deliveries[0],
6699 CommitDelivery::Pending { client_commit_id } if client_commit_id == &commit_id
6700 ));
6701 assert!(matches!(
6702 &read.deliveries[1],
6703 CommitDelivery::Unknown { client_commit_id } if client_commit_id.as_str() == "never-known"
6704 ));
6705
6706 assert!(reader
6708 .query_snapshot("SELECT * FROM does_not_exist", &[], &[])
6709 .is_err());
6710 assert_eq!(
6711 reader.local_revision().expect("revision after rollback"),
6712 revision
6713 );
6714 assert!(reader
6716 .query_snapshot("DELETE FROM tasks", &[], &[])
6717 .is_err());
6718
6719 client
6721 .conn
6722 .execute(
6723 "UPDATE _syncular_meta SET value = '+1' WHERE key = 'localRevision'",
6724 [],
6725 )
6726 .expect("corrupt revision");
6727 let failure = reader.local_revision().expect_err("corrupt revision fails");
6728 assert_eq!(failure.code, Some("sync.local_corrupt"));
6729
6730 drop(reader);
6731 drop(client);
6732 std::fs::remove_file(path).expect("remove temp database");
6733 }
6734
6735 #[test]
6736 fn snapshot_delivery_is_the_persisted_journal_view_without_retained_rows() {
6737 let path =
6738 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
6739 let schema = json!({
6740 "version": 1,
6741 "tables": [{
6742 "name": "tasks",
6743 "primaryKey": "id",
6744 "columns": [
6745 { "name": "id", "type": "string", "nullable": false },
6746 { "name": "project_id", "type": "string", "nullable": false }
6747 ],
6748 "scopes": [{ "pattern": "project:{project_id}" }]
6749 }]
6750 });
6751 let mut client = SyncClient::open_path_with_identity(
6752 Some("sidecar-delivery".to_owned()),
6753 &schema,
6754 ClientLimits::default(),
6755 path.to_str().expect("UTF-8 temp path"),
6756 )
6757 .expect("open owner");
6758 client.begin_observation("test_delivery").expect("observe");
6759 let operations = [OutboxOp {
6760 upsert: true,
6761 table: "tasks".to_owned(),
6762 row_id: "t1".to_owned(),
6763 base_version: Some(1),
6764 values: Some(Map::from_iter([
6765 ("id".into(), json!("t1")),
6766 ("project_id".into(), json!("p1")),
6767 ])),
6768 }];
6769 client
6770 .persist_commit_outcome(
6771 "conflict-commit",
6772 CommitOutcomeStatus::Conflict,
6773 &[CommitOperationOutcome::Conflict {
6774 conflict: ConflictRecord {
6775 client_commit_id: "conflict-commit".to_owned(),
6776 op_index: 0,
6777 table: "tasks".to_owned(),
6778 row_id: "t1".to_owned(),
6779 code: "sync.version_conflict".to_owned(),
6780 message: "stale base version".to_owned(),
6781 server_version: 2,
6782 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
6783 conflict_columns: vec!["project_id".to_owned()],
6784 operation: None,
6785 },
6786 }],
6787 Some(&operations),
6788 )
6789 .expect("persist conflict");
6790 client
6791 .persist_commit_outcome(
6792 "rejected-commit",
6793 CommitOutcomeStatus::Rejected,
6794 &[CommitOperationOutcome::Error {
6795 rejection: RejectionRecord {
6796 client_commit_id: "rejected-commit".to_owned(),
6797 op_index: 0,
6798 code: "sync.unknown_table".to_owned(),
6799 message: "unknown table".to_owned(),
6800 retryable: false,
6801 details: None,
6802 operation: None,
6803 },
6804 }],
6805 Some(&operations),
6806 )
6807 .expect("persist rejection");
6808 client
6809 .finish_observation(
6810 "test_delivery",
6811 ChangeAccumulator {
6812 conflicts: true,
6813 outcomes: true,
6814 ..ChangeAccumulator::default()
6815 },
6816 )
6817 .expect("commit outcomes");
6818
6819 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6820 match reader.commit_delivery("conflict-commit").expect("conflict") {
6821 CommitDelivery::Known { outcome, .. } => {
6822 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
6823 assert!(matches!(
6824 outcome.results.as_slice(),
6825 [CommitOperationOutcome::Conflict { conflict }]
6826 if conflict.code == "sync.version_conflict"
6827 && conflict.server_version == 2
6828 ));
6829 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
6830 assert!(
6831 outcome.retained_rows.is_none(),
6832 "the sidecar reports the persisted view, not owner-derived retained rows"
6833 );
6834 }
6835 other => panic!("expected a known conflict: {other:?}"),
6836 }
6837 match reader
6838 .commit_delivery("rejected-commit")
6839 .expect("rejection")
6840 {
6841 CommitDelivery::Known { outcome, .. } => {
6842 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
6843 assert!(matches!(
6844 outcome.results.as_slice(),
6845 [CommitOperationOutcome::Error { rejection }]
6846 if rejection.code == "sync.unknown_table"
6847 ));
6848 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
6849 assert!(outcome.retained_rows.is_none());
6850 }
6851 other => panic!("expected a known rejection: {other:?}"),
6852 }
6853 drop(reader);
6854 drop(client);
6855 std::fs::remove_file(path).expect("remove temp database");
6856 }
6857
6858 #[test]
6859 fn snapshot_read_isolates_a_concurrent_writer_commit() {
6860 use std::sync::mpsc::channel;
6861 let path = std::env::temp_dir().join(format!(
6862 "syncular-read-concurrent-{}.db",
6863 uuid::Uuid::new_v4()
6864 ));
6865 let schema = json!({
6866 "version": 1,
6867 "tables": [{
6868 "name": "tasks",
6869 "primaryKey": "id",
6870 "columns": [
6871 { "name": "id", "type": "string", "nullable": false },
6872 { "name": "project_id", "type": "string", "nullable": false }
6873 ],
6874 "scopes": [{ "pattern": "project:{project_id}" }]
6875 }]
6876 });
6877 let mut client = SyncClient::open_path_with_identity(
6878 Some("sidecar-concurrent".to_owned()),
6879 &schema,
6880 ClientLimits::default(),
6881 path.to_str().expect("UTF-8 temp path"),
6882 )
6883 .expect("open owner");
6884 client
6885 .mutate(vec![Mutation::Upsert {
6886 table: "tasks".to_owned(),
6887 values: Map::from_iter([
6888 ("id".to_owned(), Value::from("t1")),
6889 ("project_id".to_owned(), Value::from("one")),
6890 ]),
6891 base_version: None,
6892 }])
6893 .expect("seed row");
6894 let revision_before = client.local_revision().to_string();
6895
6896 let (ready_tx, ready_rx) = channel::<()>();
6899 let (done_tx, done_rx) = channel::<()>();
6900 let writer_path = path.to_string_lossy().into_owned();
6901 let writer_schema = schema.clone();
6902 let writer = std::thread::spawn(move || {
6903 let mut writer = SyncClient::open_path_with_identity(
6904 Some("sidecar-concurrent".to_owned()),
6905 &writer_schema,
6906 ClientLimits::default(),
6907 &writer_path,
6908 )
6909 .expect("open writer");
6910 ready_rx.recv().expect("reader signalled");
6911 writer
6912 .mutate(vec![Mutation::Upsert {
6913 table: "tasks".to_owned(),
6914 values: Map::from_iter([
6915 ("id".to_owned(), Value::from("t2")),
6916 ("project_id".to_owned(), Value::from("one")),
6917 ]),
6918 base_version: None,
6919 }])
6920 .expect("writer commit");
6921 let _ = done_tx.send(());
6922 });
6923 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
6924 SNAPSHOT_READ_HOOK.with(|slot| {
6925 *slot.borrow_mut() = Some(Box::new(move || {
6926 ready_tx.send(()).expect("signal writer");
6927 done_rx.recv().expect("writer finished");
6928 }));
6929 });
6930 let read = reader
6931 .snapshot_read(&SnapshotReadRequest {
6932 statements: vec![
6933 SnapshotStatement {
6934 sql: "SELECT count(*) AS n FROM tasks",
6935 params: &[],
6936 },
6937 SnapshotStatement {
6938 sql: "SELECT count(*) AS n FROM tasks",
6939 params: &[],
6940 },
6941 ],
6942 coverage: &[],
6943 subscriptions: Vec::new(),
6944 commit_ids: Vec::new(),
6945 })
6946 .expect("snapshot read");
6947 assert_eq!(
6948 read.queries[0][0]["n"], 1,
6949 "first statement sees the pre-commit row"
6950 );
6951 assert_eq!(
6952 read.queries[1][0]["n"], 1,
6953 "a later statement keeps the one read snapshot"
6954 );
6955 assert_eq!(
6956 read.revision, revision_before,
6957 "the read reports the snapshot revision, not the writer's"
6958 );
6959 writer.join().expect("join writer");
6960
6961 assert_ne!(
6962 reader.local_revision().expect("revision after commit"),
6963 revision_before,
6964 "the next read observes the committed revision"
6965 );
6966 let after = reader
6967 .query_snapshot("SELECT count(*) AS n FROM tasks", &[], &[])
6968 .expect("count after commit");
6969 assert_eq!(
6970 after.rows[0]["n"], 2,
6971 "the committed row is visible afterwards"
6972 );
6973
6974 drop(reader);
6975 drop(client);
6976 std::fs::remove_file(path).expect("remove temp database");
6977 }
6978
6979 #[test]
6980 fn snapshot_read_rejects_every_non_canonical_revision() {
6981 let path = std::env::temp_dir().join(format!(
6982 "syncular-read-revision-{}.db",
6983 uuid::Uuid::new_v4()
6984 ));
6985 let schema = json!({
6986 "version": 1,
6987 "tables": [{
6988 "name": "tasks",
6989 "primaryKey": "id",
6990 "columns": [
6991 { "name": "id", "type": "string", "nullable": false },
6992 { "name": "project_id", "type": "string", "nullable": false }
6993 ],
6994 "scopes": [{ "pattern": "project:{project_id}" }]
6995 }]
6996 });
6997 let client = SyncClient::open_path_with_identity(
6998 Some("sidecar-revision".to_owned()),
6999 &schema,
7000 ClientLimits::default(),
7001 path.to_str().expect("UTF-8 temp path"),
7002 )
7003 .expect("open owner");
7004 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7005 for bad in [
7006 "+1",
7007 "01",
7008 "-1",
7009 " 1",
7010 "1 ",
7011 "18446744073709551616",
7012 "1.0",
7013 "abc",
7014 "",
7015 ] {
7016 client
7017 .conn
7018 .execute(
7019 "UPDATE _syncular_meta SET value = ?1 WHERE key = 'localRevision'",
7020 rusqlite::params![bad],
7021 )
7022 .expect("corrupt revision");
7023 let failure = reader.local_revision().expect_err("corrupt marker fails");
7024 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad:?}");
7025 }
7026 for value in ["X'31'", "CAST(X'FF' AS TEXT)"] {
7027 client
7028 .conn
7029 .execute_batch(&format!(
7030 "UPDATE _syncular_meta SET value={value} WHERE key='localRevision'"
7031 ))
7032 .expect("corrupt stored revision type");
7033 assert_eq!(
7034 reader.local_revision().expect_err("invalid SQL field").code,
7035 Some("sync.local_corrupt")
7036 );
7037 }
7038 client
7040 .conn
7041 .execute("DELETE FROM _syncular_meta WHERE key = 'localRevision'", [])
7042 .expect("delete marker");
7043 assert_eq!(reader.local_revision().expect("legacy zero"), "0");
7044
7045 drop(reader);
7046 drop(client);
7047 std::fs::remove_file(path).expect("remove temp database");
7048 }
7049
7050 #[test]
7051 fn snapshot_read_rejects_corrupt_subscription_state() {
7052 let path = std::env::temp_dir().join(format!(
7053 "syncular-read-substate-{}.db",
7054 uuid::Uuid::new_v4()
7055 ));
7056 let schema = json!({
7057 "version": 1,
7058 "tables": [{
7059 "name": "tasks",
7060 "primaryKey": "id",
7061 "columns": [
7062 { "name": "id", "type": "string", "nullable": false },
7063 { "name": "project_id", "type": "string", "nullable": false }
7064 ],
7065 "scopes": [{ "pattern": "project:{project_id}" }]
7066 }]
7067 });
7068 let mut client = SyncClient::open_path_with_identity(
7069 Some("sidecar-substate".to_owned()),
7070 &schema,
7071 ClientLimits::default(),
7072 path.to_str().expect("UTF-8 temp path"),
7073 )
7074 .expect("open owner");
7075 client
7076 .subscribe(
7077 "s1".into(),
7078 "tasks".into(),
7079 vec![("project_id".into(), vec!["one".into()])],
7080 None,
7081 )
7082 .expect("subscribe");
7083 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7084 for bad in [
7085 "not json".to_owned(),
7086 "{}".to_owned(),
7087 json!({ "status": "bogus", "cursor": 0 }).to_string(),
7088 json!({ "status": "active", "cursor": "x" }).to_string(),
7089 json!({ "status": "active", "cursor": 0, "bootstrapState": 7 }).to_string(),
7090 json!({ "status": "active", "cursor": 0, "reasonCode": 7 }).to_string(),
7091 ] {
7092 client
7093 .conn
7094 .execute(
7095 "UPDATE _syncular_subscriptions SET state_json = ?1 WHERE id = 's1'",
7096 rusqlite::params![bad],
7097 )
7098 .expect("corrupt subscription state");
7099 let failure = reader
7100 .subscription_catchup("s1")
7101 .expect_err("corrupt state fails");
7102 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad}");
7103 }
7104
7105 for assignment in [
7106 "tbl=X'7461736b73'",
7107 "state_json=X'7b7d'",
7108 "state_json=CAST(X'FF' AS TEXT)",
7109 ] {
7110 client.conn.execute_batch(&format!(
7111 "UPDATE _syncular_subscriptions SET tbl='tasks',state_json='{{\"status\":\"active\",\"cursor\":0}}'; UPDATE _syncular_subscriptions SET {assignment} WHERE id='s1'"
7112 )).expect("corrupt stored subscription type");
7113 assert_eq!(
7114 reader
7115 .subscription_catchup("s1")
7116 .expect_err("invalid SQL field")
7117 .code,
7118 Some("sync.local_corrupt")
7119 );
7120 }
7121
7122 drop(reader);
7123 drop(client);
7124 std::fs::remove_file(path).expect("remove temp database");
7125 }
7126
7127 #[test]
7128 fn snapshot_read_rolls_back_a_later_invalid_statement_and_guards_every_write() {
7129 let path = std::env::temp_dir().join(format!(
7130 "syncular-read-rollback-{}.db",
7131 uuid::Uuid::new_v4()
7132 ));
7133 let schema = json!({
7134 "version": 1,
7135 "tables": [{
7136 "name": "tasks",
7137 "primaryKey": "id",
7138 "columns": [
7139 { "name": "id", "type": "string", "nullable": false },
7140 { "name": "project_id", "type": "string", "nullable": false }
7141 ],
7142 "scopes": [{ "pattern": "project:{project_id}" }]
7143 }]
7144 });
7145 let mut client = SyncClient::open_path_with_identity(
7146 Some("sidecar-rollback".to_owned()),
7147 &schema,
7148 ClientLimits::default(),
7149 path.to_str().expect("UTF-8 temp path"),
7150 )
7151 .expect("open owner");
7152 let commit_id = client
7153 .mutate(vec![Mutation::Upsert {
7154 table: "tasks".to_owned(),
7155 values: Map::from_iter([
7156 ("id".to_owned(), Value::from("t1")),
7157 ("project_id".to_owned(), Value::from("one")),
7158 ]),
7159 base_version: None,
7160 }])
7161 .expect("seed row");
7162 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7163 let revision = reader.local_revision().expect("revision");
7164
7165 let failure = reader
7168 .snapshot_read(&SnapshotReadRequest {
7169 statements: vec![
7170 SnapshotStatement {
7171 sql: "SELECT id FROM tasks",
7172 params: &[],
7173 },
7174 SnapshotStatement {
7175 sql: "SELECT * FROM does_not_exist",
7176 params: &[],
7177 },
7178 ],
7179 coverage: &[],
7180 subscriptions: Vec::new(),
7181 commit_ids: vec![commit_id.clone()],
7182 })
7183 .expect_err("second statement fails");
7184 assert_eq!(failure.code, None);
7185 assert_eq!(
7186 reader.local_revision().expect("revision after rollback"),
7187 revision,
7188 "a rolled-back read consumes no revision"
7189 );
7190 assert!(
7191 !matches!(
7192 reader
7193 .commit_delivery(&commit_id)
7194 .expect("delivery after rollback"),
7195 CommitDelivery::Unknown { .. }
7196 ),
7197 "the same reader serves the next read after releasing the connection"
7198 );
7199
7200 let guard = reader
7202 .snapshot_read(&SnapshotReadRequest {
7203 statements: vec![SnapshotStatement {
7204 sql: "DELETE FROM tasks",
7205 params: &[],
7206 }],
7207 coverage: &[],
7208 subscriptions: Vec::new(),
7209 commit_ids: Vec::new(),
7210 })
7211 .expect_err("write guard");
7212 assert!(guard.message.contains("read-only"), "{}", guard.message);
7213
7214 drop(reader);
7215 drop(client);
7216 std::fs::remove_file(path).expect("remove temp database");
7217 }
7218
7219 #[test]
7220 fn owned_query_failures_classify_order_bound_and_clear() {
7221 let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7222 rusqlite::ffi::Error::new(266),
7223 Some("private SQLite prose".to_owned()),
7224 ));
7225 assert_eq!(io.code, Some("client.storage_io"));
7226 assert_eq!(io.sqlite_code, Some(266));
7227 assert_eq!(io.message, "local SQLite storage I/O failed");
7228 let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7229 rusqlite::ffi::Error::new(26),
7230 None,
7231 ));
7232 assert_eq!(corrupt.code, Some("client.storage_corrupt"));
7233 assert!(!io.retryable());
7234 assert!(!corrupt.retryable());
7235
7236 for (raw, extended) in [(5, 5), (5 | (1 << 8), 261), (6, 6), (6 | (1 << 8), 262)] {
7239 let busy = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7240 rusqlite::ffi::Error::new(raw),
7241 Some("database is locked".to_owned()),
7242 ));
7243 assert_eq!(busy.code, Some("client.storage_busy"));
7244 assert_eq!(busy.sqlite_code, Some(extended));
7245 assert_eq!(busy.message, "local SQLite storage is busy");
7246 assert!(busy.retryable());
7247 }
7248 let unrelated = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7249 rusqlite::ffi::Error::new(1),
7250 Some("generic SQLite prose".to_owned()),
7251 ));
7252 assert_eq!(unrelated.code, None);
7253 assert!(!unrelated.retryable());
7254
7255 let mut client = client();
7256 let generic = QueryReadFailure::from("private query prose".to_owned());
7257 client.record_query_read(
7258 QueryOwner {
7259 id: "first",
7260 tables: &["tasks", "tasks"],
7261 },
7262 Some(&generic),
7263 );
7264 client.record_query_read(
7265 QueryOwner {
7266 id: "second",
7267 tables: &["tasks"],
7268 },
7269 Some(&generic),
7270 );
7271 client.record_query_read(
7272 QueryOwner {
7273 id: "first",
7274 tables: &["docs"],
7275 },
7276 Some(&io),
7277 );
7278 let failures = &client
7279 .diagnostics_snapshot(&Default::default())
7280 .expect("diagnostics")
7281 .query_failures;
7282 assert_eq!(
7283 failures
7284 .iter()
7285 .map(|entry| entry.id.as_str())
7286 .collect::<Vec<_>>(),
7287 vec!["second", "first"]
7288 );
7289 assert_eq!(failures[1].tables, vec!["docs"]);
7290 assert_eq!(failures[1].code, "client.storage_io");
7291
7292 for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
7293 client.record_query_read(
7294 QueryOwner {
7295 id: &format!("query-{index}"),
7296 tables: &["tasks"],
7297 },
7298 Some(&generic),
7299 );
7300 }
7301 let failures = &client
7302 .diagnostics_snapshot(&Default::default())
7303 .expect("bounded diagnostics")
7304 .query_failures;
7305 assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
7306 assert_eq!(failures.last().expect("last failure").id, "query-256");
7307
7308 client
7309 .query_snapshot(
7310 "SELECT id FROM tasks",
7311 &[],
7312 &[],
7313 Some(QueryOwner {
7314 id: "query-256",
7315 tables: &["tasks"],
7316 }),
7317 )
7318 .expect("successful owned read");
7319 assert!(client
7320 .diagnostics_snapshot(&Default::default())
7321 .expect("cleared diagnostics")
7322 .query_failures
7323 .iter()
7324 .all(|entry| entry.id != "query-256"));
7325 }
7326
7327 #[test]
7328 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
7329 let mut client = client();
7330 let image = Connection::open_in_memory().unwrap();
7331 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
7332 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7333 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
7334 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7335 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
7336 let table = client.schema.table("tasks").unwrap().clone();
7337 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7338 assert!(client
7339 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
7340 .is_err());
7341 assert!(client.conn.is_autocommit());
7342 assert_eq!(
7343 client
7344 .query("SELECT count(*) AS n FROM tasks", &[])
7345 .unwrap()[0]["n"],
7346 1024
7347 );
7348 assert_eq!(client.drain_change_batches().len(), 1);
7349 image
7350 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
7351 .unwrap();
7352 assert!(matches!(
7353 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
7354 Ok(2050)
7355 ));
7356 assert_eq!(
7357 client
7358 .query("SELECT count(*) AS n FROM tasks", &[])
7359 .unwrap()[0]["n"],
7360 2050
7361 );
7362 assert_eq!(client.drain_change_batches().len(), 3);
7363 }
7364
7365 #[test]
7366 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
7367 let mut client = client();
7368 let base = WindowBase {
7369 table: "tasks".into(),
7370 variable: "project_id".into(),
7371 fixed_scopes: Vec::new(),
7372 params: None,
7373 };
7374 client
7375 .set_window(&base, &["p1".into(), "p2".into()])
7376 .unwrap();
7377 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7378 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7379 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
7380 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7381 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7382 client.drain_change_batches();
7383 assert!(client.set_window(&base, &["p2".into()]).is_err());
7384 assert!(client.conn.is_autocommit());
7385 assert_eq!(
7386 client
7387 .query("SELECT count(*) AS n FROM tasks", &[])
7388 .unwrap()[0]["n"],
7389 1027
7390 );
7391 assert_eq!(client.window_state(&base).units, vec!["p2"]);
7392 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
7393 assert_eq!(client.drain_change_batches().len(), 1);
7394 client
7395 .conn
7396 .execute_batch("DROP TRIGGER interrupt_evict")
7397 .unwrap();
7398 client.drain_pending_evictions().unwrap();
7399 assert!(client.load_pending_evictions().unwrap().is_empty());
7400 assert_eq!(
7401 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
7402 "held"
7403 );
7404 assert_eq!(client.drain_change_batches().len(), 2);
7405 }
7406
7407 #[test]
7408 fn eviction_marker_failures_roll_back_window_changes() {
7409 let mut client = client();
7410 let base = WindowBase {
7411 table: "tasks".into(),
7412 variable: "project_id".into(),
7413 fixed_scopes: Vec::new(),
7414 params: None,
7415 };
7416 client.set_window(&base, &["p1".into()]).unwrap();
7417 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
7418 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
7419 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
7420 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7421 let revision = client.local_revision();
7422 assert!(client.set_window(&base, &[]).is_err());
7423 assert!(client.conn.is_autocommit());
7424 assert_eq!(client.local_revision(), revision);
7425 assert_eq!(
7426 client
7427 .query("SELECT count(*) AS n FROM tasks", &[])
7428 .unwrap()[0]["n"],
7429 2050
7430 );
7431 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7432 client.conn.execute_batch("DROP TRIGGER fail_marker;
7433 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
7434 assert!(client.set_window(&base, &[]).is_err());
7435 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
7436 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
7437 assert!(client.drain_pending_evictions().is_err());
7438 assert!(client.conn.is_autocommit());
7439 assert_eq!(
7440 client
7441 .query("SELECT count(*) AS n FROM tasks", &[])
7442 .unwrap()[0]["n"],
7443 2
7444 );
7445 assert!(client.set_window(&base, &["p1".into()]).is_err());
7446 assert!(client.conn.is_autocommit());
7447 assert!(client.window_state(&base).units.is_empty());
7448 assert!(client.subs.is_empty());
7449 assert_eq!(
7450 client
7451 .query(
7452 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
7453 &[]
7454 )
7455 .unwrap()[0]["n"],
7456 1
7457 );
7458 client
7459 .conn
7460 .execute_batch("DROP TRIGGER fail_marker")
7461 .unwrap();
7462 client.set_window(&base, &["p1".into()]).unwrap();
7463 assert_eq!(client.window_state(&base).units, vec!["p1"]);
7464 }
7465
7466 #[test]
7467 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
7468 let mut client = client();
7469 client
7470 .conn
7471 .execute(
7472 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
7473 [],
7474 )
7475 .unwrap();
7476 assert_eq!(
7477 client.drain_pending_evictions().unwrap_err(),
7478 "sync.local_corrupt: invalid pending eviction scopes"
7479 );
7480 assert!(client.conn.is_autocommit());
7481 assert!(client.enqueue_startup_sync_if_needed().is_err());
7482 }
7483
7484 #[test]
7485 fn image_chunks_reject_null_primary_keys() {
7486 let mut client = client();
7487 let image = Connection::open_in_memory().unwrap();
7488 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7489 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
7490 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
7491 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7492 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
7493 let table = client.schema.table("tasks").unwrap().clone();
7494 assert!(client
7495 .apply_sqlite_image(
7496 &image,
7497 &table,
7498 true,
7499 &[("project_id".into(), vec!["p1".into()])],
7500 1025,
7501 7,
7502 "digest"
7503 )
7504 .is_err());
7505 assert!(client.conn.is_autocommit());
7506 assert_eq!(
7507 client
7508 .query("SELECT count(*) AS n FROM tasks", &[])
7509 .unwrap()[0]["n"],
7510 0
7511 );
7512 }
7513
7514 #[test]
7515 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
7516 for images in [false, true] {
7517 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7518 "name":name,"primaryKey":"id","columns":[
7519 {"name":"id","type":"string","nullable":false},
7520 {"name":"project_id","type":"string","nullable":false},
7521 {"name":"title","type":"string","nullable":false}],
7522 "scopes":[{"pattern":"project:{project_id}"}],
7523 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
7524 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7525 })))});
7526 let mut actual =
7527 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
7528 let mut reference =
7529 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
7530 let scopes = vec![("project_id".into(), vec!["p1".into()])];
7531 for client in [&mut actual, &mut reference] {
7532 client
7533 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
7534 .unwrap();
7535 client.subs[0].effective = Some(scopes.clone());
7536 for (table, id, title) in [
7537 ("tasks", "a", "original"),
7538 ("tasks", "b", "occupied"),
7539 ("tasks", "c", "deleted"),
7540 ("notes", "n", "untouched"),
7541 ] {
7542 client
7543 .write_base_row(
7544 table,
7545 &vec![
7546 Some(ColumnValue::String(id.into())),
7547 Some(ColumnValue::String("p1".into())),
7548 Some(ColumnValue::String(title.into())),
7549 ],
7550 1,
7551 )
7552 .unwrap();
7553 }
7554 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7555 client
7556 .mutate(vec![
7557 Mutation::Upsert {
7558 table: "tasks".into(),
7559 values: Map::from_iter([
7560 ("id".into(), Value::from("a")),
7561 ("project_id".into(), Value::from("p1")),
7562 ("title".into(), Value::from("available")),
7563 ]),
7564 base_version: None,
7565 },
7566 Mutation::Delete {
7567 table: "tasks".into(),
7568 row_id: "c".into(),
7569 base_version: None,
7570 },
7571 Mutation::Upsert {
7572 table: "tasks".into(),
7573 values: Map::from_iter([
7574 ("id".into(), Value::from("local")),
7575 ("project_id".into(), Value::from("p1")),
7576 ("title".into(), Value::from("local value")),
7577 ]),
7578 base_version: None,
7579 },
7580 Mutation::Upsert {
7581 table: "notes".into(),
7582 values: Map::from_iter([
7583 ("id".into(), Value::from("n")),
7584 ("project_id".into(), Value::from("p1")),
7585 ("title".into(), Value::from("pending note")),
7586 ]),
7587 base_version: None,
7588 },
7589 ])
7590 .unwrap();
7591 }
7592 let pending = actual.pending_commit_ids();
7593 actual.overlay_rebuild_count.set(0);
7594 for (id, title) in [
7595 ("b", "available"),
7596 ("b", "occupied"),
7597 ("a", "server original"),
7598 ("d", "local value"),
7599 ("d", "released"),
7600 ("c", "redelivered"),
7601 ] {
7602 let table = actual.schema.table("tasks").unwrap().clone();
7603 let values = vec![
7604 Some(ColumnValue::String(id.into())),
7605 Some(ColumnValue::String("p1".into())),
7606 Some(ColumnValue::String(title.into())),
7607 ];
7608 if images {
7609 let image = Connection::open_in_memory().unwrap();
7610 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7611 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7612 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7613 image
7614 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
7615 .unwrap();
7616 assert!(matches!(
7617 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
7618 Ok(1)
7619 ));
7620 } else {
7621 let segment = RowsSegment {
7622 table: "tasks".into(),
7623 schema_version: 1,
7624 columns: table.wire_columns.clone(),
7625 blocks: vec![vec![ssp2::segment::SegmentRow {
7626 server_version: 2,
7627 values: values.clone(),
7628 }]],
7629 };
7630 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
7631 }
7632 reference.write_base_row("tasks", &values, 2).unwrap();
7633 reference.rebuild_overlay().expect("rebuild overlay");
7634 for table in ["tasks", "notes"] {
7635 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
7636 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
7637 }
7638 }
7639 assert_eq!(actual.pending_commit_ids(), pending);
7640 }
7641 assert_eq!(
7642 actual.overlay_rebuild_count.get(),
7643 0,
7644 "each chunk must leave unrelated tables alone"
7645 );
7646 let before = actual
7647 .query("SELECT * FROM tasks ORDER BY id", &[])
7648 .unwrap();
7649 let fts_before = actual
7650 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7651 .unwrap();
7652 let revision = actual.local_revision();
7653 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
7654 let table = actual.schema.table("tasks").unwrap().clone();
7655 let segment = RowsSegment {
7656 table: "tasks".into(),
7657 schema_version: 1,
7658 columns: table.wire_columns.clone(),
7659 blocks: vec![vec![ssp2::segment::SegmentRow {
7660 server_version: 3,
7661 values: vec![
7662 Some(ColumnValue::String("b".into())),
7663 Some(ColumnValue::String("p1".into())),
7664 Some(ColumnValue::String("fresh".into())),
7665 ],
7666 }]],
7667 };
7668 assert!(actual.apply_segment(0, &segment, false).is_err());
7669 assert!(actual.conn.is_autocommit());
7670 assert_eq!(
7671 actual
7672 .query("SELECT * FROM tasks ORDER BY id", &[])
7673 .unwrap(),
7674 before
7675 );
7676 assert_eq!(
7677 actual
7678 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
7679 .unwrap(),
7680 fts_before
7681 );
7682 assert_eq!(actual.local_revision(), revision);
7683 assert_eq!(actual.pending_commit_ids(), pending);
7684 }
7685 }
7686
7687 #[test]
7688 fn import_reconciliation_preserves_typed_pending_primary_keys() {
7689 for (kind, a, b) in [
7690 ("string", json!("a"), json!("b")),
7691 ("integer", json!(1), json!(2)),
7692 ("boolean", json!(true), json!(false)),
7693 ("json", json!("1"), json!("2")),
7694 ] {
7695 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
7696 for images in [false, true] {
7697 let mut client =
7698 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
7699 let table = client.schema.table("tasks").unwrap().clone();
7700 client
7701 .subscribe("tasks".into(), "tasks".into(), vec![], None)
7702 .unwrap();
7703 for (id, title) in [(&a, "original"), (&b, "occupied")] {
7704 client
7705 .write_base_row(
7706 "tasks",
7707 &vec![
7708 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
7709 Some(ColumnValue::String(title.into())),
7710 ],
7711 1,
7712 )
7713 .unwrap();
7714 }
7715 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
7716 client
7717 .mutate(vec![Mutation::Upsert {
7718 table: "tasks".into(),
7719 values: Map::from_iter([
7720 ("id".into(), a.clone()),
7721 ("title".into(), json!("available")),
7722 ]),
7723 base_version: None,
7724 }])
7725 .unwrap();
7726 client.overlay_rebuild_count.set(0);
7727 for (title, expected) in [("available", "original"), ("occupied", "available")] {
7728 let values = vec![
7729 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
7730 Some(ColumnValue::String(title.into())),
7731 ];
7732 if images {
7733 let image = Connection::open_in_memory().unwrap();
7734 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7735 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
7736 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
7737 image
7738 .execute(
7739 "INSERT INTO tasks VALUES (?1,?2,2)",
7740 rusqlite::params![RowParam::Cell(&values[0]), title],
7741 )
7742 .unwrap();
7743 assert!(
7744 matches!(
7745 client.apply_sqlite_image(
7746 &image,
7747 &table,
7748 false,
7749 &[],
7750 1,
7751 7,
7752 "digest"
7753 ),
7754 Ok(1)
7755 ),
7756 "{kind}, images={images}"
7757 );
7758 } else {
7759 let segment = RowsSegment {
7760 table: "tasks".into(),
7761 schema_version: 1,
7762 columns: table.wire_columns.clone(),
7763 blocks: vec![vec![ssp2::segment::SegmentRow {
7764 server_version: 2,
7765 values,
7766 }]],
7767 };
7768 assert!(
7769 matches!(client.apply_segment(0, &segment, false), Ok(1)),
7770 "{kind}"
7771 );
7772 }
7773 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
7774 let actual: String = client
7775 .conn
7776 .query_row(
7777 "SELECT title FROM tasks WHERE id=?1",
7778 [RowParam::Cell(&key)],
7779 |row| row.get(0),
7780 )
7781 .unwrap();
7782 assert_eq!(actual, expected, "{kind}, images={images}");
7783 }
7784 assert_eq!(client.overlay_rebuild_count.get(), 0);
7785 }
7786 }
7787 }
7788
7789 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
7790 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
7791 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
7792 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
7793 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
7794 })))});
7795 let mut client =
7796 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
7797 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
7798 client.rebuild_overlay().expect("rebuild overlay");
7799 client
7800 .mutate(vec![Mutation::Upsert {
7801 table: "tasks".into(),
7802 values: Map::from_iter([
7803 ("id".into(), json!("offline")),
7804 ("project_id".into(), json!("p1")),
7805 ("title".into(), json!("pending edit")),
7806 ]),
7807 base_version: None,
7808 }])
7809 .unwrap();
7810 let image = Connection::open_in_memory().unwrap();
7811 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
7812 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
7813 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
7814 image
7815 .execute(
7816 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
7817 [rows],
7818 )
7819 .unwrap();
7820 client.overlay_rebuild_count.set(0);
7821 let table = client.schema.table("tasks").unwrap().clone();
7822 (client, image, table)
7823 }
7824
7825 #[test]
7826 fn sqlite_full_image_import_retains_the_first_code_and_recovers() {
7827 let (mut client, image, table) = unique_import_fixture(1);
7828 image
7829 .execute("UPDATE tasks SET title = ?1", ["full ".repeat(32768)])
7830 .unwrap();
7831 client
7832 .conn
7833 .execute_batch("PRAGMA max_page_count = 1")
7834 .unwrap();
7835 let effective = [("project_id".into(), vec!["p1".into()])];
7836 assert!(client
7837 .apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest")
7838 .is_err());
7839 let failure = client.storage_failure.borrow().clone().unwrap();
7840 assert_eq!(failure.code, Some("client.storage_full"));
7841 assert_eq!(failure.sqlite_code, Some(13));
7842 assert!(!failure.message.contains("rollback"));
7843 assert!(client.conn.is_autocommit());
7844 client
7845 .conn
7846 .execute_batch("PRAGMA max_page_count = 1073741823")
7847 .unwrap();
7848 assert!(matches!(
7849 client.apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest"),
7850 Ok(1)
7851 ));
7852 assert!(client.conn.is_autocommit());
7853 }
7854
7855 #[test]
7856 fn authoring_storage_failures_are_structured_and_scoped() {
7857 let path = std::env::temp_dir().join(format!(
7858 "syncular-authoring-failure-{}.db",
7859 uuid::Uuid::new_v4()
7860 ));
7861 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
7862 {"name":"id","type":"string","nullable":false},
7863 {"name":"project_id","type":"string","nullable":false},
7864 {"name":"title","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
7865 let mut client = SyncClient::open_path(
7866 "authoring".into(),
7867 &schema,
7868 ClientLimits::default(),
7869 path.to_str().unwrap(),
7870 )
7871 .expect("open client");
7872 client.create_synced_tables().expect("create synced tables");
7873 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
7875
7876 let upsert = |id: &str| {
7877 vec![Mutation::Upsert {
7878 table: "tasks".into(),
7879 values: Map::from_iter([
7880 ("id".into(), json!(id)),
7881 ("project_id".into(), json!("p1")),
7882 ("title".into(), json!("first")),
7883 ]),
7884 base_version: None,
7885 }]
7886 };
7887
7888 let locker = Connection::open(&path).expect("second connection");
7889 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
7890 let busy = client.mutate(upsert("t1")).expect_err("busy write");
7891 assert_eq!(busy.code, "client.storage_busy");
7892 assert!(busy.retryable);
7893 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
7894 assert!(client.pending_commit_ids().is_empty());
7895 assert_eq!(client.local_revision(), 0);
7896
7897 let invalid = client.mutate(Vec::new()).expect_err("empty commit");
7899 assert_eq!(invalid.code, "sync.invalid_request");
7900 assert_eq!(invalid.message, "the authoring request is invalid");
7901 assert!(
7902 invalid.details.as_ref().unwrap()["legacyCause"]
7903 .as_str()
7904 .unwrap()
7905 .contains("at least one operation"),
7906 "{invalid:?}"
7907 );
7908
7909 locker.execute_batch("ROLLBACK").unwrap();
7910 let id = client.mutate(upsert("t1")).expect("retry after busy");
7911 assert_eq!(client.pending_commit_ids(), vec![id]);
7912 assert_eq!(client.local_revision(), 1);
7913
7914 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
7916 let busy_again = client.mutate(upsert("t2")).expect_err("busy write again");
7917 assert_eq!(busy_again.code, "client.storage_busy");
7918 assert!(busy_again.retryable);
7919 locker.execute_batch("ROLLBACK").unwrap();
7920
7921 let full = ClientError::from(QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7923 rusqlite::ffi::Error::new(13),
7924 Some("database or disk is full".to_owned()),
7925 )));
7926 assert_eq!(full.code, "client.storage_full");
7927 assert!(!full.retryable);
7928 assert_eq!(full.details.as_ref().unwrap()["sqliteCode"], 13);
7929
7930 drop(locker);
7931 drop(client);
7932 let _ = std::fs::remove_file(&path);
7933 }
7934
7935 #[test]
7936 fn caller_value_failures_are_invalid_request_and_leave_no_outbox_entry() {
7937 let path = std::env::temp_dir().join(format!(
7938 "syncular-caller-values-{}.db",
7939 uuid::Uuid::new_v4()
7940 ));
7941 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
7942 {"name":"id","type":"string","nullable":false},
7943 {"name":"project_id","type":"string","nullable":false},
7944 {"name":"title","type":"string","nullable":false},
7945 {"name":"priority","type":"integer","nullable":false},
7946 {"name":"payload","type":"bytes","nullable":false},
7947 {"name":"extra","type":"bytes","nullable":true}],"scopes":[{"pattern":"project:{project_id}"}]}]});
7948 let mut client = SyncClient::open_path(
7949 "caller-values".into(),
7950 &schema,
7951 ClientLimits::default(),
7952 path.to_str().unwrap(),
7953 )
7954 .expect("open client");
7955 client.create_synced_tables().expect("create synced tables");
7956
7957 let values = |id: &str| {
7958 Map::from_iter([
7959 ("id".to_owned(), json!(id)),
7960 ("project_id".to_owned(), json!("p1")),
7961 ("title".to_owned(), json!("first")),
7962 ("priority".to_owned(), json!(1)),
7963 ("payload".to_owned(), json!({"$bytes": "0a"})),
7964 ])
7965 };
7966 let upsert = |values: Map<String, Value>| {
7967 vec![Mutation::Upsert {
7968 table: "tasks".into(),
7969 values,
7970 base_version: None,
7971 }]
7972 };
7973 let seed = client.mutate(upsert(values("t1"))).expect("seed row");
7974 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
7975
7976 for (label, override_value, cause) in [
7981 ("unknown column", ("nope", json!(1)), "unknown column"),
7982 (
7983 "internal sync column",
7984 ("_sync_version", json!(1)),
7985 "internal sync column",
7986 ),
7987 ("wrong value type", ("title", json!(7)), "expected a string"),
7988 (
7989 "malformed bytes envelope",
7990 ("payload", json!({"$bytes": "zz"})),
7991 "expected exactly",
7992 ),
7993 (
7994 "odd-length bytes envelope",
7995 ("payload", json!({"$bytes": "abc"})),
7996 "expected exactly",
7997 ),
7998 (
7999 "plus-sign bytes envelope",
8000 ("payload", json!({"$bytes": "+a"})),
8001 "expected exactly",
8002 ),
8003 (
8004 "bytes envelope with a second key",
8005 ("payload", json!({"$bytes": "0a", "extra": 1})),
8006 "expected exactly",
8007 ),
8008 (
8009 "no bytes envelope",
8010 ("payload", json!({"nested": true})),
8011 "expected exactly",
8012 ),
8013 ] {
8014 let mut full = values("t2");
8015 full.insert(override_value.0.to_owned(), override_value.1);
8016 let error = client.mutate(upsert(full)).expect_err(label);
8017 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8018 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8019 assert!(!error.retryable, "{label}");
8020 let legacy = error.details.as_ref().unwrap()["legacyCause"]
8021 .as_str()
8022 .unwrap();
8023 assert!(
8024 legacy.starts_with("sync.invalid_request: ") && legacy.contains(cause),
8025 "{label}: {legacy}"
8026 );
8027 assert_eq!(client.pending_commit_ids(), vec![seed.clone()], "{label}");
8028 assert_eq!(client.local_revision(), 1, "{label}");
8029 assert_eq!(
8030 client.query("SELECT * FROM tasks", &[]).unwrap(),
8031 rows,
8032 "{label}"
8033 );
8034 }
8035
8036 let mut missing = values("t3");
8039 missing.remove("payload");
8040 let missing_error = client.mutate(upsert(missing)).expect_err("missing column");
8041 assert_eq!(missing_error.code, "sync.invalid_request");
8042 assert!(missing_error.details.as_ref().unwrap()["legacyCause"]
8043 .as_str()
8044 .unwrap()
8045 .contains("is not nullable"));
8046
8047 let mut bad_row_id = values("t4");
8050 bad_row_id.insert("id".to_owned(), json!({"nested": true}));
8051 let row_id_error = client.mutate(upsert(bad_row_id)).expect_err("bad rowId");
8052 assert_eq!(row_id_error.code, "sync.invalid_request");
8053
8054 for (label, partial, cause) in [
8056 (
8057 "patch wrong value type",
8058 ("priority", json!("high")),
8059 "expected an integer",
8060 ),
8061 ("patch unknown column", ("nope", json!(1)), "unknown column"),
8062 ] {
8063 let error = client
8064 .patch(
8065 "tasks",
8066 "t1",
8067 Map::from_iter([(partial.0.to_owned(), partial.1)]),
8068 None,
8069 )
8070 .expect_err(label);
8071 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8072 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8073 assert!(
8074 error.details.as_ref().unwrap()["legacyCause"]
8075 .as_str()
8076 .unwrap()
8077 .contains(cause),
8078 "{label}"
8079 );
8080 }
8081 assert_eq!(client.pending_commit_ids(), vec![seed.clone()]);
8082 assert_eq!(client.local_revision(), 1);
8083 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
8084
8085 for (label, payload) in [
8088 ("explicit null", json!(null)),
8089 ("empty envelope", json!({"$bytes": ""})),
8090 ("uppercase envelope", json!({"$bytes": "0AFF"})),
8091 ] {
8092 let mut accepted = values("t-ok");
8093 accepted.insert("extra".to_owned(), payload);
8094 let encoded = client.mutate(upsert(accepted)).unwrap_or_else(|error| {
8095 panic!("{label} must be accepted: {error:?}");
8096 });
8097 assert!(!encoded.is_empty(), "{label}");
8098 }
8099
8100 let next = client
8102 .mutate(upsert(values("t5")))
8103 .expect("valid after failures");
8104 assert_ne!(next, seed);
8105
8106 drop(client);
8107 let _ = std::fs::remove_file(&path);
8108 }
8109
8110 #[test]
8111 fn a_legacy_outbox_value_the_codec_refuses_is_dropped_instead_of_wedging() {
8112 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8113 {"name":"id","type":"string","nullable":false},
8114 {"name":"project_id","type":"string","nullable":false},
8115 {"name":"payload","type":"bytes","nullable":true}],
8116 "scopes":[{"pattern":"project:{project_id}"}]}]});
8117 let open = |path: &std::path::Path| {
8118 SyncClient::open_path(
8119 "legacy-values".into(),
8120 &schema,
8121 ClientLimits::default(),
8122 path.to_str().unwrap(),
8123 )
8124 .expect("open client")
8125 };
8126 let row = |id: &str| {
8127 Map::from_iter([
8128 ("id".to_owned(), json!(id)),
8129 ("project_id".to_owned(), json!("p1")),
8130 ("payload".to_owned(), json!({"$bytes": "0a"})),
8131 ])
8132 };
8133 let upsert = |values| {
8134 vec![Mutation::Upsert {
8135 table: "tasks".into(),
8136 values,
8137 base_version: None,
8138 }]
8139 };
8140 for (label, stored_cell, representable) in [
8141 ("plus-sign envelope", json!({"$bytes": "+a"}), false),
8142 ("non-hex envelope", json!({"$bytes": "zz"}), false),
8143 ("odd-length envelope", json!({"$bytes": "abc"}), false),
8144 ("no envelope", json!({"nested": true}), false),
8145 ("wrong declared type", json!("high"), true),
8149 ("stored null for a non-nullable column", json!(null), true),
8150 ] {
8151 let path = std::env::temp_dir().join(format!(
8152 "syncular-legacy-values-{}-{}.db",
8153 label.replace(' ', "-"),
8154 uuid::Uuid::new_v4()
8155 ));
8156 let mut client = open(&path);
8157 client.create_synced_tables().expect("create synced tables");
8158 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8159 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8160 drop(client);
8161
8162 let connection = Connection::open(&path).expect("raw connection");
8164 let (column, cell) = if stored_cell.is_null() {
8165 ("project_id", stored_cell.clone())
8166 } else {
8167 ("payload", stored_cell.clone())
8168 };
8169 let mut planted_values = json!({
8170 "id": "t1", "project_id": "p1", "payload": {"$bytes": "0a"}
8171 });
8172 planted_values[column] = cell;
8173 let planted = json!([{
8174 "op": "upsert",
8175 "table": "tasks",
8176 "rowId": "t1",
8177 "baseVersion": null,
8178 "values": planted_values
8179 }]);
8180 connection
8181 .execute(
8182 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8183 rusqlite::params![planted.to_string(), stale],
8184 )
8185 .expect("plant legacy intent");
8186 drop(connection);
8187
8188 let reopened = open(&path);
8191 assert_eq!(
8192 reopened.pending_commit_ids(),
8193 vec![successor.clone()],
8194 "{label}"
8195 );
8196 let rejection = reopened
8197 .rejections()
8198 .iter()
8199 .find(|rejection| rejection.client_commit_id == stale)
8200 .unwrap_or_else(|| panic!("{label}: rejection"));
8201 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8202 assert_eq!(
8203 rejection.message, "the persisted commit carries values the current codec refuses",
8204 "{label}"
8205 );
8206 assert_eq!(rejection.operation.is_some(), representable, "{label}");
8207 assert_eq!(
8208 rejection
8209 .details
8210 .as_ref()
8211 .and_then(|details| details.reason.as_deref()),
8212 Some(INVALID_STORED_VALUES_REASON),
8213 "{label}"
8214 );
8215 let journal = reopened
8217 .commit_outcome(&stale)
8218 .expect("read outcome")
8219 .expect("journal entry");
8220 assert!(matches!(journal.status, CommitOutcomeStatus::Rejected));
8221 drop(reopened);
8222 let mut reopened = open(&path);
8223 let reread = reopened
8224 .commit_outcome(&stale)
8225 .expect("read outcome")
8226 .expect("journal entry");
8227 assert!(matches!(reread.status, CommitOutcomeStatus::Rejected));
8228
8229 assert_eq!(
8232 reopened
8233 .query("SELECT id FROM tasks ORDER BY id", &[])
8234 .unwrap(),
8235 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8236 "{label}"
8237 );
8238 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8239 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8240 assert!(!reopened
8241 .drop_unencodable_outbox(false, None)
8242 .expect("idempotent prepass"));
8243 let mut transport = CountingRealtimeTransport::default();
8244 let outcome = reopened.sync(&mut transport);
8245 assert!(matches!(outcome, SyncOutcome::Failed { .. }), "{outcome:?}");
8246 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8247 let next = reopened.mutate(upsert(row("t3"))).expect("after the drop");
8248 assert_eq!(reopened.pending_commit_ids(), vec![successor, next]);
8249 assert_eq!(reopened.pending_payloads().len(), 2, "{label}");
8250 drop(reopened);
8251 let _ = std::fs::remove_file(&path);
8252 }
8253 }
8254
8255 #[test]
8256 fn two_legacy_values_on_the_same_row_recover_in_fifo_order() {
8257 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8258 {"name":"id","type":"string","nullable":false},
8259 {"name":"project_id","type":"string","nullable":false},
8260 {"name":"payload","type":"bytes","nullable":true}],
8261 "scopes":[{"pattern":"project:{project_id}"}]}]});
8262 for (label, column, planted) in [
8263 ("malformed envelope", "payload", json!({"$bytes": "zz"})),
8264 (
8265 "stored null for a non-nullable column",
8266 "project_id",
8267 json!(null),
8268 ),
8269 ("wrong declared type", "payload", json!(7)),
8270 ] {
8271 let path = std::env::temp_dir().join(format!(
8272 "syncular-shared-row-{}-{}.db",
8273 label.replace(' ', "-"),
8274 uuid::Uuid::new_v4()
8275 ));
8276 let open = |path: &std::path::Path| {
8277 SyncClient::open_path(
8278 "shared-row".into(),
8279 &schema,
8280 ClientLimits::default(),
8281 path.to_str().unwrap(),
8282 )
8283 .expect("open client")
8284 };
8285 let row = |id: &str| {
8286 Map::from_iter([
8287 ("id".to_owned(), json!(id)),
8288 ("project_id".to_owned(), json!("p1")),
8289 ("payload".to_owned(), json!({"$bytes": "0a"})),
8290 ])
8291 };
8292 let upsert = |values| {
8293 vec![Mutation::Upsert {
8294 table: "tasks".into(),
8295 values,
8296 base_version: None,
8297 }]
8298 };
8299 let mut client = open(&path);
8300 client.create_synced_tables().expect("create synced tables");
8301 let older = client.mutate(upsert(row("t1"))).expect("older commit");
8302 let newer = client.mutate(upsert(row("t1"))).expect("newer commit");
8303 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8304 drop(client);
8305
8306 let connection = Connection::open(&path).expect("raw connection");
8307 for commit_id in [&older, &newer] {
8308 let mut values = row("t1");
8309 values.insert(column.to_owned(), planted.clone());
8310 let planted_ops = json!([{
8311 "op": "upsert",
8312 "table": "tasks",
8313 "rowId": "t1",
8314 "baseVersion": null,
8315 "values": values
8316 }]);
8317 connection
8318 .execute(
8319 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8320 rusqlite::params![planted_ops.to_string(), commit_id],
8321 )
8322 .expect("plant legacy intent");
8323 }
8324 drop(connection);
8325
8326 let mut reopened = open(&path);
8328 assert_eq!(
8329 reopened.pending_commit_ids(),
8330 vec![successor.clone()],
8331 "{label}"
8332 );
8333 assert_eq!(
8334 reopened
8335 .rejections()
8336 .iter()
8337 .map(|rejection| rejection.client_commit_id.clone())
8338 .collect::<Vec<_>>(),
8339 vec![older.clone(), newer.clone()],
8340 "{label}"
8341 );
8342 assert!(
8343 reopened
8344 .rejections()
8345 .iter()
8346 .all(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE
8347 && rejection.details.as_ref().and_then(|d| d.reason.as_deref())
8348 == Some(INVALID_STORED_VALUES_REASON)),
8349 "{label}"
8350 );
8351 for commit_id in [&older, &newer] {
8352 assert!(
8353 matches!(
8354 reopened
8355 .commit_outcome(commit_id)
8356 .expect("outcome")
8357 .map(|o| o.status),
8358 Some(CommitOutcomeStatus::Rejected)
8359 ),
8360 "{label}"
8361 );
8362 }
8363 assert_eq!(
8364 reopened
8365 .query("SELECT id FROM tasks ORDER BY id", &[])
8366 .unwrap(),
8367 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8368 "{label}"
8369 );
8370 let revision = reopened.local_revision();
8371 let batches = reopened.drain_change_batches();
8372 assert!(
8373 batches.iter().any(|batch| batch.rejections_changed
8374 && batch.outcomes_changed
8375 && batch.tables.iter().any(|change| change.table == "tasks")
8376 && batch.revision == revision.to_string()),
8377 "{label}: {batches:?}"
8378 );
8379 drop(reopened);
8380
8381 let mut stable = open(&path);
8384 assert_eq!(stable.local_revision(), revision, "{label}");
8385 assert!(stable.drain_change_batches().is_empty(), "{label}");
8386 assert_eq!(stable.pending_commit_ids(), vec![successor], "{label}");
8387 drop(stable);
8388 let _ = std::fs::remove_file(&path);
8389 }
8390 }
8391
8392 #[test]
8393 fn recovery_omits_a_legacy_operation_the_journal_reader_rejects() {
8394 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8395 {"name":"id","type":"string","nullable":false},
8396 {"name":"project_id","type":"string","nullable":false},
8397 {"name":"payload","type":"bytes","nullable":true}],
8398 "scopes":[{"pattern":"project:{project_id}"}]}]});
8399 for (label, second_op) in [
8400 (
8401 "delete carrying values",
8402 json!({"op": "delete", "table": "tasks", "rowId": "t9", "values": {}}),
8403 ),
8404 (
8405 "baseVersion beyond the safe bound",
8406 json!({"op": "upsert", "table": "tasks", "rowId": "t9",
8407 "baseVersion": MAX_JS_SAFE_INTEGER + 1,
8408 "values": {"id": "t9", "project_id": "p1", "payload": {"$bytes": "0a"}}}),
8409 ),
8410 ] {
8411 let path = std::env::temp_dir().join(format!(
8412 "syncular-journal-shape-{}-{}.db",
8413 label.replace(' ', "-"),
8414 uuid::Uuid::new_v4()
8415 ));
8416 let open = |path: &std::path::Path| {
8417 SyncClient::open_path(
8418 "journal-shape".into(),
8419 &schema,
8420 ClientLimits::default(),
8421 path.to_str().unwrap(),
8422 )
8423 .expect("open client")
8424 };
8425 let values = |id: &str| {
8426 Map::from_iter([
8427 ("id".to_owned(), json!(id)),
8428 ("project_id".to_owned(), json!("p1")),
8429 ("payload".to_owned(), json!({"$bytes": "0a"})),
8430 ])
8431 };
8432 let upsert = |values| {
8433 vec![Mutation::Upsert {
8434 table: "tasks".into(),
8435 values,
8436 base_version: None,
8437 }]
8438 };
8439 let mut client = open(&path);
8440 client.create_synced_tables().expect("create synced tables");
8441 let stale = client.mutate(upsert(values("t1"))).expect("stale commit");
8442 drop(client);
8443
8444 let connection = Connection::open(&path).expect("raw connection");
8445 let planted = json!([
8449 {
8450 "op": "upsert",
8451 "table": "tasks",
8452 "rowId": "t1",
8453 "baseVersion": null,
8454 "values": {"id": "t1", "project_id": null,
8455 "payload": {"$bytes": "0a"}}
8456 },
8457 second_op
8458 ]);
8459 connection
8460 .execute(
8461 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8462 rusqlite::params![planted.to_string(), stale],
8463 )
8464 .expect("plant legacy intent");
8465 drop(connection);
8466
8467 let reopened = open(&path);
8468 let journal = reopened
8469 .commit_outcome(&stale)
8470 .unwrap_or_else(|error| panic!("{label}: journal read: {error}"))
8471 .unwrap_or_else(|| panic!("{label}: journal entry"));
8472 assert!(
8473 matches!(journal.status, CommitOutcomeStatus::Rejected),
8474 "{label}"
8475 );
8476 assert!(journal.operations.is_none(), "{label}");
8477 assert!(
8478 reopened
8479 .commit_outcomes(CommitOutcomeQuery::default())
8480 .is_ok(),
8481 "{label}: outcome list query"
8482 );
8483 assert!(reopened.pending_commit_ids().is_empty(), "{label}");
8484 drop(reopened);
8485 let _ = std::fs::remove_file(&path);
8486 }
8487 }
8488
8489 #[test]
8490 fn legacy_values_recover_at_the_recreate_and_bump_reset_boundaries() {
8491 let schema = |version: i32, extra: bool| {
8492 let mut columns = vec![
8493 json!({"name":"id","type":"string","nullable":false}),
8494 json!({"name":"project_id","type":"string","nullable":false}),
8495 json!({"name":"payload","type":"bytes","nullable":true}),
8496 ];
8497 if extra {
8498 columns.push(json!({"name":"later","type":"string","nullable":true}));
8499 }
8500 json!({"version":version,"tables":[{"name":"tasks","primaryKey":"id","columns":columns,
8501 "scopes":[{"pattern":"project:{project_id}"}]}]})
8502 };
8503 for (label, bump) in [
8504 ("same-version recreation", false),
8505 ("version-bump reset", true),
8506 ("version-only bump reset", true),
8507 ] {
8508 let path = std::env::temp_dir().join(format!(
8509 "syncular-legacy-reset-{}-{}.db",
8510 label.replace(' ', "-"),
8511 uuid::Uuid::new_v4()
8512 ));
8513 let v1 = schema(1, false);
8514 let open = |schema: &Value, path: &std::path::Path| {
8515 SyncClient::open_path(
8516 "legacy-reset".into(),
8517 schema,
8518 ClientLimits::default(),
8519 path.to_str().unwrap(),
8520 )
8521 .expect("open client")
8522 };
8523 let row = |id: &str| {
8524 Map::from_iter([
8525 ("id".to_owned(), json!(id)),
8526 ("project_id".to_owned(), json!("p1")),
8527 ("payload".to_owned(), json!({"$bytes": "0a"})),
8528 ])
8529 };
8530 let upsert = |values| {
8531 vec![Mutation::Upsert {
8532 table: "tasks".into(),
8533 values,
8534 base_version: None,
8535 }]
8536 };
8537 let mut client = open(&v1, &path);
8538 client.create_synced_tables().expect("create synced tables");
8539 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8540 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8541 drop(client);
8542
8543 let connection = Connection::open(&path).expect("raw connection");
8544 let planted = json!([{
8545 "op": "upsert",
8546 "table": "tasks",
8547 "rowId": "t1",
8548 "baseVersion": null,
8549 "values": {"id": "t1", "project_id": 7, "payload": {"$bytes": "0a"}}
8550 }]);
8551 connection
8552 .execute(
8553 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8554 rusqlite::params![planted.to_string(), stale],
8555 )
8556 .expect("plant legacy intent");
8557 drop(connection);
8558
8559 let target = if bump {
8560 schema(2, label == "version-bump reset")
8561 } else {
8562 v1.clone()
8563 };
8564 let reopened = open(&target, &path);
8565 let rejection = reopened
8566 .rejections()
8567 .iter()
8568 .find(|rejection| rejection.client_commit_id == stale)
8569 .unwrap_or_else(|| panic!("{label}: rejection"));
8570 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8571 assert_eq!(
8572 rejection
8573 .details
8574 .as_ref()
8575 .and_then(|details| details.reason.as_deref()),
8576 Some(INVALID_STORED_VALUES_REASON),
8577 "{label}"
8578 );
8579 assert!(rejection.operation.is_some(), "{label}");
8580 assert_eq!(
8581 reopened.pending_commit_ids(),
8582 vec![successor.clone()],
8583 "{label}"
8584 );
8585 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8590 if label == "version-bump reset" {
8591 assert_eq!(
8592 reopened
8593 .query("SELECT id FROM tasks ORDER BY id", &[])
8594 .unwrap(),
8595 Vec::<Map<String, Value>>::new(),
8596 "{label}"
8597 );
8598 let mut next = reopened;
8602 let revision = next.local_revision();
8603 let batches = next.drain_change_batches();
8604 assert!(
8605 batches.iter().any(|batch| batch.rejections_changed
8606 && batch.outcomes_changed
8607 && batch.tables.iter().any(|change| change.table == "tasks")
8608 && batch.revision == revision.to_string()),
8609 "{label}: {batches:?}"
8610 );
8611 next.mutate(upsert(Map::from_iter([
8612 ("id".to_owned(), json!("t3")),
8613 ("project_id".to_owned(), json!("p1")),
8614 ("payload".to_owned(), json!({"$bytes": "0b"})),
8615 ("later".to_owned(), json!("v2")),
8616 ])))
8617 .expect("v2 authoring after the reset");
8618 assert_eq!(next.pending_commit_ids().len(), 2, "{label}");
8619 assert_eq!(next.pending_payloads().len(), 2, "{label}");
8620 drop(next);
8621 } else {
8622 assert_eq!(
8623 reopened
8624 .query("SELECT id FROM tasks ORDER BY id", &[])
8625 .unwrap(),
8626 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8627 "{label}"
8628 );
8629 drop(reopened);
8630 }
8631
8632 if !bump {
8635 let mut recreated = open(&v1, &path);
8636 let fresh = recreated
8637 .mutate(upsert(row("t3")))
8638 .expect("fresh commit while open");
8639 recreated.drain_change_batches();
8640 let planted_fresh = json!([{
8641 "op": "upsert",
8642 "table": "tasks",
8643 "rowId": "t3",
8644 "baseVersion": null,
8645 "values": {"id": "t3", "project_id": "p1",
8646 "payload": {"$bytes": "zz"}}
8647 }]);
8648 let connection = Connection::open(&path).expect("raw connection");
8649 let affected = connection
8650 .execute(
8651 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8652 rusqlite::params![planted_fresh.to_string(), fresh],
8653 )
8654 .expect("plant fresh legacy intent");
8655 assert_eq!(
8656 affected, 1,
8657 "{label}: the in-place plant must hit the queued commit"
8658 );
8659 drop(connection);
8660 let revision_before = recreated.local_revision();
8661 recreated
8662 .recreate_with_schema(&v1)
8663 .unwrap_or_else(|error| panic!("{label}: recreation failed: {error}"));
8664 assert_eq!(recreated.pending_commit_ids(), vec![successor], "{label}");
8665 let fresh_rejection = recreated
8666 .rejections()
8667 .iter()
8668 .find(|rejection| rejection.client_commit_id == fresh)
8669 .expect("the in-place recovery rejection");
8670 assert_eq!(fresh_rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8671 assert_eq!(
8672 fresh_rejection
8673 .details
8674 .as_ref()
8675 .and_then(|details| details.reason.as_deref()),
8676 Some(INVALID_STORED_VALUES_REASON),
8677 "{label}"
8678 );
8679 assert!(
8680 matches!(
8681 recreated
8682 .commit_outcome(&fresh)
8683 .expect("outcome read")
8684 .map(|outcome| outcome.status),
8685 Some(CommitOutcomeStatus::Rejected)
8686 ),
8687 "{label}"
8688 );
8689 assert!(recreated.local_revision() > revision_before, "{label}");
8690 let revision = recreated.local_revision();
8691 let batches = recreated.drain_change_batches();
8692 assert_eq!(batches.len(), 1, "{label}: one recovery batch");
8693 assert!(
8694 batches.iter().any(|batch| batch.rejections_changed
8695 && batch.outcomes_changed
8696 && batch.tables.iter().any(|change| change.table == "tasks")
8697 && batch.revision == revision.to_string()),
8698 "{label}: {batches:?}"
8699 );
8700 assert_eq!(recreated.pending_payloads().len(), 1, "{label}");
8701 assert_eq!(
8702 recreated
8703 .query("SELECT id FROM tasks ORDER BY id", &[])
8704 .unwrap(),
8705 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8706 "{label}: the recovered commit leaves no projection"
8707 );
8708 drop(recreated);
8709 }
8710 let _ = std::fs::remove_file(&path);
8711 }
8712 }
8713
8714 #[test]
8715 fn legacy_values_recover_over_a_synced_authoritative_row() {
8716 let path = std::env::temp_dir().join(format!(
8717 "syncular-authoritative-row-{}.db",
8718 uuid::Uuid::new_v4()
8719 ));
8720 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8721 {"name":"id","type":"string","nullable":false},
8722 {"name":"project_id","type":"string","nullable":false},
8723 {"name":"title","type":"string","nullable":false},
8724 {"name":"payload","type":"bytes","nullable":true}],
8725 "scopes":[{"pattern":"project:{project_id}"}]}]});
8726 let open = |path: &std::path::Path| {
8727 SyncClient::open_path(
8728 "authoritative-row".into(),
8729 &schema,
8730 ClientLimits::default(),
8731 path.to_str().unwrap(),
8732 )
8733 .expect("open client")
8734 };
8735 let row = |title: &str| {
8736 Map::from_iter([
8737 ("id".to_owned(), json!("t1")),
8738 ("project_id".to_owned(), json!("p1")),
8739 ("title".to_owned(), json!(title)),
8740 ("payload".to_owned(), json!({"$bytes": "0a"})),
8741 ])
8742 };
8743 let upsert = |values| {
8744 vec![Mutation::Upsert {
8745 table: "tasks".into(),
8746 values,
8747 base_version: None,
8748 }]
8749 };
8750 let mut client = open(&path);
8751 client.create_synced_tables().expect("create synced tables");
8752 client
8755 .conn
8756 .execute_batch(
8757 "INSERT INTO _syncular_base_tasks VALUES ('t1','p1','authoritative',x'0a',1);
8758 INSERT INTO tasks VALUES ('t1','p1','authoritative',x'0a',1);",
8759 )
8760 .expect("seed authoritative row");
8761 let older = client.mutate(upsert(row("older"))).expect("older");
8762 let newer = client.mutate(upsert(row("newer"))).expect("newer");
8763 let successor = client.mutate(upsert(row("successor"))).expect("successor");
8764 drop(client);
8765
8766 let connection = Connection::open(&path).expect("raw connection");
8767 for commit_id in [&older, &newer] {
8768 let planted = json!([{
8769 "op": "upsert",
8770 "table": "tasks",
8771 "rowId": "t1",
8772 "baseVersion": null,
8773 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
8774 "payload": {"$bytes": "zz"}}
8775 }]);
8776 connection
8777 .execute(
8778 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8779 rusqlite::params![planted.to_string(), commit_id],
8780 )
8781 .expect("plant legacy intent");
8782 }
8783 drop(connection);
8784
8785 let reopened = open(&path);
8786 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8787 assert_eq!(
8788 reopened
8789 .rejections()
8790 .iter()
8791 .map(|rejection| rejection.client_commit_id.clone())
8792 .collect::<Vec<_>>(),
8793 vec![older.clone(), newer.clone()]
8794 );
8795 assert_eq!(
8798 reopened.query("SELECT id,title,_syncular_version FROM tasks", &[]),
8799 Ok(vec![Map::from_iter([
8800 ("id".to_owned(), json!("t1")),
8801 ("title".to_owned(), json!("successor")),
8802 ("_syncular_version".to_owned(), json!(1)),
8803 ])])
8804 );
8805
8806 drop(reopened);
8809 let connection = Connection::open(&path).expect("raw connection");
8810 connection
8811 .execute(
8812 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8813 rusqlite::params![
8814 json!([{
8815 "op": "upsert",
8816 "table": "tasks",
8817 "rowId": "t1",
8818 "baseVersion": null,
8819 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
8820 "payload": {"$bytes": "zz"}}
8821 }])
8822 .to_string(),
8823 successor
8824 ],
8825 )
8826 .expect("plant successor intent");
8827 drop(connection);
8828 let late = open(&path);
8829 assert!(late.pending_commit_ids().is_empty());
8830 assert_eq!(
8831 late.query("SELECT id,title,_syncular_version FROM tasks", &[]),
8832 Ok(vec![Map::from_iter([
8833 ("id".to_owned(), json!("t1")),
8834 ("title".to_owned(), json!("authoritative")),
8835 ("_syncular_version".to_owned(), json!(1)),
8836 ])])
8837 );
8838 drop(late);
8839 let _ = std::fs::remove_file(&path);
8840 }
8841
8842 #[test]
8843 fn a_failed_value_recovery_commit_restores_every_side_effect() {
8844 let path = std::env::temp_dir().join(format!(
8845 "syncular-value-recovery-commit-{}.db",
8846 uuid::Uuid::new_v4()
8847 ));
8848 let path_str = path.to_str().expect("path");
8849 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8850 {"name":"id","type":"string","nullable":false},
8851 {"name":"project_id","type":"string","nullable":false},
8852 {"name":"payload","type":"bytes","nullable":true}],
8853 "scopes":[{"pattern":"project:{project_id}"}]}]});
8854 let conn = Connection::open(path_str).expect("replica connection");
8857 let mut client = SyncClient::with_connection(
8858 "value-recovery-commit".into(),
8859 &schema,
8860 ClientLimits::default(),
8861 conn,
8862 )
8863 .expect("replica");
8864 assert_eq!(
8865 client
8866 .conn
8867 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
8868 .expect("journal mode"),
8869 "delete"
8870 );
8871 client.create_synced_tables().expect("create synced tables");
8872 let stale = client
8873 .mutate(vec![Mutation::Upsert {
8874 table: "tasks".into(),
8875 values: Map::from_iter([
8876 ("id".to_owned(), json!("t1")),
8877 ("project_id".to_owned(), json!("p1")),
8878 ("payload".to_owned(), json!({"$bytes": "0a"})),
8879 ]),
8880 base_version: None,
8881 }])
8882 .expect("queued write");
8883 client.drain_change_batches();
8884 let affected = client
8885 .conn
8886 .execute(
8887 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8888 rusqlite::params![
8889 json!([{
8890 "op": "upsert",
8891 "table": "tasks",
8892 "rowId": "t1",
8893 "baseVersion": null,
8894 "values": {"id": "t1", "project_id": "p1",
8895 "payload": {"$bytes": "zz"}}
8896 }])
8897 .to_string(),
8898 stale
8899 ],
8900 )
8901 .expect("plant legacy intent");
8902 assert_eq!(affected, 1, "the planted row must exist");
8903 let revision = client.local_revision();
8904 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
8905 let prior_outbox = client.pending_commit_ids();
8906 let prior_rejections = client.rejections().len();
8907 let prior_changes = client.change_queue.len();
8908 let prior_intents = client.sync_intent_queue.len();
8909 let prior_failed = client.failed_commits.len();
8910 let reader = Connection::open(path_str).expect("reader");
8913 reader
8914 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
8915 .expect("reader snapshot");
8916 client
8917 .conn
8918 .busy_timeout(std::time::Duration::ZERO)
8919 .expect("busy timeout");
8920 let error = client
8921 .recreate_with_schema(&schema)
8922 .expect_err("a blocked COMMIT must refuse the recovery");
8923 assert!(error.starts_with("client.storage_busy:"), "{error}");
8924 assert!(client.conn.is_autocommit(), "the transaction is released");
8925 assert_eq!(client.local_revision(), revision);
8926 assert_eq!(client.pending_commit_ids(), prior_outbox);
8927 assert_eq!(client.rejections().len(), prior_rejections);
8928 assert_eq!(client.change_queue.len(), prior_changes);
8929 assert_eq!(client.sync_intent_queue.len(), prior_intents);
8930 assert_eq!(client.failed_commits.len(), prior_failed);
8931 assert_eq!(
8932 client.query("SELECT * FROM tasks", &[]).expect("rows"),
8933 rows
8934 );
8935 assert!(
8936 client
8937 .commit_outcome(&stale)
8938 .expect("outcome read")
8939 .is_none(),
8940 "the rolled-back recovery kept no outcome"
8941 );
8942 reader.execute_batch("ROLLBACK").expect("release reader");
8944 client
8945 .recreate_with_schema(&schema)
8946 .expect("retry after the blocked commit");
8947 assert!(client.pending_commit_ids().is_empty());
8948 assert_eq!(client.rejections().len(), prior_rejections + 1);
8949 assert!(client.local_revision() > revision);
8950 let batches = client.drain_change_batches();
8951 assert!(
8952 batches.iter().any(|batch| batch.rejections_changed
8953 && batch.outcomes_changed
8954 && batch.tables.iter().any(|change| change.table == "tasks")),
8955 "{batches:?}"
8956 );
8957 drop(client);
8958 let _ = std::fs::remove_file(&path);
8959 }
8960
8961 #[cfg(feature = "e2ee")]
8962 #[test]
8963 fn a_failed_recovery_restores_retained_failed_commits() {
8964 let path = std::env::temp_dir().join(format!(
8965 "syncular-value-recovery-retention-{}.db",
8966 uuid::Uuid::new_v4()
8967 ));
8968 let path_str = path.to_str().expect("path");
8969 let schema = json!({"version":1,"tables":[{"name":"secrets","primaryKey":"id","columns":[
8970 {"name":"id","type":"string","nullable":false},
8971 {"name":"project_id","type":"string","nullable":false},
8972 {"name":"note","type":"bytes","nullable":true,"encrypted":true,"declaredType":"string"}],
8973 "scopes":[{"pattern":"project:{project_id}"}]}]});
8974 let conn = Connection::open(path_str).expect("replica connection");
8975 let mut client = SyncClient::with_connection(
8976 "value-recovery-retention".into(),
8977 &schema,
8978 ClientLimits::default(),
8979 conn,
8980 )
8981 .expect("replica");
8982 client.create_synced_tables().expect("create synced tables");
8983 client.set_retain_failed_commits(true).expect("retention");
8984 client.set_encryption(crate::values::EncryptionConfig::default());
8986 client
8987 .mutate(vec![Mutation::Upsert {
8988 table: "secrets".into(),
8989 values: Map::from_iter([
8990 ("id".to_owned(), json!("s1")),
8991 ("project_id".to_owned(), json!("p1")),
8992 ("note".to_owned(), json!("secret")),
8993 ]),
8994 base_version: None,
8995 }])
8996 .expect("queued write");
8997 client.drain_change_batches();
8998 let revision = client.local_revision();
8999 let prior_outbox = client.pending_commit_ids();
9000 let prior_rejections = client.rejections().len();
9001 let prior_failed = client.failed_commits.len();
9002 let reader = Connection::open(path_str).expect("reader");
9003 reader
9004 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9005 .expect("reader snapshot");
9006 client
9007 .conn
9008 .busy_timeout(std::time::Duration::ZERO)
9009 .expect("busy timeout");
9010 let error = client
9011 .drop_unencodable_outbox(true, None)
9012 .expect_err("a blocked COMMIT must refuse the recovery");
9013 assert!(error.starts_with("client.storage_busy:"), "{error}");
9014 assert_eq!(client.failed_commits.len(), prior_failed);
9015 assert_eq!(client.pending_commit_ids(), prior_outbox);
9016 assert_eq!(client.rejections().len(), prior_rejections);
9017 assert_eq!(client.local_revision(), revision);
9018 reader.execute_batch("ROLLBACK").expect("release reader");
9020 client
9021 .drop_unencodable_outbox(true, None)
9022 .expect("retry after the blocked commit");
9023 assert_eq!(client.failed_commits.len(), prior_failed + 1);
9024 assert!(client.pending_commit_ids().is_empty());
9025 assert_eq!(client.rejections().len(), prior_rejections + 1);
9026 drop(client);
9027 let _ = std::fs::remove_file(&path);
9028 }
9029
9030 #[test]
9031 fn client_error_maps_legacy_codes_and_static_messages() {
9032 let code_only = ClientError::from("sync.unknown_table".to_owned());
9033 assert_eq!(code_only.code, "sync.unknown_table");
9034 assert_eq!(code_only.message, "the commit targets an unknown table");
9035 assert!(code_only.details.is_none());
9036
9037 let dynamic = ClientError::from(
9038 "sync.invalid_request: table \"tasks\": patch cannot write scope column".to_owned(),
9039 );
9040 assert_eq!(dynamic.code, "sync.invalid_request");
9041 assert_eq!(dynamic.message, "the authoring request is invalid");
9042 assert!(dynamic.details.as_ref().unwrap()["legacyCause"]
9043 .as_str()
9044 .unwrap()
9045 .contains("patch cannot write scope column"));
9046
9047 let unclassified =
9048 ClientError::from("a commit must contain at least one operation".to_owned());
9049 assert_eq!(unclassified.code, "client.failed");
9050 assert_eq!(unclassified.message, "the local authoring operation failed");
9051 assert!(unclassified.details.as_ref().unwrap()["legacyCause"]
9052 .as_str()
9053 .unwrap()
9054 .contains("at least one operation"));
9055
9056 let metadata = ClientError::from(QueryReadFailure {
9058 code: None,
9059 sqlite_code: Some(1),
9060 sqlite_message: Some("generic SQLite prose".to_owned()),
9061 rollback_failure: Some(Box::new(QueryReadFailure::from(
9062 rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(1), None),
9063 ))),
9064 message: "unclassified read".to_owned(),
9065 });
9066 assert_eq!(metadata.code, "client.failed");
9067 assert_eq!(metadata.message, "the local authoring operation failed");
9068 let details = metadata.details.as_ref().unwrap();
9069 assert_eq!(details["sqliteCode"], 1);
9070 assert_eq!(details["sqliteMessage"], "generic SQLite prose");
9071 assert_eq!(details["rollbackFailure"]["sqliteCode"], 1);
9072 assert_eq!(details["legacyCause"], "unclassified read");
9073 }
9074
9075 #[test]
9076 fn schema_marker_read_keeps_storage_classification() {
9077 let busy = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9078 rusqlite::ffi::Error::new(5),
9079 Some("database is locked".to_owned()),
9080 ));
9081 assert!(busy.starts_with("client.storage_busy:"), "{busy}");
9082 let invalid = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9083 rusqlite::ffi::Error::new(1),
9084 Some("unreadable".to_owned()),
9085 ));
9086 assert_eq!(
9087 invalid,
9088 "sync.local_corrupt: persisted local schema marker is unreadable or invalid"
9089 );
9090
9091 let path =
9094 std::env::temp_dir().join(format!("syncular-marker-busy-{}.db", uuid::Uuid::new_v4()));
9095 let setup = Connection::open(&path).expect("setup connection");
9096 setup
9097 .execute_batch(
9098 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value TEXT);\n INSERT INTO _syncular_meta VALUES ('localSchemaVersion','1');",
9099 )
9100 .unwrap();
9101 let locker = Connection::open(&path).expect("locker connection");
9102 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9103 let reader = Connection::open(&path).expect("reader connection");
9104 reader.busy_timeout(std::time::Duration::ZERO).unwrap();
9105 let locked = read_local_schema_version(&reader, 1).expect_err("locked marker read");
9106 assert!(locked.starts_with("client.storage_busy:"), "{locked}");
9107 locker.execute_batch("ROLLBACK").unwrap();
9108 assert_eq!(read_local_schema_version(&reader, 1).unwrap(), Some(1));
9109 let rows = setup
9110 .query_row("SELECT count(*) FROM _syncular_meta", [], |row| {
9111 row.get::<_, i64>(0)
9112 })
9113 .unwrap();
9114 assert_eq!(rows, 1, "the guard performed no writes");
9115 drop(reader);
9116 drop(locker);
9117 drop(setup);
9118 let _ = std::fs::remove_file(&path);
9119 }
9120
9121 #[cfg(feature = "crdt-yjs")]
9122 #[test]
9123 fn crdt_authoring_pre_read_preserves_storage_classification() {
9124 let path =
9125 std::env::temp_dir().join(format!("syncular-crdt-busy-{}.db", uuid::Uuid::new_v4()));
9126 let schema = json!({"version":1,"tables":[{"name":"notes","primaryKey":"id","columns":[
9127 {"name":"id","type":"string","nullable":false},
9128 {"name":"project_id","type":"string","nullable":false},
9129 {"name":"doc","type":"crdt","nullable":true,"crdtType":"yjs-doc"}],
9130 "scopes":[{"pattern":"project:{project_id}"}]}]});
9131 let mut client = SyncClient::open_path(
9132 "crdt-busy".into(),
9133 &schema,
9134 ClientLimits::default(),
9135 path.to_str().unwrap(),
9136 )
9137 .expect("open client");
9138 client.create_synced_tables().expect("create synced tables");
9139 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
9140
9141 let locker = Connection::open(&path).expect("locker connection");
9142 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9143 let busy = client
9144 .crdt_apply_update("notes", "n1", "doc", &[])
9145 .expect_err("busy crdt pre-read");
9146 assert_eq!(busy.code, "client.storage_busy");
9147 assert!(busy.retryable);
9148 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
9149 assert!(client.pending_commit_ids().is_empty());
9150 assert_eq!(client.local_revision(), 0);
9151
9152 locker.execute_batch("ROLLBACK").unwrap();
9153 drop(locker);
9154 drop(client);
9155 let _ = std::fs::remove_file(&path);
9156 }
9157
9158 #[test]
9159 fn large_unique_import_copies_no_unrelated_tables() {
9160 let (mut client, image, table) = unique_import_fixture(10240);
9161 let pending = client.pending_commit_ids();
9162 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
9163 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
9164 assert!(matches!(
9165 client.apply_sqlite_image(
9166 &image,
9167 &table,
9168 true,
9169 &[("project_id".into(), vec!["p1".into()])],
9170 10240,
9171 7,
9172 "digest"
9173 ),
9174 Ok(10240)
9175 ));
9176 assert_eq!(client.overlay_rebuild_count.get(), 0);
9177 assert_eq!(client.pending_commit_ids(), pending);
9178 assert_eq!(
9179 client
9180 .query("SELECT count(*) AS n FROM tasks", &[])
9181 .unwrap()[0]["n"],
9182 10241
9183 );
9184 assert_eq!(
9185 client
9186 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9187 .unwrap()[0]["n"],
9188 10241
9189 );
9190 assert_eq!(
9191 client
9192 .query("SELECT count(*) AS n FROM notes_fts", &[])
9193 .unwrap()[0]["n"],
9194 8192
9195 );
9196 }
9197
9198 #[test]
9199 #[ignore = "manual scaling benchmark; see bench/README.md"]
9200 fn benchmark_unique_image_import() {
9201 let mut samples = Vec::new();
9202 for rows in [10240, 51200, 102400] {
9203 for trial in 0..4 {
9204 let (mut client, image, table) = unique_import_fixture(rows);
9205 let start = std::time::Instant::now();
9206 assert!(
9207 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
9208 );
9209 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
9210 assert_eq!(
9211 client
9212 .query("SELECT count(*) AS n FROM tasks", &[])
9213 .unwrap()[0]["n"],
9214 rows + 1
9215 );
9216 assert_eq!(
9217 client
9218 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9219 .unwrap()[0]["n"],
9220 rows + 1
9221 );
9222 assert_eq!(
9223 client
9224 .query("SELECT count(*) AS n FROM notes_fts", &[])
9225 .unwrap()[0]["n"],
9226 8192
9227 );
9228 assert_eq!(client.pending_commit_ids().len(), 1);
9229 if trial > 0 {
9230 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
9231 }
9232 }
9233 }
9234 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
9235 let output = serde_json::to_string_pretty(&artifact).unwrap();
9236 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
9237 use std::io::Write;
9238 let mut file = std::fs::OpenOptions::new()
9239 .write(true)
9240 .create_new(true)
9241 .open(path)
9242 .unwrap();
9243 file.write_all(output.as_bytes()).unwrap();
9244 } else {
9245 println!("{output}");
9246 }
9247 }
9248
9249 #[test]
9250 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
9251 let schema = json!({
9252 "version": 1,
9253 "tables": [{
9254 "name": "catalogue_codes",
9255 "primaryKey": "id",
9256 "columns": [
9257 { "name": "id", "type": "string", "nullable": false },
9258 { "name": "release_id", "type": "string", "nullable": false },
9259 { "name": "code", "type": "string", "nullable": false },
9260 { "name": "title", "type": "string", "nullable": false }
9261 ],
9262 "scopes": [{ "pattern": "release:{release_id}" }],
9263 "ftsIndexes": [{
9264 "name": "catalogue_codes_fts",
9265 "columns": ["code", "title"],
9266 "tokenize": "unicode61 remove_diacritics 2"
9267 }]
9268 }]
9269 });
9270 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
9271 .expect("FTS5 client");
9272 let insert_trigger: String = client
9273 .conn
9274 .query_row(
9275 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
9276 [],
9277 |row| row.get(0),
9278 )
9279 .expect("insert trigger");
9280 let replace_guard: String = client
9281 .conn
9282 .query_row(
9283 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
9284 [],
9285 |row| row.get(0),
9286 )
9287 .expect("replace guard");
9288 assert!(!insert_trigger.contains("DELETE FROM"));
9289 assert!(replace_guard.contains("BEFORE INSERT"));
9290 assert!(replace_guard.contains("WHEN EXISTS"));
9291 let search = |client: &SyncClient, query: &str| {
9292 client
9293 .query(
9294 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
9295 &[Value::from(query)],
9296 )
9297 .expect("FTS query")
9298 };
9299
9300 client
9301 .mutate(vec![Mutation::Upsert {
9302 table: "catalogue_codes".to_owned(),
9303 values: Map::from_iter([
9304 ("id".to_owned(), Value::from("c1")),
9305 ("release_id".to_owned(), Value::from("r1")),
9306 ("code".to_owned(), Value::from("A01")),
9307 ("title".to_owned(), Value::from("Cholera")),
9308 ]),
9309 base_version: None,
9310 }])
9311 .expect("insert code");
9312 assert_eq!(search(&client, "cholera").len(), 1);
9313 let prior = client
9314 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9315 .unwrap();
9316 client
9317 .conn
9318 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
9319 .unwrap();
9320 client.create_synced_tables().unwrap();
9321 assert_eq!(
9322 client
9323 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
9324 .unwrap(),
9325 prior
9326 );
9327 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
9328 let rendered = serde_json::to_string(&plan).unwrap();
9329 assert!(rendered.contains("COVERING INDEX"));
9330 assert!(rendered.contains("INDEX 0:="));
9331 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
9332 assert_eq!(search(&client, "cholera").len(), 1);
9333
9334 client
9335 .mutate(vec![Mutation::Upsert {
9336 table: "catalogue_codes".to_owned(),
9337 values: Map::from_iter([
9338 ("id".to_owned(), Value::from("c1")),
9339 ("release_id".to_owned(), Value::from("r1")),
9340 ("code".to_owned(), Value::from("A01")),
9341 ("title".to_owned(), Value::from("Enteric infection")),
9342 ]),
9343 base_version: None,
9344 }])
9345 .expect("update code");
9346 assert!(search(&client, "cholera").is_empty());
9347 assert_eq!(search(&client, "enteric").len(), 1);
9348
9349 client
9350 .mutate(vec![Mutation::Delete {
9351 table: "catalogue_codes".to_owned(),
9352 row_id: "c1".to_owned(),
9353 base_version: None,
9354 }])
9355 .expect("delete code");
9356 assert!(search(&client, "enteric").is_empty());
9357 }
9358
9359 #[test]
9360 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
9361 let schema = json!({
9362 "version": 1,
9363 "tables": [{
9364 "name": "patient_notes",
9365 "primaryKey": "id",
9366 "columns": [
9367 { "name": "id", "type": "string", "nullable": false },
9368 { "name": "practice_id", "type": "string", "nullable": false },
9369 { "name": "encryption_key_id", "type": "string", "nullable": false },
9370 { "name": "title", "type": "string", "nullable": false }
9371 ],
9372 "scopes": [{ "pattern": "practice:{practice_id}" }],
9373 "ftsIndexes": [{
9374 "name": "patient_notes_fts",
9375 "columns": ["title"],
9376 "tokenize": "unicode61 remove_diacritics 2"
9377 }]
9378 }]
9379 });
9380 let mut client = SyncClient::new(
9381 "local-purge-test".to_owned(),
9382 &schema,
9383 ClientLimits::default(),
9384 )
9385 .expect("local purge client");
9386 client
9387 .conn
9388 .execute(
9389 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
9390 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
9391 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
9392 [],
9393 )
9394 .expect("seed base rows");
9395 client.overlay_dirty.set(true);
9396 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
9397
9398 let note = |id: &str, key_id: &str, title: &str| {
9399 Map::from_iter([
9400 ("id".to_owned(), Value::from(id)),
9401 ("practice_id".to_owned(), Value::from("practice-1")),
9402 ("encryption_key_id".to_owned(), Value::from(key_id)),
9403 ("title".to_owned(), Value::from(title)),
9404 ])
9405 };
9406 let doomed = client
9407 .mutate(vec![
9408 Mutation::Upsert {
9409 table: "patient_notes".to_owned(),
9410 values: note("target", "key-revoked", "Target changed"),
9411 base_version: None,
9412 },
9413 Mutation::Upsert {
9414 table: "patient_notes".to_owned(),
9415 values: note("unrelated", "key-held", "Unrelated changed"),
9416 base_version: None,
9417 },
9418 ])
9419 .expect("doomed commit");
9420 let kept = client
9421 .mutate(vec![Mutation::Upsert {
9422 table: "patient_notes".to_owned(),
9423 values: note("kept", "key-held", "Kept optimistic"),
9424 base_version: None,
9425 }])
9426 .expect("kept commit");
9427 client.drain_change_batches();
9428
9429 let input = LocalDataPurgeInput {
9430 purge_id: "purge-001".to_owned(),
9431 targets: vec![LocalDataPurgeTarget {
9432 table: "patient_notes".to_owned(),
9433 selectors: BTreeMap::from([(
9434 "encryption_key_id".to_owned(),
9435 vec!["key-revoked".to_owned()],
9436 )]),
9437 }],
9438 };
9439 assert_eq!(
9440 client.purge_local_data(&input).expect("apply purge"),
9441 LocalDataPurgeResult {
9442 already_applied: false,
9443 purged_rows: 1,
9444 dropped_commits: 1,
9445 }
9446 );
9447 let rows = client
9448 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
9449 .expect("visible rows");
9450 assert_eq!(rows.len(), 2);
9451 assert_eq!(rows[0]["id"], "kept");
9452 assert_eq!(rows[1]["id"], "unrelated");
9453 assert_eq!(rows[1]["title"], "Unrelated original");
9454 let fts = client
9455 .query(
9456 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
9457 &[],
9458 )
9459 .expect("fts query");
9460 assert!(fts.is_empty());
9461 assert_eq!(client.outbox.len(), 1);
9462 assert_eq!(client.outbox[0].client_commit_id, kept);
9463 let outcome = client
9464 .commit_outcome(&doomed)
9465 .expect("read doomed outcome")
9466 .expect("doomed outcome");
9467 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
9468 match &outcome.results[0] {
9469 CommitOperationOutcome::Error { rejection } => {
9470 assert_eq!(rejection.code, "client.local_data_purged");
9471 assert_eq!(rejection.client_commit_id, doomed);
9472 }
9473 other => panic!("expected local purge rejection, got {other:?}"),
9474 }
9475 assert_eq!(client.drain_change_batches().len(), 1);
9476 assert_eq!(
9477 client.purge_local_data(&input).expect("retry purge"),
9478 LocalDataPurgeResult {
9479 already_applied: true,
9480 purged_rows: 0,
9481 dropped_commits: 0,
9482 }
9483 );
9484 let conflicting = LocalDataPurgeInput {
9485 purge_id: input.purge_id.clone(),
9486 targets: vec![LocalDataPurgeTarget {
9487 table: "patient_notes".to_owned(),
9488 selectors: BTreeMap::from([(
9489 "encryption_key_id".to_owned(),
9490 vec!["key-held".to_owned()],
9491 )]),
9492 }],
9493 };
9494 assert!(client
9495 .purge_local_data(&conflicting)
9496 .expect_err("id collision must fail")
9497 .contains("already used with a different plan"));
9498 }
9499}
9500
9501impl SubState {
9502 fn name(self) -> &'static str {
9503 match self {
9504 SubState::Active => "active",
9505 SubState::Revoked => "revoked",
9506 SubState::Failed => "failed",
9507 }
9508 }
9509
9510 fn parse(value: &str) -> Self {
9511 match value {
9512 "revoked" => Self::Revoked,
9513 "failed" => Self::Failed,
9514 _ => Self::Active,
9515 }
9516 }
9517}
9518
9519#[derive(Debug, Clone)]
9520struct Subscription {
9521 id: String,
9522 table: String,
9523 requested: Vec<(String, Vec<String>)>,
9524 params: Option<String>,
9525 cursor: i64,
9526 bootstrap_state: Option<String>,
9528 state: SubState,
9529 reason_code: Option<String>,
9530 effective: Option<Vec<(String, Vec<String>)>>,
9533 synced_once: bool,
9534}
9535
9536#[cfg(test)]
9537mod subscription_rearm_tests {
9538 use super::*;
9539 use serde_json::json;
9540
9541 fn client() -> SyncClient {
9542 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9543 {"name":"id","type":"string","nullable":false},
9544 {"name":"project_id","type":"string","nullable":false}],
9545 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[],"ftsIndexes":[]}]});
9546 SyncClient::new("rearm".into(), &schema, ClientLimits::default()).expect("client")
9547 }
9548
9549 fn scopes() -> Vec<(String, Vec<String>)> {
9550 vec![("project_id".to_owned(), vec!["p1".to_owned()])]
9551 }
9552
9553 fn store_revoked(client: &mut SyncClient) {
9555 client
9556 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9557 .expect("subscribe");
9558 let sub = &mut client.subs[0];
9559 sub.cursor = 41;
9560 sub.bootstrap_state = Some("resume".to_owned());
9561 sub.effective = Some(scopes());
9562 sub.state = SubState::Revoked;
9563 sub.reason_code = Some("sync.scope_revoked".to_owned());
9564 client.persist_sub(&client.subs[0]).expect("persist");
9565 client.sync_needed = false;
9566 client.sync_intent_queue.clear();
9567 }
9568
9569 #[test]
9570 fn resubscribing_the_same_intent_rearms_a_revoked_registration() {
9571 let mut client = client();
9572 store_revoked(&mut client);
9573
9574 client
9575 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9576 .expect("re-arm");
9577
9578 let view = client.subscription_state("sub").expect("state");
9579 assert_eq!(view.status, "active");
9580 assert_eq!(view.cursor, -1);
9581 assert!(!view.has_resume_token);
9582 assert_eq!(view.effective_scopes, None);
9583 assert_eq!(view.reason_code, None);
9584 assert!(client.sync_needed, "a re-arm asks for a pull");
9585 assert!(matches!(
9586 client.drain_sync_intents().as_slice(),
9587 [SyncIntent::Interactive]
9588 ));
9589 let stored: String = client
9590 .conn
9591 .query_row(
9592 "SELECT state_json FROM _syncular_subscriptions WHERE id = 'sub'",
9593 [],
9594 |row| row.get(0),
9595 )
9596 .expect("persisted row");
9597 let stored: Value = serde_json::from_str(&stored).expect("stored json");
9598 assert_eq!(stored["status"], json!("active"));
9599 assert_eq!(stored["cursor"], json!(-1));
9600 assert_eq!(stored["bootstrapState"], Value::Null);
9601 assert_eq!(stored["effectiveScopes"], Value::Null);
9602 assert_eq!(stored["reasonCode"], Value::Null);
9603 }
9604
9605 #[test]
9606 fn a_failed_registration_is_not_rearmed() {
9607 let mut client = client();
9608 client
9609 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9610 .expect("subscribe");
9611 client.subs[0].state = SubState::Failed;
9612 client.subs[0].reason_code = Some("sync.scope_revoked".to_owned());
9613 client.persist_sub(&client.subs[0]).expect("persist");
9614 client.sync_needed = false;
9615 client.sync_intent_queue.clear();
9616
9617 client
9618 .subscribe("sub".into(), "tasks".into(), scopes(), None)
9619 .expect("subscribe");
9620
9621 let view = client.subscription_state("sub").expect("state");
9622 assert_eq!(view.status, "failed");
9623 assert_eq!(view.reason_code.as_deref(), Some("sync.scope_revoked"));
9624 assert!(!client.sync_needed);
9625 assert!(client.drain_sync_intents().is_empty());
9626 }
9627}
9628
9629#[derive(Debug, Clone)]
9630struct OutboxOp {
9631 upsert: bool,
9632 table: String,
9633 row_id: String,
9634 base_version: Option<i64>,
9635 values: Option<Map<String, Value>>,
9639}
9640
9641impl From<&OutboxOp> for CommitOperation {
9642 fn from(operation: &OutboxOp) -> Self {
9643 Self {
9644 table: operation.table.clone(),
9645 row_id: operation.row_id.clone(),
9646 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
9647 base_version: operation.base_version,
9648 values: operation.values.clone(),
9649 }
9650 }
9651}
9652
9653#[derive(Debug, Clone)]
9654struct OutboxCommit {
9655 client_commit_id: String,
9656 ops: Vec<OutboxOp>,
9657}
9658
9659fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>)> {
9663 commit
9664 .ops
9665 .iter()
9666 .filter(|operation| operation.upsert)
9667 .map(|operation| {
9668 (
9669 operation.table.as_str(),
9670 operation
9671 .values
9672 .as_ref()
9673 .map(|values| values.keys().map(String::as_str).collect())
9674 .unwrap_or_default(),
9675 )
9676 })
9677 .collect()
9678}
9679
9680type FailedCommit = (OutboxCommit, Map<String, Value>);
9681
9682#[derive(Debug, Clone)]
9683struct CompiledLocalDataPurgeTarget {
9684 table: String,
9685 selectors: Vec<(String, Vec<String>)>,
9686}
9687
9688struct StoredCommitOutcomeRow {
9689 sequence: i64,
9690 client_commit_id: String,
9691 status: String,
9692 recorded_at_ms: i64,
9693 results_json: String,
9694 operations_json: Option<String>,
9695 resolution: String,
9696 resolved_at_ms: Option<i64>,
9697 replacement_client_commit_id: Option<String>,
9698}
9699
9700impl StoredCommitOutcomeRow {
9701 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
9702 Ok(Self {
9703 sequence: row.get(0)?,
9704 client_commit_id: row.get(1)?,
9705 status: row.get(2)?,
9706 recorded_at_ms: row.get(3)?,
9707 results_json: row.get(4)?,
9708 operations_json: row.get(5)?,
9709 resolution: row.get(6)?,
9710 resolved_at_ms: row.get(7)?,
9711 replacement_client_commit_id: row.get(8)?,
9712 })
9713 }
9714}
9715
9716enum SectionError {
9719 FailClosed,
9720 Abort(String, String),
9721}
9722
9723#[derive(Debug)]
9724pub(crate) struct RequestMeta {
9725 pushed_ids: Vec<String>,
9726 fresh: Vec<(String, bool)>,
9729 pub(crate) accept: u8,
9730 deferred_commits: usize,
9734}
9735
9736#[derive(Default)]
9737struct ChangeAccumulator {
9738 tables: BTreeMap<String, Option<BTreeSet<String>>>,
9740 windows: BTreeMap<(String, String), BTreeSet<String>>,
9741 status: bool,
9742 conflicts: bool,
9743 rejections: bool,
9744 outcomes: bool,
9745}
9746
9747impl ChangeAccumulator {
9748 fn table(&mut self, table: &str) {
9749 self.tables.insert(table.to_owned(), None);
9750 }
9751
9752 fn scope(&mut self, table: &str, key: String) {
9753 match self.tables.get_mut(table) {
9754 Some(None) => {}
9755 Some(Some(keys)) => {
9756 keys.insert(key);
9757 }
9758 None => {
9759 self.tables
9760 .insert(table.to_owned(), Some(BTreeSet::from([key])));
9761 }
9762 }
9763 }
9764
9765 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
9766 self.windows
9767 .entry((base_key.to_owned(), table.to_owned()))
9768 .or_default()
9769 .insert(unit.to_owned());
9770 }
9771
9772 fn touched(&self) -> bool {
9773 !self.tables.is_empty()
9774 || !self.windows.is_empty()
9775 || self.status
9776 || self.conflicts
9777 || self.rejections
9778 || self.outcomes
9779 }
9780}
9781
9782const PUSH_OPS_PER_REQUEST: usize = 500;
9788const MAX_LOCAL_PURGE_TARGETS: usize = 64;
9789const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
9790const MAX_LOCAL_PURGE_VALUES: usize = 128;
9791const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
9792pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
9793
9794struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
9796
9797impl OverlayDirty {
9798 fn clean() -> Self {
9799 Self(RefCell::new(Some(BTreeSet::new())))
9800 }
9801 fn get(&self) -> bool {
9802 self.0
9803 .borrow()
9804 .as_ref()
9805 .is_none_or(|tables| !tables.is_empty())
9806 }
9807 fn set(&self, dirty: bool) {
9808 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
9809 }
9810 fn table(&self, table: &str) {
9811 if let Some(tables) = self.0.borrow_mut().as_mut() {
9812 tables.insert(table.to_owned());
9813 }
9814 }
9815 fn snapshot(&self) -> Option<BTreeSet<String>> {
9816 self.0.borrow().clone()
9817 }
9818 fn restore(&self, tables: Option<BTreeSet<String>>) {
9819 *self.0.borrow_mut() = tables;
9820 }
9821}
9822
9823pub struct SyncClient {
9824 #[cfg(feature = "bench-internals")]
9825 benchmark_phases: Recorder,
9826 conn: Connection,
9827 schema: ClientSchema,
9828 client_id: String,
9829 limits: ClientLimits,
9830 subs: Vec<Subscription>,
9831 outbox: Vec<OutboxCommit>,
9832 failed_commits: Vec<FailedCommit>,
9833 retain_failed_commits: bool,
9834 conflicts: Vec<ConflictRecord>,
9835 rejections: Vec<RejectionRecord>,
9836 schema_floor: Option<SchemaFloor>,
9837 lease_state: Option<LeaseState>,
9839 stopped: bool,
9841 upgrading: bool,
9843 sync_needed: bool,
9845 realtime_policy: RealtimePolicy,
9848 realtime_state: RealtimeState,
9849 realtime_reason_code: Option<String>,
9850 realtime_retry_delay_ms: Option<u64>,
9851 presence: HashMap<String, HashMap<String, PresencePeer>>,
9853 now_ms: Option<i64>,
9856 encryption: crate::values::EncryptionConfig,
9860 security_preflight: bool,
9863 transport_enabled: bool,
9864 previous_version: Option<PreviousVersionContextConfig>,
9868 insert_sql: RefCell<HashMap<String, String>>,
9874 overlay_dirty: OverlayDirty,
9877 storage_failure: RefCell<Option<QueryReadFailure>>,
9879 #[cfg(test)]
9882 overlay_rebuild_count: Cell<usize>,
9883 #[cfg(test)]
9884 acknowledged_replay_count: Cell<usize>,
9885 #[cfg(test)]
9888 outcome_prune_count: Cell<usize>,
9889 progress: ProgressObserver,
9891 change_queue: VecDeque<ClientChangeBatch>,
9892 sync_intent_queue: VecDeque<SyncIntent>,
9893 retry_delay_ms: u64,
9895 round_retry_delay_ms: Option<u64>,
9897 active_round: Option<uuid::Uuid>,
9898 last_round: Option<DiagnosticLastRound>,
9899 last_change: Option<DiagnosticLastChange>,
9900 query_failures: Vec<DiagnosticQueryFailure>,
9902 coalesced: Option<CoalescedApply>,
9904 coalesced_failure: Option<String>,
9907}
9908
9909struct CoalescedApply {
9913 rows: usize,
9914 subs: Vec<Subscription>,
9915 outbox: Vec<OutboxCommit>,
9916 failed_commits: Vec<FailedCommit>,
9917 conflicts: Vec<ConflictRecord>,
9918 rejections: Vec<RejectionRecord>,
9919 schema_floor: Option<SchemaFloor>,
9920 lease_state: Option<LeaseState>,
9921 stopped: bool,
9922 upgrading: bool,
9923 sync_needed: bool,
9924 overlay_dirty: Option<BTreeSet<String>>,
9925 change_queue: usize,
9926 sync_intent_queue: usize,
9927 last_change: Option<DiagnosticLastChange>,
9928}
9929
9930const COALESCED_APPLY_ROWS: usize = 1024;
9931
9932pub(crate) fn quote_ident(name: &str) -> String {
9933 format!("\"{}\"", name.replace('"', "\"\""))
9934}
9935pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
9939 conn.query_row(
9940 "SELECT value FROM _syncular_meta WHERE key = ?1",
9941 rusqlite::params![key],
9942 |row| row.get::<_, String>(0),
9943 )
9944 .ok()
9945}
9946
9947pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
9948 let _ = conn.execute(
9949 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
9950 rusqlite::params![key, value],
9951 );
9952}
9953
9954pub(crate) fn meta_delete(conn: &Connection, key: &str) {
9955 let _ = conn.execute(
9956 "DELETE FROM _syncular_meta WHERE key = ?1",
9957 rusqlite::params![key],
9958 );
9959}
9960
9961fn row_id_predicate(table: &TableSchema) -> String {
9963 let key = quote_ident(&table.primary_key);
9964 let text_match = format!("CAST({key} AS TEXT) = ?1");
9965 match table.columns[table.pk_index].ty {
9966 ColumnType::String | ColumnType::Json => {
9967 format!("{key} = ?1 AND {text_match}")
9968 }
9969 ColumnType::Integer | ColumnType::Boolean => {
9973 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
9974 }
9975 _ => text_match,
9978 }
9979}
9980
9981fn is_local_operation_code_like(value: &str) -> bool {
9982 let bytes = value.as_bytes();
9983 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
9984 && bytes
9985 .iter()
9986 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
9987}
9988
9989fn base_table(name: &str) -> String {
9990 quote_ident(&format!("_syncular_base_{name}"))
9991}
9992
9993type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
9998
9999fn window_base_key(base: &WindowBase) -> String {
10000 format!(
10001 "{}\0{}\0{}",
10002 base.table,
10003 base.variable,
10004 canonical_scope_json(&base.fixed_scopes)
10005 )
10006}
10007
10008fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
10010 let mut scopes = base.fixed_scopes.clone();
10011 scopes.retain(|(k, _)| k != &base.variable);
10012 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
10013 scopes
10014}
10015
10016fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
10020 let canonical = canonical_scope_json(&unit_scopes(base, unit));
10021 let digest = Sha256::digest(canonical.as_bytes());
10022 let hex = bytes_to_hex(&digest);
10023 format!("w:{}:{}", base.table, &hex[..16])
10024}
10025
10026fn visible_table(name: &str) -> String {
10027 quote_ident(name)
10028}
10029
10030const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
10031
10032fn is_synced_table_name(name: &str) -> bool {
10035 if name.starts_with("sqlite_") {
10036 return false;
10037 }
10038 if name.starts_with("_syncular_base_") {
10039 return true; }
10041 !name.starts_with("_syncular_")
10043}
10044
10045fn blob_id_for(bytes: &[u8]) -> String {
10047 let digest = Sha256::digest(bytes);
10048 format!("sha256:{}", bytes_to_hex(&digest))
10049}
10050
10051enum RowParam<'a> {
10055 Cell(&'a Option<ColumnValue>),
10056 Version(i64),
10057}
10058
10059impl rusqlite::ToSql for RowParam<'_> {
10060 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
10061 Ok(match self {
10062 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
10063 RowParam::Cell(cell) => match cell {
10064 None => ToSqlOutput::Owned(SqlValue::Null),
10065 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
10066 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
10067 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
10068 Some(ColumnValue::Boolean(b)) => {
10069 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
10070 }
10071 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
10072 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
10073 }
10074 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
10076 ToSqlOutput::Borrowed(ValueRef::Blob(b))
10077 }
10078 },
10079 })
10080 }
10081}
10082
10083fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
10085 match param.to_sql().map_err(|error| error.to_string())? {
10086 ToSqlOutput::Owned(value) => Ok(value),
10087 ToSqlOutput::Borrowed(value) => {
10088 SqlValue::try_from(value).map_err(|error| error.to_string())
10089 }
10090 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
10091 }
10092}
10093
10094fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
10102 use ssp2::segment::ColumnType;
10103 let mismatch = || {
10104 Err(format!(
10105 "image column {:?} holds a value of the wrong type",
10106 column.name
10107 ))
10108 };
10109 match value {
10110 ValueRef::Null => {
10111 if !column.nullable {
10112 return Err(format!(
10113 "image column {:?} is NULL but not nullable",
10114 column.name
10115 ));
10116 }
10117 Ok(ToSqlOutput::Owned(SqlValue::Null))
10118 }
10119 ValueRef::Integer(i) => match column.ty {
10120 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
10121 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
10122 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
10123 _ => mismatch(),
10124 },
10125 ValueRef::Real(_) => match column.ty {
10126 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
10127 _ => mismatch(),
10128 },
10129 ValueRef::Text(t) => {
10130 std::str::from_utf8(t)
10131 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
10132 match column.ty {
10133 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
10134 Ok(ToSqlOutput::Borrowed(value))
10135 }
10136 _ => mismatch(),
10137 }
10138 }
10139 ValueRef::Blob(_) => match column.ty {
10140 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
10142 _ => mismatch(),
10143 },
10144 }
10145}
10146
10147pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
10148 use rusqlite::types::ValueRef;
10149 match value {
10150 ValueRef::Null => Value::Null,
10151 ValueRef::Integer(i) => match column.ty {
10152 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
10153 ssp2::segment::ColumnType::Float => {
10154 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
10155 }
10156 _ => Value::from(i),
10157 },
10158 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10159 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10160 ValueRef::Blob(b) => {
10161 let mut map = Map::new();
10162 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10163 Value::Object(map)
10164 }
10165 }
10166}
10167
10168fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
10172 Ok(match value {
10173 Value::Null => SqlValue::Null,
10174 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
10175 Value::Number(n) => {
10176 if let Some(i) = n.as_i64() {
10177 SqlValue::Integer(i)
10178 } else if let Some(f) = n.as_f64() {
10179 SqlValue::Real(f)
10180 } else {
10181 return Err(format!("query param number {n} is out of range"));
10182 }
10183 }
10184 Value::String(s) => SqlValue::Text(s.clone()),
10185 Value::Object(_) => {
10186 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
10187 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
10188 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
10189 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
10190 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
10191 })?)
10192 } else {
10193 return Err(
10194 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
10195 .to_owned(),
10196 );
10197 }
10198 }
10199 Value::Array(_) => return Err("query array params are not supported".to_owned()),
10200 })
10201}
10202
10203fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
10208 use rusqlite::types::ValueRef;
10209 match value {
10210 ValueRef::Null => Value::Null,
10211 ValueRef::Integer(i) => {
10212 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
10216 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
10217 Value::from(i)
10218 } else {
10219 let mut map = Map::new();
10220 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
10221 Value::Object(map)
10222 }
10223 }
10224 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
10225 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
10226 ValueRef::Blob(b) => {
10227 let mut map = Map::new();
10228 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
10229 Value::Object(map)
10230 }
10231 }
10232}
10233
10234fn query_connection(
10236 conn: &Connection,
10237 sql: &str,
10238 params: &[QueryValue],
10239) -> Result<Vec<QueryRow>, QueryReadFailure> {
10240 crate::query_guard::assert_read_only_query(sql)?;
10241 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
10242 let bound: Vec<SqlValue> = params
10243 .iter()
10244 .map(json_param_to_sql)
10245 .collect::<Result<_, _>>()?;
10246 let mut stmt = conn.prepare(&lowered_sql)?;
10247 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
10248 let bound_refs: Vec<&dyn rusqlite::ToSql> =
10249 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
10250 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
10251 let mut out = Vec::new();
10252 while let Some(row) = sql_rows.next()? {
10253 let mut record = Map::new();
10254 for (i, name) in column_names.iter().enumerate() {
10255 let value = row.get_ref(i)?;
10256 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
10257 }
10258 out.push(record);
10259 }
10260 Ok(out)
10261}
10262
10263fn persisted_window_state(
10264 conn: &Connection,
10265 base: &WindowBase,
10266) -> Result<WindowState, QueryReadFailure> {
10267 let mut stmt = conn.prepare(
10268 "SELECT windows.unit, subscriptions.state_json
10269 FROM _syncular_windows AS windows
10270 JOIN _syncular_subscriptions AS subscriptions
10271 ON subscriptions.id = windows.sub_id
10272 WHERE windows.base = ?1
10273 ORDER BY windows.unit ASC",
10274 )?;
10275 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
10276 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
10277 })?;
10278 let mut units = Vec::new();
10279 let mut pending = Vec::new();
10280 for row in rows {
10281 let (unit, raw) = row?;
10282 let state: Value = serde_json::from_str(&raw)
10283 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
10284 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
10285 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
10286 || state
10287 .get("bootstrapState")
10288 .is_some_and(|value| !value.is_null());
10289 if is_pending {
10290 pending.push(unit.clone());
10291 }
10292 units.push(unit);
10293 }
10294 Ok(WindowState { units, pending })
10295}
10296
10297#[derive(Debug, Clone, Copy)]
10299pub struct SnapshotStatement<'a> {
10300 pub sql: &'a str,
10301 pub params: &'a [Value],
10302}
10303
10304#[derive(Debug, Clone, Default)]
10308pub struct SnapshotReadRequest<'a> {
10309 pub statements: Vec<SnapshotStatement<'a>>,
10310 pub coverage: &'a [WindowCoverage],
10311 pub subscriptions: Vec<String>,
10312 pub commit_ids: Vec<String>,
10313}
10314
10315#[cfg(test)]
10316thread_local! {
10317 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
10321 std::cell::RefCell::new(None);
10322}
10323
10324#[cfg(test)]
10325fn run_snapshot_read_hook() {
10326 SNAPSHOT_READ_HOOK.with(|slot| {
10327 if let Some(hook) = slot.borrow_mut().take() {
10328 hook();
10329 }
10330 });
10331}
10332
10333fn snapshot_read_connection(
10338 conn: &Connection,
10339 request: &SnapshotReadRequest<'_>,
10340) -> Result<SnapshotRead, QueryReadFailure> {
10341 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
10342 let corrupt = |message: &str| QueryReadFailure {
10343 code: Some("sync.local_corrupt"),
10344 sqlite_code: None,
10345 sqlite_message: None,
10346 rollback_failure: None,
10347 message: message.to_owned(),
10348 };
10349 let stored_failure = |error: rusqlite::Error, message: &str| match error {
10350 rusqlite::Error::Utf8Error(..)
10351 | rusqlite::Error::InvalidColumnType(..)
10352 | rusqlite::Error::FromSqlConversionFailure(..)
10353 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
10354 _ => QueryReadFailure::from(error),
10355 };
10356 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
10357 let revision: Option<String> = conn
10360 .query_row(
10361 "SELECT value FROM _syncular_meta WHERE key = ?1",
10362 rusqlite::params![LOCAL_REVISION_KEY],
10363 |row| row.get(0),
10364 )
10365 .optional()
10366 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
10367 let revision = match revision {
10368 None => "0".to_owned(),
10369 Some(value)
10370 if value == "0"
10371 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10372 && value.bytes().all(|b| b.is_ascii_digit())
10373 && value.parse::<u64>().is_ok()) =>
10374 {
10375 value
10376 }
10377 Some(_) => return Err(corrupt("persisted local revision is invalid")),
10378 };
10379 let mut queries = Vec::with_capacity(request.statements.len());
10380 for statement in &request.statements {
10381 queries.push(query_connection(conn, statement.sql, statement.params)?);
10382 #[cfg(test)]
10383 run_snapshot_read_hook();
10384 }
10385 let mut pending = Vec::new();
10386 let mut missing = Vec::new();
10387 for requested in request.coverage {
10388 let base_key = window_base_key(&requested.base);
10389 let state = persisted_window_state(conn, &requested.base)?;
10390 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
10391 let reference = WindowUnitRef {
10392 base_key: base_key.clone(),
10393 unit: unit.clone(),
10394 };
10395 if !state.units.iter().any(|held| held == &unit) {
10396 missing.push(reference);
10397 } else if state.pending.iter().any(|held| held == &unit) {
10398 pending.push(reference);
10399 }
10400 }
10401 }
10402 let mut subscriptions = Vec::with_capacity(request.subscriptions.len());
10403 for id in &request.subscriptions {
10404 let row: Option<(String, String)> = conn
10405 .query_row(
10406 "SELECT tbl, state_json FROM _syncular_subscriptions WHERE id = ?1",
10407 rusqlite::params![id],
10408 |row| Ok((row.get(0)?, row.get(1)?)),
10409 )
10410 .optional()
10411 .map_err(|error| {
10412 stored_failure(error, "persisted subscription state is invalid")
10413 })?;
10414 let Some((table, raw)) = row else {
10415 subscriptions.push(SubscriptionCatchup::Unknown { id: id.clone() });
10416 continue;
10417 };
10418 let state: Value = serde_json::from_str(&raw)
10421 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10422 let status = match state.get("status").and_then(Value::as_str) {
10423 Some(value @ ("active" | "revoked" | "failed")) => value,
10424 _ => return Err(corrupt("persisted subscription state is invalid")),
10425 };
10426 let Some(cursor) = state.get("cursor").and_then(Value::as_i64) else {
10427 return Err(corrupt("persisted subscription state is invalid"));
10428 };
10429 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
10430 return Err(corrupt("persisted subscription state is invalid"));
10431 }
10432 let effective_scopes = state
10433 .get("effectiveScopes")
10434 .filter(|value| !value.is_null());
10435 if let Some(scopes) = effective_scopes {
10436 json_to_scope_map(scopes)
10437 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
10438 }
10439 let has_resume_token = match state.get("bootstrapState") {
10440 None | Some(Value::Null) => false,
10441 Some(Value::String(_)) => true,
10442 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10443 };
10444 let reason_code = match state.get("reasonCode") {
10447 None | Some(Value::Null) => None,
10448 Some(Value::String(value)) => Some(value.clone()),
10449 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
10450 };
10451 let active = status == "active";
10452 subscriptions.push(SubscriptionCatchup::Known {
10453 id: id.clone(),
10454 table,
10455 status: status.to_owned(),
10456 cursor,
10457 has_resume_token,
10458 bootstrap_complete: active && cursor >= 0 && !has_resume_token,
10459 known_pending_pages: active && (has_resume_token || cursor < 0),
10460 effective_scopes: effective_scopes.cloned(),
10461 reason_code,
10462 });
10463 }
10464 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
10465 for id in &request.commit_ids {
10466 let pending: bool = conn.query_row(
10467 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
10468 rusqlite::params![id],
10469 |row| row.get(0),
10470 )?;
10471 if pending {
10472 deliveries.push(CommitDelivery::Pending {
10473 client_commit_id: id.clone(),
10474 });
10475 continue;
10476 }
10477 let row = conn
10478 .query_row(
10479 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
10480 resolution, resolved_at_ms, replacement_client_commit_id
10481 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
10482 rusqlite::params![id],
10483 StoredCommitOutcomeRow::from_row,
10484 )
10485 .optional()
10486 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
10487 deliveries.push(match row {
10488 Some(row) => CommitDelivery::Known {
10489 client_commit_id: id.clone(),
10490 outcome: SyncClient::outcome_from_row(row)
10491 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
10492 },
10493 None => CommitDelivery::Unknown {
10494 client_commit_id: id.clone(),
10495 },
10496 });
10497 }
10498 Ok(SnapshotRead {
10499 revision,
10500 queries,
10501 coverage: CoverageSnapshot {
10502 complete: pending.is_empty() && missing.is_empty(),
10503 pending,
10504 missing,
10505 },
10506 subscriptions,
10507 deliveries,
10508 })
10509 })();
10510 match result {
10511 Ok(read) => {
10512 conn.execute_batch("RELEASE syncular_snapshot_read")?;
10513 Ok(read)
10514 }
10515 Err(mut error) => {
10516 if let Err(rollback) = conn
10517 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
10518 {
10519 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
10520 }
10521 Err(error)
10522 }
10523 }
10524}
10525
10526fn snapshot_connection(
10527 conn: &Connection,
10528 sql: &str,
10529 params: &[Value],
10530 coverage: &[WindowCoverage],
10531) -> Result<QuerySnapshot, QueryReadFailure> {
10532 let read = snapshot_read_connection(
10533 conn,
10534 &SnapshotReadRequest {
10535 statements: vec![SnapshotStatement { sql, params }],
10536 coverage,
10537 subscriptions: Vec::new(),
10538 commit_ids: Vec::new(),
10539 },
10540 )?;
10541 Ok(QuerySnapshot {
10542 revision: read.revision,
10543 rows: read
10544 .queries
10545 .into_iter()
10546 .next()
10547 .expect("single-query snapshot returns one result"),
10548 coverage: read.coverage,
10549 })
10550}
10551
10552pub struct FileQuerySnapshotReader {
10557 path: String,
10558 conn: Option<Connection>,
10559}
10560
10561impl FileQuerySnapshotReader {
10562 #[must_use]
10563 pub fn new(path: impl Into<String>) -> Self {
10564 Self {
10565 path: path.into(),
10566 conn: None,
10567 }
10568 }
10569
10570 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
10571 if self.conn.is_none() {
10572 let conn = Connection::open_with_flags(
10573 &self.path,
10574 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
10575 )
10576 .map_err(QueryReadFailure::from)?;
10577 conn.busy_timeout(std::time::Duration::from_millis(250))
10578 .map_err(QueryReadFailure::from)?;
10579 self.conn = Some(conn);
10580 }
10581 self.conn
10582 .as_ref()
10583 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
10584 }
10585
10586 pub fn snapshot_read(
10589 &mut self,
10590 request: &SnapshotReadRequest<'_>,
10591 ) -> Result<SnapshotRead, QueryReadFailure> {
10592 snapshot_read_connection(self.connection()?, request)
10593 }
10594
10595 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
10598 Ok(self
10599 .snapshot_read(&SnapshotReadRequest::default())?
10600 .revision)
10601 }
10602
10603 pub fn subscription_catchup(
10605 &mut self,
10606 id: &str,
10607 ) -> Result<SubscriptionCatchup, QueryReadFailure> {
10608 let read = self.snapshot_read(&SnapshotReadRequest {
10609 subscriptions: vec![id.to_owned()],
10610 ..Default::default()
10611 })?;
10612 Ok(read
10613 .subscriptions
10614 .into_iter()
10615 .next()
10616 .expect("one requested catch-up"))
10617 }
10618
10619 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
10622 let read = self.snapshot_read(&SnapshotReadRequest {
10623 commit_ids: vec![id.to_owned()],
10624 ..Default::default()
10625 })?;
10626 Ok(read
10627 .deliveries
10628 .into_iter()
10629 .next()
10630 .expect("one requested delivery"))
10631 }
10632
10633 pub fn query_snapshot(
10636 &mut self,
10637 sql: &str,
10638 params: &[Value],
10639 coverage: &[WindowCoverage],
10640 ) -> Result<QuerySnapshot, QueryReadFailure> {
10641 snapshot_connection(self.connection()?, sql, params, coverage)
10642 }
10643}
10644
10645fn validate_authority_reads(
10646 schema: &ClientSchema,
10647 declarations: &[crate::api::AuthorityReadDeclaration],
10648) -> Result<(), String> {
10649 let mut seen = BTreeSet::new();
10650 for read in declarations {
10651 let table = schema
10652 .table(&read.table)
10653 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
10654 let plain = |name: &str| {
10655 table.columns.iter().enumerate().any(|(i, c)| {
10656 c.name == name
10657 && name != "authority_version"
10658 && !name.starts_with("_sync")
10659 && !table.encrypted_columns.iter().any(|e| e.index == i)
10660 && matches!(
10661 c.ty,
10662 ssp2::segment::ColumnType::String
10663 | ssp2::segment::ColumnType::Integer
10664 | ssp2::segment::ColumnType::Float
10665 | ssp2::segment::ColumnType::Boolean
10666 | ssp2::segment::ColumnType::Json
10667 )
10668 })
10669 };
10670 if !seen.insert(&read.table)
10671 || read.columns.is_empty()
10672 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
10673 || !read.columns.contains(&table.primary_key)
10674 || !read.columns.iter().all(|c| plain(c))
10675 || read.scopes.is_empty()
10676 || !read.scopes.iter().all(|(v, values)| {
10677 table
10678 .scope_column(v)
10679 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
10680 && !values.is_empty()
10681 && values.iter().all(|v| !v.is_empty() && v != "*")
10682 })
10683 {
10684 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
10685 }
10686 }
10687 Ok(())
10688}
10689
10690fn schema_marker_read_failure(error: rusqlite::Error) -> String {
10695 let failure = QueryReadFailure::from(error);
10696 if failure.code.is_some() {
10697 failure.to_string()
10698 } else {
10699 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
10700 }
10701}
10702
10703fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
10707 if limits.outcome_retention_max_entries == Some(0) {
10708 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
10709 }
10710 if [
10711 limits.max_push_commits_per_request,
10712 limits.max_push_operations_per_request,
10713 limits.max_push_request_bytes,
10714 ]
10715 .into_iter()
10716 .flatten()
10717 .any(|value| value == 0 || value > u32::MAX as usize)
10718 {
10719 return Err(
10720 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
10721 );
10722 }
10723 Ok(())
10724}
10725
10726fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
10727 let corrupt =
10728 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
10729 let table_type: Option<String> = conn
10730 .query_row(
10731 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
10732 [],
10733 |row| row.get(0),
10734 )
10735 .optional()
10736 .map_err(schema_marker_read_failure)?;
10737 let Some(table_type) = table_type else {
10738 return Ok(None);
10739 };
10740 if table_type != "table" {
10741 return Err(corrupt());
10742 }
10743 let mut statement = conn
10747 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
10748 .map_err(schema_marker_read_failure)?;
10749 let mut rows = statement
10750 .query([LOCAL_SCHEMA_VERSION_KEY])
10751 .map_err(schema_marker_read_failure)?;
10752 let value: Option<String> = rows
10753 .next()
10754 .map_err(schema_marker_read_failure)?
10755 .map(|row| row.get(0))
10756 .transpose()
10757 .map_err(schema_marker_read_failure)?;
10758 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
10759 return Err(corrupt());
10760 }
10761 let Some(value) = value else {
10762 let has_descriptor = conn
10763 .query_row(
10764 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
10765 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
10766 |_| Ok(()),
10767 )
10768 .optional()
10769 .map_err(schema_marker_read_failure)?
10770 .is_some();
10771 if has_descriptor {
10772 return Err(corrupt());
10773 }
10774 return Ok(None);
10775 };
10776 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
10777 || !value.bytes().all(|b| b.is_ascii_digit())
10778 {
10779 return Err(corrupt());
10780 }
10781 let version = value.parse::<i32>().map_err(|_| corrupt())?;
10782 if version > requested {
10783 return Err(
10784 "client.schema_downgrade: persisted local schema is newer than the requested schema"
10785 .to_owned(),
10786 );
10787 }
10788 Ok(Some(version))
10789}
10790
10791impl SyncClient {
10792 pub fn new_with_identity(
10793 client_id: Option<String>,
10794 schema_json: &Value,
10795 limits: ClientLimits,
10796 ) -> Result<Self, String> {
10797 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
10798 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
10799 Self::with_connection(resolved, schema_json, limits, conn)
10800 }
10801
10802 pub fn new(
10803 client_id: String,
10804 schema_json: &Value,
10805 limits: ClientLimits,
10806 ) -> Result<Self, String> {
10807 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
10808 Self::with_connection(client_id, schema_json, limits, conn)
10809 }
10810
10811 pub fn open_path(
10817 client_id: String,
10818 schema_json: &Value,
10819 limits: ClientLimits,
10820 path: &str,
10821 ) -> Result<Self, String> {
10822 validate_client_limits(&limits)?;
10823 parse_schema_json(schema_json)?;
10824 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
10825 Self::with_connection(client_id, schema_json, limits, conn)
10826 }
10827
10828 pub fn open_path_with_identity(
10829 client_id: Option<String>,
10830 schema_json: &Value,
10831 limits: ClientLimits,
10832 path: &str,
10833 ) -> Result<Self, String> {
10834 validate_client_limits(&limits)?;
10835 parse_schema_json(schema_json)?;
10838 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
10839 conn.busy_timeout(std::time::Duration::from_millis(250))
10847 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
10848 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
10849 let journal_mode: String = client
10850 .conn
10851 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
10852 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
10853 if journal_mode != "wal" {
10854 return Err(
10855 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
10856 );
10857 }
10858 Ok(client)
10859 }
10860
10861 #[must_use]
10862 pub fn client_id(&self) -> &str {
10863 &self.client_id
10864 }
10865
10866 pub fn with_connection(
10873 client_id: String,
10874 schema_json: &Value,
10875 limits: ClientLimits,
10876 conn: Connection,
10877 ) -> Result<Self, String> {
10878 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
10879 }
10880
10881 fn with_connection_identity(
10884 client_id: Option<String>,
10885 schema_json: &Value,
10886 limits: ClientLimits,
10887 conn: Connection,
10888 ) -> Result<Self, String> {
10889 validate_client_limits(&limits)?;
10890 let schema = parse_schema_json(schema_json)?;
10891 validate_authority_reads(&schema, &limits.authority_reads)?;
10892 let previous_version = match limits.previous_version_context {
10895 Some(config) => {
10896 config.validate()?;
10897 Some(config)
10898 }
10899 None => None,
10900 };
10901 let mut client = SyncClient {
10902 #[cfg(feature = "bench-internals")]
10903 benchmark_phases: Recorder::default(),
10904 conn,
10905 schema,
10906 client_id: client_id
10907 .clone()
10908 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
10909 limits,
10910 subs: Vec::new(),
10911 outbox: Vec::new(),
10912 failed_commits: Vec::new(),
10913 retain_failed_commits: false,
10914 conflicts: Vec::new(),
10915 rejections: Vec::new(),
10916 schema_floor: None,
10917 lease_state: None,
10918 stopped: false,
10919 upgrading: false,
10920 sync_needed: false,
10921 realtime_policy: RealtimePolicy::Optional,
10922 realtime_state: RealtimeState::Disconnected,
10923 realtime_reason_code: None,
10924 realtime_retry_delay_ms: None,
10925 presence: HashMap::new(),
10926 now_ms: None,
10927 encryption: crate::values::EncryptionConfig::default(),
10928 security_preflight: false,
10929 transport_enabled: true,
10930 previous_version,
10931 insert_sql: RefCell::new(HashMap::new()),
10932 overlay_dirty: OverlayDirty::clean(),
10933 storage_failure: RefCell::new(None),
10934 #[cfg(test)]
10935 overlay_rebuild_count: Cell::new(0),
10936 #[cfg(test)]
10937 acknowledged_replay_count: Cell::new(0),
10938 #[cfg(test)]
10939 outcome_prune_count: Cell::new(0),
10940 progress: ProgressObserver::default(),
10941 change_queue: VecDeque::new(),
10942 sync_intent_queue: VecDeque::new(),
10943 retry_delay_ms: 250,
10944 round_retry_delay_ms: None,
10945 active_round: None,
10946 last_round: None,
10947 last_change: None,
10948 query_failures: Vec::new(),
10949 coalesced: None,
10950 coalesced_failure: None,
10951 };
10952 client
10956 .conn
10957 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
10958 client
10964 .conn
10965 .execute_batch("BEGIN IMMEDIATE")
10966 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
10967 let startup = (|| -> Result<(), String> {
10968 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
10971 client.create_bookkeeping_tables()?;
10974 match client.get_meta(CLIENT_ID_KEY) {
10975 Some(existing) => match client_id {
10976 Some(requested) if existing != requested => {
10977 return Err(format!(
10978 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
10979 ))
10980 }
10981 _ => client.client_id = existing,
10982 },
10983 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
10984 }
10985 client.restore_persisted_state()?;
10986 client.failed_commits = client.load_failed_commits()?;
10987 match marker {
10988 None => {
10989 client.create_synced_tables()?;
10990 client.persist_schema_version()?;
10991 client.save_subscription_scope_schema();
10992 }
10993 Some(version) if version == client.schema.version => {
10994 client.create_synced_tables()?;
10995 set_local_schema_descriptor(&client.conn, &client.schema)
10999 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
11000 client.save_subscription_scope_schema();
11001 }
11002 Some(_) => client.run_schema_reset()?,
11003 }
11004 client.reconcile_previous_version_at_boot()?;
11010 client.clear_satisfied_persisted_schema_floor();
11011 client.prune_unknown_subscriptions(false)?;
11012 client.drop_unencodable_outbox(false, None)?;
11021 if marker == Some(client.schema.version)
11022 && (!client.outbox.is_empty()
11023 || !client.failed_commits.is_empty()
11024 || client.has_acknowledged_rows())
11025 {
11026 client.overlay_dirty.set(true);
11030 client.rebuild_overlay()?;
11031 }
11032 client.enqueue_startup_sync_if_needed()?;
11039 Ok(())
11040 })();
11041 match startup {
11042 Ok(()) => match client.conn.execute_batch("COMMIT") {
11043 Ok(()) => Ok(client),
11044 Err(error) => {
11045 let failure = Self::sqlite_failure(&client.storage_failure, error);
11046 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
11047 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
11049 }
11050 Err(failure)
11051 }
11052 },
11053 Err(error) => {
11054 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
11055 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
11057 }
11058 Err(error)
11059 }
11060 }
11061 }
11062
11063 pub fn set_now_ms(&mut self, now_ms: i64) {
11066 self.now_ms = Some(now_ms);
11067 }
11068
11069 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
11073 self.encryption = encryption;
11074 }
11075
11076 #[must_use]
11077 pub fn security_lifecycle(&self) -> &'static str {
11078 if self.security_preflight {
11079 "preflight"
11080 } else {
11081 "active"
11082 }
11083 }
11084
11085 #[must_use]
11086 pub fn security_preflight(&self) -> bool {
11087 self.security_preflight
11088 }
11089
11090 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
11092 if self.transport_enabled != enabled {
11093 self.transport_enabled = enabled;
11094 self.sync_intent_queue.clear();
11095 self.sync_intent_queue
11096 .push_back(if enabled && !self.security_preflight {
11097 SyncIntent::Interactive
11098 } else {
11099 SyncIntent::None
11100 });
11101 }
11102 if !enabled && self.active_round.is_none() {
11104 self.disconnect_realtime(transport);
11105 }
11106 }
11107
11108 pub fn transport_enabled(&self) -> bool {
11109 self.transport_enabled
11110 }
11111
11112 pub fn begin_security_preflight(&mut self) {
11119 self.seal_security_on_teardown();
11120 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
11123 }
11124
11125 pub fn seal_security_on_teardown(&mut self) {
11135 self.cancel_sync_round();
11136 self.security_preflight = true;
11137 self.encryption = crate::values::EncryptionConfig::default();
11138 self.sync_intent_queue.clear();
11139 }
11140
11141 pub fn activate_security(
11143 &mut self,
11144 encryption: crate::values::EncryptionConfig,
11145 ) -> Result<(), String> {
11146 if !self.security_preflight {
11147 return Err(
11148 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
11149 );
11150 }
11151 self.enqueue_startup_sync_if_needed()?;
11152 self.encryption = encryption;
11153 self.security_preflight = false;
11154 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
11155 Ok(())
11156 }
11157
11158 fn clock_now_ms(&self) -> i64 {
11159 self.now_ms.unwrap_or_else(crate::wall_clock_ms)
11160 }
11161
11162 fn create_bookkeeping_tables(&self) -> Result<(), String> {
11163 self.conn
11165 .execute_batch(
11166 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
11167 seq INTEGER PRIMARY KEY AUTOINCREMENT,
11168 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
11169 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
11170 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
11171 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
11172 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
11173 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
11174 PRIMARY KEY(commit_id, idx));
11175 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
11176 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
11177 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
11178 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
11179 seq INTEGER PRIMARY KEY AUTOINCREMENT,
11180 client_commit_id TEXT NOT NULL UNIQUE,
11181 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
11182 recorded_at_ms INTEGER NOT NULL,
11183 results_json TEXT NOT NULL,
11184 operations_json TEXT,
11185 resolution TEXT NOT NULL DEFAULT 'active'
11186 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
11187 resolved_at_ms INTEGER,
11188 replacement_client_commit_id TEXT);
11189 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
11190 ON _syncular_commit_outcomes(resolution, seq);
11191 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
11192 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
11193 CREATE TABLE IF NOT EXISTS _syncular_meta (
11194 key TEXT PRIMARY KEY, value TEXT NOT NULL);
11195 CREATE TABLE IF NOT EXISTS _syncular_windows (
11196 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
11197 PRIMARY KEY (base, unit));
11198 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
11199 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
11200 effective_scopes TEXT NOT NULL);
11201 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
11202 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
11203 PRIMARY KEY(commit_id, blob_id));
11204 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
11205 ON _syncular_blob_commit_refs(blob_id);",
11206 )
11207 .map_err(|e| e.to_string())?;
11208 let _ = self
11211 .conn
11212 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
11213 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
11214 self.set_meta(LOCAL_REVISION_KEY, "0");
11215 }
11216 if self.schema_has_blobs() {
11219 let exists = self
11220 .conn
11221 .query_row(
11222 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
11223 [],
11224 |row| row.get::<_, bool>(0),
11225 )
11226 .map_err(|e| e.to_string())?;
11227 if exists {
11228 let mut statement = self
11229 .conn
11230 .prepare("PRAGMA table_info(_syncular_blobs)")
11231 .map_err(|e| e.to_string())?;
11232 let mut columns = statement
11233 .query_map([], |row| row.get::<_, String>(1))
11234 .map_err(|e| e.to_string())?
11235 .collect::<Result<Vec<_>, _>>()
11236 .map_err(|e| e.to_string())?;
11237 columns.sort();
11238 let mut expected = vec![
11239 "blob_id",
11240 "byte_length",
11241 "bytes",
11242 "created_at_ms",
11243 "media_type",
11244 ];
11245 expected.sort_unstable();
11246 if columns != expected {
11247 return Err(
11248 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
11249 );
11250 }
11251 } else {
11252 self.conn
11253 .execute_batch(
11254 "CREATE TABLE _syncular_blobs(
11255 blob_id TEXT PRIMARY KEY,
11256 bytes BLOB NOT NULL,
11257 byte_length INTEGER NOT NULL,
11258 media_type TEXT,
11259 created_at_ms INTEGER NOT NULL);",
11260 )
11261 .map_err(|e| e.to_string())?;
11262 }
11263 self.conn
11264 .execute_batch(
11265 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
11266 blob_id TEXT PRIMARY KEY,
11267 media_type TEXT,
11268 created_at_ms INTEGER NOT NULL);",
11269 )
11270 .map_err(|e| e.to_string())?;
11271 }
11272 Ok(())
11273 }
11274
11275 fn schema_has_blobs(&self) -> bool {
11277 self.schema
11278 .tables
11279 .iter()
11280 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
11281 }
11282
11283 fn get_meta(&self, key: &str) -> Option<String> {
11286 meta_get(&self.conn, key)
11287 }
11288
11289 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
11290 self.conn
11291 .query_row(
11292 "SELECT value FROM _syncular_meta WHERE key = ?1",
11293 rusqlite::params![key],
11294 |row| row.get::<_, String>(0),
11295 )
11296 .optional()
11297 .map_err(|_| {
11298 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11299 })
11300 }
11301
11302 fn persist_schema_version(&self) -> Result<(), String> {
11305 self.conn
11306 .execute(
11307 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
11308 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
11309 )
11310 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11311 set_local_schema_descriptor(&self.conn, &self.schema)
11312 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11313 }
11314
11315 fn set_meta(&self, key: &str, value: &str) {
11316 meta_set(&self.conn, key, value);
11317 }
11318
11319 fn delete_meta(&self, key: &str) {
11320 meta_delete(&self.conn, key);
11321 }
11322
11323 fn load_outbox(&self) -> Result<Vec<OutboxCommit>, String> {
11325 let mut stmt = self
11326 .conn
11327 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
11328 .map_err(|error| error.to_string())?;
11329 let rows = stmt
11330 .query_map([], |row| {
11331 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11332 })
11333 .map_err(|error| error.to_string())?;
11334 let mut commits = Vec::new();
11335 for row in rows {
11336 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
11337 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
11338 format!("invalid persisted outbox {client_commit_id:?}: {error}")
11339 })?;
11340 let mut ops = Vec::with_capacity(entries.len());
11341 for entry in entries {
11342 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
11343 ops.push(OutboxOp {
11344 upsert: op == "upsert",
11345 table: entry
11346 .get("table")
11347 .and_then(Value::as_str)
11348 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
11349 .to_owned(),
11350 row_id: entry
11351 .get("rowId")
11352 .and_then(Value::as_str)
11353 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
11354 .to_owned(),
11355 base_version: entry.get("baseVersion").and_then(Value::as_i64),
11356 values: entry.get("values").and_then(Value::as_object).cloned(),
11359 });
11360 }
11361 commits.push(OutboxCommit {
11362 client_commit_id,
11363 ops,
11364 });
11365 }
11366 Ok(commits)
11367 }
11368
11369 fn restore_persisted_state(&mut self) -> Result<(), String> {
11370 self.subs = {
11371 let mut stmt = self
11372 .conn
11373 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
11374 .map_err(|error| error.to_string())?;
11375 let rows = stmt
11376 .query_map([], |row| {
11377 Ok((
11378 row.get::<_, String>(0)?,
11379 row.get::<_, String>(1)?,
11380 row.get::<_, String>(2)?,
11381 ))
11382 })
11383 .map_err(|error| error.to_string())?;
11384 let mut subscriptions = Vec::new();
11385 for row in rows {
11386 let (id, table, raw) = row.map_err(|error| error.to_string())?;
11387 let state: Value = serde_json::from_str(&raw)
11388 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
11389 let requested = json_to_scope_map(
11390 state.get("requested").unwrap_or(&Value::Object(Map::new())),
11391 )?;
11392 let effective = state
11393 .get("effectiveScopes")
11394 .filter(|value| !value.is_null())
11395 .map(json_to_scope_map)
11396 .transpose()?;
11397 subscriptions.push(Subscription {
11398 id,
11399 table,
11400 requested,
11401 params: state
11402 .get("params")
11403 .and_then(Value::as_str)
11404 .map(str::to_owned),
11405 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
11406 bootstrap_state: state
11407 .get("bootstrapState")
11408 .and_then(Value::as_str)
11409 .map(str::to_owned),
11410 state: SubState::parse(
11411 state
11412 .get("status")
11413 .and_then(Value::as_str)
11414 .unwrap_or("active"),
11415 ),
11416 reason_code: state
11417 .get("reasonCode")
11418 .and_then(Value::as_str)
11419 .map(str::to_owned),
11420 effective,
11421 synced_once: state
11422 .get("syncedOnce")
11423 .and_then(Value::as_bool)
11424 .unwrap_or(false),
11425 });
11426 }
11427 subscriptions
11428 };
11429
11430 self.outbox = self.load_outbox()?;
11431
11432 self.prune_commit_outcomes()?;
11433 let active = self.commit_outcomes(CommitOutcomeQuery {
11434 active_only: true,
11435 ..CommitOutcomeQuery::default()
11436 })?;
11437 self.conflicts = active
11438 .iter()
11439 .flat_map(|outcome| outcome.results.iter())
11440 .filter_map(|result| match result {
11441 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
11442 _ => None,
11443 })
11444 .collect();
11445 self.rejections = active
11446 .iter()
11447 .flat_map(|outcome| outcome.results.iter())
11448 .filter_map(|result| match result {
11449 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
11450 _ => None,
11451 })
11452 .collect();
11453
11454 self.lease_state = self
11455 .get_meta(LEASE_STATE_KEY)
11456 .map(|raw| serde_json::from_str(&raw))
11457 .transpose()
11458 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
11459 self.schema_floor = self
11460 .get_meta(SCHEMA_FLOOR_KEY)
11461 .map(|raw| serde_json::from_str(&raw))
11462 .transpose()
11463 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
11464 self.stopped = self.schema_floor.is_some();
11465 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
11468 self.security_preflight = true;
11469 }
11470 Ok(())
11471 }
11472
11473 fn save_subscription_scope_schema(&self) {
11474 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
11475 .schema
11476 .tables
11477 .iter()
11478 .map(|table| {
11479 (
11480 table.name.clone(),
11481 table
11482 .scope_variables
11483 .iter()
11484 .map(|scope| {
11485 (
11486 scope.variable.clone(),
11487 [scope.prefix.clone(), scope.column.clone()],
11488 )
11489 })
11490 .collect(),
11491 )
11492 })
11493 .collect();
11494 self.set_meta(
11495 "subscriptionScopeSchema",
11496 &serde_json::to_string(&declarations).expect("scope schema serialization"),
11497 );
11498 }
11499
11500 fn prune_unknown_subscriptions(&mut self, bump: bool) -> Result<(), String> {
11502 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
11503 self.get_meta("subscriptionScopeSchema")
11504 .map(|raw| serde_json::from_str(&raw))
11505 .transpose()
11506 .map_err(|error| format!("invalid persisted scope schema: {error}"))?
11507 } else {
11508 None
11509 };
11510 let compatible = |table_name: &str, requested: &[(String, Vec<String>)]| {
11511 let Some(table) = self.schema.table(table_name) else {
11512 return false;
11513 };
11514 if requested
11515 .iter()
11516 .any(|(variable, _)| table.scope_column(variable).is_none())
11517 {
11518 return false;
11519 }
11520 if !bump {
11521 return true;
11522 }
11523 let Some(old) = previous.as_ref().and_then(|tables| tables.get(table_name)) else {
11524 return false;
11525 };
11526 let variables: BTreeSet<_> = if requested.is_empty() {
11527 old.keys()
11528 .cloned()
11529 .chain(
11530 table
11531 .scope_variables
11532 .iter()
11533 .map(|scope| scope.variable.clone()),
11534 )
11535 .collect()
11536 } else {
11537 requested
11538 .iter()
11539 .map(|(variable, _)| variable.clone())
11540 .collect()
11541 };
11542 variables.iter().all(|variable| {
11543 let current = table
11544 .scope_variables
11545 .iter()
11546 .find(|scope| &scope.variable == variable);
11547 match (old.get(variable), current) {
11548 (Some(old), Some(current)) => {
11549 old == &[current.prefix.clone(), current.column.clone()]
11550 }
11551 _ => false,
11552 }
11553 })
11554 };
11555 let mut stale_ids: BTreeSet<_> = self
11556 .subs
11557 .iter()
11558 .filter(|sub| !compatible(&sub.table, &sub.requested))
11559 .map(|sub| sub.id.clone())
11560 .collect();
11561 for (id, table, effective) in self.load_pending_evictions()? {
11562 if !compatible(&table, &effective) {
11563 stale_ids.insert(id);
11564 }
11565 }
11566 for id in &stale_ids {
11567 for sql in [
11568 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
11569 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
11570 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
11571 ] {
11572 self.conn
11573 .execute(sql, rusqlite::params![id])
11574 .map_err(|error| error.to_string())?;
11575 }
11576 }
11577 self.subs.retain(|sub| !stale_ids.contains(&sub.id));
11578 Ok(())
11579 }
11580
11581 #[must_use]
11582 pub fn local_revision(&self) -> u64 {
11583 self.get_meta(LOCAL_REVISION_KEY)
11584 .and_then(|value| value.parse().ok())
11585 .unwrap_or(0)
11586 }
11587
11588 pub fn snapshot_read(
11594 &mut self,
11595 request: &SnapshotReadRequest<'_>,
11596 owner: Option<QueryOwner<'_>>,
11597 ) -> Result<SnapshotRead, QueryReadFailure> {
11598 let result = snapshot_read_connection(&self.conn, request);
11599 if let Some(owner) = owner {
11600 self.record_query_read(owner, result.as_ref().err());
11601 }
11602 result
11603 }
11604
11605 #[must_use]
11606 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
11607 SyncStatusSnapshot {
11608 current_schema_version: self.schema.version,
11609 outbox: self.outbox.len(),
11610 upgrading: self.upgrading,
11611 lease_state: self.lease_state.clone(),
11612 schema_floor: self.schema_floor.clone(),
11613 sync_needed: self.sync_needed,
11614 previous_version_context: self.previous_version_status(),
11615 }
11616 }
11617
11618 fn previous_version_status(&self) -> PreviousVersionStatus {
11621 let Some(replica_path) = self.previous_version_replica_path() else {
11622 return PreviousVersionStatus {
11623 present: false,
11624 created_at_ms: None,
11625 };
11626 };
11627 match crate::previous_version::read_previous_version_record(&replica_path) {
11628 Ok(Some(record)) => PreviousVersionStatus {
11629 present: true,
11630 created_at_ms: Some(record.created_at_ms),
11631 },
11632 _ => PreviousVersionStatus {
11633 present: std::path::Path::new(
11634 &crate::previous_version::previous_version_container_path(&replica_path),
11635 )
11636 .exists(),
11637 created_at_ms: None,
11638 },
11639 }
11640 }
11641
11642 pub fn previous_version_snapshot(
11648 &self,
11649 spec: &PreviousVersionReadSpec,
11650 ) -> Result<PreviousVersionSnapshot, String> {
11651 let now_ms = self.clock_now_ms();
11652 crate::previous_version::previous_version_snapshot(
11653 &self.conn,
11654 self.previous_version_replica_path().as_deref(),
11655 self.previous_version.as_ref(),
11656 PreviousVersionLifecycle {
11657 lease_inactive: self.previous_version_lease_inactive(now_ms),
11658 scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
11659 coverage_complete: self.previous_version_coverage_complete(),
11660 },
11661 now_ms,
11662 self.schema.version,
11663 spec,
11664 )
11665 }
11666
11667 #[must_use]
11669 pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
11670 crate::previous_version::stored_previous_version_audit(&self.conn)
11671 }
11672
11673 pub fn previous_version_discard(
11677 &self,
11678 ) -> crate::previous_version::PreviousVersionDiscardOutcome {
11679 self.drop_previous_version()
11680 }
11681
11682 fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
11686 let absent = crate::previous_version::PreviousVersionDiscardOutcome {
11687 present: false,
11688 discarded: false,
11689 };
11690 let Some(replica_path) = self.previous_version_replica_path() else {
11691 return absent;
11692 };
11693 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
11694 .unwrap_or(absent)
11695 }
11696
11697 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
11699 let Some(lease) = &self.lease_state else {
11700 return false;
11701 };
11702 if lease.error_code.is_some() {
11703 return true;
11704 }
11705 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
11706 }
11707
11708 fn previous_version_coverage_complete(&self) -> bool {
11712 let active = self
11713 .subs
11714 .iter()
11715 .filter(|sub| sub.state == SubState::Active)
11716 .collect::<Vec<_>>();
11717 if active.is_empty() {
11718 return false;
11719 }
11720 active
11721 .iter()
11722 .all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
11723 }
11724
11725 fn previous_version_lifetime_check(&self) {
11729 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
11730 return;
11731 };
11732 let now_ms = self.clock_now_ms();
11733 if self.previous_version_lease_inactive(now_ms) {
11734 let _ = self.drop_previous_version();
11735 return;
11736 }
11737 let Some(replica_path) = self.previous_version_replica_path() else {
11738 return;
11739 };
11740 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
11741 else {
11742 return;
11743 };
11744 if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
11745 let _ = self.drop_previous_version();
11746 return;
11747 }
11748 if now_ms - record.created_at_ms > config.max_age_ms {
11749 let _ = self.drop_previous_version();
11750 return;
11751 }
11752 if self.previous_version_coverage_complete() {
11753 let _ = self.drop_previous_version();
11754 }
11755 }
11756
11757 pub fn diagnostics_snapshot(
11758 &self,
11759 request: &ClientDiagnosticsRequest,
11760 ) -> Result<ClientDiagnosticsSnapshot, String> {
11761 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
11762 return Err(format!(
11763 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
11764 ));
11765 }
11766 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
11767 for sub in &self.subs {
11768 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
11769 let complete =
11770 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
11771 let state = match sub.state {
11772 SubState::Revoked => "revoked",
11773 SubState::Failed => "failed",
11774 SubState::Active if reset => "reset",
11775 SubState::Active if complete => "complete",
11776 SubState::Active => "bootstrapping",
11777 };
11778 subscriptions.insert(
11779 sub.id.clone(),
11780 DiagnosticSubscription {
11781 id: sub.id.clone(),
11782 table: sub.table.clone(),
11783 state: state.to_owned(),
11784 complete,
11785 cursor: Some(sub.cursor),
11786 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
11787 },
11788 );
11789 }
11790 for expected in &request.expected_subscriptions {
11791 if expected.id.is_empty() || expected.table.is_empty() {
11792 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
11793 }
11794 if subscriptions
11795 .get(&expected.id)
11796 .is_some_and(|registered| registered.table != expected.table)
11797 {
11798 subscriptions.insert(
11799 expected.id.clone(),
11800 DiagnosticSubscription {
11801 id: expected.id.clone(),
11802 table: expected.table.clone(),
11803 state: "failed".to_owned(),
11804 complete: false,
11805 cursor: None,
11806 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
11807 },
11808 );
11809 } else {
11810 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
11811 DiagnosticSubscription {
11812 id: expected.id.clone(),
11813 table: expected.table.clone(),
11814 state: "unregistered".to_owned(),
11815 complete: false,
11816 cursor: None,
11817 reason_code: None,
11818 }
11819 });
11820 }
11821 }
11822 let mut ordered_subscriptions = Vec::new();
11823 let mut included = BTreeSet::new();
11824 for expected in &request.expected_subscriptions {
11825 if included.insert(expected.id.clone()) {
11826 if let Some(subscription) = subscriptions.get(&expected.id) {
11827 ordered_subscriptions.push(subscription.clone());
11828 }
11829 }
11830 }
11831 for (id, subscription) in &subscriptions {
11832 if included.insert(id.clone()) {
11833 ordered_subscriptions.push(subscription.clone());
11834 }
11835 }
11836 let subscriptions_truncated =
11837 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
11838 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
11839 let captured_at_ms = self.clock_now_ms();
11840 let lease = if let Some(error_code) = self
11841 .lease_state
11842 .as_ref()
11843 .and_then(|state| state.error_code.clone())
11844 {
11845 ClientDiagnosticsLease {
11846 state: "stopped".to_owned(),
11847 expires_at_ms: self
11848 .lease_state
11849 .as_ref()
11850 .and_then(|state| state.expires_at_ms),
11851 error_code: Some(Self::diagnostic_code(&error_code)),
11852 }
11853 } else if let Some(expires_at_ms) = self
11854 .lease_state
11855 .as_ref()
11856 .and_then(|state| state.expires_at_ms)
11857 {
11858 ClientDiagnosticsLease {
11859 state: if expires_at_ms <= captured_at_ms {
11860 "expired".to_owned()
11861 } else {
11862 "active".to_owned()
11863 },
11864 expires_at_ms: Some(expires_at_ms),
11865 error_code: None,
11866 }
11867 } else {
11868 ClientDiagnosticsLease {
11869 state: "none".to_owned(),
11870 expires_at_ms: None,
11871 error_code: None,
11872 }
11873 };
11874 let connectivity = match self.last_round.as_ref() {
11875 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
11876 Some(round) if round.status == "succeeded" => "online",
11877 Some(round)
11878 if round.status == "failed"
11879 && round
11880 .error_code
11881 .as_deref()
11882 .is_some_and(Self::retryable_transport_code) =>
11883 {
11884 "offline"
11885 }
11886 _ => "unknown",
11887 };
11888 Ok(ClientDiagnosticsSnapshot {
11889 version: CLIENT_DIAGNOSTICS_VERSION,
11890 captured_at_ms,
11891 host: ClientDiagnosticsHost {
11892 kind: "direct".to_owned(),
11893 role: "single".to_owned(),
11894 connectivity: connectivity.to_owned(),
11895 realtime: self.realtime_state,
11896 realtime_policy: self.realtime_policy,
11897 realtime_reason_code: self.realtime_reason_code.clone(),
11898 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
11899 },
11900 security_lifecycle: self.security_lifecycle().to_owned(),
11901 schema: ClientDiagnosticsSchema {
11902 current_version: self.schema.version,
11903 upgrading: self.upgrading,
11904 required_version: self
11905 .schema_floor
11906 .as_ref()
11907 .and_then(|floor| floor.required_schema_version),
11908 latest_version: self
11909 .schema_floor
11910 .as_ref()
11911 .and_then(|floor| floor.latest_schema_version),
11912 },
11913 replica: ClientDiagnosticsReplica {
11914 local_revision: self.local_revision().to_string(),
11915 sync_needed: self.sync_needed,
11916 pending_outbox: self.outbox.len(),
11917 },
11918 lease,
11919 subscriptions: ordered_subscriptions,
11920 subscriptions_truncated,
11921 last_round: self.last_round.clone(),
11922 last_change: self.last_change.clone(),
11923 storage: self.diagnostics_storage(),
11924 query_failures: self.query_failures.clone(),
11925 })
11926 }
11927
11928 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
11929 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
11930 let page_count: i64 = self
11931 .conn
11932 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
11933 let page_size: i64 = self
11934 .conn
11935 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
11936 let outbox_bytes: i64 = self
11938 .conn
11939 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
11940 .query_row([], |row| row.get(0))?;
11941 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
11942 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
11943 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
11944 let blob_bytes = if self.schema_has_blobs() {
11945 self.conn
11946 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
11947 .query_row([], |row| row.get(0))?
11948 } else {
11949 0
11950 };
11951 let pressure = self
11952 .limits
11953 .blob_cache_max_bytes
11954 .is_some_and(|limit| blob_bytes > limit);
11955 Ok(ClientDiagnosticsStorage {
11956 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
11957 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
11958 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
11959 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
11960 retained_outcome_entries: Some(outcome_entries.max(0)),
11961 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
11962 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
11963 })
11964 };
11965 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
11966 status: "unreadable".to_owned(),
11967 database_bytes_approx: None,
11968 pending_outbox_bytes_approx: None,
11969 retained_outcome_bytes_approx: None,
11970 retained_outcome_entries: None,
11971 blob_cache_bytes_approx: None,
11972 pressure_reason_code: None,
11973 })
11974 }
11975
11976 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
11977 self.change_queue.drain(..).collect()
11978 }
11979
11980 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
11981 let intents = self.sync_intent_queue.drain(..).collect();
11982 if self.transport_enabled {
11983 intents
11984 } else {
11985 vec![SyncIntent::None]
11986 }
11987 }
11988
11989 fn schedule_background_retry(&mut self) -> u64 {
11990 let delay_ms = self.retry_delay_ms;
11991 self.sync_intent_queue
11992 .push_back(SyncIntent::Background { delay_ms });
11993 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
11994 self.round_retry_delay_ms = Some(delay_ms);
11995 delay_ms
11996 }
11997
11998 fn reset_background_retry(&mut self) {
11999 self.retry_delay_ms = 250;
12000 }
12001
12002 fn retryable_transport_code(code: &str) -> bool {
12003 code == "transport.failed"
12004 || code == "transport.unavailable"
12005 || code == "sync.transport_failed"
12006 }
12007
12008 fn retryable_failure_code(code: &str) -> bool {
12013 Self::retryable_transport_code(code)
12014 || matches!(
12015 code,
12016 "sync.auth_required"
12017 | "sync.auth_lease_required"
12018 | "sync.auth_lease_revoked"
12019 | "sync.internal_error"
12020 | "sync.idempotency_cache_miss"
12021 | "sync.schema_not_ready"
12022 | "sync.segment_expired"
12023 | "sync.rate_limited"
12024 | "sync.websocket_connection_limit"
12025 )
12026 }
12027
12028 fn diagnostic_code(code: &str) -> String {
12029 let valid = !code.is_empty()
12030 && code.len() <= 96
12031 && code.contains('.')
12032 && code
12033 .bytes()
12034 .next()
12035 .is_some_and(|byte| byte.is_ascii_lowercase())
12036 && code.bytes().all(|byte| {
12037 byte.is_ascii_lowercase()
12038 || byte.is_ascii_digit()
12039 || matches!(byte, b'.' | b'_' | b'-')
12040 });
12041 if valid {
12042 code.to_owned()
12043 } else {
12044 "client.unknown_failure".to_owned()
12045 }
12046 }
12047
12048 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
12049 if self.sync_needed != value {
12050 if self.begin_observation("syncular_status").is_ok() {
12051 self.sync_needed = value;
12052 let batch = ChangeAccumulator {
12053 status: true,
12054 ..ChangeAccumulator::default()
12055 };
12056 if self.finish_observation("syncular_status", batch).is_err() {
12057 self.rollback_observation("syncular_status");
12058 }
12059 } else {
12060 self.sync_needed = value;
12061 }
12062 }
12063 if value && interactive {
12064 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12065 }
12066 }
12067
12068 fn sqlite_failure(
12069 storage_failure: &RefCell<Option<QueryReadFailure>>,
12070 error: rusqlite::Error,
12071 ) -> String {
12072 let message = error.to_string();
12073 let failure = QueryReadFailure::from(error);
12074 let code = failure.code;
12075 if code.is_some() && storage_failure.borrow().is_none() {
12076 *storage_failure.borrow_mut() = Some(failure);
12077 }
12078 match code {
12079 Some(code) => format!("{code}: {message}"),
12080 None => message,
12081 }
12082 }
12083
12084 fn begin_observation(&self, name: &str) -> Result<(), String> {
12085 if let Some(failure) = &self.coalesced_failure {
12086 return Err(failure.clone());
12087 }
12088 if self.coalesced.is_some() && self.conn.is_autocommit() {
12091 return Err("the coalesced apply transaction was rolled back".to_owned());
12092 }
12093 self.conn
12094 .execute_batch(&format!("SAVEPOINT {name}"))
12095 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12096 }
12097
12098 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
12101 if self.coalesced.is_some() || !self.conn.is_autocommit() {
12102 return Ok(());
12103 }
12104 self.conn
12105 .execute_batch("BEGIN")
12106 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12107 self.coalesced = Some(CoalescedApply {
12108 rows: 0,
12109 subs: self.subs.clone(),
12110 outbox: self.outbox.clone(),
12111 failed_commits: self.failed_commits.clone(),
12112 conflicts: self.conflicts.clone(),
12113 rejections: self.rejections.clone(),
12114 schema_floor: self.schema_floor.clone(),
12115 lease_state: self.lease_state.clone(),
12116 stopped: self.stopped,
12117 upgrading: self.upgrading,
12118 sync_needed: self.sync_needed,
12119 overlay_dirty: self.overlay_dirty.snapshot(),
12120 change_queue: self.change_queue.len(),
12121 sync_intent_queue: self.sync_intent_queue.len(),
12122 last_change: self.last_change.clone(),
12123 });
12124 Ok(())
12125 }
12126
12127 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
12130 let Some(open) = &self.coalesced else {
12131 return Ok(());
12132 };
12133 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
12134 self.commit_coalesced_apply()?;
12135 self.begin_coalesced_apply()?;
12136 }
12137 if let Some(open) = &mut self.coalesced {
12138 open.rows += rows;
12139 }
12140 Ok(())
12141 }
12142
12143 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
12148 let Some(open) = self.coalesced.take() else {
12149 return Ok(());
12150 };
12151 let committed = if self.conn.is_autocommit() {
12152 Err("the coalesced apply transaction was rolled back".to_owned())
12153 } else {
12154 self.conn
12155 .execute_batch("COMMIT")
12156 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12157 };
12158 let Err(error) = committed else {
12159 return Ok(());
12160 };
12161 if !self.conn.is_autocommit() {
12162 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12163 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12164 }
12165 }
12166 self.subs = open.subs;
12167 self.outbox = open.outbox;
12168 self.failed_commits = open.failed_commits;
12169 self.conflicts = open.conflicts;
12170 self.rejections = open.rejections;
12171 self.schema_floor = open.schema_floor;
12172 self.lease_state = open.lease_state;
12173 self.stopped = open.stopped;
12174 self.upgrading = open.upgrading;
12175 self.sync_needed = open.sync_needed;
12176 self.overlay_dirty.restore(open.overlay_dirty);
12177 self.change_queue.truncate(open.change_queue);
12178 self.sync_intent_queue.truncate(open.sync_intent_queue);
12179 self.last_change = open.last_change;
12180 self.coalesced_failure = Some(error.clone());
12181 Err(error)
12182 }
12183
12184 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
12186 let committed = self.commit_coalesced_apply();
12187 match self.coalesced_failure.take() {
12188 Some(failure) => Err(failure),
12189 None => committed,
12190 }
12191 }
12192
12193 fn rollback_observation(&self, name: &str) {
12194 if let Err(error) = self
12195 .conn
12196 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
12197 {
12198 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
12199 if failure.rollback_failure.is_none() {
12200 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
12201 }
12202 }
12203 }
12204 }
12205
12206 fn finish_observation(
12207 &mut self,
12208 name: &str,
12209 mut batch: ChangeAccumulator,
12210 ) -> Result<(), String> {
12211 #[cfg(feature = "bench-internals")]
12212 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
12213 if self.failed_commits.iter().any(|(commit, _)| {
12214 commit
12215 .ops
12216 .iter()
12217 .any(|operation| batch.tables.contains_key(&operation.table))
12218 }) {
12219 batch.outcomes = true;
12220 }
12221 if !batch.touched() {
12222 self.conn
12223 .execute_batch(&format!("RELEASE {name}"))
12224 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12225 return Ok(());
12226 }
12227 let revision = self
12228 .local_revision()
12229 .checked_add(1)
12230 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
12231 self.conn
12232 .execute(
12233 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
12234 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
12235 )
12236 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12237 let status = batch.status.then(|| self.status_snapshot());
12238 let event = ClientChangeBatch {
12239 revision: revision.to_string(),
12240 tables: batch
12241 .tables
12242 .into_iter()
12243 .map(|(table, scope_keys)| TableChange {
12244 table,
12245 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
12246 })
12247 .collect(),
12248 windows: batch
12249 .windows
12250 .into_iter()
12251 .map(|((base_key, table), units)| WindowChange {
12252 base_key,
12253 table,
12254 units: units.into_iter().collect(),
12255 })
12256 .collect(),
12257 status,
12258 conflicts_changed: batch.conflicts,
12259 rejections_changed: batch.rejections,
12260 outcomes_changed: batch.outcomes,
12261 };
12262 self.conn
12263 .execute_batch(&format!("RELEASE {name}"))
12264 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12265 let mut diagnostic_tables = event
12266 .tables
12267 .iter()
12268 .map(|entry| entry.table.clone())
12269 .collect::<BTreeSet<_>>()
12270 .into_iter()
12271 .collect::<Vec<_>>();
12272 let mut diagnostic_windows = event
12273 .windows
12274 .iter()
12275 .map(|entry| entry.table.clone())
12276 .collect::<BTreeSet<_>>()
12277 .into_iter()
12278 .collect::<Vec<_>>();
12279 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
12280 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
12281 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
12282 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
12283 self.last_change = Some(DiagnosticLastChange {
12284 revision: event.revision.clone(),
12285 recorded_at_ms: self.clock_now_ms(),
12286 tables: diagnostic_tables,
12287 windows: diagnostic_windows,
12288 domains_truncated,
12289 status_changed: event.status.is_some(),
12290 conflicts_changed: event.conflicts_changed,
12291 rejections_changed: event.rejections_changed,
12292 outcomes_changed: event.outcomes_changed,
12293 });
12294 self.change_queue.push_back(event);
12295 Ok(())
12296 }
12297
12298 fn record_scope_map(
12299 &self,
12300 batch: &mut ChangeAccumulator,
12301 table_name: &str,
12302 scopes: &[(String, Vec<String>)],
12303 ) {
12304 let Some(table) = self.schema.table(table_name) else {
12305 return;
12306 };
12307 for (variable, values) in scopes {
12308 let Some(scope) = table
12309 .scope_variables
12310 .iter()
12311 .find(|scope| &scope.variable == variable)
12312 else {
12313 continue;
12314 };
12315 for value in values {
12316 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12317 }
12318 }
12319 }
12320
12321 fn record_row_scopes(
12323 &self,
12324 batch: &mut ChangeAccumulator,
12325 table_name: &str,
12326 row_id: &str,
12327 base: bool,
12328 ) -> bool {
12329 let Some(table) = self.schema.table(table_name) else {
12330 return false;
12331 };
12332 if table.scope_variables.is_empty() {
12333 return false;
12334 }
12335 let columns = table
12336 .scope_variables
12337 .iter()
12338 .map(|scope| quote_ident(&scope.column))
12339 .collect::<Vec<_>>()
12340 .join(", ");
12341 let full_table = if base {
12342 base_table(table_name)
12343 } else {
12344 visible_table(table_name)
12345 };
12346 let sql = format!(
12347 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
12348 row_id_predicate(table)
12349 );
12350 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
12351 return false;
12352 };
12353 let values = stmt.query_row(rusqlite::params![row_id], |row| {
12354 let mut values = Vec::with_capacity(table.scope_variables.len());
12355 for index in 0..table.scope_variables.len() {
12356 values.push(row.get::<_, Option<String>>(index)?);
12357 }
12358 Ok(values)
12359 });
12360 let Ok(values) = values else {
12361 return false;
12362 };
12363 let mut recorded = false;
12364 for (scope, value) in table.scope_variables.iter().zip(values) {
12365 if let Some(value) = value {
12366 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12367 recorded = true;
12368 }
12369 }
12370 recorded
12371 }
12372
12373 fn record_commit_changes(
12374 &self,
12375 batch: &mut ChangeAccumulator,
12376 tables: &[String],
12377 changes: &[ssp2::model::Change],
12378 ) {
12379 #[cfg(feature = "bench-internals")]
12380 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
12381 for change in changes {
12382 let Some(table_name) = tables.get(change.table_index as usize) else {
12383 continue;
12384 };
12385 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
12386 if let Some(table) = self.schema.table(table_name) {
12387 for (variable, value) in &change.scopes {
12388 if let Some(scope) = table
12389 .scope_variables
12390 .iter()
12391 .find(|scope| &scope.variable == variable)
12392 {
12393 batch.scope(table_name, format!("{}:{value}", scope.prefix));
12394 precise = true;
12395 }
12396 }
12397 }
12398 if !precise {
12399 batch.table(table_name);
12400 }
12401 }
12402 }
12403
12404 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
12405 if effective.is_empty() {
12406 return false;
12407 }
12408 let Some(table) = self.schema.table(table_name) else {
12409 return false;
12410 };
12411 let mut clauses = Vec::new();
12412 let mut params = Vec::new();
12413 for (variable, values) in effective {
12414 let Some(column) = table.scope_column(variable) else {
12415 return false;
12416 };
12417 if values.is_empty() {
12418 return false;
12419 }
12420 let placeholders = values
12421 .iter()
12422 .map(|value| {
12423 params.push(SqlValue::Text(value.clone()));
12424 "?"
12425 })
12426 .collect::<Vec<_>>()
12427 .join(", ");
12428 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
12429 }
12430 let sql = format!(
12431 "SELECT 1 FROM {} WHERE {} LIMIT 1",
12432 base_table(table_name),
12433 clauses.join(" AND ")
12434 );
12435 self.conn
12436 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
12437 .is_ok()
12438 }
12439
12440 pub fn upgrading(&self) -> bool {
12442 self.upgrading
12443 }
12444
12445 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
12453 let new_schema = parse_schema_json(schema_json)?;
12454 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
12455 self.conn
12456 .execute_batch("BEGIN IMMEDIATE")
12457 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12458 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
12459 let prior_subs = self.subs.clone();
12465 let prior_outbox = self.outbox.clone();
12466 let prior_rejections = self.rejections.clone();
12467 let prior_upgrading = self.upgrading;
12468 let prior_stopped = self.stopped;
12469 let prior_schema_floor = self.schema_floor.clone();
12470 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12471 let prior_sync_needed = self.sync_needed;
12472 let prior_active_round = self.active_round;
12473 let prior_changes = self.change_queue.len();
12474 let prior_intents = self.sync_intent_queue.len();
12475 let prior_last_change = self.last_change.clone();
12476 let guarded = (|| -> Result<(), String> {
12477 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
12482 self.create_bookkeeping_tables()?;
12483 match marker {
12484 Some(version) if version < self.schema.version => self.run_schema_reset()?,
12485 None => {
12486 self.create_synced_tables()?;
12487 self.persist_schema_version()?;
12488 }
12489 _ => {}
12490 }
12491 self.outbox = self.load_outbox()?;
12496 self.drop_unencodable_outbox(false, None)?;
12501 if marker == Some(self.schema.version)
12502 && (!self.outbox.is_empty()
12503 || !self.failed_commits.is_empty()
12504 || self.has_acknowledged_rows())
12505 {
12506 self.overlay_dirty.set(true);
12507 self.rebuild_overlay()?;
12508 }
12509 self.prune_unknown_subscriptions(false)?;
12510 self.save_subscription_scope_schema();
12511 self.enqueue_startup_sync_if_needed()?;
12515 Ok(())
12516 })();
12517 let outcome = match guarded {
12518 Ok(()) => self
12519 .conn
12520 .execute_batch("COMMIT")
12521 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
12522 Err(error) => Err(error),
12523 };
12524 if let Err(error) = outcome {
12525 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12526 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
12528 }
12529 self.schema = prior_schema;
12530 self.subs = prior_subs;
12531 self.outbox = prior_outbox;
12532 self.rejections = prior_rejections;
12533 self.upgrading = prior_upgrading;
12534 self.stopped = prior_stopped;
12535 self.schema_floor = prior_schema_floor;
12536 self.overlay_dirty.restore(prior_overlay_dirty);
12537 self.sync_needed = prior_sync_needed;
12538 self.active_round = prior_active_round;
12539 self.change_queue.truncate(prior_changes);
12540 self.sync_intent_queue.truncate(prior_intents);
12541 self.last_change = prior_last_change;
12542 self.insert_sql.borrow_mut().clear();
12544 return Err(error);
12545 }
12546 Ok(())
12547 }
12548
12549 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
12550 let startup_work = !self.stopped
12551 && (!self.outbox.is_empty()
12552 || self.has_acknowledged_rows()
12553 || !self.load_pending_evictions()?.is_empty()
12554 || self.subs.iter().any(|sub| sub.state == SubState::Active));
12555 if startup_work {
12556 self.sync_needed = true;
12557 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12558 }
12559 Ok(())
12560 }
12561
12562 fn clear_satisfied_persisted_schema_floor(&mut self) {
12569 let satisfied = self
12570 .schema_floor
12571 .as_ref()
12572 .and_then(|floor| floor.required_schema_version)
12573 .is_some_and(|required| self.schema.version >= required);
12574 if !satisfied {
12575 return;
12576 }
12577 self.schema_floor = None;
12578 self.stopped = false;
12579 self.delete_meta(SCHEMA_FLOOR_KEY);
12580 }
12581
12582 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
12587 let Some(replica_path) = self.previous_version_replica_path() else {
12588 return Ok(());
12589 };
12590 let max_age_ms = self
12591 .previous_version
12592 .filter(|config| config.enabled)
12593 .map(|config| config.max_age_ms);
12594 reconcile_previous_version_at_boot(
12595 &self.conn,
12596 &replica_path,
12597 self.schema.version,
12598 max_age_ms,
12599 self.clock_now_ms(),
12600 )
12601 }
12602
12603 fn previous_version_replica_path(&self) -> Option<String> {
12607 let path = self.conn.path()?;
12608 if path.is_empty() || path.starts_with("file:") {
12609 return None;
12610 }
12611 Some(path.to_owned())
12612 }
12613
12614 fn run_schema_reset(&mut self) -> Result<(), String> {
12620 self.begin_observation("syncular_schema_reset")?;
12621 let prior_subs = self.subs.clone();
12622 let prior_outbox = self.outbox.clone();
12623 let prior_rejections = self.rejections.clone();
12624 let prior_upgrading = self.upgrading;
12625 let prior_stopped = self.stopped;
12626 let prior_schema_floor = self.schema_floor.clone();
12627 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12628 let prior_sync_needed = self.sync_needed;
12629 let prior_active_round = self.active_round;
12630 let mut batch = ChangeAccumulator::default();
12631 let result = self
12632 .run_schema_reset_observed(&mut batch, true)
12633 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
12634 if let Err(error) = result {
12635 self.rollback_observation("syncular_schema_reset");
12636 self.subs = prior_subs;
12637 self.outbox = prior_outbox;
12638 self.rejections = prior_rejections;
12639 self.upgrading = prior_upgrading;
12640 self.stopped = prior_stopped;
12641 self.schema_floor = prior_schema_floor;
12642 self.overlay_dirty.restore(prior_overlay_dirty);
12643 self.sync_needed = prior_sync_needed;
12644 self.active_round = prior_active_round;
12645 return Err(error);
12646 }
12647 Ok(())
12648 }
12649
12650 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
12651 if self.get_meta(LOG_EPOCH_KEY).is_none() {
12652 self.begin_observation("syncular_epoch_acquisition")?;
12653 let previous_sync_needed = self.sync_needed;
12654 if let Err(error) = self.conn.execute(
12655 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12656 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12657 ) {
12658 self.rollback_observation("syncular_epoch_acquisition");
12659 return Err(Self::sqlite_failure(&self.storage_failure, error));
12660 }
12661 self.sync_needed = true;
12662 let batch = ChangeAccumulator {
12663 status: true,
12664 ..ChangeAccumulator::default()
12665 };
12666 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
12667 self.rollback_observation("syncular_epoch_acquisition");
12668 self.sync_needed = previous_sync_needed;
12669 return Err(error);
12670 }
12671 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12672 return Ok(Vec::new());
12673 }
12674 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
12675 self.begin_observation("syncular_log_epoch_reset")?;
12676 let prior_subs = self.subs.clone();
12677 let prior_outbox = self.outbox.clone();
12678 let prior_rejections = self.rejections.clone();
12679 let prior_upgrading = self.upgrading;
12680 let prior_stopped = self.stopped;
12681 let prior_schema_floor = self.schema_floor.clone();
12682 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12683 let prior_sync_needed = self.sync_needed;
12684 let prior_active_round = self.active_round;
12685 let mut batch = ChangeAccumulator::default();
12686 let result = self
12687 .run_schema_reset_observed(&mut batch, false)
12688 .and_then(|()| {
12689 self.conn
12690 .execute(
12691 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12692 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
12693 )
12694 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12695 self.sync_needed = true;
12696 batch.status = true;
12697 self.finish_observation("syncular_log_epoch_reset", batch)
12698 });
12699 if let Err(error) = result {
12700 self.rollback_observation("syncular_log_epoch_reset");
12701 self.subs = prior_subs;
12702 self.outbox = prior_outbox;
12703 self.rejections = prior_rejections;
12704 self.upgrading = prior_upgrading;
12705 self.stopped = prior_stopped;
12706 self.schema_floor = prior_schema_floor;
12707 self.overlay_dirty.restore(prior_overlay_dirty);
12708 self.sync_needed = prior_sync_needed;
12709 self.active_round = prior_active_round;
12710 return Err(error);
12711 }
12712 self.sync_intent_queue.push_back(SyncIntent::Interactive);
12713 Ok(resets)
12714 }
12715
12716 fn run_schema_reset_observed(
12717 &mut self,
12718 batch: &mut ChangeAccumulator,
12719 capture: bool,
12720 ) -> Result<(), String> {
12721 read_local_schema_version(&self.conn, self.schema.version)?;
12727 self.conn
12728 .execute_batch(
12729 "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
12730 )
12731 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12732 self.cancel_sync_round();
12733 self.upgrading = true;
12734 batch.status = true;
12735 if self.previous_version.is_some_and(|config| config.enabled) {
12743 let previous_version = self
12744 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
12745 .and_then(|value| value.parse::<i32>().ok())
12746 .unwrap_or(self.schema.version);
12747 let pending = self
12748 .outbox
12749 .iter()
12750 .map(|commit| PendingCommitAudit {
12751 commit_id: &commit.client_commit_id,
12752 operations: commit_audit_operations(commit),
12753 })
12754 .collect::<Vec<_>>();
12755 crate::previous_version::write_previous_version_audit(
12756 &self.conn,
12757 &crate::previous_version::build_previous_version_audit(
12758 &self.schema,
12759 &pending,
12760 previous_version,
12761 self.schema.version,
12762 self.clock_now_ms(),
12763 ),
12764 );
12765 }
12766 if let Some(replica_path) = self.previous_version_replica_path() {
12771 sweep_previous_version_container(&self.conn, &replica_path)?;
12772 if capture {
12773 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
12774 let previous_version = self
12775 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
12776 .and_then(|value| value.parse::<i32>().ok())
12777 .unwrap_or(self.schema.version);
12778 capture_previous_version_from_replica(
12779 &self.conn,
12780 &replica_path,
12781 previous_version,
12782 self.schema.version,
12783 &config,
12784 self.clock_now_ms(),
12785 )?;
12786 }
12787 }
12788 }
12789 for table in &self.schema.tables {
12790 batch.table(&table.name);
12791 }
12792 for (base_key, unit, table) in self.load_registered_window_units() {
12793 batch.window(&base_key, &table, &unit);
12794 }
12795 if capture {
12796 self.prune_unknown_subscriptions(true)?;
12797 }
12798 self.insert_sql.borrow_mut().clear();
12800 self.overlay_dirty.set(true);
12801 let virtual_tables: Vec<String> = {
12809 let mut stmt = self
12810 .conn
12811 .prepare(
12812 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
12813 )
12814 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12815 let rows = stmt
12816 .query_map([], |row| row.get::<_, String>(0))
12817 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12818 rows.filter_map(Result::ok)
12819 .filter(|name| is_synced_table_name(name))
12820 .collect()
12821 };
12822 for name in virtual_tables {
12823 self.conn
12824 .execute_batch(&format!(
12825 "DROP TABLE IF EXISTS {}",
12826 quote_ident(&format!("_syncular_fts_{name}"))
12827 ))
12828 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12829 self.conn
12830 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
12831 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12832 }
12833 let existing: Vec<String> = {
12834 let mut stmt = self
12835 .conn
12836 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
12837 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12838 let rows = stmt
12839 .query_map([], |row| row.get::<_, String>(0))
12840 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12841 rows.filter_map(Result::ok)
12842 .filter(|name| is_synced_table_name(name))
12843 .collect()
12844 };
12845 for name in &existing {
12846 if let Some(table) = name.strip_prefix("_syncular_base_") {
12847 batch.table(table);
12848 } else if !name.starts_with("_syncular_") {
12849 batch.table(name);
12850 }
12851 }
12852 for name in existing {
12853 self.conn
12854 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
12855 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12856 }
12857 self.create_synced_tables()?;
12859 for sub in &mut self.subs {
12861 sub.cursor = -1;
12862 sub.bootstrap_state = None;
12863 sub.effective = None;
12864 sub.state = SubState::Active;
12865 sub.reason_code = None;
12866 sub.synced_once = false;
12867 }
12868 let subs = self.subs.clone();
12869 for sub in &subs {
12870 self.persist_sub(sub)?;
12871 }
12872 self.stopped = false;
12874 self.schema_floor = None;
12875 self.delete_meta(SCHEMA_FLOOR_KEY);
12876 self.persist_schema_version()?;
12878 self.save_subscription_scope_schema();
12879 self.drop_unencodable_outbox(false, Some(batch))?;
12890 self.rebuild_overlay()?;
12892 Ok(())
12893 }
12894
12895 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
12902 let incompatible = self
12903 .outbox
12904 .iter()
12905 .filter(|commit| {
12906 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
12907 })
12908 .cloned()
12909 .collect::<Vec<_>>();
12910 if incompatible.is_empty() {
12911 return Ok(false);
12912 }
12913 let prior_outbox = self.outbox.clone();
12919 let prior_rejections = self.rejections.clone();
12920 let prior_overlay_dirty = self.overlay_dirty.snapshot();
12921 let owns_transaction = self.conn.is_autocommit();
12922 self.begin_observation("syncular_outbox_drop")?;
12923 let mut batch = ChangeAccumulator::default();
12924 let dropped = (|| -> Result<(), String> {
12925 let mut rejections = Vec::new();
12926 for commit in &incompatible {
12927 let results = commit
12928 .ops
12929 .iter()
12930 .enumerate()
12931 .map(|(op_index, operation)| {
12932 let rejection = RejectionRecord {
12933 client_commit_id: commit.client_commit_id.clone(),
12934 op_index: op_index as i32,
12935 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
12936 message: "the persisted commit cannot encode under the current schema"
12937 .to_owned(),
12938 retryable: false,
12939 details: None,
12940 operation: Some(CommitOperation::from(operation)),
12941 };
12942 rejections.push(rejection.clone());
12943 CommitOperationOutcome::Error { rejection }
12944 })
12945 .collect::<Vec<_>>();
12946 self.persist_commit_outcome(
12947 &commit.client_commit_id,
12948 CommitOutcomeStatus::Rejected,
12949 &results,
12950 Some(&commit.ops),
12951 )?;
12952 self.delete_outbox_persisted(&commit.client_commit_id)?;
12953 for operation in &commit.ops {
12956 if self.schema.table(&operation.table).is_some() {
12957 batch.table(&operation.table);
12958 }
12959 }
12960 }
12961 self.prune_commit_outcomes()?;
12962 let incompatible_ids = incompatible
12963 .iter()
12964 .map(|commit| commit.client_commit_id.as_str())
12965 .collect::<BTreeSet<_>>();
12966 self.outbox
12967 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
12968 self.rejections.extend(rejections);
12969 self.overlay_dirty.set(true);
12972 self.rebuild_overlay()?;
12973 batch.status = true;
12974 batch.rejections = true;
12975 batch.outcomes = true;
12976 Ok(())
12977 })();
12978 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
12979 if let Err(error) = result {
12980 if owns_transaction {
12984 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
12985 let mut failure = self.storage_failure.borrow_mut();
12986 let failure =
12987 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
12988 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
12989 }
12990 } else {
12991 self.rollback_observation("syncular_outbox_drop");
12992 }
12993 self.outbox = prior_outbox;
12994 self.rejections = prior_rejections;
12995 self.overlay_dirty.restore(prior_overlay_dirty);
12996 return Err(error);
12997 }
12998 Ok(true)
12999 }
13000
13001 fn create_synced_tables(&self) -> Result<(), String> {
13004 for table in &self.schema.tables {
13005 for (full, index_prefix) in [
13009 (base_table(&table.name), "_syncular_base_"),
13010 (visible_table(&table.name), ""),
13011 ] {
13012 let mut cols: Vec<String> =
13013 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
13014 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
13015 let sql = format!(
13016 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
13017 cols.join(", "),
13018 quote_ident(&table.primary_key)
13019 );
13020 self.conn
13021 .execute(&sql, [])
13022 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13023 for index in &table.indexes {
13027 let unique = if index.unique { "UNIQUE " } else { "" };
13028 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
13029 let cols_sql = index
13030 .columns
13031 .iter()
13032 .map(|c| quote_ident(c))
13033 .collect::<Vec<_>>()
13034 .join(", ");
13035 let index_sql = format!(
13036 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
13037 );
13038 self.conn
13039 .execute(&index_sql, [])
13040 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13041 }
13042 }
13043 }
13044 for table in &self.schema.tables {
13047 for index in &table.fts_indexes {
13048 self.create_fts_projection(table, index)?;
13049 }
13050 }
13051 Ok(())
13052 }
13053
13054 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
13055 let count: i64 = self
13056 .conn
13057 .query_row(
13058 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
13059 rusqlite::params![index.name],
13060 |row| row.get(0),
13061 )
13062 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13063 Ok(count > 0)
13064 }
13065
13066 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
13067 for suffix in ["ai", "ad", "au"] {
13068 self.conn
13069 .execute(
13070 &format!(
13071 "DROP TRIGGER IF EXISTS {}",
13072 quote_ident(&format!("{}_{suffix}", index.name))
13073 ),
13074 [],
13075 )
13076 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13077 }
13078 Ok(())
13079 }
13080
13081 fn create_fts_triggers(
13082 &self,
13083 table: &TableSchema,
13084 index: &FtsIndexSchema,
13085 ) -> Result<(), String> {
13086 let fts = quote_ident(&index.name);
13087 let source = visible_table(&table.name);
13088 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
13089 let pk = quote_ident(&table.primary_key);
13090 let projection_columns = std::iter::once(source_id.clone())
13091 .chain(index.columns.iter().map(|column| quote_ident(column)))
13092 .collect::<Vec<_>>()
13093 .join(", ");
13094 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
13095 .chain(
13096 index
13097 .columns
13098 .iter()
13099 .map(|column| format!("new.{}", quote_ident(column))),
13100 )
13101 .collect::<Vec<_>>()
13102 .join(", ");
13103 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13104 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
13105 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
13106 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
13107 let bi = quote_ident(&format!("{}_bi", index.name));
13108 let ai = quote_ident(&format!("{}_ai", index.name));
13109 let ad = quote_ident(&format!("{}_ad", index.name));
13110 let au = quote_ident(&format!("{}_au", index.name));
13111 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
13112 let sql = format!(
13113 "DROP TRIGGER IF EXISTS {bi};
13114 DROP TRIGGER IF EXISTS {ai};
13115 DROP TRIGGER IF EXISTS {ad};
13116 DROP TRIGGER IF EXISTS {au};
13117 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
13118 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
13119 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
13120 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
13121 );
13122 self.conn
13123 .execute_batch(&sql)
13124 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13125 }
13126
13127 fn rebuild_fts_projection(
13128 &self,
13129 table: &TableSchema,
13130 index: &FtsIndexSchema,
13131 ) -> Result<(), String> {
13132 let fts = quote_ident(&index.name);
13133 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
13134 let pk = quote_ident(&table.primary_key);
13135 let indexed_columns = index
13136 .columns
13137 .iter()
13138 .map(|column| quote_ident(column))
13139 .collect::<Vec<_>>();
13140 let projection_columns = std::iter::once(source_id.clone())
13141 .chain(indexed_columns.iter().cloned())
13142 .collect::<Vec<_>>()
13143 .join(", ");
13144 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13145 let sql = format!(
13146 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
13147 columns = indexed_columns.join(", "),
13148 source = visible_table(&table.name),
13149 );
13150 self.conn
13151 .execute_batch(&sql)
13152 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13153 }
13154
13155 fn create_fts_projection(
13156 &self,
13157 table: &TableSchema,
13158 index: &FtsIndexSchema,
13159 ) -> Result<(), String> {
13160 self.begin_observation("syncular_fts_schema")?;
13161 let result = (|| {
13162 let existed = self.fts_projection_exists(index)?;
13163 let tokenizer = index.tokenize.replace('\'', "''");
13164 let columns = index
13165 .columns
13166 .iter()
13167 .map(|column| quote_ident(column))
13168 .collect::<Vec<_>>()
13169 .join(", ");
13170 let sql = format!(
13171 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
13172 fts = quote_ident(&index.name),
13173 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
13174 );
13175 self.conn.execute(&sql, []).map_err(|error| {
13176 format!(
13177 "cannot create local FTS5 projection {:?}: {error}",
13178 index.name
13179 )
13180 })?;
13181 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
13182 let mapped: bool = self
13183 .conn
13184 .query_row(
13185 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
13186 [format!("_syncular_fts_{}", index.name)],
13187 |row| row.get(0),
13188 )
13189 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13190 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13191 if !mapped && existed {
13192 self.conn
13193 .execute_batch(&format!(
13194 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
13195 quote_ident(FTS_SOURCE_ID_COLUMN),
13196 quote_ident(&index.name)
13197 ))
13198 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13199 }
13200 self.create_fts_triggers(table, index)?;
13201 if !existed {
13202 self.rebuild_fts_projection(table, index)?;
13203 }
13204 Ok(())
13205 })();
13206 if let Err(error) = result {
13207 self.rollback_observation("syncular_fts_schema");
13208 return Err(error);
13209 }
13210 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
13211 self.rollback_observation("syncular_fts_schema");
13212 return Err(error.to_string());
13213 }
13214 Ok(())
13215 }
13216
13217 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
13220 let state = serde_json::json!({
13221 "requested": scope_map_to_json(&sub.requested),
13222 "params": sub.params,
13223 "cursor": sub.cursor,
13224 "bootstrapState": sub.bootstrap_state,
13225 "status": sub.state.name(),
13226 "reasonCode": sub.reason_code,
13227 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
13228 "syncedOnce": sub.synced_once,
13229 });
13230 self.conn.execute(
13231 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
13232 rusqlite::params![sub.id, sub.table, state.to_string()],
13233 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13234 }
13235
13236 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
13237 let ops: Vec<Value> = commit
13238 .ops
13239 .iter()
13240 .map(|op| {
13241 serde_json::json!({
13242 "op": if op.upsert { "upsert" } else { "delete" },
13243 "table": op.table,
13244 "rowId": op.row_id,
13245 "baseVersion": op.base_version,
13246 "values": op.values.clone().map(Value::Object),
13247 })
13248 })
13249 .collect();
13250 self.conn
13251 .execute(
13252 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
13253 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
13254 )
13255 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13256 for operation in &commit.ops {
13257 let Some(values) = &operation.values else {
13258 continue;
13259 };
13260 let Some(table) = self.schema.table(&operation.table) else {
13261 continue;
13262 };
13263 for column in table
13264 .columns
13265 .iter()
13266 .filter(|column| column.ty == ColumnType::BlobRef)
13267 {
13268 let Some(Value::String(raw)) = values.get(&column.name) else {
13269 continue;
13270 };
13271 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
13272 continue;
13273 };
13274 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
13275 continue;
13276 };
13277 self.conn
13278 .execute(
13279 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
13280 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
13281 rusqlite::params![commit.client_commit_id, blob_id],
13282 )
13283 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13284 }
13285 }
13286 Ok(())
13287 }
13288
13289 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
13290 if let Some(commit) = self
13291 .outbox
13292 .iter()
13293 .find(|commit| commit.client_commit_id == client_commit_id)
13294 {
13295 for operation in &commit.ops {
13296 self.overlay_dirty.table(&operation.table);
13297 }
13298 }
13299
13300 self.conn
13301 .execute(
13302 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
13303 rusqlite::params![client_commit_id],
13304 )
13305 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13306 self.conn
13307 .execute(
13308 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
13309 rusqlite::params![client_commit_id],
13310 )
13311 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13312 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
13313 .map(|_| ())
13314 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13315 }
13316
13317 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
13318 match status {
13319 CommitOutcomeStatus::Applied => "applied",
13320 CommitOutcomeStatus::Cached => "cached",
13321 CommitOutcomeStatus::Conflict => "conflict",
13322 CommitOutcomeStatus::Rejected => "rejected",
13323 }
13324 }
13325
13326 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
13327 match resolution {
13328 CommitOutcomeResolution::Active => "active",
13329 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
13330 CommitOutcomeResolution::Superseded => "superseded",
13331 CommitOutcomeResolution::Dismissed => "dismissed",
13332 }
13333 }
13334
13335 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
13336 match value {
13337 "applied" => Ok(CommitOutcomeStatus::Applied),
13338 "cached" => Ok(CommitOutcomeStatus::Cached),
13339 "conflict" => Ok(CommitOutcomeStatus::Conflict),
13340 "rejected" => Ok(CommitOutcomeStatus::Rejected),
13341 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
13342 }
13343 }
13344
13345 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
13346 match value {
13347 "active" => Ok(CommitOutcomeResolution::Active),
13348 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
13349 "superseded" => Ok(CommitOutcomeResolution::Superseded),
13350 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
13351 _ => Err(format!(
13352 "invalid persisted commit outcome resolution {value:?}"
13353 )),
13354 }
13355 }
13356
13357 fn persist_commit_outcome(
13358 &self,
13359 client_commit_id: &str,
13360 status: CommitOutcomeStatus,
13361 results: &[CommitOperationOutcome],
13362 operations: Option<&[OutboxOp]>,
13363 ) -> Result<(), String> {
13364 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
13365 let operations_json = operations
13366 .map(|items| {
13367 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
13368 })
13369 .transpose()
13370 .map_err(|error| error.to_string())?;
13371 self.conn
13372 .execute(
13373 "INSERT INTO _syncular_commit_outcomes (
13374 client_commit_id, status, recorded_at_ms, results_json,
13375 operations_json, resolution
13376 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
13377 rusqlite::params![
13378 client_commit_id,
13379 Self::outcome_status_name(status),
13380 self.clock_now_ms(),
13381 results_json,
13382 operations_json
13383 ],
13384 )
13385 .map(|_| ())
13386 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13387 }
13388
13389 fn stored_cell_is_representable(cell: &Value) -> bool {
13392 cell.is_null()
13393 || cell.is_string()
13394 || cell.is_boolean()
13395 || cell.is_number()
13396 || is_canonical_bytes_envelope(cell)
13397 }
13398
13399 fn stored_row_is_representable(value: &Value) -> bool {
13400 value
13401 .as_object()
13402 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
13403 }
13404
13405 fn stored_operation_is_representable(value: &Value) -> bool {
13406 value.get("table").is_some_and(Value::is_string)
13407 && value.get("rowId").is_some_and(Value::is_string)
13408 && value.get("baseVersion").is_none_or(|v| {
13409 v.as_i64()
13410 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
13411 })
13412 && match value.get("op").and_then(Value::as_str) {
13413 Some("upsert") => value
13414 .get("values")
13415 .is_some_and(Self::stored_row_is_representable),
13416 Some("delete") => value.get("values").is_none(),
13417 _ => false,
13418 }
13419 }
13420
13421 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
13425 serde_json::to_value(CommitOperation::from(operation))
13426 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
13427 }
13428
13429 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
13430 let StoredCommitOutcomeRow {
13431 sequence,
13432 client_commit_id,
13433 status,
13434 recorded_at_ms,
13435 results_json,
13436 operations_json,
13437 resolution,
13438 resolved_at_ms,
13439 replacement_client_commit_id,
13440 } = row;
13441 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
13442 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
13443 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
13444 {
13445 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
13446 }
13447 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
13448 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
13449 for result in &stored {
13450 let nested = match result.get("status").and_then(Value::as_str) {
13451 Some("conflict") => result.get("conflict"),
13452 Some("error") => result.get("rejection"),
13453 _ => None,
13454 };
13455 if nested
13456 .and_then(|record| record.get("operation"))
13457 .is_some_and(|op| !Self::stored_operation_is_representable(op))
13458 || (result.get("status").and_then(Value::as_str) == Some("conflict")
13459 && nested
13460 .and_then(|record| record.get("serverRow"))
13461 .is_some_and(|row| !Self::stored_row_is_representable(row)))
13462 {
13463 return Err(invalid());
13464 }
13465 }
13466 let results: Vec<CommitOperationOutcome> =
13467 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
13468 let has_conflict = results
13469 .iter()
13470 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
13471 let has_error = results
13472 .iter()
13473 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
13474 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
13475 let redacted = results.is_empty() && operations_json.is_none();
13476 if match status {
13477 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
13478 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
13479 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
13480 } {
13481 return Err(invalid());
13482 }
13483 let operations: Option<Vec<CommitOperation>> = operations_json
13484 .map(|raw| {
13485 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
13486 if !stored.iter().all(Self::stored_operation_is_representable) {
13487 return Err(invalid());
13488 }
13489 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
13490 })
13491 .transpose()?;
13492 for result in &results {
13493 match result {
13494 CommitOperationOutcome::Conflict { conflict }
13495 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
13496 {
13497 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
13498 }
13499 CommitOperationOutcome::Error { rejection } => {
13500 if let Some(details) = &rejection.details {
13501 let encoded = serde_json::to_string(details).map_err(|_| {
13502 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13503 })?;
13504 RejectionDetails::parse(&encoded).map_err(|_| {
13505 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
13506 })?;
13507 }
13508 }
13509 _ => {}
13510 }
13511 }
13512 Ok(CommitOutcome {
13513 sequence,
13514 client_commit_id,
13515 status,
13516 recorded_at_ms,
13517 results,
13518 operations,
13519 retained_rows: None,
13520 resolution: Self::parse_outcome_resolution(&resolution)?,
13521 resolved_at_ms,
13522 replacement_client_commit_id,
13523 })
13524 }
13525
13526 fn unique_conflicts(
13528 &self,
13529 table: &TableSchema,
13530 values: &Map<String, Value>,
13531 base: bool,
13532 ) -> Result<Vec<RetainedUniqueConflict>, String> {
13533 let source = if base {
13534 base_table(&table.name)
13535 } else {
13536 visible_table(&table.name)
13537 };
13538 let mut conflicts = Vec::new();
13539 for index in table.indexes.iter().filter(|index| index.unique) {
13540 let mut params = vec![json_param_to_sql(
13541 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
13542 )?];
13543 for name in &index.columns {
13544 let column = table
13545 .columns
13546 .iter()
13547 .find(|column| column.name == *name)
13548 .ok_or("sync.local_corrupt")?;
13549 let cell = json_to_column_value(column, values.get(name))?;
13550 params.push(owned_sql_value(RowParam::Cell(&cell))?);
13551 }
13552 let sql = format!(
13553 "SELECT * FROM {source} WHERE {} != ? AND {}",
13554 quote_ident(&table.primary_key),
13555 index
13556 .columns
13557 .iter()
13558 .map(|column| format!("{} = ?", quote_ident(column)))
13559 .collect::<Vec<_>>()
13560 .join(" AND ")
13561 );
13562 let mut statement = self
13563 .conn
13564 .prepare_cached(&sql)
13565 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13566 let records = statement
13567 .query_map(rusqlite::params_from_iter(params), |record| {
13568 let values = table
13569 .columns
13570 .iter()
13571 .enumerate()
13572 .map(|(index, column)| {
13573 let value = sql_ref_to_json(column, record.get_ref(index)?);
13574 json_to_column_value(column, Some(&value)).map_err(|message| {
13575 rusqlite::Error::ToSqlConversionFailure(message.into())
13576 })
13577 })
13578 .collect::<Result<Row, _>>()?;
13579 Ok((values, record.get::<_, i64>(table.columns.len())?))
13580 })
13581 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13582 for record in records {
13583 let (row, version) =
13584 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13585 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
13586 conflicts.push(RetainedUniqueConflict {
13587 index: index.name.clone(),
13588 columns: index.columns.clone(),
13589 row_id,
13590 server_row: table
13591 .columns
13592 .iter()
13593 .zip(&row)
13594 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
13595 .collect(),
13596 server_version: version,
13597 });
13598 }
13599 }
13600 Ok(conflicts)
13601 }
13602
13603 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
13604 if let Some((commit, initial)) = self
13605 .failed_commits
13606 .iter()
13607 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
13608 {
13609 let mut rows = Vec::new();
13610 for operation in &commit.ops {
13611 let table = self
13612 .schema
13613 .table(&operation.table)
13614 .ok_or_else(|| "sync.unknown_table".to_owned())?;
13615 let key =
13616 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
13617 .map_err(|_| "sync.local_corrupt".to_owned())?;
13618 let base = self.stored_row(table, &operation.row_id, true)?;
13619 rows.push(RetainedCommitRow {
13620 table: operation.table.clone(),
13621 row_id: operation.row_id.clone(),
13622 unique_conflicts: if operation.upsert {
13623 self.unique_conflicts(
13624 table,
13625 initial
13626 .get(&key)
13627 .and_then(Value::as_object)
13628 .or(operation.values.as_ref())
13629 .ok_or("sync.local_corrupt")?,
13630 true,
13631 )?
13632 } else {
13633 Vec::new()
13634 },
13635 local_row: if operation.upsert {
13636 initial
13637 .get(&key)
13638 .and_then(Value::as_object)
13639 .cloned()
13640 .or_else(|| operation.values.clone())
13641 } else {
13642 None
13643 },
13644 server_row: base.as_ref().map(|(values, _)| {
13645 table
13646 .columns
13647 .iter()
13648 .zip(values)
13649 .map(|(column, value)| {
13650 (column.name.clone(), column_value_to_json(value))
13651 })
13652 .collect()
13653 }),
13654 server_version: base.map(|(_, version)| version),
13655 });
13656 }
13657 outcome.retained_rows = Some(rows);
13658 }
13659 Ok(outcome)
13660 }
13661
13662 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
13663 let row = self
13664 .conn
13665 .query_row(
13666 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
13667 resolution, resolved_at_ms, replacement_client_commit_id
13668 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
13669 rusqlite::params![client_commit_id],
13670 |row| {
13671 Ok(StoredCommitOutcomeRow {
13672 sequence: row.get(0)?,
13673 client_commit_id: row.get(1)?,
13674 status: row.get(2)?,
13675 recorded_at_ms: row.get(3)?,
13676 results_json: row.get(4)?,
13677 operations_json: row.get(5)?,
13678 resolution: row.get(6)?,
13679 resolved_at_ms: row.get(7)?,
13680 replacement_client_commit_id: row.get(8)?,
13681 })
13682 },
13683 )
13684 .optional()
13685 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13686 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
13687 .transpose()
13688 }
13689
13690 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
13691 if query.limit == Some(0) {
13692 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
13693 }
13694 let mut sql = String::from(
13695 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
13696 resolution, resolved_at_ms, replacement_client_commit_id
13697 FROM _syncular_commit_outcomes",
13698 );
13699 if query.active_only {
13700 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
13701 }
13702 sql.push_str(" ORDER BY seq DESC");
13703 if let Some(limit) = query.limit {
13704 sql.push_str(&format!(" LIMIT {limit}"));
13705 }
13706 let mut stmt = self
13707 .conn
13708 .prepare(&sql)
13709 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13710 let rows = stmt
13711 .query_map([], |row| {
13712 Ok(StoredCommitOutcomeRow {
13713 sequence: row.get(0)?,
13714 client_commit_id: row.get(1)?,
13715 status: row.get(2)?,
13716 recorded_at_ms: row.get(3)?,
13717 results_json: row.get(4)?,
13718 operations_json: row.get(5)?,
13719 resolution: row.get(6)?,
13720 resolved_at_ms: row.get(7)?,
13721 replacement_client_commit_id: row.get(8)?,
13722 })
13723 })
13724 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13725 let mut outcomes = Vec::new();
13726 for row in rows {
13727 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
13728 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
13729 )?)?);
13730 }
13731 Ok(outcomes)
13732 }
13733
13734 fn prune_commit_outcomes(&self) -> Result<(), String> {
13735 #[cfg(test)]
13736 self.outcome_prune_count
13737 .set(self.outcome_prune_count.get() + 1);
13738 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
13739 let count = self
13740 .conn
13741 .query_row(
13742 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
13743 [],
13744 |row| row.get::<_, i64>(0),
13745 )
13746 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
13747 as usize;
13748 let excess = count.saturating_sub(max_entries);
13749 if excess == 0 {
13750 return Ok(());
13751 }
13752 let mut stmt = self
13753 .conn
13754 .prepare(
13755 "SELECT seq FROM _syncular_commit_outcomes
13756 WHERE status IN ('applied', 'cached') OR resolution != 'active'
13757 ORDER BY seq ASC LIMIT ?1",
13758 )
13759 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13760 let rows = stmt
13761 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
13762 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13763 let sequences = rows
13764 .collect::<Result<Vec<_>, _>>()
13765 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13766 drop(stmt);
13767 for sequence in sequences {
13768 self.conn
13769 .execute(
13770 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
13771 rusqlite::params![sequence],
13772 )
13773 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13774 }
13775 Ok(())
13776 }
13777
13778 pub fn subscribe(
13781 &mut self,
13782 id: String,
13783 table: String,
13784 scopes: Vec<(String, Vec<String>)>,
13785 params: Option<String>,
13786 ) -> Result<(), String> {
13787 if self.schema.table(&table).is_none() {
13788 return Err(format!("unknown table {table:?}"));
13789 }
13790 let registered_at = self
13791 .subs
13792 .iter()
13793 .position(|subscription| subscription.id == id);
13794 if let Some(index) = registered_at {
13795 let existing = &self.subs[index];
13796 let same_intent = existing.table == table
13797 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
13798 && existing.params == params;
13799 if !same_intent {
13800 return Err(
13801 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
13802 .to_owned(),
13803 );
13804 }
13805 if existing.state != SubState::Revoked {
13810 return Ok(());
13811 }
13812 }
13813 let sub = Subscription {
13814 id: id.clone(),
13815 table,
13816 requested: scopes,
13817 params,
13818 cursor: -1,
13819 bootstrap_state: None,
13820 state: SubState::Active,
13821 reason_code: None,
13822 effective: None,
13823 synced_once: false,
13824 };
13825 self.persist_sub(&sub)?;
13826 match registered_at {
13827 Some(index) => self.subs[index] = sub,
13828 None => self.subs.push(sub),
13829 }
13830 if registered_at.is_some() {
13834 self.set_sync_needed(true, true);
13835 }
13836 Ok(())
13837 }
13838
13839 pub fn subscribe_many(&mut self, requests: Vec<SubscriptionRequest>) -> Result<(), String> {
13845 let prior_subs = self.subs.clone();
13846 self.begin_observation("syncular_subscribe")?;
13847 let registered = (|| {
13848 for request in requests {
13849 self.subscribe(request.id, request.table, request.scopes, request.params)?;
13850 }
13851 self.conn
13852 .execute_batch("RELEASE syncular_subscribe")
13853 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13854 })();
13855 if let Err(error) = registered {
13856 self.rollback_observation("syncular_subscribe");
13857 self.subs = prior_subs;
13858 return Err(error);
13859 }
13860 Ok(())
13861 }
13862
13863 pub fn unsubscribe(&mut self, id: &str) {
13864 if self.subs.iter().any(|s| s.id == id) {
13865 self.cancel_sync_round();
13866 }
13867 self.subs.retain(|s| s.id != id);
13868 let _ = self.conn.execute(
13869 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
13870 rusqlite::params![id],
13871 );
13872 }
13873
13874 pub fn set_window(
13882 &mut self,
13883 base: &WindowBase,
13884 units: &[String],
13885 ) -> Result<CommandEffects, String> {
13886 let table = self
13887 .schema
13888 .table(&base.table)
13889 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
13890 if table.scope_column(&base.variable).is_none() {
13891 return Err(format!(
13892 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
13893 base.table, base.variable
13894 ));
13895 }
13896 let base_key = window_base_key(base);
13897 let wanted: std::collections::HashSet<&String> = units.iter().collect();
13898 let live = self.load_window_units(&base_key);
13899 self.begin_observation("syncular_window")?;
13900 let mut batch = ChangeAccumulator::default();
13901 let mut changed = false;
13902
13903 let prior_subs = self.subs.clone();
13904 let widened = (|| {
13905 for unit in units {
13907 if live.iter().any(|(u, _)| u == unit) {
13908 continue;
13909 }
13910 let sub_id = derive_sub_id(base, unit);
13911 self.delete_pending_evict(&sub_id)?;
13912 self.insert_window_unit(&base_key, unit, &sub_id)?;
13913 self.subscribe(
13914 sub_id,
13915 base.table.clone(),
13916 unit_scopes(base, unit),
13917 base.params.clone(),
13918 )?;
13919 batch.window(&base_key, &base.table, unit);
13920 changed = true;
13921 }
13922
13923 self.finish_observation("syncular_window", batch)
13924 })();
13925 if let Err(error) = widened {
13926 self.rollback_observation("syncular_window");
13927 self.subs = prior_subs;
13928 return Err(error);
13929 }
13930 for (unit, sub_id) in live {
13931 if wanted.contains(&unit) {
13932 continue;
13933 }
13934 self.rebuild_overlay_if_dirty()?;
13935 let prior_subs = self.subs.clone();
13936 self.begin_observation("syncular_window")?;
13937 let effective = self
13938 .subs
13939 .iter()
13940 .find(|sub| sub.id == sub_id)
13941 .and_then(|sub| sub.effective.clone())
13942 .unwrap_or_else(|| unit_scopes(base, &unit));
13943 let mut batch = ChangeAccumulator::default();
13944 self.record_scope_map(&mut batch, &base.table, &effective);
13945 batch.window(&base_key, &base.table, &unit);
13946 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
13947 let remaining = match result {
13948 Ok(remaining) => remaining,
13949 Err(error) => {
13950 self.rollback_observation("syncular_window");
13951 self.subs = prior_subs;
13952 return Err(error);
13953 }
13954 };
13955 if let Err(error) = self.finish_observation("syncular_window", batch) {
13956 self.rollback_observation("syncular_window");
13957 self.subs = prior_subs;
13958 return Err(error);
13959 }
13960 if remaining {
13961 self.drain_pending_evictions()?;
13962 }
13963 changed = true;
13964 }
13965 Ok(if changed {
13966 CommandEffects::interactive()
13967 } else {
13968 CommandEffects::none()
13969 })
13970 }
13971
13972 pub fn window_state(&self, base: &WindowBase) -> WindowState {
13977 let mut units = Vec::new();
13978 let mut pending = Vec::new();
13979 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
13980 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
13981 Some(sub) => {
13982 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
13983 }
13984 None => true,
13985 };
13986 if is_pending {
13987 pending.push(unit.clone());
13988 }
13989 units.push(unit);
13990 }
13991 WindowState { units, pending }
13992 }
13993
13994 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
13995 let mut stmt = match self
13996 .conn
13997 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
13998 {
13999 Ok(stmt) => stmt,
14000 Err(_) => return Vec::new(),
14001 };
14002 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
14003 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
14004 });
14005 match rows {
14006 Ok(rows) => rows.filter_map(Result::ok).collect(),
14007 Err(_) => Vec::new(),
14008 }
14009 }
14010
14011 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
14012 let mut stmt = match self.conn.prepare(
14013 "SELECT windows.base, windows.unit, subscriptions.tbl
14014 FROM _syncular_windows AS windows
14015 JOIN _syncular_subscriptions AS subscriptions
14016 ON subscriptions.id = windows.sub_id
14017 ORDER BY windows.base, windows.unit",
14018 ) {
14019 Ok(stmt) => stmt,
14020 Err(_) => return Vec::new(),
14021 };
14022 let rows = stmt.query_map([], |row| {
14023 Ok((
14024 row.get::<_, String>(0)?,
14025 row.get::<_, String>(1)?,
14026 row.get::<_, String>(2)?,
14027 ))
14028 });
14029 match rows {
14030 Ok(rows) => rows.filter_map(Result::ok).collect(),
14031 Err(_) => Vec::new(),
14032 }
14033 }
14034
14035 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
14036 self.conn
14037 .query_row(
14038 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
14039 rusqlite::params![sub_id],
14040 |row| Ok((row.get(0)?, row.get(1)?)),
14041 )
14042 .ok()
14043 }
14044
14045 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
14046 self.conn
14047 .execute(
14048 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
14049 rusqlite::params![base_key, unit, sub_id],
14050 )
14051 .map(|_| ())
14052 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14053 }
14054
14055 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
14056 self.conn
14057 .execute(
14058 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
14059 rusqlite::params![base_key, unit],
14060 )
14061 .map(|_| ())
14062 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14063 }
14064
14065 fn evict_unit(
14072 &mut self,
14073 base_key: &str,
14074 base: &WindowBase,
14075 unit: &str,
14076 sub_id: &str,
14077 ) -> Result<bool, String> {
14078 let effective = self
14079 .subs
14080 .iter()
14081 .find(|s| s.id == sub_id)
14082 .and_then(|s| s.effective.clone())
14083 .unwrap_or_else(|| unit_scopes(base, unit));
14084 let pinned = self.pinned_row_ids(&base.table);
14085 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
14086 self.delete_window_unit(base_key, unit)?;
14087 self.conn
14088 .execute(
14089 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
14090 [sub_id],
14091 )
14092 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14093 self.subs.retain(|sub| sub.id != sub_id);
14094 if deferred {
14095 self.save_pending_evict(sub_id, &base.table, &effective)?;
14096 } else {
14097 self.delete_pending_evict(sub_id)?;
14098 }
14099 Ok(remaining)
14100 }
14101
14102 fn evict_scope_rows(
14106 &self,
14107 table_name: &str,
14108 effective: &[(String, Vec<String>)],
14109 pinned: &std::collections::HashSet<String>,
14110 ) -> Result<(bool, bool), String> {
14111 if effective.is_empty() {
14112 return Ok((false, false));
14113 }
14114 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
14115 let mut clauses = Vec::new();
14116 let mut params: Vec<SqlValue> = Vec::new();
14117 for (variable, values) in effective {
14118 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
14119 if values.is_empty() {
14120 return Ok((false, false));
14121 }
14122 let holes = values
14123 .iter()
14124 .map(|v| {
14125 params.push(SqlValue::Text(v.clone()));
14126 "?"
14127 })
14128 .collect::<Vec<_>>()
14129 .join(", ");
14130 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
14131 }
14132 let scope = clauses.join(" AND ");
14133 let scope_params = params.len();
14134 let pk = quote_ident(&table.primary_key);
14135 if !pinned.is_empty() {
14136 let holes = pinned
14137 .iter()
14138 .map(|id| {
14139 params.push(SqlValue::Text(id.clone()));
14140 "?"
14141 })
14142 .collect::<Vec<_>>()
14143 .join(", ");
14144 clauses.push(format!("{pk} NOT IN ({holes})"));
14145 }
14146 let predicate = clauses.join(" AND ");
14147 let base = base_table(table_name);
14148 let visible = visible_table(table_name);
14149 let mut ids: Vec<SqlValue> = Vec::new();
14150 for source in [&base, &visible] {
14151 let mut stmt = self
14152 .conn
14153 .prepare_cached(&format!(
14154 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
14155 ))
14156 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14157 ids = stmt
14158 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
14159 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
14160 .collect::<Result<_, _>>()
14161 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14162 if !ids.is_empty() {
14163 break;
14164 }
14165 }
14166 if !ids.is_empty() {
14167 for id in &ids {
14168 self.conn
14169 .execute(
14170 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
14171 rusqlite::params![table_name, id],
14172 )
14173 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14174 }
14175 let holes = vec!["?"; ids.len()].join(", ");
14176 for source in [&base, &visible] {
14177 self.conn
14178 .execute(
14179 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
14180 rusqlite::params_from_iter(&ids),
14181 )
14182 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14183 }
14184 }
14185 let mut remaining = false;
14186 let mut deferred = false;
14187 for source in [&base, &visible] {
14188 remaining |= self
14189 .conn
14190 .query_row(
14191 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
14192 rusqlite::params_from_iter(¶ms),
14193 |row| row.get::<_, bool>(0),
14194 )
14195 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14196 deferred |= self
14197 .conn
14198 .query_row(
14199 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
14200 rusqlite::params_from_iter(¶ms[..scope_params]),
14201 |row| row.get::<_, bool>(0),
14202 )
14203 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14204 }
14205 Ok((remaining, deferred))
14206 }
14207
14208 fn drain_pending_evictions(&mut self) -> Result<(), String> {
14210 let pending = self.load_pending_evictions()?;
14211 if pending.is_empty() {
14212 return Ok(());
14213 }
14214 self.rebuild_overlay_if_dirty()?;
14215 for (sub_id, table_name, effective) in pending {
14216 if self.schema.table(&table_name).is_none() {
14217 self.delete_pending_evict(&sub_id)?;
14218 continue;
14219 }
14220 loop {
14221 std::thread::yield_now();
14222 self.begin_observation("syncular_evict")?;
14223 let pinned = self.pinned_row_ids(&table_name);
14224 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
14225 let (remaining, deferred) = match result {
14226 Ok(result) => result,
14227 Err(error) => {
14228 self.rollback_observation("syncular_evict");
14229 return Err(error);
14230 }
14231 };
14232 if !deferred {
14233 if let Err(error) = self.delete_pending_evict(&sub_id) {
14234 self.rollback_observation("syncular_evict");
14235 return Err(error);
14236 }
14237 }
14238 let mut batch = ChangeAccumulator::default();
14239 self.record_scope_map(&mut batch, &table_name, &effective);
14240 if let Err(error) = self.finish_observation("syncular_evict", batch) {
14241 self.rollback_observation("syncular_evict");
14242 return Err(error);
14243 }
14244 if !remaining {
14245 break;
14246 }
14247 }
14248 }
14249 Ok(())
14250 }
14251
14252 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
14255 let mut pinned = std::collections::HashSet::new();
14256 for commit in self
14257 .outbox
14258 .iter()
14259 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
14260 {
14261 for op in &commit.ops {
14262 if op.table == table {
14263 pinned.insert(op.row_id.clone());
14264 }
14265 }
14266 }
14267 pinned
14268 }
14269
14270 fn save_pending_evict(
14271 &self,
14272 sub_id: &str,
14273 table: &str,
14274 effective: &[(String, Vec<String>)],
14275 ) -> Result<(), String> {
14276 self.conn.execute(
14277 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
14278 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
14279 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14280 }
14281
14282 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
14283 self.conn
14284 .execute(
14285 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
14286 rusqlite::params![sub_id],
14287 )
14288 .map(|_| ())
14289 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14290 }
14291
14292 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
14293 let mut stmt = self
14294 .conn
14295 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
14296 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14297 let rows = stmt
14298 .query_map([], |row| {
14299 Ok((
14300 row.get::<_, String>(0)?,
14301 row.get::<_, String>(1)?,
14302 row.get::<_, String>(2)?,
14303 ))
14304 })
14305 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14306 rows.map(|row| {
14307 let (sub_id, table, json) =
14308 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14309 let value: Value = serde_json::from_str(&json)
14310 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14311 let effective = json_to_scope_map(&value)
14312 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
14313 Ok((sub_id, table, effective))
14314 })
14315 .collect()
14316 }
14317
14318 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
14320 self.retain_failed_commits = enabled;
14321 for (commit, _) in &self.failed_commits {
14322 for operation in &commit.ops {
14323 self.overlay_dirty.table(&operation.table);
14324 }
14325 }
14326 self.failed_commits = self.load_failed_commits()?;
14327 for (commit, _) in &self.failed_commits {
14328 for operation in &commit.ops {
14329 self.overlay_dirty.table(&operation.table);
14330 }
14331 }
14332 Ok(())
14333 }
14334
14335 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
14336 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
14337 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14338 let rows = statement
14339 .query_map([], |row| {
14340 Ok((
14341 row.get::<_, String>(0)?,
14342 row.get::<_, String>(1)?,
14343 row.get::<_, String>(2)?,
14344 ))
14345 })
14346 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14347 rows.map(|row| {
14348 let (id, operations, initial) =
14349 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14350 let operations: Vec<CommitOperation> =
14351 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
14352 let initial =
14353 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
14354 Ok((
14355 OutboxCommit {
14356 client_commit_id: id,
14357 ops: operations
14358 .into_iter()
14359 .map(|op| OutboxOp {
14360 upsert: op.op == "upsert",
14361 table: op.table,
14362 row_id: op.row_id,
14363 base_version: op.base_version,
14364 values: op.values,
14365 })
14366 .collect(),
14367 },
14368 initial,
14369 ))
14370 })
14371 .collect()
14372 }
14373
14374 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
14375 let mut initial = Map::new();
14376 for operation in operations {
14377 let table = self
14378 .schema
14379 .table(&operation.table)
14380 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14381 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
14382 Some(row) => Some(row),
14383 None => self.visible_row(table, &operation.row_id)?,
14384 };
14385 if let Some((values, _)) = initial_row {
14386 let mut intended: Map<String, Value> = table
14387 .columns
14388 .iter()
14389 .zip(values.iter())
14390 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
14391 .collect();
14392 if let Some(patch) = &operation.values {
14393 intended.extend(patch.clone());
14394 }
14395 initial.insert(
14396 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14397 .map_err(|_| "sync.local_corrupt".to_owned())?,
14398 Value::Object(intended),
14399 );
14400 }
14401 }
14402 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
14403 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
14404 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
14405 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14406 self.failed_commits.push((
14407 OutboxCommit {
14408 client_commit_id: id.to_owned(),
14409 ops: operations.to_vec(),
14410 },
14411 initial,
14412 ));
14413 Ok(())
14414 }
14415
14416 fn drop_failed_commits(
14417 &mut self,
14418 matches: impl Fn(&str, &Map<String, Value>) -> bool,
14419 ) -> Result<Vec<OutboxCommit>, String> {
14420 let mut doomed = self.drop_acknowledged_rows(&matches)?;
14421 for (commit, initial) in &self.failed_commits {
14422 for operation in &commit.ops {
14423 let key =
14424 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
14425 .map_err(|_| "sync.local_corrupt".to_owned())?;
14426 if initial
14427 .get(&key)
14428 .and_then(Value::as_object)
14429 .is_some_and(|values| matches(&operation.table, values))
14430 {
14431 doomed.push(commit.clone());
14432 break;
14433 }
14434 }
14435 }
14436 for commit in &doomed {
14437 self.conn
14438 .execute(
14439 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
14440 rusqlite::params![commit.client_commit_id],
14441 )
14442 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14443 }
14444 for commit in &doomed {
14445 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14446 }
14447 self.conflicts.retain(|record| {
14448 !doomed
14449 .iter()
14450 .any(|commit| commit.client_commit_id == record.client_commit_id)
14451 });
14452 self.rejections.retain(|record| {
14453 !doomed
14454 .iter()
14455 .any(|commit| commit.client_commit_id == record.client_commit_id)
14456 });
14457 self.failed_commits.retain(|(commit, _)| {
14458 !doomed
14459 .iter()
14460 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
14461 });
14462 for commit in &doomed {
14463 for operation in &commit.ops {
14464 self.overlay_dirty.table(&operation.table);
14465 }
14466 }
14467 Ok(doomed)
14468 }
14469
14470 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
14474 let prior = self.storage_failure.borrow_mut().take();
14475 let outcome = (|| -> Result<String, String> {
14476 if mutations.is_empty() {
14477 return Err(
14478 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
14479 .to_owned(),
14480 );
14481 }
14482 let mut ops = Vec::with_capacity(mutations.len());
14483 for mutation in mutations {
14484 match mutation {
14485 Mutation::Patch {
14486 table,
14487 values,
14488 base_version,
14489 } => {
14490 let schema_table = self
14491 .schema
14492 .table(&table)
14493 .ok_or_else(|| "sync.unknown_table".to_owned())?;
14494 let values = normalize_values_casing(schema_table, values)?;
14495 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14496 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
14497 }
14498 Mutation::Upsert {
14499 table,
14500 values,
14501 base_version,
14502 } => {
14503 let schema_table = self.schema.table(&table).ok_or_else(|| {
14504 format!("sync.unknown_table: unknown table {table:?}")
14505 })?;
14506 let values = normalize_values_casing(schema_table, values)?;
14510 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
14511 let values = full_row_values(schema_table, values)?;
14514 self.validate_author_encode(schema_table, &row_id, &values)?;
14520 ops.push(OutboxOp {
14521 upsert: true,
14522 table,
14523 row_id,
14524 base_version,
14525 values: Some(values),
14526 });
14527 }
14528 Mutation::Delete {
14529 table,
14530 row_id,
14531 base_version,
14532 } => {
14533 if self.schema.table(&table).is_none() {
14534 return Err(format!("sync.unknown_table: unknown table {table:?}"));
14535 }
14536 ops.push(OutboxOp {
14537 upsert: false,
14538 table,
14539 row_id,
14540 base_version,
14541 values: None,
14542 });
14543 }
14544 }
14545 }
14546 self.record_outbox_commit(ops)
14547 })();
14548 self.finish_authoring(prior, outcome)
14549 }
14550
14551 fn finish_authoring(
14557 &self,
14558 prior: Option<QueryReadFailure>,
14559 outcome: Result<String, String>,
14560 ) -> Result<String, ClientError> {
14561 let fresh = self.storage_failure.borrow_mut().take();
14562 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
14563 match outcome {
14564 Ok(id) => Ok(id),
14565 Err(message) => Err(fresh
14566 .map(ClientError::from)
14567 .unwrap_or_else(|| ClientError::from(message))),
14568 }
14569 }
14570
14571 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
14572 let commit = OutboxCommit {
14573 client_commit_id: uuid::Uuid::new_v4().to_string(),
14574 ops,
14575 };
14576 self.begin_observation("syncular_mutation")?;
14577 let mut batch = ChangeAccumulator::default();
14578 for op in &commit.ops {
14579 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
14580 if let Some(values) = &op.values {
14581 if let Some(table) = self.schema.table(&op.table) {
14582 for scope in &table.scope_variables {
14583 if let Some(Value::String(value)) = values.get(&scope.column) {
14584 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
14585 precise = true;
14586 }
14587 }
14588 }
14589 }
14590 if !precise {
14591 batch.table(&op.table);
14592 }
14593 }
14594 if let Err(error) = self.persist_outbox_insert(&commit) {
14595 self.rollback_observation("syncular_mutation");
14596 return Err(error);
14597 }
14598 let was_dirty = self.overlay_dirty.snapshot();
14599 if let Err(error) = self.rebuild_overlay_if_dirty() {
14604 self.rollback_observation("syncular_mutation");
14605 self.overlay_dirty.restore(was_dirty.clone());
14606 return Err(error);
14607 }
14608 if let Some(error) = commit
14609 .ops
14610 .iter()
14611 .find_map(|op| self.apply_outbox_op(op).err())
14612 {
14613 self.rollback_observation("syncular_mutation");
14614 self.overlay_dirty.restore(was_dirty.clone());
14615 return Err(error);
14616 }
14617 let id = commit.client_commit_id.clone();
14618 self.outbox.push(commit);
14619 batch.status = true;
14620 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
14621 self.rollback_observation("syncular_mutation");
14622 self.outbox.pop();
14623 self.overlay_dirty.restore(was_dirty.clone());
14624 return Err(error);
14625 }
14626 Ok(id)
14627 }
14628
14629 pub fn patch(
14635 &mut self,
14636 table: &str,
14637 row_id: &str,
14638 partial: Map<String, Value>,
14639 base_version: Option<i64>,
14640 ) -> Result<String, ClientError> {
14641 let prior = self.storage_failure.borrow_mut().take();
14642 let outcome = self.perform_patch(table, row_id, partial, base_version);
14643 self.finish_authoring(prior, outcome)
14644 }
14645
14646 fn perform_patch(
14649 &mut self,
14650 table: &str,
14651 row_id: &str,
14652 partial: Map<String, Value>,
14653 base_version: Option<i64>,
14654 ) -> Result<String, String> {
14655 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
14656 self.record_outbox_commit(vec![operation])
14657 }
14658
14659 fn prepare_patch(
14660 &self,
14661 table: &str,
14662 row_id: &str,
14663 partial: Map<String, Value>,
14664 base_version: Option<i64>,
14665 ) -> Result<OutboxOp, String> {
14666 let schema_table = self
14667 .schema
14668 .table(table)
14669 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
14670 let partial = normalize_values_casing(schema_table, partial)?;
14671 if let Some(pk) = partial.get(&schema_table.primary_key) {
14672 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
14673 return Err(format!(
14674 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
14675 ));
14676 }
14677 }
14678 let exists: bool = self
14681 .conn
14682 .query_row(
14683 &format!(
14684 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
14685 visible_table(table),
14686 row_id_predicate(schema_table)
14687 ),
14688 rusqlite::params![row_id],
14689 |row| row.get(0),
14690 )
14691 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14692 if !exists {
14693 return Err("sync.row_missing: a sparse patch requires a local row".to_owned());
14694 }
14695 for scope in &schema_table.scope_variables {
14705 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
14706 continue;
14707 }
14708 let Some(column) = schema_table
14709 .columns
14710 .iter()
14711 .find(|column| column.name == scope.column)
14712 else {
14713 continue;
14714 };
14715 let stored = self
14716 .conn
14717 .query_row(
14718 &format!(
14719 "SELECT {} FROM {} WHERE {}",
14720 quote_ident(&scope.column),
14721 visible_table(&schema_table.name),
14722 row_id_predicate(schema_table)
14723 ),
14724 rusqlite::params![row_id],
14725 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
14726 )
14727 .optional()
14728 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14729 let incoming = json_to_column_value(column, partial.get(&scope.column))
14734 .ok()
14735 .map(|value| column_value_to_json(&value));
14736 let equal = stored.as_ref().is_some_and(|stored| {
14740 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
14741 });
14742 if !equal {
14743 return Err(format!(
14744 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
14745 scope.column
14746 ));
14747 }
14748 }
14749 let mut values = partial;
14750 for scope in &schema_table.scope_variables {
14753 values.remove(&scope.column);
14754 }
14755 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
14756 ColumnType::Integer => match row_id.parse::<i64>() {
14758 Ok(v)
14759 if v.to_string() == row_id
14760 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
14761 .contains(&v) =>
14762 {
14763 Value::from(v)
14764 }
14765 _ => {
14766 return Err(format!(
14767 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
14768 ))
14769 }
14770 },
14771 ColumnType::Boolean => match row_id {
14772 "true" => Value::Bool(true),
14773 "false" => Value::Bool(false),
14774 _ => {
14775 return Err(format!(
14776 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
14777 ))
14778 }
14779 },
14780 _ => Value::from(row_id.to_owned()),
14781 };
14782 values.insert(schema_table.primary_key.clone(), pk_value);
14783 self.validate_author_encode(schema_table, row_id, &values)?;
14789 Ok(OutboxOp {
14790 upsert: true,
14791 table: table.to_owned(),
14792 row_id: row_id.to_owned(),
14793 base_version,
14794 values: Some(values),
14795 })
14796 }
14797
14798 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
14803 let selector = self.encryption.key_id_columns.get(&table.name)?;
14804 let index = table
14805 .columns
14806 .iter()
14807 .position(|column| &column.name == selector)?;
14808 let sql = format!(
14809 "SELECT {} FROM {} WHERE {}",
14810 quote_ident(selector),
14811 visible_table(&table.name),
14812 row_id_predicate(table)
14813 );
14814 let key: Option<String> = self
14815 .conn
14816 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
14817 .ok()?;
14818 let key = key.filter(|key| !key.is_empty())?;
14819 let mut fallback: Row = vec![None; table.columns.len()];
14820 fallback[index] = Some(ColumnValue::String(key));
14821 Some(fallback)
14822 }
14823
14824 fn validate_author_encode(
14829 &self,
14830 table: &TableSchema,
14831 row_id: &str,
14832 values: &Map<String, Value>,
14833 ) -> Result<(), String> {
14834 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
14835 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
14836 result => result.map(|_| ()),
14837 }
14838 }
14839
14840 fn drop_unencodable_outbox(
14854 &mut self,
14855 with_encryption: bool,
14856 batch: Option<&mut ChangeAccumulator>,
14857 ) -> Result<bool, String> {
14858 let failures = self
14859 .outbox
14860 .iter()
14861 .filter_map(|commit| {
14862 commit
14863 .ops
14864 .iter()
14865 .find_map(|op| {
14866 let values = op.values.as_ref().filter(|_| op.upsert)?;
14867 let table = self.schema.table(&op.table)?;
14868 let fallback = self.stored_key_fallback(table, &op.row_id);
14869 match encode_sparse_row_json(
14870 table,
14871 &op.row_id,
14872 values,
14873 &self.encryption,
14874 fallback.as_ref(),
14875 ) {
14876 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
14877 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
14878 }
14879 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
14884 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
14885 OUTBOX_INCOMPATIBLE_CODE,
14886 )),
14887 _ => None,
14888 }
14889 })
14890 .map(|(error, code)| (commit.clone(), error, code))
14891 })
14892 .collect::<Vec<_>>();
14893 if failures.is_empty() {
14894 return Ok(false);
14895 }
14896 let prior_outbox = self.outbox.clone();
14902 let prior_rejections = self.rejections.clone();
14903 let prior_failed_commits = self.failed_commits.clone();
14907 let prior_changes = self.change_queue.len();
14908 let prior_intents = self.sync_intent_queue.len();
14909 let prior_last_change = self.last_change.clone();
14910 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14911 let owns_observation = batch.is_none();
14912 let owns_transaction = self.conn.is_autocommit();
14913 let mut domains = ChangeAccumulator::default();
14914 if owns_observation {
14915 self.begin_observation("syncular_value_recovery")?;
14916 }
14917 let dropped = (|| -> Result<(), String> {
14918 for (commit, message, code) in failures {
14919 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
14920 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
14921 }
14922 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
14923 let rejection = RejectionRecord {
14924 client_commit_id: commit.client_commit_id.clone(),
14925 op_index: 0,
14926 code: code.to_owned(),
14927 message,
14928 retryable: false,
14929 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
14930 field_paths: None,
14931 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
14932 required_action: None,
14933 references: None,
14934 }),
14935 operation: representable
14936 .then(|| commit.ops.first())
14937 .flatten()
14938 .map(CommitOperation::from),
14939 };
14940 self.persist_commit_outcome(
14941 &commit.client_commit_id,
14942 CommitOutcomeStatus::Rejected,
14943 &[CommitOperationOutcome::Error {
14944 rejection: rejection.clone(),
14945 }],
14946 representable.then_some(commit.ops.as_slice()),
14947 )?;
14948 self.delete_outbox_persisted(&commit.client_commit_id)?;
14949 self.outbox
14950 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
14951 self.rejections.push(rejection);
14952 for operation in &commit.ops {
14953 domains.table(operation.table.as_str());
14954 }
14955 }
14956 domains.status = true;
14957 domains.rejections = true;
14958 domains.outcomes = true;
14959 self.prune_commit_outcomes()?;
14960 self.overlay_dirty.set(true);
14962 self.rebuild_overlay()?;
14963 Ok(())
14964 })();
14965 let result = match batch {
14966 Some(batch) => {
14967 if dropped.is_ok() {
14968 for table in domains.tables.keys() {
14969 batch.table(table.as_str());
14970 }
14971 batch.status |= domains.status;
14972 batch.rejections |= domains.rejections;
14973 batch.outcomes |= domains.outcomes;
14974 }
14975 dropped
14976 }
14977 None => {
14978 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
14979 }
14980 };
14981 if let Err(error) = result {
14982 if owns_observation {
14983 if owns_transaction {
14984 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
14985 let mut failure = self.storage_failure.borrow_mut();
14986 let failure =
14987 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
14988 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
14989 }
14990 } else {
14991 self.rollback_observation("syncular_value_recovery");
14992 }
14993 }
14994 self.outbox = prior_outbox;
14995 self.rejections = prior_rejections;
14996 self.failed_commits = prior_failed_commits;
14997 self.change_queue.truncate(prior_changes);
14998 self.sync_intent_queue.truncate(prior_intents);
14999 self.last_change = prior_last_change;
15000 self.overlay_dirty.restore(prior_overlay_dirty);
15001 return Err(error);
15002 }
15003 Ok(true)
15004 }
15005
15006 pub fn pending_commit_ids(&self) -> Vec<String> {
15007 self.outbox
15008 .iter()
15009 .map(|c| c.client_commit_id.clone())
15010 .collect()
15011 }
15012
15013 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
15017 let mut payloads = Vec::new();
15018 for commit in &self.outbox {
15019 for op in &commit.ops {
15020 let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
15021 continue;
15022 };
15023 let Some(table) = self.schema.table(&op.table) else {
15024 continue;
15025 };
15026 let fallback = self.stored_key_fallback(table, &op.row_id);
15027 if let Ok(payload) = encode_sparse_row_json(
15028 table,
15029 &op.row_id,
15030 values,
15031 &self.encryption,
15032 fallback.as_ref(),
15033 ) {
15034 payloads.push(payload);
15035 }
15036 }
15037 }
15038 payloads
15039 }
15040
15041 pub fn conflicts(&self) -> &[ConflictRecord] {
15042 &self.conflicts
15043 }
15044
15045 pub fn rejections(&self) -> &[RejectionRecord] {
15046 &self.rejections
15047 }
15048
15049 pub fn resolve_commit_outcome(
15050 &mut self,
15051 input: ResolveCommitOutcomeInput,
15052 ) -> Result<CommitOutcome, String> {
15053 let current = self
15054 .commit_outcome(&input.client_commit_id)?
15055 .ok_or_else(|| {
15056 format!(
15057 "sync.outcome_not_found: no durable outcome exists for {:?}",
15058 input.client_commit_id
15059 )
15060 })?;
15061 if current.resolution != CommitOutcomeResolution::Active {
15062 return Ok(current);
15063 }
15064 if input.resolution == CommitOutcomeResolution::Active {
15065 return Err("sync.invalid_request: resolution must leave active state".to_owned());
15066 }
15067 match input.resolution {
15068 CommitOutcomeResolution::Superseded => {
15069 let replacement = input
15070 .replacement_client_commit_id
15071 .as_deref()
15072 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
15073 .ok_or_else(|| {
15074 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
15075 .to_owned()
15076 })?;
15077 let _ = replacement;
15078 }
15079 _ if input.replacement_client_commit_id.is_some() => {
15080 return Err(
15081 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
15082 .to_owned(),
15083 );
15084 }
15085 _ => {}
15086 }
15087 let allowed = match current.status {
15088 CommitOutcomeStatus::Conflict => matches!(
15089 input.resolution,
15090 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
15091 ),
15092 CommitOutcomeStatus::Rejected => {
15093 matches!(
15094 input.resolution,
15095 CommitOutcomeResolution::Superseded
15096 | CommitOutcomeResolution::ResolvedKeepServer
15097 )
15098 }
15099 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
15100 input.resolution == CommitOutcomeResolution::Dismissed
15101 }
15102 };
15103 if !allowed {
15104 return Err(format!(
15105 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
15106 input.resolution, current.status
15107 ));
15108 }
15109
15110 self.begin_observation("syncular_outcome_resolution")?;
15111 let previous_failed = self.failed_commits.clone();
15112 let previous_conflicts = self.conflicts.clone();
15113 let previous_rejections = self.rejections.clone();
15114 let result = (|| {
15115 self.conn
15116 .execute(
15117 "UPDATE _syncular_commit_outcomes
15118 SET resolution = ?1, resolved_at_ms = ?2,
15119 replacement_client_commit_id = ?3
15120 WHERE client_commit_id = ?4 AND resolution = 'active'",
15121 rusqlite::params![
15122 Self::outcome_resolution_name(input.resolution),
15123 self.clock_now_ms(),
15124 input.replacement_client_commit_id,
15125 input.client_commit_id
15126 ],
15127 )
15128 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15129 let resolved = self
15130 .commit_outcome(¤t.client_commit_id)?
15131 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
15132 self.conn
15133 .execute(
15134 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
15135 rusqlite::params![input.client_commit_id],
15136 )
15137 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15138 for (commit, _) in &self.failed_commits {
15139 if commit.client_commit_id == input.client_commit_id {
15140 for operation in &commit.ops {
15141 self.overlay_dirty.table(&operation.table);
15142 }
15143 }
15144 }
15145 self.failed_commits
15146 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
15147 self.prune_commit_outcomes()?;
15148 Ok(resolved)
15149 })();
15150 let resolved = match result {
15151 Ok(outcome) => outcome,
15152 Err(error) => {
15153 self.rollback_observation("syncular_outcome_resolution");
15154 self.failed_commits = previous_failed;
15155 self.conflicts = previous_conflicts;
15156 self.rejections = previous_rejections;
15157 return Err(error);
15158 }
15159 };
15160 self.conflicts
15161 .retain(|record| record.client_commit_id != current.client_commit_id);
15162 self.rejections
15163 .retain(|record| record.client_commit_id != current.client_commit_id);
15164 let mut batch = ChangeAccumulator {
15165 conflicts: current.status == CommitOutcomeStatus::Conflict,
15166 rejections: current.status == CommitOutcomeStatus::Rejected,
15167 outcomes: true,
15168 ..ChangeAccumulator::default()
15169 };
15170 if let Some(operations) = ¤t.operations {
15171 for operation in operations {
15172 batch.table(&operation.table);
15173 }
15174 }
15175 if let Err(error) = self
15176 .rebuild_overlay_if_dirty()
15177 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
15178 {
15179 self.rollback_observation("syncular_outcome_resolution");
15180 self.failed_commits = previous_failed;
15181 self.conflicts = previous_conflicts;
15182 self.rejections = previous_rejections;
15183 return Err(error);
15184 }
15185 Ok(resolved)
15186 }
15187
15188 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
15189 self.schema_floor.as_ref()
15190 }
15191
15192 pub fn lease_state(&self) -> Option<&LeaseState> {
15194 self.lease_state.as_ref()
15195 }
15196
15197 fn record_lease_error(&mut self, code: &str) {
15200 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
15201 return;
15202 }
15203 let mut next = self.lease_state.clone().unwrap_or_default();
15204 next.error_code = Some(code.to_owned());
15205 self.set_lease_state(Some(next));
15206 }
15207
15208 fn set_lease_state(&mut self, next: Option<LeaseState>) {
15209 if self.lease_state == next {
15210 return;
15211 }
15212 if self.begin_observation("syncular_lease").is_err() {
15213 return;
15214 }
15215 self.lease_state = next;
15216 if let Some(lease) = &self.lease_state {
15217 if let Ok(json) = serde_json::to_string(lease) {
15218 self.set_meta(LEASE_STATE_KEY, &json);
15219 }
15220 } else {
15221 self.delete_meta(LEASE_STATE_KEY);
15222 }
15223 let batch = ChangeAccumulator {
15224 status: true,
15225 ..ChangeAccumulator::default()
15226 };
15227 if self.finish_observation("syncular_lease", batch).is_err() {
15228 self.rollback_observation("syncular_lease");
15229 }
15230 }
15231
15232 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
15233 if self.schema_floor == next {
15234 return;
15235 }
15236 if self.begin_observation("syncular_schema_floor").is_err() {
15237 return;
15238 }
15239 self.schema_floor = next;
15240 self.stopped = self.schema_floor.is_some();
15241 if let Some(floor) = &self.schema_floor {
15242 if let Ok(json) = serde_json::to_string(floor) {
15243 self.set_meta(SCHEMA_FLOOR_KEY, &json);
15244 }
15245 } else {
15246 self.delete_meta(SCHEMA_FLOOR_KEY);
15247 }
15248 let batch = ChangeAccumulator {
15249 status: true,
15250 ..ChangeAccumulator::default()
15251 };
15252 if self
15253 .finish_observation("syncular_schema_floor", batch)
15254 .is_err()
15255 {
15256 self.rollback_observation("syncular_schema_floor");
15257 }
15258 }
15259
15260 fn set_upgrading(&mut self, value: bool) {
15261 if self.upgrading == value {
15262 return;
15263 }
15264 if self.begin_observation("syncular_upgrading").is_err() {
15265 return;
15266 }
15267 self.upgrading = value;
15268 let batch = ChangeAccumulator {
15269 status: true,
15270 ..ChangeAccumulator::default()
15271 };
15272 if self
15273 .finish_observation("syncular_upgrading", batch)
15274 .is_err()
15275 {
15276 self.rollback_observation("syncular_upgrading");
15277 }
15278 }
15279
15280 pub fn sync_needed(&self) -> bool {
15281 self.sync_needed
15282 }
15283
15284 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
15285 let sub = self.subs.iter().find(|s| s.id == id)?;
15286 Some(SubscriptionStateView {
15287 id: sub.id.clone(),
15288 table: sub.table.clone(),
15289 status: sub.state.name().to_owned(),
15290 cursor: sub.cursor,
15291 has_resume_token: sub.bootstrap_state.is_some(),
15292 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
15293 reason_code: sub.reason_code.clone(),
15294 })
15295 }
15296
15297 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
15298 let schema_table = self
15299 .schema
15300 .table(table)
15301 .ok_or_else(|| format!("unknown table {table:?}"))?;
15302 let sql = format!(
15303 "SELECT * FROM {} ORDER BY {} ASC",
15304 visible_table(table),
15305 quote_ident(&schema_table.primary_key)
15306 );
15307 let mut stmt = self
15308 .conn
15309 .prepare(&sql)
15310 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15311 let mut rows = stmt
15312 .query([])
15313 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15314 let mut out = Vec::new();
15315 while let Some(row) = rows
15316 .next()
15317 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15318 {
15319 let mut values = Map::new();
15320 for (i, column) in schema_table.columns.iter().enumerate() {
15321 let value = row
15322 .get_ref(i)
15323 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15324 values.insert(column.name.clone(), sql_ref_to_json(column, value));
15325 }
15326 let version: i64 = row
15327 .get(schema_table.columns.len())
15328 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15329 let row_id = match values.get(&schema_table.primary_key) {
15330 Some(Value::String(s)) => s.clone(),
15331 Some(Value::Number(n)) => n.to_string(),
15332 Some(Value::Bool(b)) => b.to_string(),
15333 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
15334 };
15335 out.push(RowState {
15336 row_id,
15337 version,
15338 values,
15339 });
15340 }
15341 Ok(out)
15342 }
15343
15344 #[cfg(feature = "crdt-yjs")]
15360 fn crdt_column_bytes(
15361 &self,
15362 table: &str,
15363 row_id: &str,
15364 column: &str,
15365 ) -> Result<Option<Vec<u8>>, String> {
15366 let schema_table = self
15367 .schema
15368 .table(table)
15369 .ok_or_else(|| format!("unknown table {table:?}"))?;
15370 let col = schema_table
15371 .columns
15372 .iter()
15373 .find(|c| c.name == column)
15374 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
15375 if col.ty != ColumnType::Crdt {
15376 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
15377 }
15378 let sql = format!(
15379 "SELECT {} FROM {} WHERE {}",
15380 quote_ident(column),
15381 visible_table(table),
15382 row_id_predicate(schema_table)
15383 );
15384 let bytes: Option<Vec<u8>> = self
15385 .conn
15386 .prepare_cached(&sql)
15387 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15388 .query_row(rusqlite::params![row_id], |row| {
15389 row.get::<_, Option<Vec<u8>>>(0)
15390 })
15391 .map_err(|e| match e {
15392 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
15393 other => Self::sqlite_failure(&self.storage_failure, other),
15394 })?;
15395 Ok(bytes)
15396 }
15397
15398 #[cfg(feature = "crdt-yjs")]
15402 pub fn crdt_text(
15403 &self,
15404 table: &str,
15405 row_id: &str,
15406 column: &str,
15407 name: &str,
15408 ) -> Result<String, String> {
15409 let bytes = self
15410 .crdt_column_bytes(table, row_id, column)?
15411 .unwrap_or_default();
15412 crate::crdt::text(&bytes, name)
15413 }
15414
15415 #[cfg(feature = "crdt-yjs")]
15419 pub fn crdt_insert_text(
15420 &mut self,
15421 table: &str,
15422 row_id: &str,
15423 column: &str,
15424 name: &str,
15425 index: u32,
15426 value: &str,
15427 ) -> Result<String, ClientError> {
15428 let prior = self.storage_failure.borrow_mut().take();
15429 let outcome = (|| -> Result<String, String> {
15430 let current = self
15431 .crdt_column_bytes(table, row_id, column)?
15432 .unwrap_or_default();
15433 let update = crate::crdt::insert_text(¤t, name, index, value)?;
15434 self.perform_crdt_push_update(table, row_id, column, &update)
15435 })();
15436 self.finish_authoring(prior, outcome)
15437 }
15438
15439 #[cfg(feature = "crdt-yjs")]
15443 pub fn crdt_delete_text(
15444 &mut self,
15445 table: &str,
15446 row_id: &str,
15447 column: &str,
15448 name: &str,
15449 index: u32,
15450 len: u32,
15451 ) -> Result<String, ClientError> {
15452 let prior = self.storage_failure.borrow_mut().take();
15453 let outcome = (|| -> Result<String, String> {
15454 let current = self
15455 .crdt_column_bytes(table, row_id, column)?
15456 .unwrap_or_default();
15457 let update = crate::crdt::delete_text(¤t, name, index, len)?;
15458 self.perform_crdt_push_update(table, row_id, column, &update)
15459 })();
15460 self.finish_authoring(prior, outcome)
15461 }
15462
15463 #[cfg(feature = "crdt-yjs")]
15468 pub fn crdt_apply_update(
15469 &mut self,
15470 table: &str,
15471 row_id: &str,
15472 column: &str,
15473 update: &[u8],
15474 ) -> Result<String, ClientError> {
15475 let prior = self.storage_failure.borrow_mut().take();
15476 let outcome = (|| -> Result<String, String> {
15477 let current = self
15478 .crdt_column_bytes(table, row_id, column)?
15479 .unwrap_or_default();
15480 let next = crate::crdt::apply_update(¤t, update)?;
15481 self.perform_crdt_push_update(table, row_id, column, &next)
15482 })();
15483 self.finish_authoring(prior, outcome)
15484 }
15485
15486 #[cfg(feature = "crdt-yjs")]
15493 fn perform_crdt_push_update(
15494 &mut self,
15495 table: &str,
15496 row_id: &str,
15497 column: &str,
15498 crdt_bytes: &[u8],
15499 ) -> Result<String, String> {
15500 let mut bytes_obj = Map::new();
15501 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
15502 let mut partial = Map::new();
15503 partial.insert(column.to_owned(), Value::Object(bytes_obj));
15504 self.perform_patch(table, row_id, partial, None)
15505 }
15506
15507 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
15522 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
15523 }
15524
15525 #[cfg(feature = "bench-internals")]
15527 #[doc(hidden)]
15528 pub fn benchmark_connection(&mut self) -> &mut Connection {
15529 &mut self.conn
15530 }
15531
15532 #[cfg(feature = "bench-internals")]
15534 #[doc(hidden)]
15535 pub fn benchmark_phases(
15536 &self,
15537 enabled: Option<bool>,
15538 reset: bool,
15539 ) -> Result<Option<Value>, String> {
15540 self.benchmark_phases.configure(enabled, reset)?;
15541 Ok(self.benchmark_phases.snapshot())
15542 }
15543
15544 pub fn authority_snapshot(&self) -> Result<Value, String> {
15546 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
15547 if self.limits.authority_reads.is_empty() {
15548 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
15549 }
15550 self.conn
15551 .execute_batch("SAVEPOINT authority_snapshot")
15552 .map_err(|e| e.to_string())?;
15553 let result = (|| {
15554 let revision = self
15555 .get_meta(LOCAL_REVISION_KEY)
15556 .ok_or("sync.local_corrupt: authority revision is missing")?;
15557 let parsed_revision = revision
15558 .parse::<u64>()
15559 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
15560 if parsed_revision.to_string() != revision {
15561 return Err("sync.local_corrupt: authority revision is invalid".into());
15562 }
15563 let mut tables = Vec::new();
15564 for read in &self.limits.authority_reads {
15565 let table = self
15566 .schema
15567 .table(&read.table)
15568 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
15569 let scalar = |column: &str, value: &Value| -> Result<String, String> {
15570 let ty = table
15571 .columns
15572 .iter()
15573 .find(|c| c.name == column)
15574 .expect("validated authority column")
15575 .ty;
15576 if ty == ColumnType::Boolean {
15577 if let Some(value) = value.as_bool() {
15578 return Ok(value.to_string());
15579 }
15580 return match value.as_i64() {
15581 Some(0) => Ok("false".into()),
15582 Some(1) => Ok("true".into()),
15583 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
15584 };
15585 }
15586 match value {
15587 Value::String(s) => Ok(s.clone()),
15588 Value::Number(n) => Ok(n.to_string()),
15589 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
15590 .get("$bigint")
15591 .and_then(Value::as_str)
15592 .map(str::to_owned)
15593 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
15594 _ => {
15595 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
15596 }
15597 }
15598 };
15599 let matches = |row: &QueryRow| -> Result<bool, String> {
15600 for (variable, values) in &read.scopes {
15601 let column = table
15602 .scope_column(variable)
15603 .expect("validated authority scope");
15604 let value = row
15605 .get(column)
15606 .ok_or("sync.local_corrupt: authority scope column is missing")?;
15607 if value.is_null() || !values.contains(&scalar(column, value)?) {
15608 return Ok(false);
15609 }
15610 }
15611 Ok(true)
15612 };
15613 let selected = read
15614 .columns
15615 .iter()
15616 .map(|c| quote_ident(c))
15617 .collect::<Vec<_>>()
15618 .join(",");
15619 let mut intent = BTreeSet::new();
15620 let mut relevant = BTreeSet::new();
15621 for row in query_connection(&self.conn,
15622 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
15623 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
15624 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
15625 intent.insert(id.clone());
15626 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
15627 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
15628 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
15629 }
15630 }
15631 let mut rows = BTreeMap::new();
15632 for full in [base_table(&read.table), visible_table(&read.table)] {
15633 for mut values in query_connection(
15634 &self.conn,
15635 &format!(
15636 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
15637 ),
15638 &[],
15639 )
15640 .map_err(|e| e.to_string())?
15641 {
15642 if !matches(&values)? {
15643 continue;
15644 }
15645 let id = scalar(
15646 &table.primary_key,
15647 values
15648 .get(&table.primary_key)
15649 .expect("selected primary key"),
15650 )?;
15651 relevant.insert(id.clone());
15652 if full == base_table(&read.table) {
15653 let version = values
15654 .remove("authority_version")
15655 .expect("selected version");
15656 let version = version
15657 .as_i64()
15658 .ok_or("sync.local_corrupt: authority server version is invalid")?;
15659 if version >= 0 {
15660 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
15661 }
15662 }
15663 }
15664 }
15665 intent.retain(|id| relevant.contains(id));
15666 let rows = rows.into_values().collect::<Vec<_>>();
15667 let mut persisted = Vec::new();
15668 for row in query_connection(
15669 &self.conn,
15670 "SELECT state_json FROM _syncular_subscriptions WHERE tbl=? ORDER BY rowid",
15671 &[Value::String(read.table.clone())],
15672 )
15673 .map_err(|e| e.message)?
15674 {
15675 let state: Value = serde_json::from_str(
15676 row.get("state_json")
15677 .and_then(Value::as_str)
15678 .ok_or("sync.local_corrupt: authority subscription state is missing")?,
15679 )
15680 .map_err(|_| "sync.local_corrupt: authority subscription state is invalid")?;
15681 let effective = state.get("effectiveScopes").and_then(Value::as_object);
15682 let status = state
15683 .get("status")
15684 .and_then(Value::as_str)
15685 .ok_or("sync.local_corrupt: authority subscription status is missing")?;
15686 let cursor = state
15687 .get("cursor")
15688 .and_then(Value::as_i64)
15689 .ok_or("sync.local_corrupt: authority subscription cursor is missing")?;
15690 let complete = effective.is_some_and(|s| s.len() == read.scopes.len())
15691 && status == "active"
15692 && cursor >= 0
15693 && state.get("bootstrapState").is_none_or(Value::is_null)
15694 && state.get("params").is_none_or(Value::is_null);
15695 let exposed = effective.map(|s| {
15696 read.scopes
15697 .keys()
15698 .map(|v| (v.clone(), s.get(v).cloned().unwrap_or(json!([]))))
15699 .collect::<Map<_, _>>()
15700 });
15701 let requested = read
15702 .scopes
15703 .keys()
15704 .map(|v| {
15705 (
15706 v.clone(),
15707 state
15708 .get("requested")
15709 .and_then(|s| s.get(v))
15710 .cloned()
15711 .unwrap_or(json!([])),
15712 )
15713 })
15714 .collect::<Map<_, _>>();
15715 persisted.push(json!({"requestedScopes": requested, "status": status, "cursor": cursor, "effectiveScopes": exposed, "complete": complete}));
15716 }
15717 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
15718 match selectors.split_first() {
15719 None => !candidates.is_empty(),
15720 Some(((variable, values), rest)) => values.iter().all(|value| {
15721 covered(
15722 rest,
15723 &candidates
15724 .iter()
15725 .copied()
15726 .filter(|p| {
15727 p["effectiveScopes"][variable.as_str()]
15728 .as_array()
15729 .is_some_and(|held| {
15730 held.contains(&Value::String(value.clone()))
15731 || held.contains(&Value::String("*".into()))
15732 })
15733 })
15734 .collect::<Vec<_>>(),
15735 )
15736 }),
15737 }
15738 }
15739 let coverage = if covered(
15740 &read.scopes.iter().collect::<Vec<_>>(),
15741 &persisted
15742 .iter()
15743 .filter(|p| p["complete"] == true)
15744 .collect::<Vec<_>>(),
15745 ) {
15746 "complete"
15747 } else if persisted.iter().any(|p| {
15748 p["status"] == "active"
15749 && read.scopes.iter().all(|(v, values)| {
15750 values.iter().any(|value| {
15751 p["requestedScopes"][v].as_array().is_some_and(|held| {
15752 held.contains(&Value::String(value.clone()))
15753 || held.contains(&Value::String("*".into()))
15754 })
15755 })
15756 })
15757 }) {
15758 "pending"
15759 } else {
15760 "missing"
15761 };
15762 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
15763 }
15764 Ok(
15765 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
15766 )
15767 })();
15768 match result {
15769 Ok(snapshot) => {
15770 self.conn
15771 .execute_batch("RELEASE authority_snapshot")
15772 .map_err(|e| e.to_string())?;
15773 Ok(snapshot)
15774 }
15775 Err(error) => {
15776 self.conn
15777 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
15778 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
15779 Err(error)
15780 }
15781 }
15782 }
15783
15784 pub fn query_snapshot(
15788 &mut self,
15789 sql: &str,
15790 params: &[QueryValue],
15791 coverage: &[WindowCoverage],
15792 owner: Option<QueryOwner<'_>>,
15793 ) -> Result<QuerySnapshot, String> {
15794 let result = snapshot_connection(&self.conn, sql, params, coverage);
15795 if let Some(owner) = owner {
15796 self.record_query_read(owner, result.as_ref().err());
15797 }
15798 result.map_err(|failure| failure.to_string())
15799 }
15800
15801 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
15805 let position = self
15806 .query_failures
15807 .iter()
15808 .position(|entry| entry.id == owner.id);
15809 let Some(failure) = failure else {
15810 if let Some(index) = position {
15811 self.query_failures.remove(index);
15812 }
15813 return;
15814 };
15815 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
15816 .into_iter()
15817 .map(str::to_owned)
15818 .collect();
15819 let code = failure.code.unwrap_or("client.query_failed").to_owned();
15820 if position.is_some_and(|index| {
15821 let entry = &self.query_failures[index];
15822 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
15823 }) {
15824 return;
15825 }
15826 if let Some(index) = position {
15827 self.query_failures.remove(index);
15828 }
15829 self.query_failures.push(DiagnosticQueryFailure {
15830 id: owner.id.to_owned(),
15831 tables,
15832 code,
15833 sqlite_code: failure.sqlite_code,
15834 at_ms: self.clock_now_ms(),
15835 });
15836 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
15837 self.query_failures.remove(0);
15838 }
15839 }
15840
15841 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
15844 #[cfg(feature = "bench-internals")]
15845 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
15846 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
15847 let max_commits = self.limits.max_push_commits_per_request;
15848 let max_ops = self
15849 .limits
15850 .max_push_operations_per_request
15851 .unwrap_or(PUSH_OPS_PER_REQUEST);
15852 let byte_cap = self.limits.max_push_request_bytes;
15853 let accept = self.limits.accept.unwrap_or(if url_capable {
15856 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
15857 } else {
15858 DEFAULT_ACCEPT
15859 });
15860 let header = Frame::ReqHeader {
15861 client_id: self.client_id.clone(),
15862 schema_version: self.schema.version,
15863 log_epoch: log_epoch.clone(),
15864 };
15865 let pull = Frame::PullHeader {
15866 limit_commits: self.limits.limit_commits.unwrap_or(0),
15867 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
15868 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
15869 accept,
15870 };
15871 let mut fresh = Vec::new();
15872 let mut sub_frames = Vec::new();
15873 for sub in &self.subs {
15874 if sub.state != SubState::Active {
15875 continue;
15876 }
15877 let mut scopes = sub.requested.clone();
15878 sort_scope_map(&mut scopes);
15879 sub_frames.push(Frame::Subscription {
15880 id: sub.id.clone(),
15881 table: sub.table.clone(),
15882 scopes,
15883 params: sub.params.clone().map(RawJson),
15884 cursor: sub.cursor,
15885 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
15886 });
15887 fresh.push((
15888 sub.id.clone(),
15889 sub.cursor < 0 && sub.bootstrap_state.is_none(),
15890 ));
15891 }
15892 let fixed_bytes = match byte_cap {
15895 Some(_) => {
15896 let mut frames = vec![header.clone(), pull.clone()];
15897 frames.extend(sub_frames.iter().cloned());
15898 encode_message(&Message {
15899 wire_version: WIRE_VERSION,
15900 msg_kind: MsgKind::Request,
15901 frames,
15902 })
15903 .len()
15904 }
15905 None => 0,
15906 };
15907 if let Some(cap) = byte_cap {
15908 if fixed_bytes > cap {
15909 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
15910 }
15911 }
15912 let base_bytes = match byte_cap {
15913 Some(_) => encode_message(&Message {
15914 wire_version: WIRE_VERSION,
15915 msg_kind: MsgKind::Request,
15916 frames: vec![header.clone(), pull.clone()],
15917 })
15918 .len(),
15919 None => 0,
15920 };
15921 let total_outbox = if log_epoch.is_some() {
15922 self.outbox.len()
15923 } else {
15924 0
15925 };
15926 let mut push_frames: Vec<Frame> = Vec::new();
15927 let mut pushed_ids = Vec::new();
15928 let mut ops_in_request = 0usize;
15929 let mut bytes_in_request = 0usize;
15930 let mut deferred_commits = 0usize;
15931 #[cfg(feature = "bench-internals")]
15932 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
15933 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
15934 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
15938 deferred_commits = total_outbox - index;
15939 break;
15940 }
15941 if ops_in_request + commit.ops.len() > max_ops {
15942 if push_frames.is_empty() {
15943 return Err(Self::push_capacity_outcome(
15944 "operations",
15945 max_ops,
15946 commit.ops.len(),
15947 Some(commit.client_commit_id.as_str()),
15948 ));
15949 }
15950 deferred_commits = total_outbox - index;
15951 break;
15952 }
15953 let operations = commit
15954 .ops
15955 .iter()
15956 .map(|op| {
15957 let payload = op.values.as_ref().and_then(|values| {
15958 let table = self.schema.table(&op.table)?;
15959 let fallback = self.stored_key_fallback(table, &op.row_id);
15968 encode_sparse_row_json(
15969 table,
15970 &op.row_id,
15971 values,
15972 &self.encryption,
15973 fallback.as_ref(),
15974 )
15975 .ok()
15976 });
15977 ssp2::model::Operation {
15978 table: op.table.clone(),
15979 row_id: op.row_id.clone(),
15980 op: if op.upsert { Op::Upsert } else { Op::Delete },
15981 base_version: op.base_version,
15982 payload,
15983 }
15984 })
15985 .collect();
15986 let frame = Frame::PushCommit {
15987 client_commit_id: commit.client_commit_id.clone(),
15988 operations,
15989 };
15990 let frame_bytes = match byte_cap {
15991 Some(_) => {
15992 encode_message(&Message {
15993 wire_version: WIRE_VERSION,
15994 msg_kind: MsgKind::Request,
15995 frames: vec![header.clone(), frame.clone(), pull.clone()],
15996 })
15997 .len()
15998 - base_bytes
15999 }
16000 None => 0,
16001 };
16002 if let Some(cap) = byte_cap {
16006 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
16007 deferred_commits = total_outbox - index;
16008 break;
16009 }
16010 }
16011 push_frames.push(frame);
16012 pushed_ids.push(commit.client_commit_id.clone());
16013 ops_in_request += commit.ops.len();
16014 bytes_in_request += frame_bytes;
16015 }
16016 #[cfg(feature = "bench-internals")]
16017 drop(outbox_phase);
16018 if let Some(cap) = byte_cap {
16019 if fixed_bytes + bytes_in_request > cap {
16020 let blocked = if push_frames.is_empty() {
16023 None
16024 } else {
16025 self.outbox.first().map(|c| c.client_commit_id.as_str())
16026 };
16027 return Err(Self::push_capacity_outcome(
16028 "bytes",
16029 cap,
16030 fixed_bytes + bytes_in_request,
16031 blocked,
16032 ));
16033 }
16034 }
16035 let mut frames = vec![header];
16036 frames.extend(push_frames);
16037 frames.push(pull);
16038 frames.extend(sub_frames);
16039 let message = Message {
16040 wire_version: WIRE_VERSION,
16041 msg_kind: MsgKind::Request,
16042 frames,
16043 };
16044 Ok((
16045 message,
16046 RequestMeta {
16047 pushed_ids,
16048 fresh,
16049 accept,
16050 deferred_commits,
16051 },
16052 ))
16053 }
16054
16055 fn push_capacity_outcome(
16057 kind: &str,
16058 limit: usize,
16059 size: usize,
16060 client_commit_id: Option<&str>,
16061 ) -> Box<SyncOutcome> {
16062 let mut details = Map::new();
16063 details.insert("kind".to_owned(), Value::from(kind));
16064 details.insert("limit".to_owned(), Value::from(limit as u64));
16065 details.insert("size".to_owned(), Value::from(size as u64));
16066 if let Some(id) = client_commit_id {
16067 details.insert("clientCommitId".to_owned(), Value::from(id));
16068 }
16069 Box::new(SyncOutcome::Failed {
16070 error_code: "client.push_request_too_large".into(),
16071 message: "the push request exceeds a configured capacity limit".into(),
16072 details: Some(Value::Object(details)),
16073 })
16074 }
16075
16076 pub fn progress(&self) -> ProgressObserver {
16080 self.progress.clone()
16081 }
16082
16083 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
16084 let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
16085 Ok(prepared) => prepared,
16086 Err(outcome) => return *outcome,
16087 };
16088 let mut next = prepared;
16093 loop {
16094 match self.apply_sync_round(next.exchange(transport)) {
16095 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
16096 crate::AppliedSyncRound::Complete {
16097 outcome, controls, ..
16098 } => {
16099 if self.transport_enabled {
16100 for text in controls {
16101 let _ = transport.realtime_send(&text);
16102 }
16103 } else {
16104 self.disconnect_realtime(transport);
16105 }
16106 return outcome;
16107 }
16108 }
16109 }
16110 }
16111
16112 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
16113 if let SyncOutcome::Failed {
16114 error_code,
16115 message,
16116 details,
16117 } = &mut outcome
16118 {
16119 if let Some(failure) = self.storage_failure.borrow().as_ref() {
16120 let code = failure
16121 .code
16122 .expect("retained storage failures are classified");
16123 *error_code = code.to_owned();
16124 *message = match code {
16125 "client.storage_busy" => "local SQLite storage is busy",
16126 "client.storage_full" => "local SQLite storage is full",
16127 "client.storage_io" => "local SQLite storage I/O failed",
16128 "client.storage_corrupt" => "local SQLite storage is corrupt",
16129 _ => unreachable!("unrecognized classified storage failure"),
16130 }
16131 .to_owned();
16132 *details = failure.details();
16133 }
16134 }
16135 let retry_delay_ms = self.round_retry_delay_ms;
16136 self.progress.update(|p| match &outcome {
16137 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
16138 if report.failed.is_empty() =>
16139 {
16140 p.state = ProgressState::Complete
16141 }
16142 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
16143 p.state = ProgressState::Failed;
16144 p.error_code = Some("sync.scope_revoked".into());
16145 }
16146 SyncOutcome::Failed { error_code, .. } => {
16147 p.state = ProgressState::Failed;
16148 p.error_code = Some(Self::diagnostic_code(error_code));
16149 p.retry_delay_ms = retry_delay_ms;
16150 }
16151 SyncOutcome::RealtimeUnavailable { .. } => {
16152 p.state = ProgressState::Failed;
16153 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
16154 p.retry_delay_ms = retry_delay_ms;
16155 }
16156 });
16157 let completed_at_ms = self.clock_now_ms();
16158 self.last_round = Some(match &outcome {
16159 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
16160 status: "succeeded".to_owned(),
16161 started_at_ms,
16162 completed_at_ms,
16163 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16164 counters: Some(DiagnosticRoundCounters {
16165 pushed: report.pushed,
16166 applied: report.applied.len(),
16167 rejected: report.rejected.len(),
16168 retryable: report.retryable.len(),
16169 conflicts: report.conflicts,
16170 commits_applied: report.commits_applied,
16171 segment_rows_applied: report.segment_rows_applied,
16172 bootstrapping: report.bootstrapping.len(),
16173 resets: report.resets.len(),
16174 revoked: report.revoked.len(),
16175 failed: report.failed.len(),
16176 deferred_commits: report.deferred_commits,
16177 }),
16178 error_code: None,
16179 },
16180 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
16181 status: "failed".to_owned(),
16182 started_at_ms,
16183 completed_at_ms,
16184 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16185 counters: None,
16186 error_code: Some(Self::diagnostic_code(error_code)),
16187 },
16188 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
16189 status: "failed".to_owned(),
16190 started_at_ms,
16191 completed_at_ms,
16192 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
16193 counters: None,
16194 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
16195 },
16196 });
16197 outcome
16198 }
16199
16200 pub fn prepare_sync_round(
16203 &mut self,
16204 url_capable: bool,
16205 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
16206 if !self.transport_enabled {
16207 return Err(Box::new(SyncOutcome::Failed {
16208 error_code: "sync.offline".into(),
16209 message: "transport gate is closed".into(),
16210 details: None,
16211 }));
16212 }
16213 if self.active_round.is_some() {
16214 return Err(Box::new(SyncOutcome::Failed {
16215 error_code: "client.round_in_progress".into(),
16216 message: "a sync round is already in flight".into(),
16217 details: None,
16218 }));
16219 }
16220 self.storage_failure.borrow_mut().take();
16221 self.progress.start();
16222 self.round_retry_delay_ms = None;
16223 let started_at_ms = self.clock_now_ms();
16224 let prepared = (|| {
16225 if self.stopped {
16226 return Err(Box::new(SyncOutcome::Ok(SyncReport {
16227 schema_floor: self.schema_floor.clone(),
16228 ..SyncReport::default()
16229 })));
16230 }
16231 if self.security_preflight {
16232 return Err(Box::new(SyncOutcome::Failed {
16233 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
16234 message: "security preflight must complete before syncing".into(),
16235 details: None,
16236 }));
16237 }
16238 self.set_sync_needed(false, false);
16239 self.drain_pending_evictions()
16240 .and_then(|_| self.drop_incompatible_outbox())
16241 .and_then(|_| self.drop_unencodable_outbox(true, None))
16242 .map_err(|message| {
16243 Box::new(SyncOutcome::Failed {
16244 error_code: "storage.failed".into(),
16245 message,
16246 details: None,
16247 })
16248 })?;
16249 let uploads = if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
16250 self.pending_blob_uploads().map_err(|e| {
16251 Box::new(SyncOutcome::Failed {
16252 error_code: e.code,
16253 message: e.message,
16254 details: e.details,
16255 })
16256 })?
16257 } else {
16258 Vec::new()
16259 };
16260 if self.realtime_state != RealtimeState::Connected
16261 && self.realtime_policy == RealtimePolicy::Required
16262 {
16263 let retry_delay_ms = self.schedule_background_retry();
16264 self.realtime_retry_delay_ms = Some(retry_delay_ms);
16265 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
16266 state: self.realtime_state,
16267 reason_code: self.realtime_reason_code.clone(),
16268 retry_delay_ms,
16269 }));
16270 }
16271 let (message, meta) = self.build_request(url_capable)?;
16272 let id = uuid::Uuid::new_v4();
16273 self.active_round = Some(id);
16274 Ok(crate::PreparedSyncRound {
16275 id,
16276 started_at_ms,
16277 message,
16278 meta,
16279 uploads: uploads.into(),
16280 realtime: self.realtime_state == RealtimeState::Connected,
16281 scopes: self
16282 .subs
16283 .iter()
16284 .map(|s| (s.id.clone(), canonical_scope_json(&s.requested)))
16285 .collect(),
16286 #[cfg(feature = "bench-internals")]
16287 benchmark_phases: self.benchmark_phases.clone(),
16288 progress: self.progress.clone(),
16289 fixed_now: self.now_ms,
16290 round_deadline_at: None,
16291 })
16292 })();
16293 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
16294 }
16295
16296 pub fn cancel_sync_round(&mut self) {
16299 self.active_round = None;
16300 }
16301
16302 pub fn apply_sync_round(
16305 &mut self,
16306 completed: crate::CompletedSyncRound,
16307 ) -> crate::AppliedSyncRound {
16308 let prepared = completed.prepared;
16309 if self.active_round != Some(prepared.id) || self.security_preflight {
16310 return crate::AppliedSyncRound::Complete {
16311 outcome: SyncOutcome::Failed {
16312 error_code: "client.round_cancelled".into(),
16313 message: "sync round belongs to a released client context".into(),
16314 details: None,
16315 },
16316 controls: Vec::new(),
16317 more: false,
16318 bootstrap_advanced: false,
16319 };
16320 }
16321 self.storage_failure.borrow_mut().take();
16322 let (response, mut downloads, transport_failed) = match completed.exchange {
16323 crate::round::ExchangeResult::Upload { id, result } => {
16324 let result = result.and_then(|()| {
16325 self.conn
16326 .execute(
16327 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
16328 rusqlite::params![id],
16329 )
16330 .map(|_| ())
16331 .map_err(|error| {
16332 TransportError::new(
16333 "client.failed",
16334 Self::sqlite_failure(&self.storage_failure, error),
16335 )
16336 })
16337 });
16338 match result {
16339 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
16340 Err(TransportError {
16341 code,
16342 message,
16343 details,
16344 }) => {
16345 self.active_round = None;
16346 if Self::retryable_failure_code(&code) {
16347 self.schedule_background_retry();
16348 }
16349 let outcome = self.finish_sync_round(
16350 prepared.started_at_ms,
16351 SyncOutcome::Failed {
16352 error_code: code,
16353 message,
16354 details,
16355 },
16356 );
16357 return crate::AppliedSyncRound::Complete {
16358 outcome,
16359 controls: Vec::new(),
16360 more: false,
16361 bootstrap_advanced: false,
16362 };
16363 }
16364 }
16365 }
16366 crate::round::ExchangeResult::Reply {
16367 response,
16368 downloads,
16369 transport_failed,
16370 } => (response, downloads, transport_failed),
16371 };
16372 self.active_round = None;
16373 let before = self
16374 .subs
16375 .iter()
16376 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16377 .collect::<Vec<_>>();
16378 let outcome = (|| {
16379 let response = match response {
16380 Ok(response) => response,
16381 Err(TransportError {
16382 code,
16383 message,
16384 details,
16385 }) => {
16386 if transport_failed {
16387 if prepared.realtime {
16388 self.set_realtime_state(RealtimeState::Lost, Some(&code));
16389 }
16390 self.record_lease_error(&code);
16391 }
16392 if Self::retryable_failure_code(&code) {
16393 let delay = self.schedule_background_retry();
16394 if prepared.realtime && transport_failed {
16395 self.realtime_retry_delay_ms = Some(delay);
16396 }
16397 }
16398 return SyncOutcome::Failed {
16399 error_code: code,
16400 message,
16401 details,
16402 };
16403 }
16404 };
16405 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
16406 if let SyncOutcome::Ok(report) = &mut outcome {
16407 report.deferred_commits = prepared.meta.deferred_commits;
16408 }
16409 match &outcome {
16410 SyncOutcome::Ok(_) => self.reset_background_retry(),
16411 SyncOutcome::Failed { error_code, .. }
16412 if Self::retryable_failure_code(error_code) =>
16413 {
16414 self.schedule_background_retry();
16415 }
16416 _ => {}
16417 }
16418 if prepared.meta.deferred_commits > 0 {
16419 self.set_sync_needed(true, true);
16420 }
16421 self.previous_version_lifetime_check();
16422 outcome
16423 })();
16424 let more = self.transport_enabled
16425 && matches!(&outcome, SyncOutcome::Ok(report)
16426 if report.schema_floor.is_none()
16427 && (!report.bootstrapping.is_empty()
16428 || report.commits_applied > 0
16429 || report.segment_rows_applied > 0
16430 || !report.resets.is_empty()
16431 || report.deferred_commits > 0
16432 || self.sync_needed));
16433 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
16434 && before
16435 != self
16436 .subs
16437 .iter()
16438 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
16439 .collect::<Vec<_>>();
16440 crate::AppliedSyncRound::Complete {
16441 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
16442 controls: if self.transport_enabled {
16443 downloads.controls
16444 } else {
16445 Vec::new()
16446 },
16447 more,
16448 bootstrap_advanced,
16449 }
16450 }
16451
16452 pub fn sync_until_idle(
16453 &mut self,
16454 transport: &mut dyn Transport,
16455 max_rounds: Option<u32>,
16456 ) -> SyncOutcome {
16457 if max_rounds == Some(0) {
16460 return SyncOutcome::Failed {
16461 details: None,
16462 error_code: "sync.invalid_request".into(),
16463 message: "maxRounds must be a positive integer".into(),
16464 };
16465 }
16466 let rounds = max_rounds.unwrap_or(20);
16467 let mut aggregate = SyncReport::default();
16468 let mut spent = 0;
16469 while spent < rounds {
16470 spent += 1;
16471 let before = self
16472 .subs
16473 .iter()
16474 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16475 .collect::<Vec<_>>();
16476 match self.sync(transport) {
16477 SyncOutcome::Failed {
16478 error_code,
16479 message,
16480 details,
16481 } => {
16482 return SyncOutcome::Failed {
16483 details,
16484 error_code,
16485 message,
16486 };
16487 }
16488 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
16489 SyncOutcome::BudgetExhausted(_) => {
16490 unreachable!("sync() never exhausts a round budget")
16491 }
16492 SyncOutcome::Ok(report) => {
16493 if max_rounds.is_none()
16494 && report.segment_rows_applied > 0
16495 && !report.bootstrapping.is_empty()
16496 && before
16497 != self
16498 .subs
16499 .iter()
16500 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
16501 .collect::<Vec<_>>()
16502 {
16503 spent = 0;
16504 }
16505 let stop = !self.transport_enabled() || report.schema_floor.is_some();
16508 aggregate.merge(&report);
16509 if stop {
16510 return SyncOutcome::Ok(aggregate);
16511 }
16512 let more = !report.bootstrapping.is_empty()
16518 || report.commits_applied > 0
16519 || report.segment_rows_applied > 0
16520 || !report.resets.is_empty()
16521 || report.deferred_commits > 0
16522 || self.sync_needed;
16523 if !more {
16524 return SyncOutcome::Ok(aggregate);
16525 }
16526 }
16527 }
16528 }
16529 SyncOutcome::BudgetExhausted(aggregate)
16532 }
16533
16534 fn process_response(
16535 &mut self,
16536 transport: &mut dyn Transport,
16537 response: Message,
16538 meta: &RequestMeta,
16539 ) -> SyncOutcome {
16540 #[cfg(feature = "bench-internals")]
16541 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
16542 let mut report = SyncReport {
16543 pushed: meta.pushed_ids.len() as u32,
16544 ..SyncReport::default()
16545 };
16546 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
16547 HashMap::new();
16548 let pushed_ids = meta
16549 .pushed_ids
16550 .iter()
16551 .map(String::as_str)
16552 .collect::<HashSet<_>>();
16553 let mut last_final_push_result_id: Option<String> = None;
16554 for frame in &response.frames {
16555 match frame {
16556 Frame::PushResultDetails {
16557 client_commit_id,
16558 entries,
16559 } => {
16560 let details = rejection_details_by_commit
16561 .entry(client_commit_id.clone())
16562 .or_default();
16563 for entry in entries {
16564 let parsed = match RejectionDetails::parse(&entry.details.0) {
16565 Ok(value) => value,
16566 Err(message) => {
16567 return SyncOutcome::Failed {
16568 details: None,
16569 error_code: "sync.invalid_request".to_owned(),
16570 message,
16571 };
16572 }
16573 };
16574 details.insert(entry.op_index, parsed);
16575 }
16576 }
16577 Frame::PushResult {
16578 client_commit_id,
16579 status,
16580 results,
16581 ..
16582 } if pushed_ids.contains(client_commit_id.as_str())
16583 && Self::push_result_is_final(*status, results) =>
16584 {
16585 last_final_push_result_id = Some(client_commit_id.clone());
16586 }
16587 _ => {}
16588 }
16589 }
16590 let mut frames = response.frames.into_iter().peekable();
16591 if response.wire_version < 2 {
16592 return SyncOutcome::Failed {
16593 details: None,
16594 error_code: "client.invalid_host_response".to_owned(),
16595 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
16596 };
16597 }
16598 match frames.next() {
16599 Some(Frame::RespHeader {
16600 required_schema_version,
16601 latest_schema_version,
16602 log_epoch,
16603 reset_required,
16604 }) => {
16605 let Some(log_epoch) = log_epoch else {
16606 return SyncOutcome::Failed {
16607 details: None,
16608 error_code: "client.invalid_host_response".to_owned(),
16609 message: "response header omits logEpoch".to_owned(),
16610 };
16611 };
16612 let Some(reset_required) = reset_required else {
16613 return SyncOutcome::Failed {
16614 details: None,
16615 error_code: "client.invalid_host_response".to_owned(),
16616 message: "response header omits resetRequired".to_owned(),
16617 };
16618 };
16619 if let Some(required) = required_schema_version {
16620 let floor = SchemaFloor {
16623 required_schema_version: Some(required),
16624 latest_schema_version,
16625 };
16626 self.set_schema_floor(Some(floor.clone()));
16627 report.schema_floor = Some(floor);
16628 return SyncOutcome::Ok(report);
16629 }
16630 if reset_required {
16631 if frames.next().is_some() {
16632 return SyncOutcome::Failed {
16633 details: None,
16634 error_code: "client.invalid_host_response".to_owned(),
16635 message: "log-epoch reset response contains body frames".to_owned(),
16636 };
16637 }
16638 match self.run_log_epoch_reset(&log_epoch) {
16639 Ok(resets) => {
16640 report.resets = resets;
16641 return SyncOutcome::Ok(report);
16642 }
16643 Err(message) => {
16644 return SyncOutcome::Failed {
16645 details: None,
16646 error_code: "sync.local_corrupt".to_owned(),
16647 message,
16648 };
16649 }
16650 }
16651 }
16652 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
16653 return SyncOutcome::Failed {
16654 details: None,
16655 error_code: "client.invalid_host_response".to_owned(),
16656 message: "server changed logEpoch without requiring a reset".to_owned(),
16657 };
16658 }
16659 }
16660 _ => {
16661 return SyncOutcome::Failed {
16662 details: None,
16663 error_code: "sync.invalid_request".to_owned(),
16664 message: "response does not start with RESP_HEADER".to_owned(),
16665 };
16666 }
16667 }
16668
16669 if let Err(message) = self.begin_coalesced_apply() {
16670 return SyncOutcome::Failed {
16671 details: None,
16672 error_code: "storage.failed".into(),
16673 message,
16674 };
16675 }
16676 let mut failure: Option<(String, String)> = None;
16677 while let Some(frame) = frames.next() {
16678 match frame {
16679 frame @ Frame::PushResult { .. } => {
16680 let successful = matches!(
16681 &frame,
16682 Frame::PushResult {
16683 status: PushStatus::Applied | PushStatus::Cached,
16684 ..
16685 }
16686 );
16687 let mut results = vec![frame];
16688 if successful {
16689 while matches!(
16690 frames.peek(),
16691 Some(Frame::PushResult {
16692 status: PushStatus::Applied | PushStatus::Cached,
16693 ..
16694 })
16695 ) {
16696 if let Some(next) = frames.next() {
16697 results.push(next);
16698 }
16699 }
16700 }
16701 if self
16702 .handle_push_results(
16703 &results,
16704 &pushed_ids,
16705 &rejection_details_by_commit,
16706 &mut report,
16707 last_final_push_result_id.as_deref(),
16708 )
16709 .is_err()
16710 {
16711 failure = Some((
16712 "client.outcome_persistence_failed".to_owned(),
16713 "local commit outcome could not be persisted".to_owned(),
16714 ));
16715 break;
16716 }
16717 }
16718 Frame::PushResultDetails { .. } => {}
16719 Frame::SubStart {
16720 id,
16721 status,
16722 reason_code,
16723 effective_scopes,
16724 bootstrap: _,
16725 } => {
16726 let mut body = Vec::new();
16727 let mut sub_end: Option<(i64, Option<String>)> = None;
16728 for inner in frames.by_ref() {
16729 match inner {
16730 Frame::SubEnd {
16731 next_cursor,
16732 bootstrap_state,
16733 } => {
16734 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
16735 break;
16736 }
16737 Frame::Unknown { .. } => {}
16738 other => body.push(other),
16739 }
16740 }
16741 if let Err(SectionError::Abort(code, message)) = self.process_section(
16742 transport,
16743 &id,
16744 status,
16745 &reason_code,
16746 effective_scopes,
16747 body,
16748 sub_end,
16749 meta,
16750 &mut report,
16751 ) {
16752 failure = Some((code, message));
16753 break;
16754 }
16755 }
16756 Frame::Lease {
16757 lease_id,
16758 expires_at_ms,
16759 } => {
16760 self.set_lease_state(Some(LeaseState {
16763 lease_id: Some(lease_id),
16764 expires_at_ms: Some(expires_at_ms),
16765 error_code: None,
16766 }));
16767 }
16768 Frame::Error { code, message, .. } => {
16769 failure = Some((code, message));
16772 break;
16773 }
16774 Frame::Unknown { .. } => {}
16775 _ => {
16776 failure = Some((
16777 "sync.invalid_request".to_owned(),
16778 "unexpected frame in response".to_owned(),
16779 ));
16780 break;
16781 }
16782 }
16783 }
16784
16785 let applied = match failure {
16786 Some(failure) => Err(failure),
16787 None => self
16792 .rebuild_overlay_if_dirty()
16793 .and_then(|()| self.drain_pending_evictions())
16794 .map_err(|message| ("storage.failed".to_owned(), message)),
16795 };
16796 if let Err(message) = self.finish_coalesced_apply() {
16799 return SyncOutcome::Failed {
16800 details: None,
16801 error_code: "storage.failed".into(),
16802 message,
16803 };
16804 }
16805 if let Err((error_code, message)) = applied {
16806 return SyncOutcome::Failed {
16807 details: None,
16808 error_code,
16809 message,
16810 };
16811 }
16812 self.ack_after_pull(transport);
16813 if self.upgrading && report.bootstrapping.is_empty() {
16816 self.set_upgrading(false);
16817 }
16818 SyncOutcome::Ok(report)
16819 }
16820
16821 fn handle_push_results(
16824 &mut self,
16825 frames: &[Frame],
16826 pushed_ids: &HashSet<&str>,
16827 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
16828 report: &mut SyncReport,
16829 last_final_push_result_id: Option<&str>,
16830 ) -> Result<(), String> {
16831 self.begin_observation("syncular_push_results")?;
16832 let previous_failed = self.failed_commits.clone();
16833 let previous_sync_needed = self.sync_needed;
16834 let conflict_count = self.conflicts.len();
16835 let rejection_count = self.rejections.len();
16836 let applied_count = report.applied.len();
16837 let rejected_count = report.rejected.len();
16838 let retryable_count = report.retryable.len();
16839 let was_dirty = self.overlay_dirty.snapshot();
16840 let mut removed = Vec::new();
16841 let mut batch = ChangeAccumulator::default();
16842 let persisted = (|| {
16843 for frame in frames {
16844 let Frame::PushResult {
16845 client_commit_id,
16846 status,
16847 results,
16848 commit_seq,
16849 } = frame
16850 else {
16851 return Err("unexpected acknowledgement frame".to_owned());
16852 };
16853 if !pushed_ids.contains(client_commit_id.as_str()) {
16854 continue;
16855 }
16856 let Some(index) = self
16857 .outbox
16858 .iter()
16859 .position(|commit| commit.client_commit_id == *client_commit_id)
16860 else {
16861 continue;
16862 };
16863 let status = *status;
16864 let rejection_details = rejection_details_by_commit.get(client_commit_id);
16865 let operations = self.outbox[index].ops.clone();
16866 match status {
16867 PushStatus::Applied | PushStatus::Cached => {
16868 let sequence =
16871 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
16872 let affected: HashSet<_> = operations
16873 .iter()
16874 .map(|op| (op.table.as_str(), op.row_id.as_str()))
16875 .collect();
16876 let prior = self.acknowledged_operations(Some(&affected), None)?;
16877 for (op_index, operation) in operations.iter().enumerate() {
16878 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
16879 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16880 if delivered {
16881 continue;
16882 }
16883 let table = self
16884 .schema
16885 .table(&operation.table)
16886 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16887 let mut intended: Map<String, Value> = self
16888 .stored_row(table, &operation.row_id, true)?
16889 .map(|(row, _)| {
16890 table
16891 .columns
16892 .iter()
16893 .zip(&row)
16894 .map(|(column, value)| {
16895 (column.name.clone(), column_value_to_json(value))
16896 })
16897 .collect()
16898 })
16899 .unwrap_or_default();
16900 let mut scope_image = intended.clone();
16901 for op in prior
16902 .iter()
16903 .chain(
16904 self.failed_commits
16905 .iter()
16906 .flat_map(|(commit, _)| &commit.ops),
16907 )
16908 .chain(
16909 self.outbox
16910 .iter()
16911 .take(index)
16912 .flat_map(|commit| &commit.ops),
16913 )
16914 .chain(operations.iter().take(op_index + 1))
16915 {
16916 if op.table != operation.table || op.row_id != operation.row_id {
16917 continue;
16918 }
16919 if let Some(values) = &op.values {
16920 intended.extend(values.clone());
16921 } else {
16922 scope_image = intended.clone();
16923 intended.clear();
16924 }
16925 }
16926 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
16927 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
16928 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
16929 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
16930 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16931 }
16932 let journal_results = results
16933 .iter()
16934 .map(|result| {
16935 let op_index = match result {
16936 OpResult::Applied { op_index }
16937 | OpResult::Conflict { op_index, .. }
16938 | OpResult::Error { op_index, .. } => *op_index,
16939 };
16940 CommitOperationOutcome::Applied { op_index }
16941 })
16942 .collect::<Vec<_>>();
16943 let outcome_status = if status == PushStatus::Applied {
16944 CommitOutcomeStatus::Applied
16945 } else {
16946 CommitOutcomeStatus::Cached
16947 };
16948 let persisted = self
16949 .persist_commit_outcome(
16950 client_commit_id,
16951 outcome_status,
16952 &journal_results,
16953 None,
16954 )
16955 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
16956 persisted?;
16957 for operation in &operations {
16958 self.overlay_dirty.table(&operation.table);
16959 }
16960 self.sync_needed = true;
16961 batch.status = true;
16962 batch.outcomes = true;
16963 }
16964 PushStatus::Rejected => {
16965 if results.iter().any(|result| {
16966 matches!(
16967 result,
16968 OpResult::Error {
16969 code,
16970 retryable: true,
16971 ..
16972 } if code == "sync.idempotency_cache_miss"
16973 )
16974 }) {
16975 report.retryable.push(client_commit_id.to_owned());
16978 continue;
16979 }
16980
16981 let mut journal_results = Vec::with_capacity(results.len());
16982 let mut conflicts = Vec::new();
16983 let mut rejections = Vec::new();
16984 for result in results {
16985 match result {
16986 OpResult::Applied { op_index } => {
16987 journal_results.push(CommitOperationOutcome::Applied {
16988 op_index: *op_index,
16989 });
16990 }
16991 OpResult::Conflict {
16992 op_index,
16993 code,
16994 message,
16995 server_version,
16996 server_row,
16997 conflict_columns: conflict_columns_bitmap,
16998 } => {
16999 let operation = operations
17000 .get(*op_index as usize)
17001 .map(CommitOperation::from);
17002 let (table, row_id) = operation
17003 .as_ref()
17004 .map(|op| (op.table.clone(), op.row_id.clone()))
17005 .unwrap_or_default();
17006 let server_row_json = self
17007 .schema
17008 .table(&table)
17009 .and_then(|t| {
17010 decode_row_bytes(t, server_row, &self.encryption)
17011 .ok()
17012 .map(|row| (t, row))
17013 })
17014 .map(|(t, row)| {
17015 let mut map = Map::new();
17016 for (i, column) in t.columns.iter().enumerate() {
17017 map.insert(
17018 column.name.clone(),
17019 column_value_to_json(
17020 row.get(i).unwrap_or(&None),
17021 ),
17022 );
17023 }
17024 map
17025 })
17026 .unwrap_or_default();
17027 let conflict_columns = self
17030 .schema
17031 .table(&table)
17032 .map(|t| {
17033 t.columns
17034 .iter()
17035 .enumerate()
17036 .filter(|(index, _)| {
17037 conflict_columns_bitmap
17038 .get(index / 8)
17039 .is_some_and(|byte| {
17040 byte & (1 << (index % 8)) != 0
17041 })
17042 })
17043 .map(|(_, column)| column.name.clone())
17044 .collect::<Vec<_>>()
17045 })
17046 .unwrap_or_default();
17047 let conflict = ConflictRecord {
17048 client_commit_id: client_commit_id.to_owned(),
17049 op_index: *op_index,
17050 table,
17051 row_id,
17052 code: code.clone(),
17053 message: message.clone(),
17054 server_version: *server_version,
17055 server_row: server_row_json,
17056 conflict_columns,
17057 operation,
17058 };
17059 journal_results.push(CommitOperationOutcome::Conflict {
17060 conflict: conflict.clone(),
17061 });
17062 conflicts.push(conflict);
17063 }
17064 OpResult::Error {
17065 op_index,
17066 code,
17067 message,
17068 retryable,
17069 } => {
17070 let rejection = RejectionRecord {
17071 client_commit_id: client_commit_id.to_owned(),
17072 op_index: *op_index,
17073 code: code.clone(),
17074 message: message.clone(),
17075 retryable: *retryable,
17076 details: rejection_details
17077 .and_then(|details| details.get(op_index))
17078 .cloned(),
17079 operation: operations
17080 .get(*op_index as usize)
17081 .map(CommitOperation::from),
17082 };
17083 journal_results.push(CommitOperationOutcome::Error {
17084 rejection: rejection.clone(),
17085 });
17086 rejections.push(rejection);
17087 }
17088 }
17089 }
17090 if self.retain_failed_commits {
17091 self.retain_failed_commit(client_commit_id, &operations)?;
17092 }
17093 let outcome_status = if conflicts.is_empty() {
17094 CommitOutcomeStatus::Rejected
17095 } else {
17096 CommitOutcomeStatus::Conflict
17097 };
17098 let persisted = self
17099 .persist_commit_outcome(
17100 client_commit_id,
17101 outcome_status,
17102 &journal_results,
17103 Some(&operations),
17104 )
17105 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
17106 persisted?;
17107 batch.conflicts = !conflicts.is_empty();
17108 batch.rejections = !rejections.is_empty();
17109 batch.status = true;
17110 batch.outcomes = true;
17111 self.conflicts.extend(conflicts);
17112 self.rejections.extend(rejections);
17113 for operation in &operations {
17114 self.overlay_dirty.table(&operation.table);
17115 }
17116 }
17117 }
17118 removed.push((index, self.outbox.remove(index)));
17119 if batch.status {
17120 for operation in &operations {
17121 if !self.record_row_scopes(
17122 &mut batch,
17123 &operation.table,
17124 &operation.row_id,
17125 false,
17126 ) {
17127 batch.table(&operation.table);
17128 }
17129 }
17130 }
17131 match status {
17132 PushStatus::Applied | PushStatus::Cached => {
17133 report.applied.push(client_commit_id.to_owned())
17134 }
17135 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
17136 }
17137 }
17138 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
17139 self.prune_commit_outcomes()?;
17140 }
17141 self.rebuild_overlay_if_dirty()?;
17142 self.finish_observation("syncular_push_results", batch)
17143 })();
17144 if let Err(error) = persisted {
17145 self.rollback_observation("syncular_push_results");
17146 self.failed_commits = previous_failed;
17147 self.sync_needed = previous_sync_needed;
17148 for (index, commit) in removed.into_iter().rev() {
17149 self.outbox.insert(index, commit);
17150 }
17151 self.conflicts.truncate(conflict_count);
17152 self.rejections.truncate(rejection_count);
17153 self.overlay_dirty.restore(was_dirty.clone());
17154 report.applied.truncate(applied_count);
17155 report.rejected.truncate(rejected_count);
17156 report.retryable.truncate(retryable_count);
17157 return Err(error);
17158 }
17159 if !previous_sync_needed && self.sync_needed {
17160 self.sync_intent_queue.push_back(SyncIntent::Interactive);
17161 }
17162 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
17163 Ok(())
17164 }
17165
17166 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
17167 status != PushStatus::Rejected
17168 || !results.iter().any(|result| {
17169 matches!(
17170 result,
17171 OpResult::Error {
17172 code,
17173 retryable: true,
17174 ..
17175 } if code == "sync.idempotency_cache_miss"
17176 )
17177 })
17178 }
17179
17180 #[allow(clippy::too_many_arguments)]
17183 fn process_section(
17184 &mut self,
17185 transport: &mut dyn Transport,
17186 id: &str,
17187 status: SubStatus,
17188 reason_code: &str,
17189 effective_scopes: Vec<(String, Vec<String>)>,
17190 body: Vec<Frame>,
17191 sub_end: Option<(i64, Option<String>)>,
17192 meta: &RequestMeta,
17193 report: &mut SyncReport,
17194 ) -> Result<(), SectionError> {
17195 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
17196 return Ok(()); };
17198 if status != SubStatus::Active && sub_end.is_none() {
17199 return Err(SectionError::Abort(
17200 "sync.invalid_request".into(),
17201 "subscription section without SUB_END".into(),
17202 ));
17203 }
17204 match status {
17205 SubStatus::Revoked => {
17206 self.begin_observation("syncular_revocation")
17207 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17208 let previous = self.subs[sub_index].clone();
17209 let was_dirty = self.overlay_dirty.snapshot();
17210 let previous_outbox = self.outbox.clone();
17211 let previous_failed = self.failed_commits.clone();
17212 let previous_rejections = self.rejections.clone();
17213 let previous_conflicts = self.conflicts.clone();
17214 let revoked_count = report.revoked.len();
17215 let failed_count = report.failed.len();
17216 let result = (|| {
17217 let mut batch = ChangeAccumulator::default();
17218 let registered = self.window_unit_by_sub_id(id);
17219 let (table, effective) = {
17221 let sub = &self.subs[sub_index];
17222 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17223 };
17224 let scoped_table = self.schema.table(&table).cloned().ok_or_else(|| {
17225 SectionError::Abort(
17226 "sync.unknown_table".to_owned(),
17227 "revoked scope references an unknown table".to_owned(),
17228 )
17229 })?;
17230 let dropped = self
17231 .drop_failed_commits(|name, values| {
17232 name == table
17233 && !effective.is_empty()
17234 && effective.iter().all(|(variable, allowed)| {
17235 scoped_table
17236 .scope_column(variable)
17237 .and_then(|column| values.get(column))
17238 .and_then(Value::as_str)
17239 .is_some_and(|value| {
17240 allowed.iter().any(|allowed| allowed == value)
17241 })
17242 })
17243 })
17244 .map_err(|message| {
17245 SectionError::Abort("storage.failed".to_owned(), message)
17246 })?;
17247 for commit in dropped {
17248 batch.conflicts = true;
17249 batch.rejections = true;
17250 batch.outcomes = true;
17251 for operation in commit.ops {
17252 batch.table(&operation.table);
17253 }
17254 }
17255 let purged = self.purge_scope_rows(&table, &effective);
17256 match purged {
17257 Ok(()) => {
17258 self.record_scope_map(&mut batch, &table, &effective);
17259 let sub = &mut self.subs[sub_index];
17260 sub.state = SubState::Revoked;
17261 sub.reason_code = Some(if reason_code.is_empty() {
17262 "sync.scope_revoked".to_owned()
17263 } else {
17264 reason_code.to_owned()
17265 });
17266 report.revoked.push(id.to_owned());
17267 let doomed_effective = effective;
17268 let sub_table = table;
17269 self.persist_sub(&self.subs[sub_index].clone())
17270 .map_err(|message| {
17271 SectionError::Abort("storage.failed".into(), message)
17272 })?;
17273 let dropped = self
17274 .drop_doomed_outbox(&sub_table, &doomed_effective)
17275 .map_err(|message| {
17276 SectionError::Abort("storage.failed".to_owned(), message)
17277 })?;
17278 if dropped {
17279 batch.status = true;
17280 batch.rejections = true;
17281 batch.outcomes = true;
17282 }
17283 self.delete_unreferenced_cached_blobs().map_err(|message| {
17286 SectionError::Abort("storage.failed".into(), message)
17287 })?;
17288 }
17289 Err(()) => {
17290 let sub = &mut self.subs[sub_index];
17293 sub.state = SubState::Failed;
17294 sub.reason_code = Some("sync.scope_revoked".to_owned());
17295 report.failed.push(id.to_owned());
17296 self.persist_sub(&self.subs[sub_index].clone())
17297 .map_err(|message| {
17298 SectionError::Abort("storage.failed".into(), message)
17299 })?;
17300 }
17301 }
17302 if let Some((base_key, unit)) = registered {
17303 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17304 }
17305 self.rebuild_overlay_if_dirty()
17306 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17307 self.finish_observation("syncular_revocation", batch)
17308 .map_err(|message| {
17309 SectionError::Abort("storage.failed".to_owned(), message)
17310 })?;
17311 Ok(())
17312 })();
17313 if result.is_err() {
17314 self.rollback_observation("syncular_revocation");
17315 self.subs[sub_index] = previous;
17316 self.overlay_dirty.restore(was_dirty.clone());
17317 self.outbox = previous_outbox;
17318 self.failed_commits = previous_failed;
17319 self.rejections = previous_rejections;
17320 self.conflicts = previous_conflicts;
17321 report.revoked.truncate(revoked_count);
17322 report.failed.truncate(failed_count);
17323 }
17324 result
17325 }
17326 SubStatus::Reset => {
17327 self.begin_observation("syncular_reset")
17328 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
17329 let previous = self.subs[sub_index].clone();
17330 let was_dirty = self.overlay_dirty.snapshot();
17331 let reset_count = report.resets.len();
17332 let result = (|| {
17333 let mut batch = ChangeAccumulator::default();
17334 let registered = self.window_unit_by_sub_id(id);
17335 let sub = &mut self.subs[sub_index];
17338 sub.cursor = -1;
17339 sub.bootstrap_state = None;
17340 sub.reason_code = Some(if reason_code.is_empty() {
17341 "sync.cursor_expired".to_owned()
17342 } else {
17343 reason_code.to_owned()
17344 });
17345 report.resets.push(id.to_owned());
17346 self.persist_sub(&self.subs[sub_index].clone())
17347 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17348 if let Some((base_key, unit)) = registered {
17349 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17350 }
17351 self.finish_observation("syncular_reset", batch)
17352 .map_err(|message| {
17353 SectionError::Abort("storage.failed".to_owned(), message)
17354 })?;
17355 Ok(())
17356 })();
17357 if result.is_err() {
17358 self.rollback_observation("syncular_reset");
17359 self.subs[sub_index] = previous;
17360 self.overlay_dirty.restore(was_dirty.clone());
17361 report.resets.truncate(reset_count);
17362 }
17363 result
17364 }
17365 SubStatus::Active => {
17366 let fresh = meta
17367 .fresh
17368 .iter()
17369 .find(|(fid, _)| fid == id)
17370 .map(|(_, fresh)| *fresh)
17371 .unwrap_or(false);
17372 let previous = self.subs[sub_index].clone();
17373 self.subs[sub_index].effective = Some(effective_scopes);
17374 let outcome = self
17375 .apply_section_body(transport, sub_index, body, fresh, meta, report)
17376 .and_then(|()| {
17377 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
17378 SectionError::Abort(
17379 "sync.invalid_request".into(),
17380 "subscription section without SUB_END".into(),
17381 )
17382 })?;
17383 self.apply_sub_end(sub_index, cursor, bootstrap_state)
17384 .map_err(|message| {
17385 SectionError::Abort("storage.failed".into(), message)
17386 })
17387 });
17388 match outcome {
17389 Ok(()) => {
17390 if self.subs[sub_index].bootstrap_state.is_some() {
17391 report.bootstrapping.push(id.to_owned());
17392 }
17393 Ok(())
17394 }
17395 Err(SectionError::FailClosed) => {
17396 self.subs[sub_index] = previous.clone();
17397 self.begin_observation("syncular_section_failure")
17398 .map_err(|message| {
17399 SectionError::Abort("storage.failed".into(), message)
17400 })?;
17401 let persisted = (|| {
17402 let mut batch = ChangeAccumulator::default();
17403 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
17404 batch.window(&base_key, &self.subs[sub_index].table, &unit);
17405 }
17406 self.subs[sub_index].state = SubState::Failed;
17407 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
17408 self.persist_sub(&self.subs[sub_index])?;
17409 self.finish_observation("syncular_section_failure", batch)
17410 })();
17411 if let Err(message) = persisted {
17412 self.rollback_observation("syncular_section_failure");
17413 self.subs[sub_index] = previous;
17414 return Err(SectionError::Abort("storage.failed".into(), message));
17415 }
17416 report.failed.push(id.to_owned());
17417 Ok(())
17418 }
17419 Err(error) => {
17420 self.subs[sub_index] = previous;
17423 Err(error)
17424 }
17425 }
17426 }
17427 }
17428 }
17429
17430 fn apply_sub_end(
17431 &mut self,
17432 sub_index: usize,
17433 cursor: i64,
17434 bootstrap_state: Option<String>,
17435 ) -> Result<(), String> {
17436 #[cfg(feature = "bench-internals")]
17437 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
17438 let previous = self.subs[sub_index].clone();
17439 self.begin_observation("syncular_sub_end")?;
17440 let persisted = (|| {
17441 let mut batch = ChangeAccumulator::default();
17442 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
17443 && cursor >= 0
17444 && bootstrap_state.is_none();
17445 if completed {
17446 let table = self
17447 .schema
17448 .table(&previous.table)
17449 .ok_or_else(|| "sync.unknown_table".to_owned())?;
17450 let effective = previous.effective.as_deref().unwrap_or(&[]);
17451 let mut statement = self.conn.prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
17452 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17453 let rows = statement
17454 .query_map(rusqlite::params![previous.table, cursor], |row| {
17455 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
17456 })
17457 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17458 let mut covered = HashSet::new();
17459 for row in rows {
17460 let (id, intent) =
17461 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17462 let values: Map<String, Value> = serde_json::from_str(&intent)
17463 .map_err(|_| "sync.local_corrupt".to_owned())?;
17464 if !effective.is_empty()
17465 && effective.iter().all(|(variable, allowed)| {
17466 table
17467 .scope_column(variable)
17468 .and_then(|column| values.get(column))
17469 .is_some_and(|value| {
17470 allowed.contains(
17471 &value
17472 .as_str()
17473 .map_or_else(|| value.to_string(), str::to_owned),
17474 )
17475 })
17476 })
17477 {
17478 covered.insert(id);
17479 }
17480 }
17481 drop(statement);
17482 for id in covered {
17483 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![previous.table, id, cursor])
17484 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
17485 self.overlay_dirty.table(&previous.table);
17486 batch.table(&previous.table);
17487 }
17488 self.rebuild_overlay_if_dirty()?;
17489 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
17490 batch.window(&base_key, &previous.table, &unit);
17491 }
17492 }
17493 let sub = &mut self.subs[sub_index];
17494 sub.cursor = cursor;
17495 sub.bootstrap_state = bootstrap_state;
17496 sub.reason_code = None;
17499 sub.synced_once = true;
17500 self.persist_sub(&self.subs[sub_index])?;
17501 self.finish_observation("syncular_sub_end", batch)
17502 })();
17503 if persisted.is_err() {
17504 self.rollback_observation("syncular_sub_end");
17505 self.subs[sub_index] = previous;
17506 }
17507 persisted
17508 }
17509
17510 fn apply_section_body(
17511 &mut self,
17512 transport: &mut dyn Transport,
17513 sub_index: usize,
17514 body: Vec<Frame>,
17515 fresh: bool,
17516 meta: &RequestMeta,
17517 report: &mut SyncReport,
17518 ) -> Result<(), SectionError> {
17519 let mut saw_segment = false;
17520 for frame in body {
17521 match frame {
17522 Frame::Commit {
17523 tables,
17524 changes,
17525 commit_seq,
17526 ..
17527 } => {
17528 self.apply_commit_frame(&tables, &changes, Some(commit_seq))
17529 .map_err(|(c, m)| SectionError::Abort(c, m))?;
17530 report.commits_applied += 1;
17531 }
17532 Frame::SegmentInline { payload } => {
17533 let segment = decode_rows_segment(&payload)
17534 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
17535 self.progress.update(|p| {
17536 p.phase = ProgressPhase::Import;
17537 p.subscription_id = Some(self.subs[sub_index].id.clone());
17538 p.table = Some(self.subs[sub_index].table.clone());
17539 p.segment_id = None;
17540 p.bytes_received = payload.len() as u64;
17541 p.bytes_total = Some(payload.len() as u64);
17542 p.rows_processed = 0;
17543 p.rows_total =
17544 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
17545 });
17546 let first = !saw_segment;
17547 saw_segment = true;
17548 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17549 report.segment_rows_applied += applied;
17550 }
17551 Frame::SegmentRef {
17552 segment_id,
17553 byte_length,
17554 media_type,
17555 table,
17556 row_count,
17557 as_of_commit_seq,
17558 scope_digest,
17559 row_cursor,
17560 next_row_cursor,
17561 url,
17562 url_expires_at_ms,
17563 ..
17564 } => {
17565 let advertised = match media_type {
17568 MediaType::Rows => {
17569 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
17570 || meta.accept & ACCEPT_INLINE_ROWS != 0
17571 }
17572 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
17573 };
17574 if !advertised {
17575 return Err(SectionError::Abort(
17576 "sync.invalid_request".to_owned(),
17577 format!(
17578 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
17579 media_type.name()
17580 ),
17581 ));
17582 }
17583 self.progress.update(|p| {
17584 p.phase = ProgressPhase::Download;
17585 p.subscription_id = Some(self.subs[sub_index].id.clone());
17586 p.table = Some(table.clone());
17587 p.segment_id = Some(segment_id.clone());
17588 p.bytes_received = 0;
17589 p.bytes_total = u64::try_from(byte_length).ok();
17590 p.rows_processed = 0;
17591 p.rows_total = u64::try_from(row_count).ok();
17592 });
17593 let observer = self.progress.clone();
17594 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
17595 let bytes = if let Some(url) = url {
17596 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
17601 return Err(SectionError::Abort(
17602 "sync.invalid_request".to_owned(),
17603 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
17604 .to_owned(),
17605 ));
17606 }
17607 if url_expires_at_ms.is_some_and(|exp| {
17609 exp <= transport
17610 .segment_fetch_started_at(&url)
17611 .unwrap_or_else(|| self.clock_now_ms())
17612 }) {
17613 return Err(SectionError::Abort(
17614 "sync.segment_expired".to_owned(),
17615 format!(
17616 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
17617 ),
17618 ));
17619 }
17620 transport
17621 .fetch_url(&url, &mut on_progress)
17622 .map_err(|e| SectionError::Abort(e.code, e.message))?
17623 } else {
17624 let requested_scopes_json =
17625 canonical_scope_json(&self.subs[sub_index].requested);
17626 transport
17627 .download_segment(
17628 &SegmentRequest {
17629 segment_id: segment_id.clone(),
17630 table,
17631 requested_scopes_json,
17632 },
17633 &mut on_progress,
17634 )
17635 .map_err(|e| SectionError::Abort(e.code, e.message))?
17636 };
17637 on_progress(bytes.len() as u64);
17639 let digest = Sha256::digest(&bytes);
17640 let expected = segment_id
17641 .strip_prefix("sha256:")
17642 .unwrap_or(segment_id.as_str());
17643 if bytes_to_hex(&digest) != expected {
17644 self.schedule_background_retry();
17646 return Err(SectionError::Abort(
17647 "sync.invalid_request".to_owned(),
17648 "segment bytes do not match the content address (§5.1)".to_owned(),
17649 ));
17650 }
17651 self.progress.update(|p| p.phase = ProgressPhase::Import);
17652 if media_type == MediaType::Sqlite {
17653 if row_cursor.is_some() || next_row_cursor.is_some() {
17656 return Err(SectionError::Abort(
17657 "sync.invalid_request".to_owned(),
17658 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
17659 .to_owned(),
17660 ));
17661 }
17662 let first = !saw_segment;
17663 saw_segment = true;
17664 let applied = self.apply_sqlite_segment(
17665 sub_index,
17666 &bytes,
17667 fresh && first,
17668 row_count,
17669 as_of_commit_seq,
17670 &scope_digest,
17671 )?;
17672 report.segment_rows_applied += applied;
17673 } else {
17674 let segment = decode_rows_segment(&bytes).map_err(|e| {
17675 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
17676 })?;
17677 let first = row_cursor.is_none();
17678 saw_segment = true;
17679 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
17680 report.segment_rows_applied += applied;
17681 }
17682 }
17683 Frame::Unknown { .. } => {}
17684 Frame::Error { code, message, .. } => {
17685 return Err(SectionError::Abort(code, message))
17686 }
17687 _ => {
17688 return Err(SectionError::Abort(
17689 "sync.invalid_request".to_owned(),
17690 "unexpected frame inside a subscription section".to_owned(),
17691 ));
17692 }
17693 }
17694 }
17695 Ok(())
17696 }
17697
17698 fn apply_commit_frame(
17700 &mut self,
17701 tables: &[String],
17702 changes: &[ssp2::model::Change],
17703 commit_seq: Option<i64>,
17704 ) -> Result<(), (String, String)> {
17705 #[cfg(feature = "bench-internals")]
17706 let _phase = self.benchmark_phases.start(Phase::CommitApply);
17707 let was_dirty = self.overlay_dirty.snapshot();
17708 self.coalesce_unit(changes.len())
17709 .and_then(|()| self.begin_observation("syncular_commit"))
17710 .map_err(|message| ("storage.failed".into(), message))?;
17711 let applied = (|| {
17712 let mut batch = ChangeAccumulator::default();
17713 self.record_commit_changes(&mut batch, tables, changes);
17714 if let Some(sequence) = commit_seq {
17715 for change in changes {
17716 let table = tables.get(change.table_index as usize).ok_or_else(|| {
17717 (
17718 "sync.invalid_request".to_owned(),
17719 "change tableIndex out of range".to_owned(),
17720 )
17721 })?;
17722 if self
17723 .outbox
17724 .iter()
17725 .flat_map(|commit| &commit.ops)
17726 .any(|op| op.table == *table && op.row_id == change.row_id)
17727 {
17728 self.conn.execute("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)", rusqlite::params![table, change.row_id, sequence])
17729 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
17730 }
17731 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![table, change.row_id, sequence])
17732 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
17733 }
17734 }
17735 self.apply_commit_changes(tables, changes)?;
17736 self.rebuild_overlay_if_dirty()
17737 .map_err(|message| ("storage.failed".into(), message))?;
17738 self.finish_observation("syncular_commit", batch)
17739 .map_err(|message| ("storage.failed".into(), message))
17740 })();
17741 if applied.is_err() {
17742 self.rollback_observation("syncular_commit");
17743 self.overlay_dirty.restore(was_dirty.clone());
17744 }
17745 applied
17746 }
17747
17748 fn apply_commit_changes(
17749 &mut self,
17750 tables: &[String],
17751 changes: &[ssp2::model::Change],
17752 ) -> Result<(), (String, String)> {
17753 let mirror_visible = !self.overlay_dirty.get()
17758 && self.failed_commits.is_empty()
17759 && ((self.outbox.is_empty() && !self.has_acknowledged_rows())
17760 || changes.iter().all(|change| {
17761 tables
17762 .get(change.table_index as usize)
17763 .and_then(|name| self.schema.table(name))
17764 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
17765 }));
17766 for change in changes {
17767 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
17768 (
17769 "sync.invalid_request".to_owned(),
17770 "change tableIndex out of range".to_owned(),
17771 )
17772 })?;
17773 let table = self.schema.table(table_name).ok_or_else(|| {
17774 (
17775 "sync.schema_mismatch".to_owned(),
17776 format!("change targets unknown table {table_name:?}"),
17777 )
17778 })?;
17779 match change.op {
17780 Op::Upsert => {
17781 let payload = change.row.as_ref().ok_or_else(|| {
17782 (
17783 "sync.invalid_request".to_owned(),
17784 "upsert change without row payload".to_owned(),
17785 )
17786 })?;
17787 let row = {
17789 #[cfg(feature = "bench-internals")]
17790 let _phase = self.benchmark_phases.start(Phase::RowDecode);
17791 decode_row_bytes(table, payload, &self.encryption)
17792 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
17793 };
17794 let version = change.row_version.unwrap_or(0);
17795 let table_name = table.name.clone();
17796 self.write_base_row(&table_name, &row, version)
17797 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17798 if mirror_visible {
17799 self.write_row(&visible_table(&table_name), &table_name, &row, version)
17800 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17801 }
17802 }
17803 Op::Delete => {
17804 self.delete_base_row(table_name, &change.row_id)
17805 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
17806 if mirror_visible {
17807 let sql = format!(
17808 "DELETE FROM {} WHERE {}",
17809 visible_table(table_name),
17810 row_id_predicate(table)
17811 );
17812 self.conn
17813 .prepare_cached(&sql)
17814 .and_then(|mut statement| {
17815 statement.execute(rusqlite::params![change.row_id])
17816 })
17817 .map_err(|error| {
17818 ("sync.invalid_request".to_owned(), error.to_string())
17819 })?;
17820 }
17821 }
17822 }
17823 }
17824 if mirror_visible {
17825 {
17826 let changed_rows: HashSet<(&str, &str)> = changes
17827 .iter()
17828 .map(|change| {
17829 (
17830 tables[change.table_index as usize].as_str(),
17831 change.row_id.as_str(),
17832 )
17833 })
17834 .collect();
17835 let acknowledged = self
17836 .acknowledged_operations(Some(&changed_rows), None)
17837 .map_err(|error| ("storage.failed".to_owned(), error))?;
17838 self.apply_outbox_ops(&acknowledged)
17839 .map_err(|message| ("storage.failed".into(), message))?;
17840 self.apply_outbox_ops(
17841 self.outbox
17842 .iter()
17843 .flat_map(|commit| &commit.ops)
17844 .filter(|op| {
17845 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
17846 }),
17847 )
17848 .map_err(|message| ("storage.failed".into(), message))?;
17849 }
17850 self.overlay_dirty.set(false);
17851 }
17852 Ok(())
17853 }
17854
17855 fn apply_segment(
17860 &mut self,
17861 sub_index: usize,
17862 segment: &RowsSegment,
17863 first_fresh_page: bool,
17864 ) -> Result<u32, SectionError> {
17865 let (sub_table, effective) = {
17866 let sub = &self.subs[sub_index];
17867 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
17868 };
17869 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
17870 SectionError::Abort(
17871 "sync.schema_mismatch".to_owned(),
17872 format!("subscription table {sub_table:?} is not in the client schema"),
17873 )
17874 })?;
17875 let matches = segment.table == table.name
17880 && segment.schema_version == self.schema.version
17881 && segment.columns.len() == table.wire_columns.len()
17882 && segment
17883 .columns
17884 .iter()
17885 .zip(table.wire_columns.iter())
17886 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
17887 if !matches {
17888 return Err(SectionError::Abort(
17889 "sync.schema_mismatch".to_owned(),
17890 "segment column table does not match the generated schema (§5.2)".to_owned(),
17891 ));
17892 }
17893 let mut applied = 0u32;
17894 let empty = Vec::new();
17895 let blocks = if segment.blocks.is_empty() {
17896 std::slice::from_ref(&empty)
17897 } else {
17898 &segment.blocks
17899 };
17900 for (index, block) in blocks.iter().enumerate() {
17901 let clear = first_fresh_page && index == 0;
17902 if !clear && block.is_empty() {
17903 continue;
17904 }
17905 let was_dirty = self.overlay_dirty.snapshot();
17906 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
17907 && self
17908 .outbox
17909 .iter()
17910 .flat_map(|commit| &commit.ops)
17911 .any(|op| op.table == table.name);
17912 let mirror_visible = !reconcile_pending
17913 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
17914 self.coalesce_unit(block.len())
17915 .and_then(|()| self.begin_observation("syncular_segment_block"))
17916 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17917 let outcome = (|| {
17918 self.rebuild_overlay_if_dirty()
17919 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17920 let mut changed_keys = Vec::new();
17921 let mut batch = ChangeAccumulator::default();
17922 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
17923 batch.table(&table.name);
17924 }
17925 if clear {
17926 self.purge_scope_rows(&table.name, &effective)
17927 .map_err(|()| SectionError::FailClosed)?;
17928 }
17929 for (row_index, row) in block.iter().enumerate() {
17930 let decrypted;
17931 let values = if table.has_encrypted_columns() {
17932 let mut values = row.values.clone();
17933 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
17934 .map_err(|message| {
17935 SectionError::Abort("client.decrypt_failed".into(), message)
17936 })?;
17937 decrypted = values;
17938 &decrypted
17939 } else {
17940 &row.values
17941 };
17942 self.write_base_row(&table.name, values, row.server_version)
17943 .map_err(|message| {
17944 SectionError::Abort("sync.invalid_request".into(), message)
17945 })?;
17946 if reconcile_pending {
17947 changed_keys.push(
17948 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
17949 |message| SectionError::Abort("storage.failed".into(), message),
17950 )?,
17951 );
17952 }
17953 if mirror_visible {
17954 self.write_row(
17955 &visible_table(&table.name),
17956 &table.name,
17957 values,
17958 row.server_version,
17959 )
17960 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17961 }
17962 let processed = u64::from(applied) + row_index as u64 + 1;
17963 if processed.is_multiple_of(1024) {
17964 self.progress.update(|p| p.rows_processed = processed);
17965 }
17966 }
17967 if reconcile_pending {
17968 self.reconcile_imported_rows(&table, &mut changed_keys)
17969 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17970 }
17971 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
17972 self.apply_outbox_ops(
17973 self.outbox
17974 .iter()
17975 .flat_map(|commit| &commit.ops)
17976 .filter(|op| op.table == table.name),
17977 )
17978 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17979 self.overlay_dirty.set(false);
17980 } else {
17981 self.rebuild_overlay_if_dirty()
17982 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
17983 }
17984 self.finish_observation("syncular_segment_block", batch)
17985 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
17986 })();
17987 if let Err(error) = outcome {
17988 self.rollback_observation("syncular_segment_block");
17989 self.overlay_dirty.restore(was_dirty.clone());
17990 return Err(error);
17991 }
17992 applied += block.len() as u32;
17993 self.progress
17994 .update(|p| p.rows_processed = u64::from(applied));
17995 std::thread::yield_now();
17996 }
17997 self.progress
17998 .update(|p| p.rows_processed = u64::from(applied));
17999 Ok(applied)
18000 }
18001
18002 fn apply_sqlite_segment(
18010 &mut self,
18011 sub_index: usize,
18012 bytes: &[u8],
18013 first_fresh_page: bool,
18014 row_count: i64,
18015 as_of_commit_seq: i64,
18016 scope_digest: &str,
18017 ) -> Result<u32, SectionError> {
18018 let invalid = |detail: &str| {
18019 SectionError::Abort(
18020 "sync.invalid_request".to_owned(),
18021 format!("sqlite segment rejected: {detail} (§5.3)"),
18022 )
18023 };
18024 let (sub_table, effective) = {
18025 let sub = &self.subs[sub_index];
18026 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
18027 };
18028 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
18029 SectionError::Abort(
18030 "sync.schema_mismatch".to_owned(),
18031 format!("subscription table {sub_table:?} is not in the client schema"),
18032 )
18033 })?;
18034
18035 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
18036 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
18037 let img = match rusqlite::Connection::open_with_flags(
18038 &path,
18039 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
18040 ) {
18041 Ok(conn) => conn,
18042 Err(_) => {
18043 let _ = std::fs::remove_file(&path);
18044 return Err(invalid("bytes do not open as a SQLite database"));
18045 }
18046 };
18047 let outcome = self.apply_sqlite_image(
18048 &img,
18049 &table,
18050 first_fresh_page,
18051 &effective,
18052 row_count,
18053 as_of_commit_seq,
18054 scope_digest,
18055 );
18056 drop(img);
18057 let _ = std::fs::remove_file(&path);
18058 outcome
18059 }
18060
18061 #[allow(clippy::too_many_arguments)]
18062 fn apply_sqlite_image(
18063 &mut self,
18064 img: &rusqlite::Connection,
18065 table: &crate::schema::TableSchema,
18066 first_fresh_page: bool,
18067 effective: &[(String, Vec<String>)],
18068 row_count: i64,
18069 as_of_commit_seq: i64,
18070 scope_digest: &str,
18071 ) -> Result<u32, SectionError> {
18072 let invalid = |detail: String| {
18073 SectionError::Abort(
18074 "sync.invalid_request".to_owned(),
18075 format!("sqlite segment rejected: {detail} (§5.3)"),
18076 )
18077 };
18078
18079 type MetaRow = (i64, String, i64, i64, String, i64, i64);
18082 let meta: MetaRow = img
18083 .query_row(
18084 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
18085 \"scopeDigest\", \"rowCount\",
18086 (SELECT count(*) FROM _syncular_segment)
18087 FROM _syncular_segment",
18088 [],
18089 |row| {
18090 Ok((
18091 row.get(0)?,
18092 row.get(1)?,
18093 row.get(2)?,
18094 row.get(3)?,
18095 row.get(4)?,
18096 row.get(5)?,
18097 row.get(6)?,
18098 ))
18099 },
18100 )
18101 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
18102 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
18103 if meta_count != 1 {
18104 return Err(invalid(format!(
18105 "_syncular_segment must contain exactly one row, found {meta_count}"
18106 )));
18107 }
18108 if format != 1 {
18109 return Err(invalid(format!("format {format}")));
18110 }
18111 if meta_table != table.name {
18112 return Err(invalid(format!("image table {meta_table:?}")));
18113 }
18114 if schema_version != i64::from(self.schema.version) {
18115 return Err(invalid(format!("schemaVersion {schema_version}")));
18116 }
18117 if pin != as_of_commit_seq {
18118 return Err(invalid(format!("asOfCommitSeq {pin}")));
18119 }
18120 if digest != scope_digest {
18121 return Err(invalid("scopeDigest mismatch".to_owned()));
18122 }
18123 if meta_rows != row_count {
18124 return Err(invalid(format!("rowCount {meta_rows}")));
18125 }
18126
18127 let mut names: Vec<String> = Vec::new();
18129 let mut primary_keys: Vec<String> = Vec::new();
18130 {
18131 let mut stmt = img
18132 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
18133 .map_err(|_| invalid("image data table missing".to_owned()))?;
18134 let mut rows = stmt
18135 .query([])
18136 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
18137 while let Some(row) = rows
18138 .next()
18139 .map_err(|_| invalid("image data table unreadable".to_owned()))?
18140 {
18141 if row
18142 .get::<_, i64>(5)
18143 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
18144 > 0
18145 {
18146 primary_keys.push(row.get(1).map_err(|error| {
18147 invalid(Self::sqlite_failure(&self.storage_failure, error))
18148 })?);
18149 }
18150 names.push(
18151 row.get::<_, String>(1)
18152 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
18153 );
18154 }
18155 }
18156 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
18157 expected.push("_syncular_version");
18158 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
18159 return Err(SectionError::Abort(
18160 "sync.schema_mismatch".to_owned(),
18161 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
18162 ));
18163 }
18164
18165 if primary_keys != vec![table.primary_key.clone()] {
18166 return Err(invalid(
18167 "image primary key does not match generated schema".into(),
18168 ));
18169 }
18170 let actual_count: i64 = img
18171 .query_row(
18172 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
18173 [],
18174 |row| row.get(0),
18175 )
18176 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18177 if actual_count != row_count {
18178 return Err(invalid("image row count does not match descriptor".into()));
18179 }
18180 let insert = self.insert_row_sql(&base_table(&table.name), table);
18181 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
18182 let column_list = names
18183 .iter()
18184 .map(|n| quote_ident(n))
18185 .collect::<Vec<_>>()
18186 .join(", ");
18187 let mut stmt = img
18188 .prepare(&format!(
18189 "SELECT {column_list} FROM {} ORDER BY {}",
18190 quote_ident(&table.name),
18191 quote_ident(&table.primary_key)
18192 ))
18193 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18194 let mut rows = stmt
18195 .query([])
18196 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
18197 let mut applied = 0u32;
18198 loop {
18199 let clear = applied == 0 && first_fresh_page;
18200 let was_dirty = self.overlay_dirty.snapshot();
18201 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
18202 && self
18203 .outbox
18204 .iter()
18205 .flat_map(|commit| &commit.ops)
18206 .any(|op| op.table == table.name);
18207 let mirror_visible = !reconcile_pending
18208 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
18209 self.coalesce_unit(1024)
18211 .and_then(|()| self.begin_observation("syncular_image_chunk"))
18212 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18213 let result = (|| {
18214 self.rebuild_overlay_if_dirty()
18215 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18216 let mut changed_keys = Vec::new();
18217 let mut batch = ChangeAccumulator::default();
18218 if clear {
18219 if self.scoped_rows_exist(&table.name, effective) {
18220 batch.table(&table.name);
18221 }
18222 self.purge_scope_rows(&table.name, effective)
18223 .map_err(|()| SectionError::FailClosed)?;
18224 }
18225 let mut processed = 0u32;
18226 {
18227 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
18228 invalid(Self::sqlite_failure(&self.storage_failure, error))
18229 })?;
18230 let mut mirror = if mirror_visible {
18231 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
18232 invalid(Self::sqlite_failure(&self.storage_failure, error))
18233 })?)
18234 } else {
18235 None
18236 };
18237 for _ in 0..1024 {
18238 let Some(row) = rows.next().map_err(|error| {
18239 invalid(Self::sqlite_failure(&self.storage_failure, error))
18240 })?
18241 else {
18242 break;
18243 };
18244 for (i, column) in table.columns.iter().enumerate() {
18245 let cell = row.get_ref(i).map_err(|error| {
18246 invalid(Self::sqlite_failure(&self.storage_failure, error))
18247 })?;
18248 let param = image_cell_param(column, cell).map_err(invalid)?;
18249 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18250 invalid(Self::sqlite_failure(&self.storage_failure, error))
18251 })?;
18252 if let Some(mirror) = &mut mirror {
18253 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
18254 invalid(Self::sqlite_failure(&self.storage_failure, error))
18255 })?;
18256 }
18257 }
18258 let version_index = table.columns.len();
18259 let version: i64 = row.get(version_index).map_err(|error| {
18260 invalid(Self::sqlite_failure(&self.storage_failure, error))
18261 })?;
18262 if version < 1 {
18263 return Err(invalid("image row version must be positive".into()));
18264 }
18265 ins.raw_bind_parameter(version_index + 1, version)
18266 .map_err(|error| {
18267 invalid(Self::sqlite_failure(&self.storage_failure, error))
18268 })?;
18269 ins.raw_execute().map_err(|error| {
18270 invalid(Self::sqlite_failure(&self.storage_failure, error))
18271 })?;
18272 if let Some(mirror) = &mut mirror {
18273 mirror
18274 .raw_bind_parameter(version_index + 1, version)
18275 .map_err(|error| {
18276 invalid(Self::sqlite_failure(&self.storage_failure, error))
18277 })?;
18278 mirror.raw_execute().map_err(|error| {
18279 invalid(Self::sqlite_failure(&self.storage_failure, error))
18280 })?;
18281 }
18282 if reconcile_pending {
18283 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
18284 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
18285 )?);
18286 }
18287 processed += 1;
18288 }
18289 }
18290 if processed > 0 {
18291 batch.table(&table.name);
18292 self.overlay_dirty.table(&table.name);
18293 }
18294 if reconcile_pending {
18295 self.reconcile_imported_rows(table, &mut changed_keys)
18296 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18297 }
18298 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
18299 self.apply_outbox_ops(
18300 self.outbox
18301 .iter()
18302 .flat_map(|commit| &commit.ops)
18303 .filter(|op| op.table == table.name),
18304 )
18305 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18306 self.overlay_dirty.set(false);
18307 } else {
18308 self.rebuild_overlay_if_dirty()
18309 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18310 }
18311 self.finish_observation("syncular_image_chunk", batch)
18312 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
18313 Ok(processed)
18314 })();
18315 let processed = match result {
18316 Ok(processed) => processed,
18317 Err(error) => {
18318 self.rollback_observation("syncular_image_chunk");
18319 self.overlay_dirty.restore(was_dirty.clone());
18320 return Err(error);
18321 }
18322 };
18323 applied += processed;
18324 self.progress
18325 .update(|p| p.rows_processed = u64::from(applied));
18326 std::thread::yield_now();
18327 if i64::from(applied) == actual_count {
18328 break;
18329 }
18330 }
18331 Ok(applied)
18332 }
18333
18334 fn reconcile_imported_rows(
18338 &self,
18339 table: &TableSchema,
18340 keys: &mut Vec<SqlValue>,
18341 ) -> Result<(), String> {
18342 let visible = visible_table(&table.name);
18343 let base = base_table(&table.name);
18344 let pk = quote_ident(&table.primary_key);
18345 let predicate = row_id_predicate(table);
18346 let mut lookup = self.conn.prepare_cached(&format!(
18347 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
18348 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18349 let mut deleted = HashSet::new();
18350 for op in self
18351 .outbox
18352 .iter()
18353 .flat_map(|commit| &commit.ops)
18354 .filter(|op| op.table == table.name)
18355 {
18356 if op.upsert {
18357 if let Some(values) = &op.values {
18358 let key = json_to_column_value(
18359 &table.columns[table.pk_index],
18360 values.get(&table.primary_key),
18361 )?;
18362 keys.push(owned_sql_value(RowParam::Cell(&key))?);
18363 }
18364 } else if deleted.insert(&op.row_id) {
18365 keys.extend(
18366 lookup
18367 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
18368 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
18369 .collect::<Result<Vec<_>, _>>()
18370 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18371 );
18372 }
18373 }
18374 for chunk in keys.chunks(400) {
18377 let holes = vec!["?"; chunk.len()].join(",");
18378 self.conn
18379 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
18380 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18381 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18382 }
18383 let updates = table
18384 .columns
18385 .iter()
18386 .map(|column| column.name.as_str())
18387 .chain(std::iter::once("_syncular_version"))
18388 .filter(|name| *name != table.primary_key)
18389 .map(|name| {
18390 let name = quote_ident(name);
18391 format!("{name}=excluded.{name}")
18392 })
18393 .collect::<Vec<_>>()
18394 .join(",");
18395 for chunk in keys.chunks(400) {
18396 let holes = vec!["?"; chunk.len()].join(",");
18397 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
18398 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
18399 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18400 }
18401 Ok(())
18402 }
18403
18404 fn compile_local_data_purge(
18407 &self,
18408 input: &LocalDataPurgeInput,
18409 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
18410 let invalid = |message: String| format!("sync.invalid_request: {message}");
18411 if input.purge_id.is_empty()
18412 || input.purge_id.len() > 128
18413 || !is_local_operation_code_like(&input.purge_id)
18414 {
18415 return Err(invalid(
18416 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
18417 ));
18418 }
18419 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
18420 return Err(invalid(format!(
18421 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
18422 )));
18423 }
18424
18425 let mut deduplicated: BTreeMap<
18426 String,
18427 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
18428 > = BTreeMap::new();
18429 for target in &input.targets {
18430 let table = self.schema.table(&target.table).ok_or_else(|| {
18431 invalid(format!(
18432 "local purge names unknown table {:?}",
18433 target.table
18434 ))
18435 })?;
18436 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
18437 return Err(invalid(format!(
18438 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
18439 target.table
18440 )));
18441 }
18442 let mut selectors = Vec::with_capacity(target.selectors.len());
18443 let mut canonical_selectors = BTreeMap::new();
18444 for (column_name, raw_values) in &target.selectors {
18445 let Some((column_index, column)) = table
18446 .columns
18447 .iter()
18448 .enumerate()
18449 .find(|(_, column)| column.name == *column_name)
18450 else {
18451 return Err(invalid(format!(
18452 "local purge target {:?} names unknown column {:?}",
18453 target.table, column_name
18454 )));
18455 };
18456 let encrypted = table
18457 .encrypted_columns
18458 .iter()
18459 .any(|candidate| candidate.index == column_index);
18460 if column.ty != ColumnType::String || encrypted {
18461 return Err(invalid(format!(
18462 "local purge selector {:?}.{:?} must be a plaintext string column",
18463 target.table, column_name
18464 )));
18465 }
18466 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
18467 return Err(invalid(format!(
18468 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
18469 target.table, column_name
18470 )));
18471 }
18472 let mut values = raw_values.clone();
18473 values.sort();
18474 values.dedup();
18475 if values.iter().any(|value| {
18476 value.is_empty()
18477 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
18478 || !is_local_operation_code_like(value)
18479 }) {
18480 return Err(invalid(format!(
18481 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
18482 )));
18483 }
18484 selectors.push((column_name.clone(), values.clone()));
18485 canonical_selectors.insert(column_name.clone(), values);
18486 }
18487 let canonical = LocalDataPurgeTarget {
18488 table: target.table.clone(),
18489 selectors: canonical_selectors,
18490 };
18491 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18492 deduplicated.insert(
18493 key,
18494 (
18495 CompiledLocalDataPurgeTarget {
18496 table: target.table.clone(),
18497 selectors,
18498 },
18499 canonical,
18500 ),
18501 );
18502 }
18503 let targets = deduplicated
18504 .values()
18505 .map(|(compiled, _)| compiled.clone())
18506 .collect::<Vec<_>>();
18507 let canonical = deduplicated
18508 .values()
18509 .map(|(_, target)| target.clone())
18510 .collect::<Vec<_>>();
18511 let canonical_plan =
18512 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
18513 Ok((targets, canonical_plan))
18514 }
18515
18516 fn local_purge_base_row_ids(
18517 &self,
18518 targets: &[CompiledLocalDataPurgeTarget],
18519 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
18520 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
18521 for target in targets {
18522 let table = self
18523 .schema
18524 .table(&target.table)
18525 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
18526 let mut clauses = Vec::with_capacity(target.selectors.len());
18527 let mut params = Vec::new();
18528 for (column, values) in &target.selectors {
18529 clauses.push(format!(
18530 "{} IN ({})",
18531 quote_ident(column),
18532 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
18533 ));
18534 params.extend(values.iter().cloned().map(SqlValue::Text));
18535 }
18536 let sql = format!(
18537 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
18538 quote_ident(&table.primary_key),
18539 base_table(&target.table),
18540 clauses.join(" AND ")
18541 );
18542 let mut statement = self
18543 .conn
18544 .prepare(&sql)
18545 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18546 let rows = statement
18547 .query_map(rusqlite::params_from_iter(params), |row| {
18548 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
18549 render_row_id_json(Some(&value))
18550 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
18551 })
18552 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18553 let ids = by_table.entry(target.table.clone()).or_default();
18554 for row in rows {
18555 ids.insert(
18556 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
18557 );
18558 }
18559 }
18560 Ok(by_table)
18561 }
18562
18563 fn local_purge_values_match(
18564 target: &CompiledLocalDataPurgeTarget,
18565 values: &Map<String, Value>,
18566 ) -> bool {
18567 target.selectors.iter().all(|(column, allowed)| {
18568 values
18569 .get(column)
18570 .and_then(Value::as_str)
18571 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
18572 })
18573 }
18574
18575 pub fn purge_local_data(
18579 &mut self,
18580 input: &LocalDataPurgeInput,
18581 ) -> Result<LocalDataPurgeResult, String> {
18582 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
18583 let meta_key = format!("localPurge:{}", input.purge_id);
18584 let applied_plan = self
18585 .conn
18586 .query_row(
18587 "SELECT value FROM _syncular_meta WHERE key = ?1",
18588 rusqlite::params![meta_key],
18589 |row| row.get::<_, String>(0),
18590 )
18591 .optional()
18592 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18593 if let Some(applied_plan) = applied_plan {
18594 if applied_plan != canonical_plan {
18595 return Err(format!(
18596 "sync.invalid_request: local purge id {:?} was already used with a different plan",
18597 input.purge_id
18598 ));
18599 }
18600 return Ok(LocalDataPurgeResult {
18601 already_applied: true,
18602 purged_rows: 0,
18603 dropped_commits: 0,
18604 });
18605 }
18606
18607 let _ = self.drop_previous_version();
18611
18612 let prior_outbox = self.outbox.clone();
18613 let prior_failed = self.failed_commits.clone();
18614 let prior_rejections = self.rejections.clone();
18615 let prior_conflicts = self.conflicts.clone();
18616 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18617 self.begin_observation("syncular_local_purge")?;
18618 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
18619 let row_ids = self.local_purge_base_row_ids(&targets)?;
18620 let retained_dropped = self.drop_failed_commits(|table, values| {
18621 targets.iter().any(|target| {
18622 target.table == table && Self::local_purge_values_match(target, values)
18623 })
18624 })?;
18625 let doomed = self
18626 .outbox
18627 .iter()
18628 .filter(|commit| {
18629 commit.ops.iter().any(|operation| {
18630 let matching_targets = targets
18631 .iter()
18632 .filter(|target| target.table == operation.table)
18633 .collect::<Vec<_>>();
18634 if matching_targets.is_empty() {
18635 return false;
18636 }
18637 if row_ids
18638 .get(&operation.table)
18639 .is_some_and(|ids| ids.contains(&operation.row_id))
18640 {
18641 return true;
18642 }
18643 operation.values.as_ref().is_some_and(|values| {
18644 matching_targets
18645 .iter()
18646 .any(|target| Self::local_purge_values_match(target, values))
18647 })
18648 })
18649 })
18650 .cloned()
18651 .collect::<Vec<_>>();
18652 let doomed_ids = doomed
18653 .iter()
18654 .map(|commit| commit.client_commit_id.clone())
18655 .collect::<BTreeSet<_>>();
18656 let mut batch = ChangeAccumulator::default();
18657 for commit in &retained_dropped {
18658 batch.conflicts = true;
18659 batch.rejections = true;
18660 batch.outcomes = true;
18661 for operation in &commit.ops {
18662 batch.table(&operation.table);
18663 }
18664 }
18665 let mut rejections = Vec::new();
18666 for commit in &doomed {
18667 for operation in &commit.ops {
18668 batch.table(&operation.table);
18669 }
18670 let results = commit
18671 .ops
18672 .iter()
18673 .enumerate()
18674 .map(|(op_index, operation)| {
18675 let rejection = RejectionRecord {
18676 client_commit_id: commit.client_commit_id.clone(),
18677 op_index: op_index as i32,
18678 code: "client.local_data_purged".to_owned(),
18679 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
18680 retryable: false,
18681 details: None,
18682 operation: Some(CommitOperation::from(operation)),
18683 };
18684 rejections.push(rejection.clone());
18685 CommitOperationOutcome::Error { rejection }
18686 })
18687 .collect::<Vec<_>>();
18688 self.persist_commit_outcome(
18689 &commit.client_commit_id,
18690 CommitOutcomeStatus::Rejected,
18691 &results,
18692 Some(&commit.ops),
18693 )?;
18694 self.delete_outbox_persisted(&commit.client_commit_id)?;
18695 }
18696 if !doomed.is_empty() {
18697 self.outbox
18698 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
18699 self.rejections.extend(rejections);
18700 self.prune_commit_outcomes()?;
18701 batch.status = true;
18702 batch.rejections = true;
18703 batch.outcomes = true;
18704 }
18705
18706 let mut purged_rows = 0usize;
18707 for (table, ids) in &row_ids {
18708 if ids.is_empty() {
18709 continue;
18710 }
18711 batch.table(table);
18712 for row_id in ids {
18713 self.delete_base_row(table, row_id)?;
18714 purged_rows += 1;
18715 }
18716 }
18717 self.rebuild_overlay_if_dirty()?;
18718 self.delete_unreferenced_cached_blobs()?;
18719 self.conn
18720 .execute(
18721 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
18722 rusqlite::params![meta_key, canonical_plan],
18723 )
18724 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18725 Ok((
18726 batch,
18727 LocalDataPurgeResult {
18728 already_applied: false,
18729 purged_rows,
18730 dropped_commits: doomed.len(),
18731 },
18732 ))
18733 })();
18734 let (batch, result) = match applied {
18735 Ok(value) => value,
18736 Err(error) => {
18737 self.rollback_observation("syncular_local_purge");
18738 self.outbox = prior_outbox;
18739 self.failed_commits = prior_failed;
18740 self.rejections = prior_rejections;
18741 self.conflicts = prior_conflicts;
18742 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18743 return Err(error);
18744 }
18745 };
18746 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
18747 self.rollback_observation("syncular_local_purge");
18748 self.outbox = prior_outbox;
18749 self.failed_commits = prior_failed;
18750 self.rejections = prior_rejections;
18751 self.conflicts = prior_conflicts;
18752 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18753 return Err(error);
18754 }
18755 self.cancel_sync_round();
18756 Ok(result)
18757 }
18758
18759 pub fn rebootstrap_local_data(
18765 &mut self,
18766 input: &LocalDataRebootstrapInput,
18767 ) -> Result<LocalDataRebootstrapResult, String> {
18768 if input.rebootstrap_id.is_empty()
18769 || input.rebootstrap_id.len() > 128
18770 || !is_local_operation_code_like(&input.rebootstrap_id)
18771 {
18772 return Err(
18773 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
18774 .to_owned(),
18775 );
18776 }
18777 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
18778 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
18779 let (retained_commits, reset_subscriptions) =
18780 decode_local_rebootstrap_receipt(&persisted)?;
18781 return Ok(LocalDataRebootstrapResult {
18782 already_applied: true,
18783 retained_commits,
18784 reset_subscriptions,
18785 });
18786 }
18787 if self.stopped || self.schema_floor.is_some() {
18788 return Err(
18789 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
18790 .to_owned(),
18791 );
18792 }
18793
18794 let retained_commits = self.outbox.len();
18795 let reset_subscriptions = self.subs.len();
18796 let prior_subs = self.subs.clone();
18797 let prior_upgrading = self.upgrading;
18798 let prior_stopped = self.stopped;
18799 let prior_schema_floor = self.schema_floor.clone();
18800 let prior_overlay_dirty = self.overlay_dirty.snapshot();
18801 let prior_sync_needed = self.sync_needed;
18802 let prior_sync_intents = self.sync_intent_queue.clone();
18803 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
18804
18805 let prior_active_round = self.active_round;
18806 self.begin_observation("syncular_local_rebootstrap")?;
18807 let mut batch = ChangeAccumulator::default();
18808 let applied = (|| -> Result<(), String> {
18809 self.run_schema_reset_observed(&mut batch, false)?;
18810 self.conn
18811 .execute(
18812 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
18813 rusqlite::params![meta_key, receipt],
18814 )
18815 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18816 self.sync_needed = true;
18817 self.sync_intent_queue.push_back(SyncIntent::Interactive);
18818 batch.status = true;
18819 Ok(())
18820 })();
18821 if let Err(error) = applied {
18822 self.rollback_observation("syncular_local_rebootstrap");
18823 self.active_round = prior_active_round;
18824 self.subs = prior_subs;
18825 self.upgrading = prior_upgrading;
18826 self.stopped = prior_stopped;
18827 self.schema_floor = prior_schema_floor;
18828 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18829 self.sync_needed = prior_sync_needed;
18830 self.sync_intent_queue = prior_sync_intents;
18831 return Err(error);
18832 }
18833 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
18834 self.rollback_observation("syncular_local_rebootstrap");
18835 self.active_round = prior_active_round;
18836 self.subs = prior_subs;
18837 self.upgrading = prior_upgrading;
18838 self.stopped = prior_stopped;
18839 self.schema_floor = prior_schema_floor;
18840 self.overlay_dirty.restore(prior_overlay_dirty.clone());
18841 self.sync_needed = prior_sync_needed;
18842 self.sync_intent_queue = prior_sync_intents;
18843 return Err(error);
18844 }
18845
18846 Ok(LocalDataRebootstrapResult {
18847 already_applied: false,
18848 retained_commits,
18849 reset_subscriptions,
18850 })
18851 }
18852
18853 fn purge_scope_rows(
18858 &mut self,
18859 table_name: &str,
18860 effective: &[(String, Vec<String>)],
18861 ) -> Result<(), ()> {
18862 if effective.is_empty() {
18863 return Ok(());
18864 }
18865 let table = self.schema.table(table_name).ok_or(())?.clone();
18866 let mut clauses = Vec::new();
18867 let mut params: Vec<SqlValue> = Vec::new();
18868 for (variable, values) in effective {
18869 let column = table.scope_column(variable).ok_or(())?;
18870 let placeholders: Vec<String> = values
18871 .iter()
18872 .map(|v| {
18873 params.push(SqlValue::Text(v.clone()));
18874 "?".to_owned()
18875 })
18876 .collect();
18877 clauses.push(format!(
18878 "{} IN ({})",
18879 quote_ident(column),
18880 placeholders.join(", ")
18881 ));
18882 }
18883 let sql = format!(
18884 "DELETE FROM {} WHERE {}",
18885 base_table(table_name),
18886 clauses.join(" AND ")
18887 );
18888 self.overlay_dirty.table(table_name);
18889 self.conn
18890 .execute(&sql, rusqlite::params_from_iter(¶ms))
18891 .map_err(|_| ())?;
18892 self.conn
18893 .execute(
18894 &format!(
18895 "DELETE FROM {} WHERE {}",
18896 visible_table(table_name),
18897 clauses.join(" AND ")
18898 ),
18899 rusqlite::params_from_iter(¶ms),
18900 )
18901 .map_err(|_| ())?;
18902 Ok(())
18903 }
18904
18905 fn drop_doomed_outbox(
18908 &mut self,
18909 table_name: &str,
18910 effective: &[(String, Vec<String>)],
18911 ) -> Result<bool, String> {
18912 if effective.is_empty() {
18913 return Ok(false);
18914 }
18915 let Some(table) = self.schema.table(table_name).cloned() else {
18916 return Ok(false);
18917 };
18918 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
18919 for (variable, values) in effective {
18920 match table.scope_column(variable) {
18921 Some(column) => mappings.push((column, values)),
18922 None => return Ok(false), }
18924 }
18925 let doomed: Vec<OutboxCommit> = self
18926 .outbox
18927 .iter()
18928 .filter(|commit| {
18929 commit.ops.iter().any(|op| {
18930 op.upsert
18931 && op.table == table_name
18932 && op.values.as_ref().is_some_and(|values| {
18933 mappings
18934 .iter()
18935 .all(|(column, allowed)| match values.get(*column) {
18936 Some(Value::String(s)) => allowed.contains(s),
18937 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
18938 Some(Value::Bool(b)) => allowed.contains(&b.to_string()),
18939 _ => false,
18940 })
18941 })
18942 })
18943 })
18944 .cloned()
18945 .collect();
18946 if doomed.is_empty() {
18947 return Ok(false);
18948 }
18949 let mut rejections = Vec::new();
18950 for commit in &doomed {
18951 let results = commit
18952 .ops
18953 .iter()
18954 .enumerate()
18955 .map(|(op_index, operation)| {
18956 let rejection = RejectionRecord {
18957 client_commit_id: commit.client_commit_id.clone(),
18958 op_index: op_index as i32,
18959 code: "sync.scope_revoked".to_owned(),
18960 message: "the commit was dropped because its effective scope was revoked"
18961 .to_owned(),
18962 retryable: false,
18963 details: None,
18964 operation: Some(CommitOperation::from(operation)),
18965 };
18966 rejections.push(rejection.clone());
18967 CommitOperationOutcome::Error { rejection }
18968 })
18969 .collect::<Vec<_>>();
18970 self.persist_commit_outcome(
18971 &commit.client_commit_id,
18972 CommitOutcomeStatus::Rejected,
18973 &results,
18974 Some(&commit.ops),
18975 )?;
18976 self.delete_outbox_persisted(&commit.client_commit_id)?;
18977 }
18978 self.prune_commit_outcomes()?;
18979 let doomed_ids = doomed
18980 .iter()
18981 .map(|commit| commit.client_commit_id.as_str())
18982 .collect::<BTreeSet<_>>();
18983 self.outbox
18984 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
18985 self.rejections.extend(rejections);
18986 Ok(true)
18987 }
18988
18989 pub fn upload_blob(
18996 &mut self,
18997 bytes: &[u8],
18998 media_type: Option<String>,
18999 name: Option<String>,
19000 ) -> Result<Value, String> {
19001 let blob_id = blob_id_for(bytes);
19002 let now = self.clock_now_ms();
19003 let tx = self
19004 .conn
19005 .transaction()
19006 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19007 tx.execute(
19008 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
19009 ON CONFLICT(blob_id) DO NOTHING",
19010 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
19011 )
19012 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19013 tx.execute(
19014 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
19015 rusqlite::params![blob_id, media_type, now],
19016 )
19017 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19018 tx.commit()
19019 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19020 self.enforce_blob_cache_cap();
19021 let mut obj = Map::new();
19022 obj.insert("blobId".to_owned(), Value::from(blob_id));
19023 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
19024 if let Some(mt) = media_type {
19025 obj.insert("mediaType".to_owned(), Value::from(mt));
19026 }
19027 if let Some(n) = name {
19028 obj.insert("name".to_owned(), Value::from(n));
19029 }
19030 Ok(Value::Object(obj))
19031 }
19032
19033 pub fn fetch_blob_bytes(
19037 &mut self,
19038 transport: &mut dyn Transport,
19039 blob_id_or_ref: &str,
19040 ) -> Result<FetchedBlob, (String, String)> {
19041 let simple = |m: String| ("client.failed".to_owned(), m);
19042 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
19043 blob_id_or_ref.to_owned()
19044 } else {
19045 let value: Value = serde_json::from_str(blob_id_or_ref)
19046 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
19047 value
19048 .get("blobId")
19049 .and_then(Value::as_str)
19050 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
19051 .to_owned()
19052 };
19053 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
19054 return Ok(cached);
19055 }
19056 if !self.transport_enabled {
19057 return Err(("sync.offline".into(), "transport gate is closed".into()));
19058 }
19059 #[cfg(feature = "bench-internals")]
19065 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
19066 let bytes = match transport
19067 .blob_download(&blob_id)
19068 .map_err(|e| (e.code, e.message))?
19069 {
19070 BlobDownload::Bytes(bytes) => bytes,
19071 BlobDownload::Url {
19072 url,
19073 url_expires_at_ms,
19074 } => {
19075 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
19077 return Err((
19078 "sync.segment_expired".to_owned(),
19079 format!(
19080 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
19081 ),
19082 ));
19083 }
19084 transport
19085 .fetch_blob_url(&url)
19086 .map_err(|e| (e.code, e.message))?
19087 }
19088 };
19089 #[cfg(feature = "bench-internals")]
19090 drop(download_phase);
19091 #[cfg(feature = "bench-internals")]
19092 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
19093 if blob_id_for(&bytes) != blob_id {
19095 return Err(simple(format!(
19096 "blob content address mismatch for {blob_id}"
19097 )));
19098 }
19099 #[cfg(feature = "bench-internals")]
19100 drop(validate_phase);
19101 #[cfg(feature = "bench-internals")]
19102 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
19103 let now = self.clock_now_ms();
19104 self.conn
19105 .execute(
19106 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
19107 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
19108 )
19109 .map_err(|e| simple(e.to_string()))?;
19110 self.conn
19111 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
19112 .map_err(|e| simple(e.to_string()))?;
19113 self.enforce_blob_cache_cap();
19114 #[cfg(feature = "bench-internals")]
19115 drop(insert_phase);
19116 self.get_cached_blob_bytes(&blob_id)
19117 .map_err(simple)?
19118 .ok_or_else(|| simple("blob cache write failed".to_owned()))
19119 }
19120
19121 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
19122 #[cfg(feature = "bench-internals")]
19123 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
19124 let mut stmt = self
19125 .conn
19126 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
19127 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19128 let mut rows = stmt
19129 .query(rusqlite::params![blob_id])
19130 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19131 if let Some(row) = rows
19132 .next()
19133 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
19134 {
19135 let bytes: Vec<u8> = row
19136 .get(0)
19137 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19138 let byte_length: i64 = row
19139 .get(1)
19140 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19141 let media_type: Option<String> = row
19142 .get(2)
19143 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19144 return Ok(Some(FetchedBlob {
19145 blob_id: blob_id.to_owned(),
19146 byte_length,
19147 bytes,
19148 media_type,
19149 }));
19150 }
19151 Ok(None)
19152 }
19153
19154 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
19156 let pending: Vec<(String, Option<String>)> = {
19157 let mut stmt = self
19158 .conn
19159 .prepare(
19160 "SELECT blob_id, media_type FROM (
19161 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
19162 UNION ALL
19163 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
19164 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
19165 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
19166 ) ORDER BY created_at_ms, blob_id",
19167 )
19168 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19169 let rows = stmt
19170 .query_map([], |row| {
19171 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
19172 })
19173 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19174 rows.collect::<Result<_, _>>()
19175 .map_err(|error| match error {
19176 rusqlite::Error::InvalidColumnType(..)
19177 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
19178 "sync.local_corrupt",
19179 "Pending blob upload metadata is invalid",
19180 ),
19181 _ => TransportError::new("client.failed", error.to_string()),
19182 })?
19183 };
19184 let mut uploads = Vec::new();
19185 for (blob_id, media_type) in pending {
19186 let bytes = (|| {
19187 let stored: Option<(SqlValue, SqlValue)> = self
19188 .conn
19189 .query_row(
19190 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
19191 rusqlite::params![blob_id],
19192 |row| Ok((row.get(0)?, row.get(1)?)),
19193 )
19194 .optional()
19195 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19196 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
19197 return Err(TransportError::new(
19198 "sync.local_corrupt",
19199 "Pending blob upload body is missing or corrupt",
19200 ));
19201 };
19202 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
19203 return Err(TransportError::new(
19204 "sync.local_corrupt",
19205 "Pending blob upload body is missing or corrupt",
19206 ));
19207 }
19208 Ok(bytes)
19209 })();
19210 let failed = bytes.is_err();
19211 uploads.push(crate::round::PendingUpload {
19212 id: blob_id,
19213 bytes,
19214 media_type,
19215 });
19216 if failed {
19217 break;
19218 }
19219 }
19220 Ok(uploads)
19221 }
19222
19223 #[cfg(test)]
19224 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
19225 for upload in self.pending_blob_uploads()? {
19226 self.upload_one(
19227 transport,
19228 &upload.id,
19229 &upload.bytes?,
19230 upload.media_type.as_deref(),
19231 )?;
19232 self.conn
19233 .execute(
19234 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
19235 rusqlite::params![upload.id],
19236 )
19237 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
19238 }
19239 Ok(())
19240 }
19241
19242 #[cfg(test)]
19249 fn upload_one(
19250 &self,
19251 transport: &mut dyn Transport,
19252 blob_id: &str,
19253 bytes: &[u8],
19254 media_type: Option<&str>,
19255 ) -> Result<(), TransportError> {
19256 crate::round::upload_one(transport, blob_id, bytes, media_type, self.now_ms)
19257 }
19258
19259 fn visible_blob_ids_sql(&self) -> String {
19260 let mut selects = Vec::new();
19261 for table in &self.schema.tables {
19262 for column in table
19263 .columns
19264 .iter()
19265 .filter(|column| column.ty == ColumnType::BlobRef)
19266 {
19267 let identifier = quote_ident(&column.name);
19268 selects.push(format!(
19269 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
19270 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
19271 AND json_type({identifier}, '$.blobId') = 'text'",
19272 quote_ident(&table.name),
19273 ));
19274 }
19275 }
19276 if selects.is_empty() {
19277 "SELECT NULL AS blob_id WHERE 0".to_owned()
19278 } else {
19279 selects.join(" UNION ")
19280 }
19281 }
19282
19283 fn enforce_blob_cache_cap(&self) {
19286 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
19287 return;
19288 };
19289 let mut total: i64 = self
19290 .conn
19291 .query_row(
19292 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
19293 [],
19294 |row| row.get(0),
19295 )
19296 .unwrap_or(0);
19297 if total <= max_bytes {
19298 return;
19299 }
19300 let query = format!(
19301 "SELECT blob_id, byte_length FROM _syncular_blobs
19302 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19303 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19304 AND blob_id NOT IN ({})
19305 ORDER BY created_at_ms ASC, blob_id ASC",
19306 self.visible_blob_ids_sql(),
19307 );
19308 let candidates: Vec<(String, i64)> = {
19309 let Ok(mut stmt) = self.conn.prepare(&query) else {
19310 return;
19311 };
19312 let Ok(rows) = stmt.query_map([], |row| {
19313 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
19314 }) else {
19315 return;
19316 };
19317 rows.filter_map(Result::ok).collect()
19318 };
19319 for (blob_id, byte_length) in candidates {
19320 if total <= max_bytes {
19321 break;
19322 }
19323 let _ = self.conn.execute(
19324 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
19325 rusqlite::params![blob_id],
19326 );
19327 total -= byte_length;
19328 }
19329 }
19330
19331 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
19332 #[cfg(feature = "bench-internals")]
19333 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
19334 if !self.schema_has_blobs() {
19335 return Ok(());
19336 }
19337 self.rebuild_overlay_if_dirty()?;
19338 self.conn
19339 .execute(
19340 &format!(
19341 "DELETE FROM _syncular_blobs
19342 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
19343 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
19344 AND blob_id NOT IN ({})",
19345 self.visible_blob_ids_sql(),
19346 ),
19347 [],
19348 )
19349 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19350 Ok(())
19351 }
19352
19353 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
19358 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
19359 return sql.clone();
19360 }
19361 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
19362 columns.push(quote_ident("_syncular_version"));
19363 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
19364 let primary_key = quote_ident(&table.primary_key);
19365 let updates = columns
19366 .iter()
19367 .filter(|column| **column != primary_key)
19368 .map(|column| format!("{column}=excluded.{column}"))
19369 .collect::<Vec<_>>()
19370 .join(", ");
19371 let sql = format!(
19372 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
19373 columns.join(", "),
19374 placeholders.join(", ")
19375 );
19376 self.insert_sql
19377 .borrow_mut()
19378 .insert(full_table.to_owned(), sql.clone());
19379 sql
19380 }
19381
19382 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
19383 self.overlay_dirty.table(table_name);
19384 self.write_row(&base_table(table_name), table_name, row, version)
19385 }
19386
19387 fn write_row(
19388 &self,
19389 full_table: &str,
19390 table_name: &str,
19391 row: &Row,
19392 version: i64,
19393 ) -> Result<(), String> {
19394 #[cfg(feature = "bench-internals")]
19395 let _phase = self.benchmark_phases.start(Phase::RowWrite);
19396 let table = self
19397 .schema
19398 .table(table_name)
19399 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19400 let sql = self.insert_row_sql(full_table, table);
19401 let mut stmt = self
19402 .conn
19403 .prepare_cached(&sql)
19404 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19405 let params = row
19406 .iter()
19407 .map(RowParam::Cell)
19408 .chain(std::iter::once(RowParam::Version(version)));
19409 stmt.execute(rusqlite::params_from_iter(params))
19410 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19411 Ok(())
19412 }
19413
19414 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
19415 let table = self
19416 .schema
19417 .table(table_name)
19418 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
19419 self.overlay_dirty.table(table_name);
19420 let sql = format!(
19421 "DELETE FROM {} WHERE {}",
19422 base_table(table_name),
19423 row_id_predicate(table)
19424 );
19425 let mut stmt = self
19426 .conn
19427 .prepare_cached(&sql)
19428 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19429 stmt.execute(rusqlite::params![row_id])
19430 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19431 Ok(())
19432 }
19433
19434 fn has_acknowledged_rows(&self) -> bool {
19435 self.conn
19436 .query_row(
19437 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
19438 [],
19439 |row| row.get(0),
19440 )
19441 .expect("read acknowledged intent")
19442 }
19443
19444 fn acknowledged_operations(
19445 &self,
19446 rows: Option<&HashSet<(&str, &str)>>,
19447 tables: Option<&BTreeSet<&str>>,
19448 ) -> Result<Vec<OutboxOp>, String> {
19449 let keys = rows.map(|rows| {
19450 serde_json::to_string(
19451 &rows
19452 .iter()
19453 .map(|(table, id)| serde_json::to_string(&(table, id)).expect("row key"))
19454 .collect::<Vec<_>>(),
19455 )
19456 .expect("row keys")
19457 });
19458 let mut statement = self
19459 .conn
19460 .prepare("SELECT op_json,intent_json FROM _syncular_acked_rows WHERE (?1 IS NULL OR json_array(tbl,id) IN(SELECT value FROM json_each(?1))) AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx")
19461 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19462 let rows = statement
19463 .query_map(
19464 rusqlite::params![
19465 keys,
19466 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
19467 ],
19468 |row| {
19469 #[cfg(test)]
19470 self.acknowledged_replay_count
19471 .set(self.acknowledged_replay_count.get() + 1);
19472 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
19473 },
19474 )
19475 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19476 rows.map(|row| {
19477 let (op, intent) =
19478 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19479 let op: CommitOperation =
19480 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
19481 let table = self
19482 .schema
19483 .table(&op.table)
19484 .ok_or_else(|| "sync.unknown_table".to_owned())?;
19485 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
19486 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
19487 } else {
19488 op.values
19489 };
19490 Ok(OutboxOp {
19491 table: op.table,
19492 row_id: op.row_id,
19493 upsert: op.op == "upsert",
19494 base_version: op.base_version,
19495 values,
19496 })
19497 })
19498 .collect()
19499 }
19500
19501 fn drop_acknowledged_rows(
19502 &self,
19503 matches: impl Fn(&str, &Map<String, Value>) -> bool,
19504 ) -> Result<Vec<OutboxCommit>, String> {
19505 let mut statement = self
19506 .conn
19507 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
19508 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19509 let rows = statement
19510 .query_map([], |row| {
19511 Ok((
19512 row.get::<_, String>(0)?,
19513 row.get::<_, String>(1)?,
19514 row.get::<_, String>(2)?,
19515 row.get::<_, String>(3)?,
19516 ))
19517 })
19518 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19519 let mut doomed = HashSet::new();
19520 for row in rows {
19521 let (id, table, intent, row_id) =
19522 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19523 let values =
19524 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
19525 let base_matches = self
19526 .schema
19527 .table(&table)
19528 .map(|schema| self.stored_row(schema, &row_id, true))
19529 .transpose()?
19530 .flatten()
19531 .is_some_and(|(base, _)| {
19532 let schema = self.schema.table(&table).expect("acknowledged table");
19533 let values = schema
19534 .columns
19535 .iter()
19536 .zip(base.iter())
19537 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
19538 .collect();
19539 matches(&table, &values)
19540 });
19541 if matches(&table, &values) || base_matches {
19542 doomed.insert(id);
19543 }
19544 }
19545 let mut dropped = Vec::new();
19546 for id in doomed {
19547 let mut statement = self
19548 .conn
19549 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
19550 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19551 let operations = statement
19552 .query_map([&id], |row| row.get::<_, String>(0))
19553 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19554 let mut ops = Vec::new();
19555 for operation in operations {
19556 let json = operation
19557 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19558 let op: CommitOperation =
19559 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
19560 self.overlay_dirty.table(&op.table);
19561 ops.push(OutboxOp {
19562 table: op.table,
19563 row_id: op.row_id,
19564 upsert: op.op == "upsert",
19565 base_version: op.base_version,
19566 values: op.values,
19567 });
19568 }
19569 dropped.push(OutboxCommit {
19570 client_commit_id: id.clone(),
19571 ops,
19572 });
19573 self.conn
19574 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
19575 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19576 }
19577 Ok(dropped)
19578 }
19579
19580 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
19583 if self.overlay_dirty.get() {
19584 self.rebuild_overlay()?;
19585 }
19586 Ok(())
19587 }
19588
19589 fn rebuild_overlay(&mut self) -> Result<(), String> {
19593 #[cfg(feature = "bench-internals")]
19594 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
19595 #[cfg(test)]
19596 self.overlay_rebuild_count
19597 .set(self.overlay_rebuild_count.get() + 1);
19598 let dirty = self.overlay_dirty.snapshot();
19599 let tables: Vec<_> = self
19600 .schema
19601 .tables
19602 .iter()
19603 .filter(|table| {
19604 dirty
19605 .as_ref()
19606 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
19607 })
19608 .cloned()
19609 .collect();
19610 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
19611 self.begin_observation("syncular_overlay")?;
19612 let rebuilt = (|| {
19613 for table in &tables {
19614 for index in &table.fts_indexes {
19615 self.drop_fts_triggers(index)?;
19616 }
19617 let visible = visible_table(&table.name);
19618 let base = base_table(&table.name);
19619 self.conn
19620 .execute_batch(&format!(
19621 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
19622 ))
19623 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19624 }
19625 let acknowledged = self.acknowledged_operations(None, Some(&names))?;
19626 self.apply_outbox_ops(
19627 acknowledged
19628 .iter()
19629 .filter(|op| names.contains(op.table.as_str())),
19630 )?;
19631 for (commit, initial) in &self.failed_commits {
19632 for operation in &commit.ops {
19633 if !names.contains(operation.table.as_str()) || !operation.upsert {
19634 continue;
19635 }
19636 if let Some(table) = self.schema.table(&operation.table) {
19637 if self.visible_row(table, &operation.row_id)?.is_none()
19638 && operation.values.as_ref().is_some_and(|values| {
19639 table
19640 .columns
19641 .iter()
19642 .all(|column| values.contains_key(&column.name))
19643 })
19644 {
19645 let key = serde_json::to_string(&(
19646 operation.table.clone(),
19647 operation.row_id.clone(),
19648 ))
19649 .expect("row key serialization");
19650 if let Some(Value::Object(values)) = initial.get(&key) {
19651 self.apply_outbox_ops(&[OutboxOp {
19652 values: Some(values.clone()),
19653 ..operation.clone()
19654 }])?;
19655 }
19656 }
19657 }
19658 }
19659 self.apply_outbox_ops(
19660 commit
19661 .ops
19662 .iter()
19663 .filter(|op| names.contains(op.table.as_str())),
19664 )?;
19665 }
19666 for commit in &self.outbox {
19667 self.apply_outbox_ops(
19668 commit
19669 .ops
19670 .iter()
19671 .filter(|op| names.contains(op.table.as_str())),
19672 )?;
19673 }
19674 for table in &tables {
19675 for index in &table.fts_indexes {
19676 self.rebuild_fts_projection(table, index)?;
19677 self.create_fts_triggers(table, index)?;
19678 }
19679 }
19680 self.conn
19681 .execute_batch("RELEASE syncular_overlay")
19682 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19683 Ok(())
19684 })();
19685 if rebuilt.is_err() {
19686 self.conn
19687 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
19688 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19689 } else {
19690 self.overlay_dirty.set(false);
19691 }
19692 rebuilt
19693 }
19694
19695 fn apply_outbox_ops<'a>(
19700 &self,
19701 ops: impl IntoIterator<Item = &'a OutboxOp>,
19702 ) -> Result<(), String> {
19703 #[cfg(feature = "bench-internals")]
19704 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
19705 for op in ops {
19706 #[cfg(feature = "bench-internals")]
19707 if let Some(phase) = &mut phase {
19708 phase.unit();
19709 }
19710 if op.upsert {
19711 if let (Some(table), Some(values)) = (
19712 self.schema
19713 .table(&op.table)
19714 .filter(|table| table.indexes.iter().any(|index| index.unique)),
19715 op.values.as_ref(),
19716 ) {
19717 let mut intended: Map<String, Value> = self
19718 .visible_row(table, &op.row_id)?
19719 .map(|(row, _)| {
19720 table
19721 .columns
19722 .iter()
19723 .zip(&row)
19724 .map(|(column, value)| {
19725 (column.name.clone(), column_value_to_json(value))
19726 })
19727 .collect()
19728 })
19729 .unwrap_or_default();
19730 intended.extend(values.clone());
19731 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
19732 continue;
19733 }
19734 }
19735 }
19736 self.apply_outbox_op(op)?;
19737 }
19738 Ok(())
19739 }
19740
19741 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
19742 let Some(table) = self.schema.table(&op.table) else {
19743 return Ok(());
19744 };
19745 let visible = visible_table(&table.name);
19746 if !op.upsert {
19747 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
19748 self.conn
19749 .prepare_cached(&sql)
19750 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
19751 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19752 return Ok(());
19753 }
19754 let Some(values) = op.values.as_ref() else {
19755 return Ok(());
19756 };
19757 let full = table
19762 .columns
19763 .iter()
19764 .all(|column| values.contains_key(&column.name));
19765 let local = self.visible_row(table, &op.row_id)?;
19766 if local.is_none() && !full {
19767 return Ok(());
19768 }
19769 let mut row: Row = Vec::with_capacity(table.columns.len());
19770 for (index, column) in table.columns.iter().enumerate() {
19771 let incoming = match values.get(&column.name) {
19772 Some(raw) => json_to_column_value(
19773 if raw.get("$bytes").is_some()
19774 && table
19775 .encrypted_columns
19776 .iter()
19777 .any(|encrypted| encrypted.index == index)
19778 {
19779 &table.wire_columns[index]
19780 } else {
19781 column
19782 },
19783 Some(raw),
19784 ),
19785 None => Ok(local
19786 .as_ref()
19787 .and_then(|(values, _)| values.get(index).cloned())
19788 .flatten()),
19789 };
19790 row.push(incoming.map_err(|_| {
19791 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
19792 })?);
19793 }
19794 let version = local.as_ref().map_or(-1, |(_, version)| *version);
19795 self.write_row(&visible, &table.name, &row, version)
19796 .map_err(|error| {
19797 if error.starts_with("UNIQUE constraint failed:") {
19798 "sync.constraint_violation: local write violates a unique constraint".to_owned()
19799 } else {
19800 error
19801 }
19802 })
19803 }
19804
19805 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
19808 self.stored_row(table, row_id, false)
19809 }
19810
19811 fn stored_row(
19812 &self,
19813 table: &TableSchema,
19814 row_id: &str,
19815 base: bool,
19816 ) -> Result<Option<(Row, i64)>, String> {
19817 let sql = format!(
19818 "SELECT * FROM {} WHERE {}",
19819 if base {
19820 base_table(&table.name)
19821 } else {
19822 visible_table(&table.name)
19823 },
19824 row_id_predicate(table)
19825 );
19826 let mut stmt = self
19827 .conn
19828 .prepare_cached(&sql)
19829 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19830 let mut rows = stmt
19831 .query(rusqlite::params![row_id])
19832 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19833 let Some(row) = rows
19834 .next()
19835 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
19836 else {
19837 return Ok(None);
19838 };
19839 let mut values = Vec::with_capacity(table.columns.len());
19840 for (index, column) in table.columns.iter().enumerate() {
19841 let raw = row
19842 .get_ref(index)
19843 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19844 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
19845 if table
19846 .encrypted_columns
19847 .iter()
19848 .any(|encrypted| encrypted.index == index)
19849 {
19850 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
19851 continue;
19852 }
19853 }
19854 let json = sql_ref_to_json(column, raw);
19855 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
19856 "sync.local_corrupt: persisted local row value is invalid".to_owned()
19857 })?);
19858 }
19859 let version: i64 = row
19860 .get(table.columns.len())
19861 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19862 Ok(Some((values, version)))
19863 }
19864
19865 #[must_use]
19869 pub fn realtime_policy(&self) -> RealtimePolicy {
19870 self.realtime_policy
19871 }
19872
19873 #[must_use]
19875 pub fn realtime_state(&self) -> RealtimeState {
19876 self.realtime_state
19877 }
19878
19879 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
19882 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
19883 return Err(
19884 "sync.invalid_request: disconnect realtime before changing the realtime policy"
19885 .to_owned(),
19886 );
19887 }
19888 self.realtime_policy = policy;
19889 match policy {
19890 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
19891 RealtimePolicy::Required | RealtimePolicy::Optional
19892 if self.realtime_state == RealtimeState::Disabled =>
19893 {
19894 self.set_realtime_state(RealtimeState::Disconnected, None);
19895 }
19896 RealtimePolicy::Required | RealtimePolicy::Optional => {}
19897 }
19898 Ok(())
19899 }
19900
19901 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
19904 self.realtime_state = state;
19905 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
19906 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
19907 self.realtime_reason_code = None;
19908 self.realtime_retry_delay_ms = None;
19909 }
19910 }
19911
19912 #[must_use]
19913 fn realtime_connected(&self) -> bool {
19914 self.realtime_state == RealtimeState::Connected
19915 }
19916
19917 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
19918 if !self.transport_enabled {
19919 return Err("sync.offline: transport gate is closed".into());
19920 }
19921 if self.realtime_policy == RealtimePolicy::Off {
19922 return Err(
19923 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
19924 );
19925 }
19926 if self.realtime_connected() {
19927 return Ok(());
19928 }
19929 self.realtime_state = RealtimeState::Connecting;
19930 match transport.realtime_connect_for_client(&self.client_id) {
19931 Ok(()) => {
19932 self.set_realtime_state(RealtimeState::Connected, None);
19933 Ok(())
19934 }
19935 Err(error) => {
19936 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
19937 Err(format!("{}: {}", error.code, error.message))
19938 }
19939 }
19940 }
19941
19942 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
19943 self.cancel_sync_round();
19944 if self.realtime_state == RealtimeState::Disconnected
19947 || self.realtime_state == RealtimeState::Disabled
19948 {
19949 return;
19950 }
19951 let _ = transport.realtime_close();
19952 self.set_realtime_state(
19953 if self.realtime_policy == RealtimePolicy::Off {
19954 RealtimeState::Disabled
19955 } else {
19956 RealtimeState::Disconnected
19957 },
19958 None,
19959 );
19960 self.presence.clear(); }
19962
19963 pub fn set_presence(
19969 &mut self,
19970 transport: &mut dyn Transport,
19971 scope_key: &str,
19972 doc: Option<&Value>,
19973 ) -> Result<(), String> {
19974 if !self.transport_enabled {
19975 return Err("sync.offline: transport gate is closed".into());
19976 }
19977 if !self.realtime_connected() {
19978 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
19979 }
19980 let text = encode_presence_publish(scope_key, doc);
19981 transport
19982 .realtime_send(&text)
19983 .map_err(|e| format!("{}: {}", e.code, e.message))
19984 }
19985
19986 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
19988 self.presence
19989 .get(scope_key)
19990 .map(|peers| peers.values().cloned().collect())
19991 .unwrap_or_default()
19992 }
19993
19994 fn apply_presence(
19996 &mut self,
19997 scope_key: String,
19998 kind: Option<PresenceKind>,
19999 actor_id: Option<String>,
20000 client_id: Option<String>,
20001 doc: Option<Value>,
20002 error: Option<String>,
20003 ) {
20004 if error.is_some() {
20007 return;
20008 }
20009 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
20010 return;
20011 };
20012 let peer_key = format!("{actor_id} {client_id}");
20013 match kind {
20014 PresenceKind::Leave => {
20015 if let Some(peers) = self.presence.get_mut(&scope_key) {
20016 peers.remove(&peer_key);
20017 if peers.is_empty() {
20018 self.presence.remove(&scope_key);
20019 }
20020 }
20021 }
20022 _ => {
20023 let doc = match doc {
20024 Some(Value::Object(_)) => doc.unwrap(),
20025 _ => return,
20026 };
20027 self.presence.entry(scope_key).or_default().insert(
20028 peer_key,
20029 PresencePeer {
20030 actor_id,
20031 client_id,
20032 doc,
20033 },
20034 );
20035 }
20036 }
20037 }
20038
20039 pub fn on_realtime_text(&mut self, text: &str) {
20041 if self.stopped || !self.transport_enabled || self.security_preflight {
20042 return;
20043 }
20044 match parse_control(text) {
20045 Ok(ControlMessage::Hello { requires_sync, .. }) => {
20046 if requires_sync {
20047 self.set_sync_needed(true, true);
20049 }
20050 }
20051 Ok(ControlMessage::Presence {
20052 scope_key,
20053 kind,
20054 actor_id,
20055 client_id,
20056 doc,
20057 error,
20058 ..
20059 }) => {
20060 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
20061 }
20062 Ok(ControlMessage::Wake { .. }) => {
20063 self.set_sync_needed(true, true);
20065 }
20066 _ => {}
20067 }
20068 }
20069
20070 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
20074 let mut transport = crate::round::DownloadResults::default();
20075 self.on_realtime_binary(&mut transport, bytes);
20076 transport.controls
20077 }
20078
20079 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
20080 if self.stopped || !self.transport_enabled || self.security_preflight {
20081 return;
20082 }
20083 #[cfg(feature = "bench-internals")]
20084 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
20085 let message = match decode_message(bytes) {
20086 Ok(m) if m.msg_kind == MsgKind::Response => m,
20087 _ => {
20088 self.set_sync_needed(true, true);
20089 return;
20090 }
20091 };
20092 #[cfg(feature = "bench-internals")]
20093 drop(decode_phase);
20094 let mut frames = message.frames.into_iter();
20095 let mut applied_cursor: Option<i64> = None;
20096 let mut any_covered = false;
20097 let mut dropped = self.begin_coalesced_apply().is_err();
20098 while let Some(frame) = frames.next() {
20099 let Frame::SubStart {
20100 id,
20101 status,
20102 effective_scopes,
20103 ..
20104 } = frame
20105 else {
20106 continue;
20107 };
20108 let mut body = Vec::new();
20109 let mut next_cursor: Option<i64> = None;
20110 for inner in frames.by_ref() {
20111 match inner {
20112 Frame::SubEnd {
20113 next_cursor: nc, ..
20114 } => {
20115 next_cursor = Some(nc);
20116 break;
20117 }
20118 Frame::Unknown { .. } => {}
20119 other => body.push(other),
20120 }
20121 }
20122 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
20123 dropped = true;
20124 continue;
20125 };
20126 let sub = &self.subs[sub_index];
20127 if status != SubStatus::Active
20130 || sub.state != SubState::Active
20131 || sub.bootstrap_state.is_some()
20132 || !sub.synced_once
20133 {
20134 dropped = true;
20135 continue;
20136 }
20137 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
20138 any_covered = true;
20140 continue;
20141 }
20142 let previous_effective = self.subs[sub_index].effective.clone();
20143 let previous_cursor = self.subs[sub_index].cursor;
20144 self.subs[sub_index].effective = Some(effective_scopes);
20145 let mut failed = false;
20146 for inner in body {
20147 match inner {
20148 Frame::Commit {
20149 tables,
20150 changes,
20151 commit_seq,
20152 ..
20153 } => {
20154 if self
20155 .apply_commit_frame(&tables, &changes, Some(commit_seq))
20156 .is_err()
20157 {
20158 failed = true;
20159 break;
20160 }
20161 }
20162 _ => {
20163 failed = true;
20164 break;
20165 }
20166 }
20167 }
20168 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
20169 self.apply_sub_end(sub_index, cursor, None)
20170 .ok()
20171 .map(|()| cursor)
20172 });
20173 let Some(next_cursor) = committed else {
20174 self.subs[sub_index].effective = previous_effective;
20175 self.subs[sub_index].cursor = previous_cursor;
20176 dropped = true;
20177 continue;
20178 };
20179 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
20180 }
20181 if self.finish_coalesced_apply().is_err() {
20184 applied_cursor = None;
20185 dropped = true;
20186 }
20187 if let Some(cursor) = applied_cursor {
20188 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
20190 let _ = transport.realtime_send(&ack);
20191 } else if !any_covered || dropped {
20192 self.set_sync_needed(true, true);
20194 }
20195 }
20196
20197 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
20201 if !self.realtime_connected() {
20202 return;
20203 }
20204 let floor = self
20205 .subs
20206 .iter()
20207 .filter(|s| {
20208 s.state == SubState::Active
20209 && s.bootstrap_state.is_none()
20210 && s.synced_once
20211 && s.cursor >= 0
20212 })
20213 .map(|s| s.cursor)
20214 .min();
20215 if let Some(cursor) = floor {
20216 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
20217 let _ = transport.realtime_send(&ack);
20218 }
20219 }
20220}
20221
20222#[cfg(test)]
20223mod previous_version_wiring_tests {
20224 use super::*;
20225 use serde_json::{json, Value};
20226
20227 fn previous_version_schema(version: i32) -> Value {
20228 json!({
20229 "version": version,
20230 "tables": [{
20231 "name": "tasks",
20232 "primaryKey": "id",
20233 "columns": [
20234 { "name": "id", "type": "string", "nullable": false },
20235 { "name": "note", "type": "string", "nullable": true }
20236 ],
20237 "scopes": []
20238 }]
20239 })
20240 }
20241
20242 fn previous_version_limits() -> crate::api::ClientLimits {
20243 crate::api::ClientLimits {
20244 previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
20245 enabled: true,
20246 ..Default::default()
20247 }),
20248 ..Default::default()
20249 }
20250 }
20251
20252 fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
20254 previous_version_limits()
20255 .previous_version_context
20256 .expect("configured")
20257 }
20258
20259 fn recapture_previous_version(client: &SyncClient) {
20262 let replica_path = client
20263 .previous_version_replica_path()
20264 .expect("file-backed replica");
20265 let outcome = crate::previous_version::capture_previous_version_from_replica(
20266 &client.conn,
20267 &replica_path,
20268 2,
20269 2,
20270 &previous_version_config(),
20271 1,
20272 )
20273 .expect("capture");
20274 assert!(matches!(
20275 outcome,
20276 crate::previous_version::CaptureOutcome::Captured(_)
20277 ));
20278 }
20279
20280 fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
20281 Subscription {
20282 id: "wiring-sub".to_owned(),
20283 table: "tasks".to_owned(),
20284 requested: Vec::new(),
20285 params: None,
20286 cursor,
20287 bootstrap_state: bootstrap_state.map(str::to_owned),
20288 state,
20289 reason_code: None,
20290 effective: None,
20291 synced_once: false,
20292 }
20293 }
20294
20295 #[test]
20296 fn schema_downgrade_refuses_all_opens_and_recreate_without_mutation() {
20297 let path =
20298 std::env::temp_dir().join(format!("syncular-downgrade-{}.db", uuid::Uuid::new_v4()));
20299 let path_str = path.to_str().expect("temp path");
20300 let schema = previous_version_schema(3);
20301 let mut older = previous_version_schema(2);
20302 older["tables"][0]["columns"] =
20303 json!([{ "name": "id", "type": "string", "nullable": false }]);
20304 let mut client = SyncClient::open_path(
20305 "downgrade".into(),
20306 &previous_version_schema(1),
20307 previous_version_limits(),
20308 path_str,
20309 )
20310 .expect("fresh");
20311 client
20312 .mutate(vec![Mutation::Upsert {
20313 table: "tasks".into(),
20314 values: Map::from_iter([
20315 ("id".into(), json!("accepted")),
20316 ("note".into(), json!("old")),
20317 ]),
20318 base_version: None,
20319 }])
20320 .expect("v1 write");
20321 client
20322 .recreate_with_schema(&schema)
20323 .expect("upgrade captures context");
20324 let commit = client
20325 .mutate(vec![Mutation::Upsert {
20326 table: "tasks".into(),
20327 values: Map::from_iter([
20328 ("id".into(), json!("queued")),
20329 ("note".into(), json!("v3-only")),
20330 ]),
20331 base_version: None,
20332 }])
20333 .expect("queued v3 write");
20334 let changes = client.conn.total_changes();
20335 let outbox = client.pending_commit_ids();
20336 assert!(outbox.contains(&commit));
20337 let descriptor = client.get_meta("localSchemaDescriptor");
20338 let container = format!(
20339 "{path_str}{}",
20340 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
20341 );
20342 let context = std::fs::read(&container).expect("captured context");
20343 assert!(client
20344 .recreate_with_schema(&older)
20345 .expect_err("refuse recreate")
20346 .starts_with("client.schema_downgrade:"));
20347 assert_eq!(client.schema.version, 3);
20348 assert_eq!(client.conn.total_changes(), changes);
20349 assert_eq!(client.pending_commit_ids(), outbox);
20350 assert_eq!(client.get_meta("localSchemaDescriptor"), descriptor);
20351 assert_eq!(
20352 client
20353 .conn
20354 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20355 row.get::<_, String>(0)
20356 })
20357 .expect("queued row"),
20358 "v3-only"
20359 );
20360 assert_eq!(std::fs::read(&container).expect("context remains"), context);
20361 drop(client);
20362 let before = std::fs::read(&path).expect("replica bytes");
20363 for entry in 0..3 {
20366 let error = match entry {
20367 0 => SyncClient::open_path(
20368 "downgrade".into(),
20369 &older,
20370 previous_version_limits(),
20371 path_str,
20372 ),
20373 1 => SyncClient::open_path_with_identity(
20374 None,
20375 &older,
20376 previous_version_limits(),
20377 path_str,
20378 ),
20379 _ => SyncClient::with_connection(
20380 "downgrade".into(),
20381 &older,
20382 previous_version_limits(),
20383 Connection::open(&path).expect("connection"),
20384 ),
20385 }
20386 .err()
20387 .expect("refuse open");
20388 assert!(error.starts_with("client.schema_downgrade:"));
20389 let conn = Connection::open(&path).expect("inspect");
20390 assert_eq!(
20391 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20392 .expect("journal mode"),
20393 "delete"
20394 );
20395 drop(conn);
20396 assert_eq!(std::fs::read(&path).expect("replica unchanged"), before);
20397 assert_eq!(
20398 std::fs::read(&container).expect("context unchanged"),
20399 context
20400 );
20401 }
20402 let reopened = SyncClient::open_path(
20403 "downgrade".into(),
20404 &schema,
20405 previous_version_limits(),
20406 path_str,
20407 )
20408 .expect("compatible reopen");
20409 assert_eq!(reopened.pending_commit_ids(), outbox);
20410 assert_eq!(
20411 reopened
20412 .conn
20413 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20414 row.get::<_, String>(0)
20415 })
20416 .expect("queued row remains"),
20417 "v3-only"
20418 );
20419 reopened.delete_meta(LOCAL_SCHEMA_VERSION_KEY);
20420 reopened.delete_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20421 drop(reopened);
20422 let legacy = SyncClient::open_path(
20423 "downgrade".into(),
20424 &schema,
20425 previous_version_limits(),
20426 path_str,
20427 )
20428 .expect("legacy no-marker reopen");
20429 assert_eq!(legacy.pending_commit_ids(), outbox);
20430 assert!(!legacy.upgrading());
20431 assert_eq!(
20432 legacy
20433 .conn
20434 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
20435 row.get::<_, String>(0)
20436 })
20437 .expect("legacy row remains"),
20438 "v3-only"
20439 );
20440 drop(legacy);
20441 std::fs::remove_file(path).expect("remove replica");
20442 let _ = std::fs::remove_file(container);
20443 }
20444
20445 #[test]
20446 fn schema_marker_errors_refuse_open_and_recreate_without_mutation() {
20447 for marker in [
20448 "''",
20449 "'bad'",
20450 "'0'",
20451 "'-1'",
20452 "'2.5'",
20453 "'03'",
20454 "'3\n'",
20455 "' 3'",
20456 "'+3'",
20457 "'3e0'",
20458 "'2147483648'",
20459 "NULL",
20460 "X'33'",
20461 ] {
20462 let conn = Connection::open_in_memory().expect("connection");
20463 conn.execute_batch(&format!("CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value); INSERT INTO _syncular_meta VALUES ('localSchemaVersion', {marker});")).expect("invalid marker fixture");
20464 let error = SyncClient::with_connection(
20465 "marker".into(),
20466 &previous_version_schema(3),
20467 ClientLimits::default(),
20468 conn,
20469 )
20470 .err()
20471 .expect("refuse marker");
20472 assert!(
20473 error.starts_with("sync.local_corrupt:"),
20474 "{marker}: {error}"
20475 );
20476 }
20477 for fault in [
20478 "UPDATE _syncular_meta SET value = 'bad' WHERE key = 'localSchemaVersion'",
20479 "ALTER TABLE _syncular_meta RENAME COLUMN value TO unreadable_value",
20480 ] {
20481 let mut client = SyncClient::new(
20482 "marker".into(),
20483 &previous_version_schema(3),
20484 ClientLimits::default(),
20485 )
20486 .expect("fresh");
20487 let commit = client
20488 .mutate(vec![Mutation::Upsert {
20489 table: "tasks".into(),
20490 values: Map::from_iter([("id".into(), json!("queued"))]),
20491 base_version: None,
20492 }])
20493 .expect("queued write");
20494 client.conn.execute_batch(fault).expect("marker fault");
20495 let changes = client.conn.total_changes();
20496 let error = client
20497 .recreate_with_schema(&previous_version_schema(4))
20498 .expect_err("refuse corrupt marker");
20499 assert!(error.starts_with("sync.local_corrupt:"));
20500 assert_eq!(client.conn.total_changes(), changes);
20501 assert_eq!(client.schema.version, 3);
20502 assert_eq!(client.pending_commit_ids(), vec![commit]);
20503 }
20504 for setup in [
20505 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY)",
20506 "CREATE VIEW _syncular_meta AS SELECT 'localSchemaVersion' AS key, '3' AS value",
20507 ] {
20508 let path =
20509 std::env::temp_dir().join(format!("syncular-marker-{}.db", uuid::Uuid::new_v4()));
20510 let conn = Connection::open(&path).expect("connection");
20511 conn.execute_batch(setup)
20512 .expect("unreadable metadata fixture");
20513 drop(conn);
20514 let before = std::fs::read(&path).expect("before");
20515 let error = SyncClient::open_path_with_identity(
20516 None,
20517 &previous_version_schema(3),
20518 ClientLimits::default(),
20519 path.to_str().expect("path"),
20520 )
20521 .err()
20522 .expect("refuse unreadable marker");
20523 assert!(error.starts_with("sync.local_corrupt:"));
20524 assert_eq!(std::fs::read(&path).expect("after"), before);
20525 std::fs::remove_file(path).expect("remove fixture");
20526 }
20527 }
20528
20529 std::thread_local! {
20530 static CONCURRENT_UPGRADE: RefCell<Option<Connection>> =
20533 const { RefCell::new(None) };
20534 }
20535
20536 fn commit_concurrent_upgrade(attempts: i32) -> bool {
20537 CONCURRENT_UPGRADE.with(|slot| {
20538 if let Some(banner) = slot.borrow_mut().take() {
20539 banner
20540 .execute_batch(
20541 "UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion'; COMMIT",
20542 )
20543 .expect("commit the concurrent upgrade");
20544 }
20545 });
20546 attempts < 10
20547 }
20548
20549 fn journal_mode(conn: &Connection) -> String {
20550 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
20551 .expect("journal mode")
20552 }
20553
20554 fn temp_replica(label: &str) -> std::path::PathBuf {
20555 std::env::temp_dir().join(format!("syncular-{label}-{}.db", uuid::Uuid::new_v4()))
20556 }
20557
20558 #[test]
20559 fn file_identity_constructor_requires_the_returned_wal_mode() {
20560 for path in [":memory:", ""] {
20561 let error = SyncClient::open_path_with_identity(
20562 None,
20563 &previous_version_schema(1),
20564 ClientLimits::default(),
20565 path,
20566 )
20567 .err()
20568 .expect("temporary SQLite databases cannot enter WAL mode");
20569 assert_eq!(
20570 error,
20571 "sync.invalid_request: file-backed client requires WAL journal mode"
20572 );
20573 }
20574 let path = temp_replica("confirmed-wal");
20575 let client = SyncClient::open_path_with_identity(
20576 None,
20577 &previous_version_schema(1),
20578 ClientLimits::default(),
20579 path.to_str().unwrap(),
20580 )
20581 .unwrap();
20582 assert_eq!(journal_mode(&client.conn), "wal");
20583 drop(client);
20584 std::fs::remove_file(path).unwrap();
20585 }
20586
20587 #[test]
20588 fn non_positive_generated_schema_versions_are_refused_before_storage() {
20589 for version in [json!(0), json!(-1), json!(i32::MIN), json!(2147483648i64)] {
20590 let mut schema = previous_version_schema(1);
20591 schema["version"] = version.clone();
20592 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20593 .err()
20594 .expect("refuse an out-of-range generated version");
20595 assert_eq!(
20596 error,
20597 "sync.invalid_request: generated schema version must be an integer in 1..=2147483647"
20598 );
20599 let path = temp_replica("bad-version");
20601 let path_str = path.to_str().expect("path");
20602 for identity in [false, true] {
20603 let error = if identity {
20604 SyncClient::open_path_with_identity(
20605 None,
20606 &schema,
20607 ClientLimits::default(),
20608 path_str,
20609 )
20610 } else {
20611 SyncClient::open_path(
20612 "bad-version".into(),
20613 &schema,
20614 ClientLimits::default(),
20615 path_str,
20616 )
20617 }
20618 .err()
20619 .expect("refuse before storage");
20620 assert!(
20621 error.starts_with("sync.invalid_request:"),
20622 "{version}: {error}"
20623 );
20624 assert!(
20625 !path.exists(),
20626 "{version}: a refused constructor creates no replica file"
20627 );
20628 }
20629 }
20630 let mut schema = previous_version_schema(1);
20632 schema
20633 .as_object_mut()
20634 .expect("schema object")
20635 .remove("version");
20636 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
20637 .err()
20638 .expect("refuse a missing version");
20639 assert!(error.starts_with("sync.invalid_request:"), "{error}");
20640 }
20641
20642 #[test]
20643 fn a_duplicated_schema_marker_is_refused_without_a_reset() {
20644 let path = temp_replica("duplicate-marker");
20645 let path_str = path.to_str().expect("path");
20646 {
20647 let conn = Connection::open(path_str).expect("fixture");
20650 conn.execute_batch(
20651 "CREATE TABLE _syncular_meta(key TEXT, value);
20652 INSERT INTO _syncular_meta VALUES
20653 ('localSchemaVersion', '1'),
20654 ('localSchemaVersion', '2'),
20655 ('clientId', 'duplicate');",
20656 )
20657 .expect("duplicate marker fixture");
20658 }
20659 let error = SyncClient::open_path_with_identity(
20660 None,
20661 &previous_version_schema(3),
20662 ClientLimits::default(),
20663 path_str,
20664 )
20665 .err()
20666 .expect("refuse a duplicated marker");
20667 assert!(error.starts_with("sync.local_corrupt:"), "{error}");
20668 let probe = Connection::open(path_str).expect("probe");
20669 assert_eq!(
20670 probe
20671 .query_row(
20672 "SELECT COUNT(*) FROM _syncular_meta WHERE key = 'localSchemaVersion'",
20673 [],
20674 |row| row.get::<_, i64>(0),
20675 )
20676 .expect("marker rows"),
20677 2,
20678 "the duplicated rows are left for the operator, not collapsed"
20679 );
20680 assert_eq!(
20681 probe
20682 .query_row(
20683 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_syncular_%' AND name NOT LIKE 'sqlite_%'",
20684 [],
20685 |row| row.get::<_, i64>(0),
20686 )
20687 .expect("synced tables"),
20688 0,
20689 "a duplicated marker never drives the reset that recreates v3 tables"
20690 );
20691 assert_eq!(
20692 journal_mode(&probe),
20693 "delete",
20694 "the refusal leaves the journal mode alone"
20695 );
20696 drop(probe);
20697 std::fs::remove_file(path).expect("remove replica");
20698 }
20699
20700 #[test]
20701 fn a_refused_open_does_not_persist_wal_or_mutate_the_replica() {
20702 let path = temp_replica("refused-wal");
20703 let path_str = path.to_str().expect("path");
20704 drop(
20707 SyncClient::open_path(
20708 "owner".into(),
20709 &previous_version_schema(2),
20710 ClientLimits::default(),
20711 path_str,
20712 )
20713 .expect("v2 replica"),
20714 );
20715 let before = std::fs::read(&path).expect("before");
20716 let error = SyncClient::open_path_with_identity(
20717 Some("intruder".into()),
20718 &previous_version_schema(2),
20719 ClientLimits::default(),
20720 path_str,
20721 )
20722 .err()
20723 .expect("refuse the identity mismatch");
20724 assert!(error.starts_with("client.identity_mismatch:"), "{error}");
20725 let probe = Connection::open(path_str).expect("probe");
20726 assert_eq!(
20727 journal_mode(&probe),
20728 "delete",
20729 "a refused open must not persist WAL"
20730 );
20731 assert_eq!(std::fs::read(&path).expect("after"), before);
20732 assert_eq!(
20733 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20734 Some("2")
20735 );
20736 drop(probe);
20737 std::fs::remove_file(path).expect("remove replica");
20738 }
20739
20740 #[test]
20741 fn a_concurrent_marker_upgrade_refuses_a_stale_open() {
20742 let path = temp_replica("marker-race");
20743 let path_str = path.to_str().expect("path");
20744 drop(
20745 SyncClient::open_path_with_identity(
20746 Some("stale".into()),
20747 &previous_version_schema(1),
20748 ClientLimits::default(),
20749 path_str,
20750 )
20751 .expect("v1 replica"),
20752 );
20753 let banner = Connection::open(path_str).expect("banner");
20757 banner
20758 .execute_batch(
20759 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
20760 )
20761 .expect("pending upgrade");
20762 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
20763 let conn = Connection::open(path_str).expect("stale connection");
20764 conn.busy_handler(Some(commit_concurrent_upgrade))
20765 .expect("busy handler");
20766 let error = SyncClient::with_connection(
20767 "stale".into(),
20768 &previous_version_schema(1),
20769 ClientLimits::default(),
20770 conn,
20771 )
20772 .err()
20773 .expect("a concurrent upgrade must refuse the stale open");
20774 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
20775 let probe = Connection::open(path_str).expect("probe");
20776 assert_eq!(
20777 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20778 Some("2"),
20779 "the concurrent upgrade survives"
20780 );
20781 drop(probe);
20782 std::fs::remove_file(path).expect("remove replica");
20783 }
20784
20785 #[test]
20786 fn a_concurrent_marker_upgrade_refuses_a_stale_recreate() {
20787 let path = temp_replica("recreate-race");
20788 let path_str = path.to_str().expect("path");
20789 let mut client = SyncClient::open_path_with_identity(
20790 Some("stale".into()),
20791 &previous_version_schema(1),
20792 ClientLimits::default(),
20793 path_str,
20794 )
20795 .expect("v1 replica");
20796 client
20797 .mutate(vec![Mutation::Upsert {
20798 table: "tasks".into(),
20799 values: Map::from_iter([("id".into(), json!("queued"))]),
20800 base_version: None,
20801 }])
20802 .expect("queued write");
20803 let commit = client.pending_commit_ids();
20804 let banner = Connection::open(path_str).expect("banner");
20805 banner
20806 .execute_batch(
20807 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
20808 )
20809 .expect("pending upgrade");
20810 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
20811 client
20812 .conn
20813 .busy_handler(Some(commit_concurrent_upgrade))
20814 .expect("busy handler");
20815 let error = client
20816 .recreate_with_schema(&previous_version_schema(1))
20817 .expect_err("a concurrent upgrade must refuse the stale recreate");
20818 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
20819 assert_eq!(client.schema.version, 1, "the compiled schema is restored");
20820 assert_eq!(
20821 client.pending_commit_ids(),
20822 commit,
20823 "the outbox is preserved"
20824 );
20825 let probe = Connection::open(path_str).expect("probe");
20826 assert_eq!(
20827 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
20828 Some("2")
20829 );
20830 drop(probe);
20831 drop(client);
20832 std::fs::remove_file(path).expect("remove replica");
20833 }
20834
20835 #[test]
20836 fn failed_schema_record_writes_roll_back_recreation() {
20837 for key in [
20838 LOCAL_SCHEMA_VERSION_KEY,
20839 crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY,
20840 ] {
20841 let mut client = SyncClient::new(
20842 "schema-record".into(),
20843 &previous_version_schema(1),
20844 ClientLimits::default(),
20845 )
20846 .expect("v1");
20847 let marker = client.get_meta(LOCAL_SCHEMA_VERSION_KEY);
20848 let descriptor = client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
20849 client.conn.execute_batch(&format!("CREATE TRIGGER refuse_schema_record BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{key}' BEGIN SELECT RAISE(ABORT, 'refuse schema record'); END")).expect("fault");
20850 client
20851 .recreate_with_schema(&previous_version_schema(2))
20852 .expect_err("both record writes must propagate failure");
20853 assert_eq!(client.schema.version, 1);
20854 assert_eq!(client.get_meta(LOCAL_SCHEMA_VERSION_KEY), marker);
20855 assert_eq!(
20856 client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY),
20857 descriptor
20858 );
20859 client
20860 .conn
20861 .execute_batch("DROP TRIGGER refuse_schema_record")
20862 .expect("remove fault");
20863 client
20864 .recreate_with_schema(&previous_version_schema(2))
20865 .expect("retry");
20866 assert_eq!(client.schema.version, 2);
20867 }
20868 }
20869
20870 #[test]
20871 fn a_failed_outer_commit_restores_every_reset_side_effect() {
20872 let path = temp_replica("outer-commit");
20873 let path_str = path.to_str().expect("path");
20874 let conn = Connection::open(path_str).expect("replica connection");
20877 let mut client = SyncClient::with_connection(
20878 "outer-commit".into(),
20879 &previous_version_schema(1),
20880 previous_version_limits(),
20881 conn,
20882 )
20883 .expect("v1 replica");
20884 assert_eq!(journal_mode(&client.conn), "delete");
20885 let commit = client
20886 .mutate(vec![Mutation::Upsert {
20887 table: "tasks".into(),
20888 values: Map::from_iter([
20889 ("id".into(), json!("queued")),
20890 ("note".into(), json!("dropped-by-v2")),
20891 ]),
20892 base_version: None,
20893 }])
20894 .expect("queued write");
20895 client.drain_change_batches();
20896 let mut dropped = previous_version_schema(2);
20898 dropped["tables"][0]["columns"] =
20899 json!([{ "name": "id", "type": "string", "nullable": false }]);
20900 let marker = meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY);
20903 let revision = client.local_revision();
20904 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
20905 let columns: i64 = client
20906 .conn
20907 .query_row(
20908 "SELECT count(*) FROM pragma_table_info('tasks')",
20909 [],
20910 |row| row.get(0),
20911 )
20912 .expect("column count");
20913 let rebuilds = client.overlay_rebuild_count.get();
20914 let prior_subs = client.subs.len();
20915 let prior_changes = client.change_queue.len();
20916 let prior_intents = client.sync_intent_queue.len();
20917 let prior_last_change = client.last_change.clone();
20918 let prior_overlay_dirty = client.overlay_dirty.get();
20919 let prior_sync_needed = client.sync_needed;
20920 let prior_upgrading = client.upgrading;
20921 let prior_stopped = client.stopped;
20922 let prior_schema_floor = client.schema_floor.clone();
20923 let prior_active_round = client.active_round;
20924 let reader = Connection::open(path_str).expect("reader");
20927 reader
20928 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
20929 .expect("reader snapshot");
20930 client
20931 .conn
20932 .busy_timeout(std::time::Duration::ZERO)
20933 .expect("busy timeout");
20934 let error = client
20935 .recreate_with_schema(&dropped)
20936 .expect_err("a blocked COMMIT must refuse the recreation");
20937 assert!(error.starts_with("client.storage_busy:"), "{error}");
20938 assert!(
20939 client.overlay_rebuild_count.get() > rebuilds,
20940 "the reset ran to its overlay rebuild inside the transaction"
20941 );
20942 assert!(client.conn.is_autocommit(), "the transaction is released");
20943 assert_eq!(meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY), marker);
20945 assert_eq!(client.local_revision(), revision);
20946 assert_eq!(
20947 client.query("SELECT * FROM tasks", &[]).expect("rows"),
20948 rows
20949 );
20950 assert_eq!(
20951 client
20952 .conn
20953 .query_row(
20954 "SELECT count(*) FROM pragma_table_info('tasks')",
20955 [],
20956 |row| { row.get::<_, i64>(0) }
20957 )
20958 .expect("column count"),
20959 columns,
20960 "the previous table layout survives the rollback"
20961 );
20962 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
20963 assert_eq!(client.schema.version, 1, "compiled schema");
20966 assert_eq!(client.subs.len(), prior_subs);
20967 assert_eq!(client.change_queue.len(), prior_changes);
20968 assert_eq!(client.sync_intent_queue.len(), prior_intents);
20969 assert_eq!(client.last_change, prior_last_change);
20970 assert_eq!(client.overlay_dirty.get(), prior_overlay_dirty);
20971 assert_eq!(client.sync_needed, prior_sync_needed);
20972 assert_eq!(client.upgrading, prior_upgrading);
20973 assert_eq!(client.stopped, prior_stopped);
20974 assert_eq!(client.schema_floor, prior_schema_floor);
20975 assert_eq!(client.active_round, prior_active_round);
20976 assert!(
20977 client.rejections.is_empty(),
20978 "the dropped-column rejection is rolled back too"
20979 );
20980 let container = crate::previous_version::previous_version_container_path(path_str);
20983 assert!(std::path::Path::new(&container).exists());
20984 drop(reader);
20987 let snapshot = client
20988 .previous_version_snapshot(&PreviousVersionReadSpec {
20989 table: "tasks".into(),
20990 ..Default::default()
20991 })
20992 .expect("refuse stale capture");
20993 assert!(!snapshot.available);
20994 assert_eq!(snapshot.current_version, 1);
20995 assert!(!std::path::Path::new(&container).exists());
20996 client
21001 .conn
21002 .busy_timeout(std::time::Duration::from_secs(1))
21003 .expect("busy timeout");
21004 client
21005 .recreate_with_schema(&dropped)
21006 .expect("the recreation succeeds once the reader releases");
21007 assert_eq!(client.schema.version, 2);
21008 assert_eq!(
21009 client.pending_commit_ids(),
21010 vec![commit.clone()],
21011 "the reset kept the incompatible commit pending"
21012 );
21013 assert!(
21014 client.rejections.is_empty(),
21015 "no rejection exists before the send-time prepass"
21016 );
21017 client
21019 .prepare_sync_round(false)
21020 .expect("the send prepass runs during round preparation");
21021 assert!(client.pending_commit_ids().is_empty());
21022 assert!(
21023 client
21024 .rejections
21025 .iter()
21026 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE),
21027 "the dropped-column commit surfaced as a rejection at send time"
21028 );
21029 client.previous_version_discard();
21030 drop(client);
21031 std::fs::remove_file(path).expect("remove replica");
21032 }
21033
21034 #[test]
21035 fn the_send_time_incompatible_drop_is_atomic_and_publishes_once() {
21036 let v1 = previous_version_schema(1);
21037 let mut v2 = previous_version_schema(2);
21038 v2["tables"][0]["columns"] = json!([{ "name": "id", "type": "string", "nullable": false }]);
21040 let mut client = SyncClient::new("drop".to_string(), &v1, ClientLimits::default()).unwrap();
21041 let dropped = client
21042 .mutate(vec![Mutation::Upsert {
21043 table: "tasks".into(),
21044 values: Map::from_iter([
21045 ("id".into(), json!("dropped")),
21046 ("note".into(), json!("gone")),
21047 ]),
21048 base_version: None,
21049 }])
21050 .unwrap();
21051 let survivor = client
21055 .mutate(vec![Mutation::Delete {
21056 table: "tasks".into(),
21057 row_id: "gone".into(),
21058 base_version: None,
21059 }])
21060 .unwrap();
21061 client
21062 .recreate_with_schema(&v2)
21063 .expect("the reset preserves the outbox");
21064 assert_eq!(
21065 client.pending_commit_ids(),
21066 vec![dropped.clone(), survivor.clone()],
21067 "the reset preserves both commits"
21068 );
21069 assert!(client.rejections.is_empty(), "the reset classifies nothing");
21070 client.drain_change_batches();
21071 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21072 assert_eq!(rows.len(), 1, "only the dropped upsert has a visible row");
21073
21074 for fault in [
21080 "outcome-insert",
21081 "outbox-delete",
21082 "overlay-rebuild",
21083 "publish-revision",
21084 ] {
21085 client.set_sync_needed(false, false);
21086 client.drain_change_batches();
21087 let revision = client.local_revision();
21088 let trigger = match fault {
21089 "outcome-insert" => {
21090 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21091 }
21092 "outbox-delete" => {
21093 "CREATE TRIGGER fail_drop BEFORE DELETE ON _syncular_outbox BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21094 }
21095 "publish-revision" => {
21100 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21101 }
21102 _ => {
21103 "CREATE TRIGGER fail_drop BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT,'drop fault'); END"
21104 }
21105 };
21106 client.conn.execute_batch(trigger).unwrap();
21107 let outcome = client
21108 .prepare_sync_round(false)
21109 .err()
21110 .unwrap_or_else(|| panic!("{fault}: a fault in the drop must fail the round"));
21111 assert!(
21112 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "storage.failed"),
21113 "{fault}: {outcome:?}"
21114 );
21115 client.conn.execute_batch("DROP TRIGGER fail_drop").unwrap();
21116 assert!(client.conn.is_autocommit(), "{fault}: no leaked savepoint");
21117 assert_eq!(
21118 client.pending_commit_ids(),
21119 vec![dropped.clone(), survivor.clone()],
21120 "{fault}: the outbox is restored"
21121 );
21122 assert!(
21123 client.rejections.is_empty(),
21124 "{fault}: no in-memory rejection survives"
21125 );
21126 assert!(
21127 client.commit_outcome(&dropped).unwrap().is_none(),
21128 "{fault}: the durable outcome rolled back"
21129 );
21130 assert_eq!(client.local_revision(), revision, "{fault}: revision");
21131 assert_eq!(
21132 client.query("SELECT * FROM tasks", &[]).unwrap(),
21133 rows,
21134 "{fault}: the visible projection is unchanged"
21135 );
21136 assert!(
21137 client.drain_change_batches().is_empty(),
21138 "{fault}: nothing is published"
21139 );
21140 }
21141
21142 client
21145 .conn
21146 .execute_batch(
21147 "BEGIN; INSERT INTO _syncular_meta(key,value) VALUES ('outerProbe','kept');
21148 CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes
21149 BEGIN SELECT RAISE(ABORT,'nested drop fault'); END;",
21150 )
21151 .unwrap();
21152 assert!(client.drop_incompatible_outbox().is_err());
21153 assert!(!client.conn.is_autocommit());
21154 assert_eq!(client.get_meta("outerProbe").as_deref(), Some("kept"));
21155 assert_eq!(
21156 client.pending_commit_ids(),
21157 vec![dropped.clone(), survivor.clone()]
21158 );
21159 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21160 client
21161 .conn
21162 .execute_batch("DROP TRIGGER fail_drop; COMMIT")
21163 .unwrap();
21164
21165 client.set_meta(LOG_EPOCH_KEY, "epoch");
21170 client.set_sync_needed(false, false);
21171 client.drain_change_batches();
21172 let revision = client.local_revision();
21173 let prepared = client.prepare_sync_round(false).expect("the drop succeeds");
21174 assert_eq!(client.pending_commit_ids(), vec![survivor.clone()]);
21175 assert_eq!(
21176 prepared.meta.pushed_ids,
21177 vec![survivor.clone()],
21178 "the incompatible commit is classified before any operation is sent"
21179 );
21180 let rejection = client
21181 .rejections
21182 .iter()
21183 .find(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE)
21184 .expect("a sync.outbox_incompatible rejection");
21185 assert_eq!(rejection.client_commit_id, dropped);
21186 assert!(
21187 !rejection.retryable,
21188 "a schema-incompatible commit is final"
21189 );
21190 assert_eq!(
21191 client
21192 .commit_outcome(&dropped)
21193 .unwrap()
21194 .expect("a durable rejected outcome")
21195 .status,
21196 CommitOutcomeStatus::Rejected
21197 );
21198 assert_eq!(client.local_revision(), revision + 1, "one revision");
21199 let batches = client.drain_change_batches();
21200 assert_eq!(batches.len(), 1, "one change batch");
21201 assert!(
21202 batches[0].rejections_changed
21203 && batches[0].outcomes_changed
21204 && batches[0].status.is_some(),
21205 "{:?}",
21206 batches[0]
21207 );
21208 assert!(
21209 batches[0].tables.iter().any(|entry| entry.table == "tasks"),
21210 "{:?}",
21211 batches[0]
21212 );
21213 let visible = client.query("SELECT * FROM tasks", &[]).unwrap();
21214 assert!(
21215 visible.is_empty(),
21216 "the dropped projection is undone: {visible:?}"
21217 );
21218
21219 let path = temp_replica("outbox-drop-commit");
21223 let path_str = path.to_str().expect("path");
21224 let conn = Connection::open(path_str).expect("replica connection");
21225 let mut client =
21226 SyncClient::with_connection("drop-commit".into(), &v1, ClientLimits::default(), conn)
21227 .expect("v1 replica");
21228 let dropped = client
21229 .mutate(vec![Mutation::Upsert {
21230 table: "tasks".into(),
21231 values: Map::from_iter([
21232 ("id".into(), json!("dropped")),
21233 ("note".into(), json!("gone")),
21234 ]),
21235 base_version: None,
21236 }])
21237 .unwrap();
21238 client
21239 .recreate_with_schema(&v2)
21240 .expect("the reset preserves the outbox");
21241 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21242 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
21243 assert_eq!(rows.len(), 1, "the optimistic row is visible");
21244 client.set_sync_needed(false, false);
21245 client.drain_change_batches();
21246 let revision = client.local_revision();
21247 let reader = Connection::open(path_str).expect("reader");
21248 reader
21249 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
21250 .expect("reader snapshot");
21251 client
21252 .conn
21253 .busy_timeout(std::time::Duration::ZERO)
21254 .expect("busy timeout");
21255 let outcome = client
21256 .prepare_sync_round(false)
21257 .err()
21258 .expect("a blocked COMMIT must fail the round");
21259 assert!(
21260 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "client.storage_busy"),
21261 "{outcome:?}"
21262 );
21263 assert!(client.conn.is_autocommit(), "the savepoint is released");
21264 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
21265 assert!(client.rejections.is_empty());
21266 assert!(client.commit_outcome(&dropped).unwrap().is_none());
21267 assert_eq!(client.local_revision(), revision);
21268 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
21269 assert!(client.drain_change_batches().is_empty());
21270 drop(reader);
21272 client
21273 .conn
21274 .busy_timeout(std::time::Duration::from_secs(1))
21275 .expect("busy timeout");
21276 client
21277 .prepare_sync_round(false)
21278 .expect("the drop succeeds after the reader releases");
21279 assert!(client.pending_commit_ids().is_empty());
21280 assert!(client
21281 .rejections
21282 .iter()
21283 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE));
21284 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
21285 drop(client);
21286 std::fs::remove_file(path).expect("remove replica");
21287 }
21288
21289 #[test]
21290 fn previous_version_read_reasons_come_from_live_client_state() {
21291 use crate::previous_version::{
21292 PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
21293 };
21294 let path = std::env::temp_dir()
21295 .join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
21296 .to_string_lossy()
21297 .into_owned();
21298 let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
21299 {
21300 let mut setup = SyncClient::open_path(
21301 "wiring".to_owned(),
21302 &previous_version_schema(1),
21303 previous_version_limits(),
21304 &path,
21305 )
21306 .expect("v1 install");
21307 setup
21308 .mutate(vec![Mutation::Upsert {
21309 table: "tasks".to_owned(),
21310 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21311 base_version: None,
21312 }])
21313 .expect("seed a row");
21314 }
21315 let mut client = SyncClient::open_path(
21316 "wiring".to_owned(),
21317 &previous_version_schema(2),
21318 previous_version_limits(),
21319 &path,
21320 )
21321 .expect("v2 bump captures");
21322 assert!(std::path::Path::new(&container).exists());
21323 let spec = PreviousVersionReadSpec {
21324 table: "tasks".to_owned(),
21325 ..Default::default()
21326 };
21327
21328 recapture_previous_version(&client);
21330 client.lease_state = Some(LeaseState {
21331 lease_id: None,
21332 expires_at_ms: None,
21333 error_code: Some("lease.expired".to_owned()),
21334 });
21335 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21336 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21337 assert!(!snapshot.available);
21338 assert!(!std::path::Path::new(&container).exists());
21339
21340 recapture_previous_version(&client);
21342 client.set_now_ms(1_000);
21343 client.lease_state = Some(LeaseState {
21344 lease_id: Some("lease".to_owned()),
21345 expires_at_ms: Some(1_000),
21346 error_code: None,
21347 });
21348 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21349 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
21350 assert!(!std::path::Path::new(&container).exists());
21351 client.lease_state = None;
21352
21353 recapture_previous_version(&client);
21355 client.subs.push(subscription(SubState::Revoked, 0, None));
21356 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21357 assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
21358 assert!(!std::path::Path::new(&container).exists());
21359 client.subs.clear();
21360
21361 recapture_previous_version(&client);
21364 client
21365 .subs
21366 .push(subscription(SubState::Active, 0, Some("resume")));
21367 assert!(!client.previous_version_coverage_complete());
21368 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21369 assert!(snapshot.available);
21370 assert_eq!(snapshot.previous_version, Some(2));
21371 assert_eq!(snapshot.rows.len(), 1);
21372
21373 recapture_previous_version(&client);
21374 client.subs[0].bootstrap_state = None;
21375 assert!(client.previous_version_coverage_complete());
21376 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
21377 assert_eq!(
21378 snapshot.reason,
21379 Some(PreviousVersionReason::CoverageComplete)
21380 );
21381 assert!(!std::path::Path::new(&container).exists());
21382 client.subs.clear();
21383
21384 assert!(!client.previous_version_coverage_complete());
21386
21387 drop(client);
21388 let _ = std::fs::remove_file(&path);
21389 }
21390
21391 #[test]
21398 fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
21399 let path = std::env::temp_dir()
21400 .join(format!(
21401 "syncular-prev-lifetime-{}.db",
21402 uuid::Uuid::new_v4()
21403 ))
21404 .to_string_lossy()
21405 .into_owned();
21406 let container = format!(
21407 "{path}{}",
21408 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
21409 );
21410 let exists = || std::path::Path::new(&container).exists();
21411 {
21412 let mut setup = SyncClient::open_path(
21413 "lifetime".to_owned(),
21414 &previous_version_schema(1),
21415 previous_version_limits(),
21416 &path,
21417 )
21418 .expect("v1 install");
21419 setup
21420 .mutate(vec![Mutation::Upsert {
21421 table: "tasks".to_owned(),
21422 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
21423 base_version: None,
21424 }])
21425 .expect("seed a row");
21426 }
21427 let mut client = SyncClient::open_path(
21428 "lifetime".to_owned(),
21429 &previous_version_schema(2),
21430 previous_version_limits(),
21431 &path,
21432 )
21433 .expect("v2 bump captures");
21434 client.set_now_ms(1_000);
21437 assert!(exists());
21438
21439 client.lease_state = Some(LeaseState {
21441 lease_id: None,
21442 expires_at_ms: None,
21443 error_code: Some("lease.expired".to_owned()),
21444 });
21445 client.previous_version_lifetime_check();
21446 assert!(!exists());
21447 client.lease_state = None;
21448
21449 recapture_previous_version(&client);
21452 client.subs.push(subscription(SubState::Active, 0, None));
21453 client.previous_version_lifetime_check();
21454 assert!(!exists());
21455 client.subs.clear();
21456
21457 recapture_previous_version(&client);
21459 client.subs.push(subscription(SubState::Revoked, 0, None));
21460 client.previous_version_lifetime_check();
21461 assert!(!exists());
21462 client.subs.clear();
21463
21464 recapture_previous_version(&client);
21467 client
21468 .subs
21469 .push(subscription(SubState::Active, 0, Some("resume")));
21470 client.previous_version_lifetime_check();
21471 assert!(exists());
21472 client.subs.clear();
21473
21474 drop(client);
21475 let _ = std::fs::remove_file(&path);
21476 let _ = std::fs::remove_file(&container);
21477 let _ = std::fs::remove_file(format!("{container}-wal"));
21478 let _ = std::fs::remove_file(format!("{container}-shm"));
21479 }
21480}