1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17
18use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
19use rusqlite::{Connection, OpenFlags, OptionalExtension};
20use serde::{Deserialize, Serialize};
21use serde_json::{json, Map, Value};
22use sha2::{Digest, Sha256};
23use ssp2::decode::WIRE_VERSION;
24use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
25use ssp2::primitives::RawJson;
26use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
27use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
28
29use crate::api::{
30 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
31 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
32 ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects, CommitDelivery,
33 CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
34 CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
35 CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
36 DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticSubscription, FetchedBlob,
37 LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
38 LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
39 PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
40 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
41 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SubscriptionCatchup,
42 SubscriptionRequest, SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport,
43 SyncStatusSnapshot, TableChange, WindowBase, WindowChange, WindowCoverage, WindowState,
44 WindowUnitRef, CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
45 MAX_DIAGNOSTIC_QUERY_FAILURES,
46};
47use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
48#[cfg(feature = "bench-internals")]
49use crate::bench::{Phase, Recorder};
50use crate::previous_version::{
51 capture_previous_version_from_replica, first_incompatibility,
52 reconcile_previous_version_at_boot, set_local_schema_descriptor,
53 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
54 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
55};
56use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
57use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
58use crate::values::{
59 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
60 encode_sparse_row_json, full_row_values, is_canonical_bytes_envelope, json_to_column_value,
61 json_to_scope_map, normalize_values_casing, render_row_id_json, scope_map_to_json,
62 sort_scope_map,
63};
64
65const DEFAULT_ACCEPT: u8 = 0b0111;
69const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
70const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
71const ACCEPT_SQLITE: u8 = 1 << 2;
72const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
73const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
74
75fn recommended_action_of(details: &Option<Value>) -> Option<String> {
77 details
78 .as_ref()?
79 .get("recommendedAction")?
80 .as_str()
81 .map(str::to_owned)
82}
83
84fn coalesced_delete_keys(
92 frames: &[Frame],
93 mut applies: impl FnMut(&str, SubStatus, Option<i64>) -> bool,
94) -> HashSet<(i64, String, String)> {
95 let mut sections = Vec::new();
96 let mut open: Option<(usize, String, SubStatus)> = None;
97 for (index, frame) in frames.iter().enumerate() {
98 match frame {
99 Frame::SubStart { id, status, .. } => {
100 open = Some((index, id.clone(), *status));
101 }
102 Frame::SubEnd { next_cursor, .. } => {
103 if let Some((start, id, status)) = open.take() {
104 sections.push((start, index, id, status, Some(*next_cursor)));
105 }
106 }
107 _ => {}
108 }
109 }
110 type Key = (i64, String, String);
111 let mut ops: HashMap<Key, (HashSet<usize>, HashSet<usize>)> = HashMap::new();
112 let mut both_in_frame: HashSet<Key> = HashSet::new();
113 let mut frame_ordinal = 0usize;
114 for (start, end, id, status, next_cursor) in sections {
115 if !applies(&id, status, next_cursor) {
116 continue;
117 }
118 for frame in &frames[start + 1..end] {
119 let Frame::Commit {
120 commit_seq,
121 tables,
122 changes,
123 ..
124 } = frame
125 else {
126 continue;
127 };
128 frame_ordinal += 1;
129 let mut local: HashMap<(String, String), (bool, bool)> = HashMap::new();
130 for change in changes {
131 let Some(table) = tables.get(change.table_index as usize) else {
132 continue;
133 };
134 let key = (*commit_seq, table.clone(), change.row_id.clone());
135 let entry = ops.entry(key).or_default();
136 let local_entry = local
137 .entry((table.clone(), change.row_id.clone()))
138 .or_default();
139 match change.op {
140 Op::Upsert => {
141 entry.0.insert(frame_ordinal);
142 local_entry.0 = true;
143 }
144 Op::Delete => {
145 entry.1.insert(frame_ordinal);
146 local_entry.1 = true;
147 }
148 }
149 }
150 for ((table, row_id), (has_upsert, has_delete)) in local {
151 if has_upsert && has_delete {
152 both_in_frame.insert((*commit_seq, table, row_id));
153 }
154 }
155 }
156 }
157 ops.into_iter()
158 .filter_map(|(key, (upserts, deletes))| {
159 (!both_in_frame.contains(&key) && !upserts.is_empty() && !deletes.is_empty())
160 .then_some(key)
161 })
162 .collect()
163}
164
165pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
167const LOCAL_REVISION_KEY: &str = "localRevision";
168const CLIENT_ID_KEY: &str = "clientId";
169const LEASE_STATE_KEY: &str = "leaseState";
170const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
171pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
172const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
177const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
178const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
179const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
182const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
185const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
186const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
188 "the persisted commit carries values the current codec refuses";
189const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
191
192#[derive(Debug, Serialize, Deserialize)]
193#[serde(rename_all = "camelCase", deny_unknown_fields)]
194struct PersistedLocalDataRebootstrapReceipt {
195 version: u8,
196 retained_commits: u64,
197 reset_subscriptions: u64,
198}
199
200fn invalid_local_rebootstrap_receipt() -> String {
201 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
202}
203
204fn encode_local_rebootstrap_receipt(
205 retained_commits: usize,
206 reset_subscriptions: usize,
207) -> Result<String, String> {
208 let retained_commits =
209 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
210 let reset_subscriptions =
211 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
212 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
213 return Err(invalid_local_rebootstrap_receipt());
214 }
215 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
216 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
217 retained_commits,
218 reset_subscriptions,
219 })
220 .map_err(|_| invalid_local_rebootstrap_receipt())
221}
222
223fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
224 if value == "v1" {
226 return Ok((0, 0));
227 }
228 let receipt: PersistedLocalDataRebootstrapReceipt =
229 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
230 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
231 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
232 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
233 {
234 return Err(invalid_local_rebootstrap_receipt());
235 }
236 Ok((
237 usize::try_from(receipt.retained_commits)
238 .map_err(|_| invalid_local_rebootstrap_receipt())?,
239 usize::try_from(receipt.reset_subscriptions)
240 .map_err(|_| invalid_local_rebootstrap_receipt())?,
241 ))
242}
243
244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
245enum SubState {
246 Active,
247 Revoked,
248 Failed,
249}
250
251#[cfg(test)]
252mod observation_tests {
253 use super::*;
254 use crate::native_transport::HostTransport;
255 use crate::values::encode_row_json;
256 use crate::BlobUploadGrant;
257 use serde_json::json;
258
259 #[test]
260 fn authority_policy_refuses_encrypted_and_undeclared_columns_before_open() {
261 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
262 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "secret", "type": "bytes", "encrypted": true, "declaredType": "string", "nullable": true}
263 ]}]});
264 for name in [
265 "secret",
266 "missing",
267 "*",
268 "id FROM authority; DELETE FROM authority",
269 "_syncular_version",
270 ] {
271 let limits = ClientLimits {
272 authority_reads: vec![crate::api::AuthorityReadDeclaration {
273 table: "authority".into(),
274 columns: vec!["id".into(), "actor_id".into(), name.into()],
275 scopes: [("actor_id".into(), vec!["a".into()])].into(),
276 }],
277 ..Default::default()
278 };
279 let error = SyncClient::new("authority".into(), &schema, limits)
280 .err()
281 .expect("forbidden policy");
282 assert!(
283 error.starts_with("client.authority_read_forbidden:"),
284 "{error}"
285 );
286 }
287 }
288
289 #[test]
290 fn authority_snapshot_observes_one_file_transaction_during_concurrent_writes() {
291 let path = std::env::temp_dir().join(format!(
292 "syncular-authority-snapshot-{}.db",
293 uuid::Uuid::new_v4()
294 ));
295 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
296 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "revision", "type": "integer", "nullable": false}
297 ]}]});
298 let limits = ClientLimits {
299 authority_reads: vec![crate::api::AuthorityReadDeclaration {
300 table: "authority".into(),
301 columns: vec!["id".into(), "actor_id".into(), "revision".into()],
302 scopes: [("actor_id".into(), vec!["a".into()])].into(),
303 }],
304 ..Default::default()
305 };
306 let mut client =
307 SyncClient::open_path("authority".into(), &schema, limits, path.to_str().unwrap())
308 .unwrap();
309 client.begin_security_preflight();
310 let update = |conn: &Connection, revision: i64| {
311 conn.execute_batch("BEGIN IMMEDIATE").unwrap();
312 conn.execute(
313 "INSERT OR REPLACE INTO _syncular_base_authority VALUES('one','a',?1,1)",
314 [revision],
315 )
316 .unwrap();
317 conn.execute(
318 "UPDATE _syncular_meta SET value=?1 WHERE key='localRevision'",
319 [revision.to_string()],
320 )
321 .unwrap();
322 conn.execute("INSERT OR REPLACE INTO _syncular_subscriptions VALUES('s','authority',?1)", [json!({"requested": {"actor_id": ["a"]}, "effectiveScopes": {"actor_id": ["a"]}, "status": "active", "cursor": revision}).to_string()]).unwrap();
323 conn.execute_batch("COMMIT").unwrap();
324 };
325 update(&client.conn, 1);
326 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
327 let writer_barrier = barrier.clone();
328 let writer_path = path.clone();
329 let writer = std::thread::spawn(move || {
330 let conn = Connection::open(writer_path).unwrap();
331 conn.busy_timeout(std::time::Duration::from_secs(5))
332 .unwrap();
333 writer_barrier.wait();
334 for revision in 2..=100 {
335 update(&conn, revision);
336 }
337 });
338 barrier.wait();
339 for _ in 0..100 {
340 let snapshot = client.authority_snapshot().unwrap();
341 let revision = snapshot["revision"]
342 .as_str()
343 .unwrap()
344 .parse::<i64>()
345 .unwrap();
346 assert_eq!(
347 snapshot["tables"][0]["rows"][0]["values"]["revision"],
348 revision
349 );
350 assert_eq!(snapshot["tables"][0]["persisted"][0]["cursor"], revision);
351 assert_eq!(snapshot["complete"], true);
352 }
353 writer.join().unwrap();
354 assert_eq!(client.authority_snapshot().unwrap()["revision"], "100");
355 drop(client);
356 let _ = std::fs::remove_file(path);
357 }
358
359 fn client() -> SyncClient {
360 SyncClient::new(
361 "retry-test".to_owned(),
362 &json!({
363 "version": 1,
364 "tables": [{
365 "name": "tasks",
366 "primaryKey": "id",
367 "columns": [
368 { "name": "id", "type": "string", "nullable": false },
369 { "name": "project_id", "type": "string", "nullable": false }
370 ],
371 "scopes": [{ "pattern": "project:{project_id}" }]
372 }]
373 }),
374 ClientLimits::default(),
375 )
376 .expect("test client")
377 }
378
379 fn round_error(outcome: SyncOutcome) -> TransportError {
380 match outcome {
381 SyncOutcome::Failed {
382 error_code,
383 message,
384 details,
385 ..
386 } => TransportError {
387 code: error_code,
388 message,
389 details,
390 },
391 other => TransportError::new(
392 "client.failed",
393 format!("unexpected round outcome: {other:?}"),
394 ),
395 }
396 }
397
398 fn flush_blob_uploads(
401 client: &mut SyncClient,
402 transport: &mut dyn Transport,
403 ) -> Result<(), TransportError> {
404 let mut prepared = match client.prepare_sync_round(false) {
405 Ok(prepared) => prepared,
406 Err(outcome) => return Err(round_error(*outcome)),
407 };
408 loop {
409 if prepared.uploads.is_empty() {
410 client.cancel_sync_round();
411 return Ok(());
412 }
413 let completed = crate::round::test_support::drive_round(prepared, transport);
414 prepared = match client.apply_sync_round(completed) {
415 crate::AppliedSyncRound::Continue(prepared) => prepared,
416 crate::AppliedSyncRound::Complete { outcome, .. } => {
417 return match outcome {
418 SyncOutcome::Ok(_) => Ok(()),
419 other => Err(round_error(other)),
420 };
421 }
422 };
423 }
424 }
425
426 #[test]
427 fn subscribe_many_registers_all_or_nothing() {
428 let request = |id: &str, project: &str| SubscriptionRequest {
429 id: id.to_owned(),
430 table: "tasks".to_owned(),
431 scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
432 params: None,
433 };
434 let mut client = client();
435 client
436 .subscribe_many(vec![request("a", "p1"), request("b", "p2")])
437 .unwrap();
438 let persisted = |client: &SyncClient| -> i64 {
439 client
440 .conn
441 .query_row("SELECT count(*) FROM _syncular_subscriptions", [], |row| {
442 row.get(0)
443 })
444 .unwrap()
445 };
446 assert_eq!(persisted(&client), 2);
447 let error = client
449 .subscribe_many(vec![request("c", "p3"), request("b", "p9")])
450 .unwrap_err();
451 assert!(error.starts_with("client.subscription_intent_mismatch:"));
452 assert!(client.subscription_state("c").is_none());
453 assert_eq!(persisted(&client), 2);
454 assert!(client.conn.is_autocommit());
455 }
456
457 #[cfg(feature = "e2ee")]
458 #[test]
459 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
460 let schema = json!({
461 "version": 1,
462 "tables": [{
463 "name": "secrets",
464 "primaryKey": "id",
465 "columns": [
466 { "name": "id", "type": "string", "nullable": false },
467 { "name": "encryption_key_id", "type": "string", "nullable": true },
468 { "name": "note", "type": "bytes", "nullable": true,
469 "encrypted": true, "declaredType": "string" }
470 ],
471 "scopes": []
472 }]
473 });
474 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
475 .expect("test client");
476 let mut config = crate::values::EncryptionConfig::default();
477 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
478 config
479 .key_id_columns
480 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
481 client.set_encryption(config);
482 client
485 .write_base_row(
486 "secrets",
487 &vec![Some(ColumnValue::String("ghost".into())), None, None],
488 1,
489 )
490 .unwrap();
491 client.rebuild_overlay().expect("rebuild overlay");
492 let commit_id = client
493 .patch(
494 "secrets",
495 "ghost",
496 Map::from_iter([("note".to_owned(), json!("x"))]),
497 None,
498 )
499 .expect("patch records without an author-time encrypt failure");
500 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
501 assert!(client.drop_unencodable_outbox(true, None).unwrap());
503 assert!(client.pending_commit_ids().is_empty());
504 let rejections = client.rejections();
505 assert_eq!(rejections.len(), 1);
506 assert_eq!(rejections[0].code, "client.encrypt_failed");
507 assert_eq!(rejections[0].client_commit_id, commit_id);
508 assert_eq!(
509 client
510 .commit_outcome(&commit_id)
511 .unwrap()
512 .map(|outcome| outcome.status),
513 Some(CommitOutcomeStatus::Rejected)
514 );
515 }
516
517 #[test]
518 fn patch_scope_read_preserves_sqlite_failure() {
519 let mut client = client();
520 client
521 .conn
522 .execute_batch(
523 "DROP TABLE tasks;
524 CREATE VIEW tasks AS SELECT 't1' AS id, abs(-9223372036854775808) AS project_id;",
525 )
526 .unwrap();
527 let error = client
528 .patch(
529 "tasks",
530 "t1",
531 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
532 None,
533 )
534 .expect_err("the scope read must fail");
535 assert_ne!(error.code, "sync.invalid_request");
536 assert_eq!(error.code, "client.failed");
537 assert!(error.details.as_ref().unwrap()["legacyCause"]
538 .as_str()
539 .unwrap()
540 .contains("integer overflow"));
541 assert!(client.pending_commit_ids().is_empty());
542 assert!(client.conn.is_autocommit());
543 }
544
545 #[test]
546 fn patch_with_a_scope_column_matches_the_stored_local_row() {
547 let mut client = client();
548 client.create_synced_tables().unwrap();
549 client
550 .conn
551 .execute(
552 "INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
553 [],
554 )
555 .unwrap();
556
557 client
560 .patch(
561 "tasks",
562 "t1",
563 Map::from_iter([
564 ("project_id".to_owned(), json!("p1")),
565 ("id".to_owned(), json!("t1")),
566 ]),
567 None,
568 )
569 .expect("an equal scope column is a no-op");
570 let values = client.outbox[0].ops[0]
571 .values
572 .clone()
573 .expect("the patch records values");
574 assert_eq!(values.get("id"), Some(&json!("t1")));
575 assert!(!values.contains_key("project_id"));
576
577 let differing = client.patch(
579 "tasks",
580 "t1",
581 Map::from_iter([("project_id".to_owned(), json!("p2"))]),
582 None,
583 );
584 let differing = differing.expect_err("a differing scope column is rejected");
585 assert_eq!(differing.code, "sync.invalid_request");
586 assert_eq!(differing.message, "the authoring request is invalid");
587 assert!(
588 differing
589 .details
590 .as_ref()
591 .and_then(|details| details["legacyCause"].as_str())
592 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
593 "{differing:?}"
594 );
595
596 let absent = client.patch(
599 "tasks",
600 "ghost",
601 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
602 None,
603 );
604 let absent = absent.expect_err("an absent local row is rejected");
605 assert_eq!(absent.code, "sync.row_missing");
606 assert!(absent.message.contains("requires a local row"));
607 assert_eq!(client.pending_commit_ids().len(), 1);
608 }
609
610 #[test]
611 fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
612 let schema = json!({
613 "version": 1,
614 "tables": [
615 {
616 "name": "tenants",
617 "primaryKey": "tenant_id",
618 "columns": [
619 { "name": "tenant_id", "type": "string", "nullable": false },
620 { "name": "body", "type": "string", "nullable": false }
621 ],
622 "scopes": [{ "pattern": "tenant:{tenant_id}" }]
623 },
624 {
625 "name": "buckets",
626 "primaryKey": "id",
627 "columns": [
628 { "name": "id", "type": "string", "nullable": false },
629 { "name": "bucket", "type": "float", "nullable": false },
630 { "name": "body", "type": "string", "nullable": false }
631 ],
632 "scopes": [{ "pattern": "bucket:{bucket}" }]
633 }
634 ]
635 });
636 let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
637 .expect("test client");
638 client.create_synced_tables().unwrap();
639
640 client
642 .conn
643 .execute(
644 "INSERT INTO tenants (tenant_id, body, _syncular_version) VALUES ('t1', 'seed', 1)",
645 [],
646 )
647 .unwrap();
648 client
649 .patch(
650 "tenants",
651 "t1",
652 Map::from_iter([
653 ("tenant_id".to_owned(), json!("t1")),
654 ("body".to_owned(), json!("created")),
655 ]),
656 None,
657 )
658 .expect("the primary-key scope column is proven equal by construction");
659
660 client
662 .conn
663 .execute(
664 "INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
665 [],
666 )
667 .unwrap();
668 client
669 .patch(
670 "buckets",
671 "b1",
672 Map::from_iter([
673 ("bucket".to_owned(), json!(2)),
674 ("body".to_owned(), json!("coerced")),
675 ]),
676 None,
677 )
678 .expect("the stored REAL value coerces from a supplied integer");
679 let values = client.outbox[1].ops[0]
680 .values
681 .clone()
682 .expect("the patch records values");
683 assert!(!values.contains_key("bucket"));
684
685 let differing = client.patch(
687 "buckets",
688 "b1",
689 Map::from_iter([("bucket".to_owned(), json!(3))]),
690 None,
691 );
692 let differing = differing.expect_err("a differing scope value is rejected");
693 assert_eq!(differing.code, "sync.invalid_request");
694 assert_eq!(differing.message, "the authoring request is invalid");
695 assert!(
696 differing
697 .details
698 .as_ref()
699 .and_then(|details| details["legacyCause"].as_str())
700 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
701 "{differing:?}"
702 );
703 }
704
705 #[cfg(feature = "e2ee")]
706 #[test]
707 fn stored_key_fallback_resolves_an_integer_primary_key() {
708 let schema = json!({
709 "version": 1,
710 "tables": [{
711 "name": "patients",
712 "primaryKey": "id",
713 "columns": [
714 { "name": "id", "type": "integer", "nullable": false },
715 { "name": "encryption_key_id", "type": "string", "nullable": true },
716 { "name": "note", "type": "bytes", "nullable": true,
717 "encrypted": true, "declaredType": "string" }
718 ],
719 "scopes": []
720 }]
721 });
722 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
723 .expect("test client");
724 client.create_synced_tables().unwrap();
725 client
726 .conn
727 .execute(
728 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
729 VALUES (5, 'k1', NULL, 1)",
730 [],
731 )
732 .unwrap();
733 let mut config = crate::values::EncryptionConfig::default();
734 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
735 config
736 .key_id_columns
737 .insert("patients".to_owned(), "encryption_key_id".to_owned());
738 client.set_encryption(config);
739 let table = client.schema.table("patients").unwrap();
740 let fallback = client
741 .stored_key_fallback(table, "5")
742 .expect("the stored integer key resolves the fallback");
743 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
744 }
745
746 #[test]
747 fn patch_types_an_integer_primary_key_from_its_canonical_row_id() {
748 let schema = json!({
749 "version": 1,
750 "tables": [{
751 "name": "counters",
752 "primaryKey": "id",
753 "columns": [
754 { "name": "id", "type": "integer", "nullable": false },
755 { "name": "title", "type": "string", "nullable": false }
756 ],
757 "scopes": []
758 }]
759 });
760 let mut client = SyncClient::new("int-patch".into(), &schema, ClientLimits::default())
761 .expect("test client");
762 client
763 .write_base_row(
764 "counters",
765 &vec![
766 Some(ColumnValue::Integer(42)),
767 Some(ColumnValue::String("before".into())),
768 ],
769 1,
770 )
771 .unwrap();
772 client.rebuild_overlay().expect("rebuild overlay");
773 client
774 .patch(
775 "counters",
776 "42",
777 Map::from_iter([
778 ("id".to_owned(), json!(42)),
779 ("title".to_owned(), json!("after")),
780 ]),
781 None,
782 )
783 .expect("an integer key that renders as the rowId is not a change");
784 let rows = client.query("SELECT id, title FROM counters", &[]).unwrap();
785 assert_eq!(Value::from(rows), json!([{"id": 42, "title": "after"}]));
786 for row_id in ["042", "+42", "42.0"] {
787 let error = client
788 .patch(
789 "counters",
790 row_id,
791 Map::from_iter([("title".to_owned(), json!("x"))]),
792 None,
793 )
794 .expect_err("a non-canonical integer rowId names no row");
795 assert_eq!(error.code, "sync.row_missing", "{row_id}");
796 }
797 }
798
799 #[test]
800 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
801 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
802 {"name":"id","type":"string","nullable":false},
803 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
804 for state in ["absent", "cached", "pinned"] {
805 let path = std::env::temp_dir().join(format!(
806 "syncular-blob-stage-{}.sqlite",
807 uuid::Uuid::new_v4()
808 ));
809 let mut client = SyncClient::open_path(
810 "stage-test".into(),
811 &schema,
812 Default::default(),
813 path.to_str().unwrap(),
814 )
815 .unwrap();
816 let bytes = b"atomic stage";
817 if state != "absent" {
818 client
819 .upload_blob(bytes, Some("text/plain".into()), None)
820 .unwrap();
821 if state == "cached" {
822 client
823 .conn
824 .execute("DELETE FROM _syncular_blob_uploads", [])
825 .unwrap();
826 }
827 }
828 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
829 client
830 .conn
831 .execute_batch(
832 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
833 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
834 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
835 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
836 )
837 .unwrap();
838 let error = client
839 .upload_blob(bytes, Some("application/octet-stream".into()), None)
840 .unwrap_err();
841 assert!(error.contains("injected stage failure"), "{state}: {error}");
842 assert!(client.conn.is_autocommit());
843 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
844 drop(client);
845 let mut reopened = SyncClient::open_path(
846 "stage-test".into(),
847 &schema,
848 Default::default(),
849 path.to_str().unwrap(),
850 )
851 .unwrap();
852 let durable = reopened
853 .query("SELECT * FROM _syncular_blobs", &[])
854 .unwrap();
855 reopened
856 .conn
857 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
858 .unwrap();
859 let retried = reopened.upload_blob(bytes, None, None).unwrap();
860 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
861 assert_eq!(
862 reopened
863 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
864 .unwrap(),
865 vec![Map::from_iter([(
866 "blob_id".into(),
867 Value::from(blob_id_for(bytes)),
868 )])]
869 );
870 drop(reopened);
871 std::fs::remove_file(&path).unwrap();
872 assert_eq!(durable, bodies, "{state}: reopened body state");
873 assert_eq!(immediate, bodies, "{state}: immediate body state");
874 }
875 }
876
877 #[test]
878 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
879 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
880 {"name":"id","type":"string","nullable":false},
881 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
882 for fault in [
883 "missing",
884 "body-type",
885 "length",
886 "hash",
887 "pin-metadata",
888 "read",
889 "pin-delete",
890 ] {
891 let path = std::env::temp_dir().join(format!(
892 "syncular-blob-upload-{}.sqlite",
893 uuid::Uuid::new_v4()
894 ));
895 let mut client = SyncClient::open_path(
896 "upload-test".into(),
897 &schema,
898 Default::default(),
899 path.to_str().unwrap(),
900 )
901 .unwrap();
902 let bytes = b"durable upload";
903 let ref_value = client
904 .upload_blob(bytes, Some("text/plain".into()), None)
905 .unwrap();
906 let commit = client
907 .mutate(vec![Mutation::Upsert {
908 table: "attachments".into(),
909 values: Map::from_iter([
910 ("id".into(), Value::from("pending")),
911 (
912 "file".into(),
913 Value::from(format!(
914 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
915 blob_id_for(bytes),
916 bytes.len()
917 )),
918 ),
919 ]),
920 base_version: None,
921 }])
922 .unwrap();
923 client
924 .conn
925 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
926 .unwrap();
927 client.conn.execute_batch(match fault {
928 "missing" => "DELETE FROM _syncular_blobs",
929 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
930 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
931 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
932 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
933 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
934 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
935 }).unwrap();
936 let state = if fault == "read" {
937 Vec::new()
938 } else {
939 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
940 };
941 client.set_meta(LOG_EPOCH_KEY, "epoch");
942 let mut transport = CountingRealtimeTransport::default();
943 let error = flush_blob_uploads(&mut client, &mut transport).unwrap_err();
944 assert_eq!(
945 error.code,
946 if matches!(fault, "read" | "pin-delete") {
947 "client.failed"
948 } else {
949 "sync.local_corrupt"
950 },
951 "{fault}"
952 );
953 assert_eq!(
954 transport.blob_uploads.len(),
955 usize::from(fault == "pin-delete"),
956 "{fault}: corrupt bytes must never be uploaded"
957 );
958 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
959 if fault != "read" {
960 assert_eq!(
961 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
962 state
963 );
964 } else {
965 client
966 .conn
967 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
968 .unwrap();
969 }
970 drop(client);
971 let mut reopened = SyncClient::open_path(
972 "upload-test".into(),
973 &schema,
974 Default::default(),
975 path.to_str().unwrap(),
976 )
977 .unwrap();
978 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
979 if fault != "read" {
980 assert_eq!(
981 reopened
982 .query("SELECT * FROM _syncular_blobs", &[])
983 .unwrap(),
984 state
985 );
986 }
987 if fault == "pin-delete" {
988 reopened
989 .conn
990 .execute_batch("DROP TRIGGER fail_pin_delete")
991 .unwrap();
992 }
993 if fault == "pin-metadata" {
994 reopened
995 .conn
996 .execute(
997 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
998 [],
999 )
1000 .unwrap();
1001 }
1002 reopened
1003 .conn
1004 .execute_batch(
1005 "DELETE FROM _syncular_blobs;
1006 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
1007 )
1008 .unwrap();
1009 transport.blob_uploads.clear();
1010 flush_blob_uploads(&mut reopened, &mut transport).unwrap();
1011 assert_eq!(
1012 transport.blob_uploads,
1013 [(blob_id_for(bytes), bytes.to_vec())]
1014 );
1015 assert_eq!(
1016 reopened
1017 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
1018 .unwrap(),
1019 Vec::<Map<String, Value>>::new()
1020 );
1021 assert_eq!(reopened.pending_commit_ids(), [commit]);
1022 assert_eq!(
1023 reopened
1024 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
1025 .unwrap()
1026 .unwrap()
1027 .bytes,
1028 bytes
1029 );
1030 drop(reopened);
1031 std::fs::remove_file(path).unwrap();
1032 }
1033 }
1034
1035 fn coalesced_fixture(sections: usize, commits: usize, rows: usize) -> (SyncClient, Message) {
1039 let mut client = client();
1040 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1041 let mut response = Message {
1042 wire_version: WIRE_VERSION,
1043 msg_kind: MsgKind::Response,
1044 frames: vec![Frame::RespHeader {
1045 required_schema_version: None,
1046 latest_schema_version: None,
1047 log_epoch: Some("epoch-1".into()),
1048 reset_required: Some(false),
1049 }],
1050 };
1051 let mut seq = 0;
1052 for section in 0..sections {
1053 let project = format!("p{section}");
1054 let scopes = vec![("project_id".to_owned(), vec![project.clone()])];
1055 client
1056 .subscribe(format!("s{section}"), "tasks".into(), scopes.clone(), None)
1057 .unwrap();
1058 let sub = client.subs.last_mut().unwrap();
1059 sub.cursor = 0;
1060 sub.synced_once = true;
1061 sub.effective = Some(scopes.clone());
1062 let sub = sub.clone();
1063 client.persist_sub(&sub).unwrap();
1064 response.frames.push(Frame::SubStart {
1065 id: format!("s{section}"),
1066 status: SubStatus::Active,
1067 reason_code: "".into(),
1068 effective_scopes: scopes,
1069 bootstrap: false,
1070 });
1071 for _ in 0..commits {
1072 seq += 1;
1073 let changes = (0..rows)
1074 .map(|row| {
1075 let id = format!("{project}-{seq}-{row}");
1076 let values = Map::from_iter([
1077 ("id".into(), Value::from(id.clone())),
1078 ("project_id".into(), Value::from(project.clone())),
1079 ]);
1080 ssp2::model::Change {
1081 table_index: 0,
1082 row_id: id.clone(),
1083 op: Op::Upsert,
1084 row_version: Some(seq),
1085 scopes: vec![("project_id".into(), project.clone())],
1086 row: Some(
1087 encode_row_json(
1088 client.schema.table("tasks").unwrap(),
1089 &id,
1090 &values,
1091 &client.encryption,
1092 )
1093 .unwrap(),
1094 ),
1095 }
1096 })
1097 .collect();
1098 response.frames.push(Frame::Commit {
1099 commit_seq: seq,
1100 actor_id: "writer".into(),
1101 created_at_ms: 1,
1102 tables: vec!["tasks".into()],
1103 changes,
1104 });
1105 }
1106 response.frames.push(Frame::SubEnd {
1107 next_cursor: seq,
1108 bootstrap_state: None,
1109 });
1110 }
1111 client.drain_change_batches();
1112 (client, response)
1113 }
1114
1115 fn count_commits(client: &SyncClient) -> std::sync::Arc<std::sync::atomic::AtomicUsize> {
1116 let commits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
1117 let counter = commits.clone();
1118 client
1119 .conn
1120 .commit_hook(Some(move || {
1121 counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1122 false
1123 }))
1124 .expect("commit hook");
1125 commits
1126 }
1127
1128 #[test]
1129 fn a_response_of_many_small_sections_commits_once() {
1130 for realtime in [false, true] {
1131 let (mut client, response) = coalesced_fixture(70, 1, 1);
1132 client.realtime_state = RealtimeState::Connected;
1133 client.realtime_reason_code = None;
1134 let mut transport = CountingRealtimeTransport::default();
1135 let (_, meta) = client.build_request(false).unwrap();
1136 let commits = count_commits(&client);
1137 if realtime {
1138 client.on_realtime_binary(&mut transport, &encode_message(&response));
1139 } else {
1140 assert!(matches!(
1141 client.process_response(&mut transport, response, &meta),
1142 SyncOutcome::Ok(_)
1143 ));
1144 }
1145 assert_eq!(
1146 commits.load(std::sync::atomic::Ordering::SeqCst),
1147 1,
1148 "{realtime}"
1149 );
1150 assert!(client.conn.is_autocommit());
1151 assert!(client.subs.iter().all(|sub| sub.cursor > 0));
1152 let revisions = client
1154 .drain_change_batches()
1155 .into_iter()
1156 .filter(|batch| !batch.tables.is_empty())
1157 .map(|batch| batch.revision)
1158 .collect::<Vec<_>>();
1159 assert_eq!(
1160 revisions,
1161 (1..=70).map(|r: u64| r.to_string()).collect::<Vec<_>>()
1162 );
1163 assert_eq!(client.query("SELECT id FROM tasks", &[]).unwrap().len(), 70);
1164 assert_eq!(
1166 transport
1167 .messages
1168 .iter()
1169 .filter(|m| m.contains("\"ack\""))
1170 .count(),
1171 1,
1172 "{realtime}"
1173 );
1174 }
1175 }
1176
1177 #[test]
1178 fn a_response_past_the_row_bound_commits_per_bound() {
1179 let (mut client, response) = coalesced_fixture(1, 3, 600);
1181 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1182 let (_, meta) = client.build_request(false).unwrap();
1183 let commits = count_commits(&client);
1184 assert!(matches!(
1185 client.process_response(&mut transport, response, &meta),
1186 SyncOutcome::Ok(_)
1187 ));
1188 assert_eq!(commits.load(std::sync::atomic::Ordering::SeqCst), 3);
1189 assert_eq!(
1190 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1191 1800
1192 );
1193 assert_eq!(client.subs[0].cursor, 3);
1194 }
1195
1196 #[test]
1197 fn a_failed_coalesced_commit_restores_memory_and_publishes_nothing() {
1198 let (mut client, response) = coalesced_fixture(2, 2, 600);
1202 client
1203 .conn
1204 .execute_batch(
1205 "PRAGMA foreign_keys = ON; CREATE TABLE fail_parent(id INTEGER PRIMARY KEY); CREATE TABLE fail_child(id INTEGER REFERENCES fail_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_commit AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'p0-1-0' BEGIN INSERT INTO fail_child VALUES(1); END",
1206 )
1207 .unwrap();
1208 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1209 let (_, meta) = client.build_request(false).unwrap();
1210 let subs = client.subs.clone();
1211 let revision = client.local_revision();
1212 let outcome = client.process_response(&mut transport, response.clone(), &meta);
1213 assert!(
1214 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == "storage.failed"),
1215 "{outcome:?}"
1216 );
1217 assert!(client.conn.is_autocommit());
1218 assert!(client.coalesced.is_none() && client.coalesced_failure.is_none());
1219 assert_eq!(client.local_revision(), revision);
1220 assert!(client
1221 .drain_change_batches()
1222 .iter()
1223 .all(|batch| batch.tables.is_empty()));
1224 assert!(client
1225 .query("SELECT id FROM tasks", &[])
1226 .unwrap()
1227 .is_empty());
1228 assert_eq!(
1229 client.subs.iter().map(|s| s.cursor).collect::<Vec<_>>(),
1230 subs.iter().map(|s| s.cursor).collect::<Vec<_>>()
1231 );
1232 client
1234 .conn
1235 .execute_batch("DROP TRIGGER fail_commit")
1236 .unwrap();
1237 let (_, meta) = client.build_request(false).unwrap();
1238 assert!(matches!(
1239 client.process_response(&mut transport, response, &meta),
1240 SyncOutcome::Ok(_)
1241 ));
1242 assert_eq!(
1243 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1244 2400
1245 );
1246 }
1247
1248 #[test]
1249 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
1250 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1251 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
1252 "scopes":[{"pattern":"project:{project_id}"}]}]});
1253 for realtime in [false, true] {
1254 for fault in [
1255 "none",
1256 "row",
1257 "revision",
1258 "commit",
1259 "cursor",
1260 "error",
1261 "missing-end",
1262 ] {
1263 if realtime && fault == "missing-end" {
1264 continue;
1265 }
1266 let path = std::env::temp_dir()
1267 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
1268 let mut client = SyncClient::open_path(
1269 "frame-test".into(),
1270 &schema,
1271 Default::default(),
1272 path.to_str().unwrap(),
1273 )
1274 .unwrap();
1275 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1276 client
1277 .subscribe(
1278 "tasks".into(),
1279 "tasks".into(),
1280 vec![("project_id".into(), vec!["p1".into()])],
1281 None,
1282 )
1283 .unwrap();
1284 client.subs[0].cursor = 0;
1285 client.subs[0].synced_once = true;
1286 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1287 client.persist_sub(&client.subs[0]).unwrap();
1288 client.realtime_state = RealtimeState::Connected;
1289 client.realtime_reason_code = None;
1290 client.drain_change_batches();
1291 let mut transport = CountingRealtimeTransport::default();
1292 let (_, meta) = client.build_request(false).unwrap();
1293 let mut response = Message {
1294 wire_version: WIRE_VERSION,
1295 msg_kind: MsgKind::Response,
1296 frames: vec![
1297 Frame::RespHeader {
1298 required_schema_version: None,
1299 latest_schema_version: None,
1300 log_epoch: Some("epoch-1".into()),
1301 reset_required: Some(false),
1302 },
1303 Frame::SubStart {
1304 id: "tasks".into(),
1305 status: SubStatus::Active,
1306 reason_code: "".into(),
1307 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1308 bootstrap: false,
1309 },
1310 ],
1311 };
1312 for (index, id) in ["first", "second"].iter().enumerate() {
1313 let values = Map::from_iter([
1314 ("id".into(), Value::from(*id)),
1315 ("project_id".into(), Value::from("p1")),
1316 ]);
1317 response.frames.push(Frame::Commit {
1318 commit_seq: index as i64 + 1,
1319 actor_id: "writer".into(),
1320 created_at_ms: 1,
1321 tables: vec!["tasks".into()],
1322 changes: vec![ssp2::model::Change {
1323 table_index: 0,
1324 row_id: (*id).into(),
1325 op: Op::Upsert,
1326 row_version: Some(index as i64 + 1),
1327 scopes: vec![("project_id".into(), "p1".into())],
1328 row: Some(
1329 encode_row_json(
1330 client.schema.table("tasks").unwrap(),
1331 id,
1332 &values,
1333 &client.encryption,
1334 )
1335 .unwrap(),
1336 ),
1337 }],
1338 });
1339 }
1340 response.frames.push(Frame::SubEnd {
1341 next_cursor: 2,
1342 bootstrap_state: None,
1343 });
1344 let valid = response.clone();
1345 match fault {
1346 "row" => {
1347 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
1348 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
1349 },
1350 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
1351 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
1352 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
1353 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
1354 "missing-end" => {response.frames.truncate(3);},
1355 _ => {}
1356 }
1357 if realtime {
1358 client.on_realtime_binary(&mut transport, &encode_message(&response));
1359 } else {
1360 assert_eq!(
1361 matches!(
1362 client.process_response(&mut transport, response, &meta),
1363 SyncOutcome::Ok(_)
1364 ),
1365 fault == "none",
1366 "{fault}"
1367 );
1368 }
1369 assert!(
1370 client.conn.is_autocommit(),
1371 "no open transaction after {realtime}/{fault}"
1372 );
1373 let count = match fault {
1377 "none" | "cursor" => 2,
1378 "commit" => 0,
1379 _ => 1,
1380 };
1381 let expected_revision = count + u64::from(realtime && fault != "none");
1382 assert_eq!(
1383 client.local_revision(),
1384 expected_revision,
1385 "{realtime}/{fault}"
1386 );
1387 let batches = client.drain_change_batches();
1388 assert_eq!(
1389 batches.len(),
1390 expected_revision as usize,
1391 "{realtime}/{fault}"
1392 );
1393 let row_batches: Vec<_> = batches
1394 .iter()
1395 .filter(|batch| !batch.tables.is_empty())
1396 .collect();
1397 assert_eq!(row_batches.len(), count as usize);
1398 assert_eq!(
1399 row_batches
1400 .iter()
1401 .map(|b| b.revision.clone())
1402 .collect::<Vec<_>>(),
1403 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
1404 );
1405 for table in ["tasks", "_syncular_base_tasks"] {
1406 assert_eq!(
1407 client
1408 .conn
1409 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
1410 .get::<_, u64>(0))
1411 .unwrap(),
1412 count,
1413 "{realtime}/{fault}/{table}"
1414 );
1415 }
1416 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1417 assert_eq!(
1418 transport
1419 .messages
1420 .iter()
1421 .filter(|m| m.contains("\"ack\""))
1422 .count(),
1423 usize::from(fault == "none")
1424 );
1425 if matches!(fault, "revision" | "commit" | "cursor") {
1426 client
1427 .conn
1428 .execute_batch("DROP TRIGGER fail_frame")
1429 .unwrap();
1430 }
1431 drop(client);
1432 let mut reopened = SyncClient::open_path(
1433 "frame-test".into(),
1434 &schema,
1435 Default::default(),
1436 path.to_str().unwrap(),
1437 )
1438 .unwrap();
1439 assert_eq!(reopened.local_revision(), expected_revision);
1440 assert_eq!(
1441 reopened
1442 .query("SELECT id FROM tasks ORDER BY id", &[])
1443 .unwrap()
1444 .len(),
1445 count as usize
1446 );
1447 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1448 let (_, meta) = reopened.build_request(false).unwrap();
1449 assert!(matches!(
1450 reopened.process_response(&mut transport, valid, &meta),
1451 SyncOutcome::Ok(_)
1452 ));
1453 assert_eq!(
1454 reopened
1455 .query("SELECT id FROM tasks ORDER BY id", &[])
1456 .unwrap()
1457 .len(),
1458 2
1459 );
1460 assert_eq!(reopened.subs[0].cursor, 2);
1461 drop(reopened);
1462 std::fs::remove_file(path).unwrap();
1463 }
1464 }
1465 }
1466
1467 #[test]
1468 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
1469 for status in [SubStatus::Reset, SubStatus::Revoked] {
1470 for fault in ["subscription", "revision"] {
1471 let mut client = client();
1472 client
1473 .subscribe(
1474 "tasks".into(),
1475 "tasks".into(),
1476 vec![("project_id".into(), vec!["p1".into()])],
1477 None,
1478 )
1479 .unwrap();
1480 client.subs[0].cursor = 41;
1481 client.subs[0].synced_once = true;
1482 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1483 client.persist_sub(&client.subs[0]).unwrap();
1484 client
1485 .mutate(vec![Mutation::Upsert {
1486 table: "tasks".into(),
1487 values: Map::from_iter([
1488 ("id".into(), Value::from("pending")),
1489 ("project_id".into(), Value::from("p1")),
1490 ]),
1491 base_version: None,
1492 }])
1493 .unwrap();
1494 let ids = client.pending_commit_ids();
1495 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
1496 let revision = client.local_revision();
1497 client.drain_change_batches();
1498 let (_, meta) = client.build_request(false).unwrap();
1499 client.conn.execute_batch(if fault == "subscription" {
1500 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
1501 } else {
1502 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
1503 }).unwrap();
1504 let mut transport = CountingRealtimeTransport::default();
1505 let mut report = SyncReport::default();
1506 if status == SubStatus::Reset {
1508 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
1509 }
1510 let result = client.process_section(
1511 &mut transport,
1512 "tasks",
1513 status,
1514 "sync.scope_revoked",
1515 vec![],
1516 vec![],
1517 Some((42, None)),
1518 &meta,
1519 &mut report,
1520 &HashSet::new(),
1521 );
1522 assert!(result.is_err(), "{status:?}/{fault}");
1523 assert!(client.conn.is_autocommit());
1524 assert_eq!(client.subs[0].cursor, 41);
1525 assert_eq!(client.subs[0].state, SubState::Active);
1526 assert_eq!(client.pending_commit_ids(), ids);
1527 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
1528 assert!(client.rejections.is_empty());
1529 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
1530 assert_eq!(client.local_revision(), revision);
1531 assert!(client.drain_change_batches().is_empty());
1532 assert!(report.revoked.is_empty() && report.resets.is_empty());
1533 }
1534 }
1535 }
1536
1537 #[test]
1538 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
1539 let mut client = client();
1540 client
1541 .subscribe(
1542 "tasks".into(),
1543 "tasks".into(),
1544 vec![("project_id".into(), vec!["p1".into()])],
1545 None,
1546 )
1547 .unwrap();
1548 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1549 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
1550 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
1551 let row = |id: &str| ssp2::segment::SegmentRow {
1552 server_version: 1,
1553 values: vec![
1554 Some(ColumnValue::String(id.into())),
1555 Some(ColumnValue::String("p1".into())),
1556 ],
1557 };
1558 let segment = RowsSegment {
1559 table: "tasks".into(),
1560 schema_version: 1,
1561 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
1562 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
1563 };
1564 assert!(client.apply_segment(0, &segment, true).is_err());
1565 assert!(client.conn.is_autocommit());
1566 assert_eq!(
1567 client
1568 .query("SELECT id FROM tasks ORDER BY id", &[])
1569 .unwrap(),
1570 vec![Map::from_iter([("id".into(), Value::from("first"))])]
1571 );
1572 assert_eq!(client.local_revision(), 1);
1573 assert_eq!(client.drain_change_batches().len(), 1);
1574 assert_eq!(client.subs[0].cursor, -1);
1575 client
1576 .conn
1577 .execute_batch("DROP TRIGGER fail_block")
1578 .unwrap();
1579 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
1580 assert_eq!(client.local_revision(), 3);
1581 assert_eq!(client.drain_change_batches().len(), 2);
1582 }
1583
1584 struct SegmentBatchTransport {
1588 response: Message,
1589 bodies: BTreeMap<String, Vec<u8>>,
1590 batches: Vec<Vec<String>>,
1591 single_fetches: usize,
1592 }
1593
1594 impl Transport for SegmentBatchTransport {
1595 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1596 Ok(encode_message(&self.response))
1597 }
1598
1599 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1600 self.sync(request)
1601 }
1602
1603 fn download_segment(
1604 &mut self,
1605 _request: &SegmentRequest,
1606 _on_progress: &mut dyn FnMut(u64),
1607 ) -> Result<Vec<u8>, TransportError> {
1608 self.single_fetches += 1;
1609 Err(TransportError::new(
1610 "sync.transport_failed",
1611 "per-descriptor fetch",
1612 ))
1613 }
1614
1615 fn download_segments(
1616 &mut self,
1617 fetches: &[crate::SegmentFetch],
1618 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1619 ) -> Vec<Result<Vec<u8>, TransportError>> {
1620 self.batches.push(
1621 fetches
1622 .iter()
1623 .map(|fetch| match fetch {
1624 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
1625 crate::SegmentFetch::Url(url) => url.clone(),
1626 })
1627 .collect(),
1628 );
1629 fetches
1630 .iter()
1631 .enumerate()
1632 .map(|(index, fetch)| {
1633 let crate::SegmentFetch::Direct(request) = fetch else {
1634 panic!("the descriptors are direct-endpoint requests")
1635 };
1636 let bytes = self.bodies[&request.segment_id].clone();
1637 on_progress(index, bytes.len() as u64);
1638 Ok(bytes)
1639 })
1640 .collect()
1641 }
1642
1643 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1644 Ok(())
1645 }
1646
1647 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1648 Ok(())
1649 }
1650
1651 fn realtime_close(&mut self) -> Result<(), TransportError> {
1652 Ok(())
1653 }
1654 }
1655
1656 #[test]
1657 fn a_response_fetches_all_its_segment_pages_in_one_ordered_batch() {
1658 let mut client = client();
1659 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1660 client
1661 .subscribe(
1662 "tasks".into(),
1663 "tasks".into(),
1664 vec![("project_id".into(), vec!["p1".into()])],
1665 None,
1666 )
1667 .unwrap();
1668 client.subs[0].cursor = 0;
1669 client.subs[0].synced_once = true;
1670 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1671 client.persist_sub(&client.subs[0]).unwrap();
1672 client.drain_change_batches();
1673 let columns = client.schema.table("tasks").unwrap().wire_columns.clone();
1674
1675 let pages = ["one", "two", "three", "four"].map(|id| {
1679 let bytes = ssp2::segment::encode_rows_segment(&RowsSegment {
1680 table: "tasks".into(),
1681 schema_version: 1,
1682 columns: columns.clone(),
1683 blocks: vec![vec![ssp2::segment::SegmentRow {
1684 server_version: 1,
1685 values: vec![
1686 Some(ColumnValue::String(id.into())),
1687 Some(ColumnValue::String("p1".into())),
1688 ],
1689 }]],
1690 });
1691 (
1692 format!("sha256:{}", bytes_to_hex(&Sha256::digest(&bytes))),
1693 bytes,
1694 )
1695 });
1696
1697 let mut frames = vec![
1698 Frame::RespHeader {
1699 required_schema_version: None,
1700 latest_schema_version: None,
1701 log_epoch: Some("epoch-1".into()),
1702 reset_required: Some(false),
1703 },
1704 Frame::SubStart {
1705 id: "tasks".into(),
1706 status: SubStatus::Active,
1707 reason_code: String::new(),
1708 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1709 bootstrap: false,
1710 },
1711 ];
1712 for (segment_id, bytes) in &pages {
1713 frames.push(Frame::SegmentRef {
1714 segment_id: segment_id.clone(),
1715 media_type: MediaType::Rows,
1716 table: "tasks".into(),
1717 byte_length: bytes.len() as i64,
1718 row_count: 1,
1719 as_of_commit_seq: 1,
1720 scope_digest: "digest".into(),
1721 row_cursor: None,
1722 next_row_cursor: None,
1723 url: None,
1724 url_expires_at_ms: None,
1725 });
1726 }
1727 frames.push(Frame::SubEnd {
1728 next_cursor: 1,
1729 bootstrap_state: None,
1730 });
1731 let mut transport = SegmentBatchTransport {
1732 response: Message {
1733 wire_version: WIRE_VERSION,
1734 msg_kind: MsgKind::Response,
1735 frames,
1736 },
1737 bodies: pages
1738 .iter()
1739 .map(|(segment_id, bytes)| (segment_id.clone(), bytes.clone()))
1740 .collect(),
1741 batches: Vec::new(),
1742 single_fetches: 0,
1743 };
1744
1745 let outcome = client.sync(&mut transport);
1746 assert!(matches!(outcome, SyncOutcome::Ok(_)), "{outcome:?}");
1747 assert_eq!(
1748 transport.batches,
1749 vec![pages
1750 .iter()
1751 .map(|(segment_id, _)| segment_id.clone())
1752 .collect::<Vec<_>>()],
1753 "one batch per response, in wire order"
1754 );
1755 assert_eq!(
1756 transport.single_fetches, 0,
1757 "the batch seam replaces the per-descriptor calls"
1758 );
1759 assert_eq!(
1760 client
1761 .query("SELECT id FROM tasks ORDER BY id", &[])
1762 .unwrap(),
1763 ["four", "one", "three", "two"]
1764 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
1765 .to_vec()
1766 );
1767 assert_eq!(client.subs[0].cursor, 1);
1768 }
1769
1770 #[test]
1771 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
1772 assert_eq!(
1773 client().diagnostics_storage().blob_cache_bytes_approx,
1774 Some(0)
1775 );
1776 let client = SyncClient::new(
1777 "diagnostic-storage".to_owned(),
1778 &json!({
1779 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
1780 "columns": [{"name": "id", "type": "string", "nullable": false},
1781 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
1782 }),
1783 ClientLimits {
1784 blob_cache_max_bytes: Some(5),
1785 ..Default::default()
1786 },
1787 )
1788 .unwrap();
1789 let empty = client.diagnostics_storage();
1790 assert_eq!(empty.status, "healthy");
1791 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
1792 assert_eq!(empty.retained_outcome_entries, Some(0));
1793 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
1794 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
1795 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
1796 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
1797 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
1798 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
1799 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
1800 let written = client.diagnostics_storage();
1801 assert_eq!(written.status, "pressure");
1802 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
1803 assert_eq!(written.retained_outcome_entries, Some(2));
1804 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
1805 assert_eq!(written.blob_cache_bytes_approx, Some(8));
1806 assert_eq!(
1807 written.pressure_reason_code.as_deref(),
1808 Some("client.blob_cache_over_limit")
1809 );
1810 client
1811 .conn
1812 .execute_batch(
1813 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
1814 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
1815 )
1816 .unwrap();
1817 let staged = client.diagnostics_storage();
1818 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
1819 assert_eq!(staged.retained_outcome_entries, Some(0));
1820 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
1821 assert_eq!(staged.status, "healthy");
1822 client.conn.execute_batch("ROLLBACK").unwrap();
1823 assert_eq!(client.diagnostics_storage(), written);
1824 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
1825 let unreadable = client.diagnostics_storage();
1826 assert_eq!(unreadable.status, "unreadable");
1827 assert!(unreadable.database_bytes_approx.is_none());
1828 assert!(unreadable.pending_outbox_bytes_approx.is_none());
1829 assert!(unreadable.retained_outcome_entries.is_none());
1830 assert!(unreadable.retained_outcome_bytes_approx.is_none());
1831 assert!(unreadable.blob_cache_bytes_approx.is_none());
1832 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
1833 let restored = client.diagnostics_storage();
1834 assert_eq!(restored.status, written.status);
1835 assert_eq!(
1836 restored.retained_outcome_bytes_approx,
1837 written.retained_outcome_bytes_approx
1838 );
1839 client.conn.set_prepared_statement_cache_capacity(0);
1840 assert_eq!(client.diagnostics_storage(), restored);
1841 }
1842
1843 struct ServerErrorTransport {
1845 code: &'static str,
1846 retryable: bool,
1847 }
1848
1849 impl Transport for ServerErrorTransport {
1850 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1851 Ok(encode_message(&Message {
1852 wire_version: WIRE_VERSION,
1853 msg_kind: MsgKind::Response,
1854 frames: vec![
1855 Frame::RespHeader {
1856 required_schema_version: None,
1857 latest_schema_version: None,
1858 log_epoch: Some("epoch-1".to_owned()),
1859 reset_required: Some(false),
1860 },
1861 Frame::Error {
1862 code: self.code.to_owned(),
1863 message: "server error".to_owned(),
1864 category: "internal".to_owned(),
1865 retryable: self.retryable,
1866 recommended_action: "retryLater".to_owned(),
1867 details: None,
1868 },
1869 ],
1870 }))
1871 }
1872
1873 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1874 self.sync(request)
1875 }
1876
1877 fn download_segment(
1878 &mut self,
1879 _request: &SegmentRequest,
1880 _on_progress: &mut dyn FnMut(u64),
1881 ) -> Result<Vec<u8>, TransportError> {
1882 Err(TransportError::new("sync.transport_failed", "offline"))
1883 }
1884
1885 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1886 Ok(())
1887 }
1888
1889 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1890 Ok(())
1891 }
1892
1893 fn realtime_close(&mut self) -> Result<(), TransportError> {
1894 Ok(())
1895 }
1896 }
1897
1898 #[test]
1899 fn a_retryable_in_band_failure_keeps_the_socket_connected_and_still_asks_for_a_retry() {
1900 let mut client = client();
1901 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1902 let mut transport = ServerErrorTransport {
1903 code: "sync.schema_not_ready",
1904 retryable: true,
1905 };
1906 client.connect_realtime(&mut transport).expect("connect");
1907 assert!(matches!(
1908 client.sync(&mut transport),
1909 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.schema_not_ready"
1910 ));
1911 assert_eq!(
1912 client.realtime_state(),
1913 RealtimeState::Connected,
1914 "the socket delivered the answer, so it is not lost"
1915 );
1916 assert!(
1917 matches!(
1918 client.drain_sync_intents().as_slice(),
1919 [SyncIntent::Background { delay_ms: 250 }]
1920 ),
1921 "the retry deadline is a sync round and no reconnect satisfies it"
1922 );
1923 assert_eq!(
1924 client.progress().snapshot().unwrap().retry_delay_ms,
1925 Some(250)
1926 );
1927 }
1928
1929 #[test]
1930 fn retryable_server_error_schedules_a_background_retry() {
1931 let mut client = client();
1932 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1933 let mut transport = ServerErrorTransport {
1934 code: "sync.internal_error",
1935 retryable: true,
1936 };
1937 assert!(matches!(
1938 client.sync(&mut transport),
1939 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
1940 ));
1941 assert!(matches!(
1942 client.drain_sync_intents().as_slice(),
1943 [SyncIntent::Background { delay_ms: 250 }]
1944 ));
1945 assert_eq!(
1946 client.progress().snapshot().unwrap().retry_delay_ms,
1947 Some(250)
1948 );
1949 let mut transport = ServerErrorTransport {
1950 code: "sync.invalid_request",
1951 retryable: false,
1952 };
1953 assert!(matches!(
1954 client.sync(&mut transport),
1955 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
1956 ));
1957 assert!(client.drain_sync_intents().is_empty());
1958 let progress = client.progress().snapshot().unwrap();
1959 assert_eq!(progress.state, ProgressState::Failed);
1960 assert_eq!(progress.retry_delay_ms, None);
1961 }
1962
1963 #[test]
1964 fn sync_until_idle_reports_budget_exhaustion_or_failure() {
1965 let exhausted = SyncOutcome::BudgetExhausted(SyncReport::default()).to_json();
1968 assert_eq!(exhausted["ok"], json!(true));
1969 assert_eq!(exhausted["budgetExhausted"], json!(true));
1970 assert!(exhausted["report"].is_object());
1971
1972 let mut client = client();
1973 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1974 let mut idle_transport = HostTransport::new_from_config(&json!({})).unwrap();
1976 let zero = client.sync_until_idle(&mut idle_transport, Some(0));
1977 assert!(
1978 matches!(zero, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
1979 "{zero:?}"
1980 );
1981 let mut erroring = ServerErrorTransport {
1983 code: "sync.internal_error",
1984 retryable: false,
1985 };
1986 let failed = client.sync_until_idle(&mut erroring, Some(3));
1987 assert!(
1988 matches!(failed, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1989 "{failed:?}"
1990 );
1991 }
1992
1993 struct CauseTransport(&'static str);
1995
1996 impl Transport for CauseTransport {
1997 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1998 let mut error = TransportError::new("transport.failed", "no route");
1999 error.details = Some(serde_json::json!({ "causeKind": self.0 }));
2000 Err(error)
2001 }
2002
2003 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2004 self.sync(request)
2005 }
2006
2007 fn download_segment(
2008 &mut self,
2009 _request: &SegmentRequest,
2010 _on_progress: &mut dyn FnMut(u64),
2011 ) -> Result<Vec<u8>, TransportError> {
2012 Err(TransportError::new("sync.transport_failed", "offline"))
2013 }
2014
2015 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2016 Ok(())
2017 }
2018
2019 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2020 Ok(())
2021 }
2022
2023 fn realtime_close(&mut self) -> Result<(), TransportError> {
2024 Ok(())
2025 }
2026 }
2027
2028 #[test]
2029 fn an_unreachable_transport_schedules_no_retry() {
2030 for cause in ["connect", "network"] {
2033 let mut client = client();
2034 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2035 let mut transport = CauseTransport(cause);
2036 assert!(matches!(
2037 client.sync(&mut transport),
2038 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.failed"
2039 ));
2040 assert!(
2041 client.drain_sync_intents().is_empty(),
2042 "{cause} scheduled a retry"
2043 );
2044 }
2045
2046 let mut client = client();
2048 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2049 let mut transport = CauseTransport("timeout");
2050 assert!(matches!(
2051 client.sync(&mut transport),
2052 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.failed"
2053 ));
2054 assert!(matches!(
2055 client.drain_sync_intents().as_slice(),
2056 [SyncIntent::Background { delay_ms: 250 }]
2057 ));
2058 }
2059
2060 #[test]
2061 fn sync_until_idle_reports_its_total_deadline() {
2062 assert_eq!(SyncClient::SYNC_IDLE_TIMEOUT_MS, 60_000);
2063 let mut client = client();
2064 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2065 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2066
2067 let invalid = client.sync_until_idle_with_deadline(&mut transport, None, Some(0));
2069 assert!(
2070 matches!(invalid, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
2071 "{invalid:?}"
2072 );
2073
2074 let now = client.clock_now_ms();
2077 let outcome = client.sync_until_idle_bounded(&mut transport, None, now, Some(now));
2078 let SyncOutcome::Failed {
2079 error_code,
2080 details,
2081 recommended_action,
2082 ..
2083 } = outcome
2084 else {
2085 panic!("expected the idle deadline, got {outcome:?}");
2086 };
2087 assert_eq!(error_code, "sync.idle_timeout");
2088 assert_eq!(recommended_action.as_deref(), Some("retryLater"));
2089 let details = details.expect("deadline details");
2090 assert_eq!(details["timeoutMs"], json!(0));
2091 assert_eq!(details["rounds"], json!(0));
2092 }
2093
2094 struct ScriptedSyncTransport {
2097 responses: std::collections::VecDeque<Message>,
2098 calls: usize,
2099 }
2100
2101 impl Transport for ScriptedSyncTransport {
2102 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2103 self.calls += 1;
2104 let response = self.responses.pop_front().expect("scripted response");
2105 Ok(encode_message(&response))
2106 }
2107
2108 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2109 self.sync(request)
2110 }
2111
2112 fn download_segment(
2113 &mut self,
2114 _request: &SegmentRequest,
2115 _on_progress: &mut dyn FnMut(u64),
2116 ) -> Result<Vec<u8>, TransportError> {
2117 Err(TransportError::new("sync.transport_failed", "offline"))
2118 }
2119
2120 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2121 Ok(())
2122 }
2123
2124 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2125 Ok(())
2126 }
2127
2128 fn realtime_close(&mut self) -> Result<(), TransportError> {
2129 Ok(())
2130 }
2131 }
2132
2133 #[test]
2134 fn sync_until_idle_success_then_failure_keeps_earlier_work() {
2135 let path = std::env::temp_dir().join(format!(
2136 "syncular-budget-script-{}.sqlite",
2137 uuid::Uuid::new_v4()
2138 ));
2139 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
2140 {"name":"id","type":"string","nullable":false},
2141 {"name":"project_id","type":"string","nullable":false}],
2142 "scopes":[{"pattern":"project:{project_id}"}]}]});
2143 let mut client = SyncClient::open_path(
2144 "budget-script".into(),
2145 &schema,
2146 ClientLimits::default(),
2147 path.to_str().unwrap(),
2148 )
2149 .expect("open client");
2150 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2151 client
2152 .subscribe(
2153 "tasks".into(),
2154 "tasks".into(),
2155 vec![("project_id".into(), vec!["p1".into()])],
2156 None,
2157 )
2158 .unwrap();
2159 client.subs[0].cursor = 0;
2160 client.subs[0].synced_once = true;
2161 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
2162 client.persist_sub(&client.subs[0]).unwrap();
2163 client.drain_change_batches();
2164
2165 let row = encode_row_json(
2166 client.schema.table("tasks").unwrap(),
2167 "kept",
2168 &Map::from_iter([
2169 ("id".into(), json!("kept")),
2170 ("project_id".into(), json!("p1")),
2171 ]),
2172 &client.encryption,
2173 )
2174 .unwrap();
2175 let success = Message {
2176 wire_version: WIRE_VERSION,
2177 msg_kind: MsgKind::Response,
2178 frames: vec![
2179 Frame::RespHeader {
2180 required_schema_version: None,
2181 latest_schema_version: None,
2182 log_epoch: Some("epoch-1".into()),
2183 reset_required: Some(false),
2184 },
2185 Frame::SubStart {
2186 id: "tasks".into(),
2187 status: SubStatus::Active,
2188 reason_code: String::new(),
2189 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
2190 bootstrap: false,
2191 },
2192 Frame::Commit {
2193 commit_seq: 1,
2194 created_at_ms: 1,
2195 actor_id: "writer".into(),
2196 tables: vec!["tasks".into()],
2197 changes: vec![ssp2::model::Change {
2198 table_index: 0,
2199 row_id: "kept".into(),
2200 op: Op::Upsert,
2201 row_version: Some(1),
2202 scopes: vec![("project_id".into(), "p1".into())],
2203 row: Some(row),
2204 }],
2205 },
2206 Frame::SubEnd {
2207 next_cursor: 1,
2208 bootstrap_state: None,
2209 },
2210 ],
2211 };
2212 let failure = Message {
2213 wire_version: WIRE_VERSION,
2214 msg_kind: MsgKind::Response,
2215 frames: vec![
2216 Frame::RespHeader {
2217 required_schema_version: None,
2218 latest_schema_version: None,
2219 log_epoch: Some("epoch-1".into()),
2220 reset_required: Some(false),
2221 },
2222 Frame::Error {
2223 code: "sync.internal_error".into(),
2224 message: "second round failed".into(),
2225 category: "internal".into(),
2226 retryable: false,
2227 recommended_action: "retryLater".into(),
2228 details: None,
2229 },
2230 ],
2231 };
2232 let mut transport = ScriptedSyncTransport {
2233 responses: std::collections::VecDeque::from([success, failure]),
2234 calls: 0,
2235 };
2236 let outcome = client.sync_until_idle(&mut transport, Some(2));
2237 let SyncOutcome::Failed {
2238 error_code,
2239 recommended_action,
2240 ..
2241 } = outcome
2242 else {
2243 panic!("expected the in-band failure");
2244 };
2245 assert_eq!(error_code, "sync.internal_error");
2246 assert_eq!(recommended_action.as_deref(), Some("retryLater"));
2247 assert_eq!(
2248 transport.calls, 2,
2249 "one successful round then the failing round"
2250 );
2251 assert_eq!(
2252 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
2253 json!("kept"),
2254 "the earlier round's durable work survives the later failure"
2255 );
2256 drop(client);
2257 let _ = std::fs::remove_file(&path);
2258 }
2259
2260 #[test]
2261 fn sync_until_idle_stops_on_a_schema_floor() {
2262 let mut client = client();
2263 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2264 let floor = Message {
2265 wire_version: WIRE_VERSION,
2266 msg_kind: MsgKind::Response,
2267 frames: vec![Frame::RespHeader {
2268 required_schema_version: Some(2),
2269 latest_schema_version: Some(2),
2270 log_epoch: Some("epoch-1".into()),
2271 reset_required: Some(false),
2272 }],
2273 };
2274 let mut transport = ScriptedSyncTransport {
2275 responses: std::collections::VecDeque::from([floor.clone(), floor]),
2276 calls: 0,
2277 };
2278 let outcome = client.sync_until_idle(&mut transport, Some(3));
2279 match outcome {
2280 SyncOutcome::Ok(report) => assert_eq!(
2281 report
2282 .schema_floor
2283 .as_ref()
2284 .and_then(|floor| floor.required_schema_version),
2285 Some(2),
2286 "the stop retains the latest schema floor"
2287 ),
2288 other => panic!("expected a schema-floor stop, got {other:?}"),
2289 }
2290 assert_eq!(
2291 transport.calls, 1,
2292 "a schema floor stops the budget loop after one round"
2293 );
2294 }
2295
2296 struct FailureScriptTransport {
2299 rounds: std::collections::VecDeque<Result<Vec<u8>, TransportError>>,
2300 requests: Vec<Message>,
2301 segment_error: Option<TransportError>,
2302 }
2303
2304 impl Transport for FailureScriptTransport {
2305 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2306 self.requests
2307 .push(decode_message(request).expect("the client encodes a legal request"));
2308 self.rounds.pop_front().expect("scripted round")
2309 }
2310
2311 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2312 self.sync(request)
2313 }
2314
2315 fn download_segment(
2316 &mut self,
2317 _request: &SegmentRequest,
2318 _on_progress: &mut dyn FnMut(u64),
2319 ) -> Result<Vec<u8>, TransportError> {
2320 Err(self
2321 .segment_error
2322 .clone()
2323 .unwrap_or_else(|| TransportError::new("transport.timeout", "segment stalled")))
2324 }
2325
2326 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2327 Ok(())
2328 }
2329
2330 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2331 Ok(())
2332 }
2333
2334 fn realtime_close(&mut self) -> Result<(), TransportError> {
2335 Ok(())
2336 }
2337 }
2338
2339 fn completed_response(frames: Vec<Frame>) -> Vec<u8> {
2341 let mut all = vec![Frame::RespHeader {
2342 required_schema_version: None,
2343 latest_schema_version: None,
2344 log_epoch: Some("epoch-1".into()),
2345 reset_required: Some(false),
2346 }];
2347 all.extend(frames);
2348 encode_message(&Message {
2349 wire_version: WIRE_VERSION,
2350 msg_kind: MsgKind::Response,
2351 frames: all,
2352 })
2353 }
2354
2355 fn carried_round_failures(request: &Message) -> Vec<String> {
2356 request
2357 .frames
2358 .iter()
2359 .filter_map(|frame| match frame {
2360 Frame::RoundFailure { code } => Some(code.clone()),
2361 _ => None,
2362 })
2363 .collect()
2364 }
2365
2366 fn failure_transport(
2367 rounds: impl IntoIterator<Item = Result<Vec<u8>, TransportError>>,
2368 ) -> FailureScriptTransport {
2369 FailureScriptTransport {
2370 rounds: rounds.into_iter().collect(),
2371 requests: Vec::new(),
2372 segment_error: None,
2373 }
2374 }
2375
2376 #[test]
2377 fn transport_diagnostics_bound_evidence_and_clear_it_after_recovery() {
2378 let secret = "https://user:password@example.invalid/private?token=secret";
2379 for (code, details, expected_cause, expected_status, connectivity, retry) in [
2380 (
2381 "transport.failed",
2382 json!({"causeKind":"connect", "secret":secret}),
2383 Some(crate::TransportCauseKind::Connect),
2384 None,
2385 "offline",
2386 None,
2387 ),
2388 (
2389 "transport.failed",
2390 json!({"causeKind":"status", "httpStatus":500, "secret":secret}),
2391 Some(crate::TransportCauseKind::Status),
2392 Some(500),
2393 "online",
2394 Some(250),
2395 ),
2396 (
2397 "sync.auth_required",
2398 json!({"causeKind":"status", "httpStatus":401, "secret":secret}),
2399 Some(crate::TransportCauseKind::Status),
2400 Some(401),
2401 "online",
2402 Some(250),
2403 ),
2404 (
2405 "transport.timeout",
2406 json!({"causeKind":"timeout", "secret":secret}),
2407 Some(crate::TransportCauseKind::Timeout),
2408 None,
2409 "unknown",
2410 Some(250),
2411 ),
2412 (
2413 "transport.failed",
2414 json!({"causeKind":"unknown", "secret":secret}),
2415 Some(crate::TransportCauseKind::Unknown),
2416 None,
2417 "unknown",
2418 Some(250),
2419 ),
2420 (
2421 "transport.failed",
2422 json!({"causeKind":secret, "httpStatus":500}),
2423 None,
2424 None,
2425 "unknown",
2426 Some(250),
2427 ),
2428 (
2429 "transport.failed",
2430 json!({"causeKind":"status", "httpStatus":500.5}),
2431 Some(crate::TransportCauseKind::Status),
2432 None,
2433 "unknown",
2434 Some(250),
2435 ),
2436 (
2437 "transport.failed",
2438 json!({"causeKind":"connect", "httpStatus":500}),
2439 Some(crate::TransportCauseKind::Connect),
2440 None,
2441 "offline",
2442 None,
2443 ),
2444 ] {
2445 let mut client = client();
2446 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2447 let mut error = TransportError::new(code, secret);
2448 error.details = Some(details);
2449 let mut transport = failure_transport([Err(error), Ok(completed_response(vec![]))]);
2450 assert!(matches!(
2451 client.sync(&mut transport),
2452 SyncOutcome::Failed { .. }
2453 ));
2454 let progress = client.progress().snapshot().unwrap();
2455 assert_eq!(progress.cause_kind, expected_cause);
2456 assert_eq!(progress.http_status, expected_status);
2457 assert_eq!(progress.retry_delay_ms, retry);
2458 let snapshot = client
2459 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2460 .unwrap();
2461 assert_eq!(snapshot.host.connectivity, connectivity);
2462 let last_round = snapshot.last_round.as_ref().unwrap();
2463 assert_eq!(last_round.cause_kind, expected_cause);
2464 assert_eq!(last_round.http_status, expected_status);
2465 assert!(!serde_json::to_string(&snapshot).unwrap().contains(secret));
2466 assert!(!serde_json::to_string(&progress).unwrap().contains(secret));
2467
2468 assert!(matches!(client.sync(&mut transport), SyncOutcome::Ok(_)));
2469 let progress = client.progress().snapshot().unwrap();
2470 assert_eq!(progress.cause_kind, None);
2471 assert_eq!(progress.http_status, None);
2472 let snapshot = client
2473 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2474 .unwrap();
2475 assert_eq!(snapshot.host.connectivity, "online");
2476 let last_round = snapshot.last_round.unwrap();
2477 assert_eq!(last_round.cause_kind, None);
2478 assert_eq!(last_round.http_status, None);
2479 }
2480 }
2481
2482 #[test]
2483 fn a_round_without_a_complete_response_reports_its_code_on_every_following_request() {
2484 let mut client = client();
2485 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2486 let mut transport = failure_transport([
2487 Err(TransportError::new("transport.timeout", "round timed out")),
2488 Ok(completed_response(Vec::new())),
2489 Ok(completed_response(Vec::new())),
2490 ]);
2491 for _ in 0..3 {
2492 client.sync(&mut transport);
2493 }
2494 assert!(carried_round_failures(&transport.requests[0]).is_empty());
2495 assert_eq!(
2496 carried_round_failures(&transport.requests[1]),
2497 vec!["transport.timeout"]
2498 );
2499 assert!(
2500 matches!(transport.requests[1].frames[0], Frame::ReqHeader { .. })
2501 && matches!(transport.requests[1].frames[1], Frame::RoundFailure { .. }),
2502 "the report sits immediately after REQ_HEADER"
2503 );
2504 assert!(
2505 carried_round_failures(&transport.requests[2]).is_empty(),
2506 "a complete response ends the report"
2507 );
2508 }
2509
2510 #[test]
2511 fn a_newer_round_failure_replaces_the_kept_code() {
2512 let mut client = client();
2513 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2514 let mut transport = failure_transport([
2515 Err(TransportError::new("transport.timeout", "round timed out")),
2516 Err(TransportError::new("transport.failed", "socket closed")),
2517 Ok(completed_response(Vec::new())),
2518 Ok(completed_response(Vec::new())),
2519 ]);
2520 for _ in 0..4 {
2521 client.sync(&mut transport);
2522 }
2523 assert_eq!(
2524 carried_round_failures(&transport.requests[1]),
2525 vec!["transport.timeout"]
2526 );
2527 assert_eq!(
2528 carried_round_failures(&transport.requests[2]),
2529 vec!["transport.failed"]
2530 );
2531 assert!(carried_round_failures(&transport.requests[3]).is_empty());
2532 }
2533
2534 #[test]
2535 fn an_in_band_error_completes_the_round_and_is_never_reported() {
2536 let mut client = client();
2537 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2538 let mut transport = failure_transport([
2539 Err(TransportError::new("transport.timeout", "round timed out")),
2540 Ok(completed_response(vec![Frame::Error {
2541 code: "sync.internal_error".into(),
2542 message: "server error".into(),
2543 category: "internal".into(),
2544 retryable: false,
2545 recommended_action: "retryLater".into(),
2546 details: None,
2547 }])),
2548 Ok(completed_response(Vec::new())),
2549 ]);
2550 client.sync(&mut transport);
2551 assert!(matches!(
2552 client.sync(&mut transport),
2553 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
2554 ));
2555 client.sync(&mut transport);
2556 assert_eq!(
2557 carried_round_failures(&transport.requests[1]),
2558 vec!["transport.timeout"]
2559 );
2560 assert!(
2561 carried_round_failures(&transport.requests[2]).is_empty(),
2562 "the server already recorded an in-band ERROR"
2563 );
2564 }
2565
2566 #[test]
2567 fn a_silent_segment_fetch_is_retryable_and_reported_on_the_next_request() {
2568 let mut client = client();
2569 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2570 client
2571 .subscribe(
2572 "tasks".into(),
2573 "tasks".into(),
2574 vec![("project_id".into(), vec!["p1".into()])],
2575 None,
2576 )
2577 .unwrap();
2578 client.subs[0].cursor = 0;
2579 client.subs[0].synced_once = true;
2580 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
2581 client.persist_sub(&client.subs[0]).unwrap();
2582 client.drain_change_batches();
2583 client.drain_sync_intents();
2584 let mut transport = failure_transport([
2585 Ok(completed_response(vec![
2586 Frame::SubStart {
2587 id: "tasks".into(),
2588 status: SubStatus::Active,
2589 reason_code: String::new(),
2590 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
2591 bootstrap: false,
2592 },
2593 Frame::SegmentRef {
2594 segment_id: "sha256:stalled".into(),
2595 media_type: MediaType::Rows,
2596 table: "tasks".into(),
2597 byte_length: 1,
2598 row_count: 1,
2599 as_of_commit_seq: 1,
2600 scope_digest: "digest".into(),
2601 row_cursor: None,
2602 next_row_cursor: None,
2603 url: None,
2604 url_expires_at_ms: None,
2605 },
2606 Frame::SubEnd {
2607 next_cursor: 1,
2608 bootstrap_state: None,
2609 },
2610 ])),
2611 Ok(completed_response(Vec::new())),
2612 Ok(completed_response(Vec::new())),
2613 ]);
2614 let mut segment_error = TransportError::new("transport.timeout", "private response body");
2615 segment_error.details = Some(json!({
2616 "causeKind": "status",
2617 "httpStatus": 500,
2618 "url": "https://example.invalid/private"
2619 }));
2620 transport.segment_error = Some(segment_error);
2621
2622 assert!(matches!(
2623 client.sync(&mut transport),
2624 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.timeout"
2625 ));
2626 let progress = client.progress().snapshot().unwrap();
2627 assert_eq!(progress.cause_kind, Some(crate::TransportCauseKind::Status));
2628 assert_eq!(progress.http_status, Some(500));
2629 let diagnostics = client
2630 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2631 .unwrap();
2632 assert_eq!(diagnostics.host.connectivity, "online");
2633 let round = diagnostics.last_round.as_ref().unwrap();
2634 assert_eq!(round.cause_kind, Some(crate::TransportCauseKind::Status));
2635 assert_eq!(round.http_status, Some(500));
2636 assert!(!serde_json::to_string(&diagnostics)
2637 .unwrap()
2638 .contains("private"));
2639 assert!(!serde_json::to_string(&progress)
2640 .unwrap()
2641 .contains("private"));
2642 assert!(
2643 matches!(
2644 client.drain_sync_intents().as_slice(),
2645 [SyncIntent::Background { delay_ms: 250 }]
2646 ),
2647 "a silent transfer asks the host for a Background retry"
2648 );
2649 client.sync(&mut transport);
2650 client.sync(&mut transport);
2651 assert!(carried_round_failures(&transport.requests[0]).is_empty());
2652 assert_eq!(
2653 carried_round_failures(&transport.requests[1]),
2654 vec!["transport.timeout"],
2655 "the round's own response was complete, so the fetch failure rides the next request"
2656 );
2657 assert!(
2658 carried_round_failures(&transport.requests[2]).is_empty(),
2659 "a complete response ends the report"
2660 );
2661 }
2662
2663 #[test]
2664 fn a_code_the_frame_grammar_rejects_is_not_attached() {
2665 let mut client = client();
2666 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2667 let mut transport = failure_transport([
2668 Err(TransportError::new("offline", "no dotted code")),
2669 Ok(completed_response(Vec::new())),
2670 ]);
2671 client.sync(&mut transport);
2672 client.sync(&mut transport);
2673 assert!(carried_round_failures(&transport.requests[1]).is_empty());
2674 }
2675
2676 #[test]
2677 fn sync_report_merge_schema_floor_is_latest_state() {
2678 let mut aggregate = SyncReport {
2679 schema_floor: Some(SchemaFloor {
2680 required_schema_version: Some(2),
2681 latest_schema_version: Some(2),
2682 }),
2683 ..SyncReport::default()
2684 };
2685 aggregate.merge(&SyncReport::default());
2686 assert!(
2687 aggregate.schema_floor.is_none(),
2688 "a later round without a floor clears it"
2689 );
2690 }
2691
2692 #[test]
2693 fn push_capacity_limits_are_hard_and_keep_intent() {
2694 let mut c = client();
2696 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2697 c.limits.max_push_operations_per_request = Some(1);
2698 let id = c
2699 .mutate(vec![
2700 Mutation::Upsert {
2701 table: "tasks".into(),
2702 values: Map::from_iter([
2703 ("id".into(), json!("a")),
2704 ("project_id".into(), json!("p1")),
2705 ]),
2706 base_version: None,
2707 },
2708 Mutation::Upsert {
2709 table: "tasks".into(),
2710 values: Map::from_iter([
2711 ("id".into(), json!("b")),
2712 ("project_id".into(), json!("p1")),
2713 ]),
2714 base_version: None,
2715 },
2716 ])
2717 .unwrap();
2718 let error = c.build_request(false).unwrap_err();
2719 let SyncOutcome::Failed {
2720 error_code,
2721 details,
2722 ..
2723 } = *error
2724 else {
2725 panic!("expected a capacity failure");
2726 };
2727 assert_eq!(error_code, "client.push_request_too_large");
2728 let details = details.expect("capacity details");
2729 assert_eq!(details["kind"], json!("operations"));
2730 assert_eq!(details["limit"], json!(1));
2731 assert_eq!(details["size"], json!(2));
2732 assert_eq!(details["clientCommitId"], json!(id));
2733 assert_eq!(c.pending_commit_ids(), vec![id]);
2734
2735 let mut c = client();
2737 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2738 c.limits.max_push_commits_per_request = Some(1);
2739 let first = c
2740 .mutate(vec![Mutation::Upsert {
2741 table: "tasks".into(),
2742 values: Map::from_iter([
2743 ("id".into(), json!("one")),
2744 ("project_id".into(), json!("p1")),
2745 ]),
2746 base_version: None,
2747 }])
2748 .unwrap();
2749 let second = c
2750 .mutate(vec![Mutation::Upsert {
2751 table: "tasks".into(),
2752 values: Map::from_iter([
2753 ("id".into(), json!("two")),
2754 ("project_id".into(), json!("p1")),
2755 ]),
2756 base_version: None,
2757 }])
2758 .unwrap();
2759 let (_, meta) = c.build_request(false).unwrap();
2760 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2761 assert_eq!(meta.deferred_commits, 1);
2762 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2763 }
2764
2765 #[test]
2766 fn push_request_byte_cap_defers_the_prefix_boundary_and_errors_the_head() {
2767 let mut c = client();
2770 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2771 let first = c
2772 .mutate(vec![Mutation::Upsert {
2773 table: "tasks".into(),
2774 values: Map::from_iter([
2775 ("id".into(), json!("one")),
2776 ("project_id".into(), json!("p1")),
2777 ]),
2778 base_version: None,
2779 }])
2780 .unwrap();
2781 let second = c
2782 .mutate(vec![Mutation::Upsert {
2783 table: "tasks".into(),
2784 values: Map::from_iter([
2785 ("id".into(), json!("two")),
2786 ("project_id".into(), json!("p1")),
2787 ]),
2788 base_version: None,
2789 }])
2790 .unwrap();
2791 let (message, meta) = c.build_request(false).unwrap();
2792 assert_eq!(meta.pushed_ids.len(), 2);
2793 let full = encode_message(&message).len();
2794 c.limits.max_push_request_bytes = Some(full - 1);
2795 let (_, meta) = c.build_request(false).unwrap();
2796 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2797 assert_eq!(meta.deferred_commits, 1);
2798 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2799
2800 let mut c = client();
2803 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2804 let huge = c
2805 .mutate(vec![Mutation::Upsert {
2806 table: "tasks".into(),
2807 values: Map::from_iter([
2808 ("id".into(), json!("x".repeat(20_000))),
2809 ("project_id".into(), json!("p1")),
2810 ]),
2811 base_version: None,
2812 }])
2813 .unwrap();
2814 let (message, meta) = c.build_request(false).unwrap();
2815 assert_eq!(meta.pushed_ids, vec![huge.clone()]);
2816 let with_huge = encode_message(&message).len();
2817 c.limits.max_push_request_bytes = Some(with_huge - 1);
2818 let error = c.build_request(false).unwrap_err();
2819 let SyncOutcome::Failed {
2820 error_code,
2821 details,
2822 ..
2823 } = *error
2824 else {
2825 panic!("expected a capacity failure");
2826 };
2827 assert_eq!(error_code, "client.push_request_too_large");
2828 let details = details.expect("capacity details");
2829 assert_eq!(details["kind"], json!("bytes"));
2830 assert_eq!(details["limit"], json!((with_huge - 1) as u64));
2831 assert_eq!(details["size"], json!(with_huge as u64));
2832 assert_eq!(details["clientCommitId"], json!(huge));
2833 assert_eq!(c.pending_commit_ids(), vec![huge]);
2834
2835 let mut c = client();
2837 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2838 c.limits.max_push_request_bytes = Some(1);
2839 let error = c.build_request(false).unwrap_err();
2840 let SyncOutcome::Failed { details, .. } = *error else {
2841 panic!("expected a capacity failure");
2842 };
2843 let details = details.expect("capacity details");
2844 assert_eq!(details["kind"], json!("bytes"));
2845 assert!(details.get("clientCommitId").is_none(), "{details}");
2846 }
2847
2848 #[test]
2849 fn push_limits_validate_at_construction() {
2850 for limits in [
2851 ClientLimits {
2852 max_push_commits_per_request: Some(0),
2853 ..Default::default()
2854 },
2855 ClientLimits {
2856 max_push_operations_per_request: Some(0),
2857 ..Default::default()
2858 },
2859 ClientLimits {
2860 max_push_request_bytes: Some(0),
2861 ..Default::default()
2862 },
2863 ClientLimits {
2864 max_push_request_bytes: Some(u32::MAX as usize + 1),
2865 ..Default::default()
2866 },
2867 ] {
2868 let schema = json!({"version":1,"tables":[]});
2869 let error = SyncClient::with_connection(
2870 "bad-limits".into(),
2871 &schema,
2872 limits,
2873 Connection::open_in_memory().unwrap(),
2874 )
2875 .err();
2876 assert_eq!(
2877 error.as_deref(),
2878 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2879 );
2880 }
2881 }
2882
2883 #[test]
2884 fn open_path_rejects_invalid_push_limits_before_storage() {
2885 for identity in [false, true] {
2886 let path = std::env::temp_dir().join(format!(
2887 "syncular-bad-limits-{}.sqlite",
2888 uuid::Uuid::new_v4()
2889 ));
2890 let schema = json!({"version":1,"tables":[]});
2891 let limits = ClientLimits {
2892 max_push_request_bytes: Some(0),
2893 ..Default::default()
2894 };
2895 let error = if identity {
2896 SyncClient::open_path_with_identity(None, &schema, limits, path.to_str().unwrap())
2897 } else {
2898 SyncClient::open_path("bad-limits".into(), &schema, limits, path.to_str().unwrap())
2899 }
2900 .err();
2901 assert_eq!(
2902 error.as_deref(),
2903 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2904 );
2905 assert!(
2906 !path.exists(),
2907 "an invalid limit is rejected before the database file is created"
2908 );
2909 }
2910 }
2911
2912 #[test]
2913 fn failed_progress_retry_delay_doubles_to_the_cap() {
2914 let mut client = client();
2915 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2916 let mut transport = ServerErrorTransport {
2917 code: "sync.internal_error",
2918 retryable: true,
2919 };
2920 let delays: Vec<Option<u64>> = (0..9)
2921 .map(|_| {
2922 client.sync(&mut transport);
2923 client.progress().snapshot().unwrap().retry_delay_ms
2924 })
2925 .collect();
2926 assert_eq!(
2927 delays,
2928 [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
2929 );
2930 }
2931
2932 #[test]
2933 fn background_retry_deadlines_back_off_and_reset() {
2934 let mut client = client();
2935 client.schedule_background_retry();
2936 assert!(matches!(
2937 client.drain_sync_intents().as_slice(),
2938 [SyncIntent::Background { delay_ms: 250 }]
2939 ));
2940 client.schedule_background_retry();
2941 assert!(matches!(
2942 client.drain_sync_intents().as_slice(),
2943 [SyncIntent::Background { delay_ms: 500 }]
2944 ));
2945 client.reset_background_retry();
2946 client.schedule_background_retry();
2947 assert!(matches!(
2948 client.drain_sync_intents().as_slice(),
2949 [SyncIntent::Background { delay_ms: 250 }]
2950 ));
2951 }
2952
2953 #[test]
2954 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
2955 let mut client = client();
2956 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2957 client
2958 .subscribe(
2959 "epoch-tasks".to_owned(),
2960 "tasks".to_owned(),
2961 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2962 None,
2963 )
2964 .expect("subscribe");
2965 client.drain_sync_intents();
2966
2967 client
2968 .run_log_epoch_reset("epoch-2")
2969 .expect("reset partition log epoch");
2970
2971 assert!(matches!(
2972 client.drain_sync_intents().as_slice(),
2973 [SyncIntent::Interactive]
2974 ));
2975 }
2976
2977 #[derive(Default)]
2978 struct CountingRealtimeTransport {
2979 connects: usize,
2980 closes: usize,
2981 messages: Vec<String>,
2982 blob_uploads: Vec<(String, Vec<u8>)>,
2983 blob_download: Option<Vec<u8>>,
2984 sync_calls: usize,
2985 realtime_sync_calls: usize,
2986 refuse_connect: bool,
2987 }
2988
2989 impl Transport for CountingRealtimeTransport {
2990 fn blob_upload(
2991 &mut self,
2992 blob_id: &str,
2993 bytes: &[u8],
2994 _media_type: Option<&str>,
2995 ) -> Result<(), TransportError> {
2996 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
2997 Ok(())
2998 }
2999
3000 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
3001 self.blob_download
3002 .clone()
3003 .map(BlobDownload::Bytes)
3004 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
3005 }
3006
3007 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
3008 self.sync_calls += 1;
3009 Err(TransportError::new("sync.transport_failed", "offline"))
3010 }
3011
3012 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
3013 self.realtime_sync_calls += 1;
3014 Err(TransportError::new("sync.transport_failed", "offline"))
3015 }
3016
3017 fn download_segment(
3018 &mut self,
3019 _request: &SegmentRequest,
3020 _on_progress: &mut dyn FnMut(u64),
3021 ) -> Result<Vec<u8>, TransportError> {
3022 Err(TransportError::new("sync.transport_failed", "offline"))
3023 }
3024
3025 fn realtime_connect(&mut self) -> Result<(), TransportError> {
3026 self.connects += 1;
3027 if self.refuse_connect {
3028 return Err(TransportError::new("transport.failed", "refused"));
3029 }
3030 Ok(())
3031 }
3032
3033 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
3034 self.messages.push(text.to_owned());
3035 Ok(())
3036 }
3037
3038 fn realtime_close(&mut self) -> Result<(), TransportError> {
3039 self.closes += 1;
3040 Ok(())
3041 }
3042 }
3043
3044 #[test]
3045 fn blob_results_own_downloaded_bytes() {
3046 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
3047 {"name":"id","type":"string","nullable":false},
3048 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
3049 let bytes = b"owned typed result".to_vec();
3050 let blob_id = blob_id_for(&bytes);
3051 let mut transport = CountingRealtimeTransport {
3052 blob_download: Some(bytes.clone()),
3053 ..Default::default()
3054 };
3055 let result = {
3056 let mut client =
3057 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
3058 .unwrap();
3059 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
3060 assert_eq!(result.blob_id, blob_id);
3061 assert_eq!(result.byte_length, bytes.len() as i64);
3062 assert_eq!(result.bytes, bytes);
3063 assert_eq!(result.media_type, None);
3064 assert_eq!(
3065 client
3066 .conn
3067 .query_row(
3068 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
3069 rusqlite::params![blob_id],
3070 |row| row.get::<_, i64>(0),
3071 )
3072 .unwrap(),
3073 0
3074 );
3075 result
3076 };
3077 assert_eq!(result.bytes, bytes);
3078
3079 let mut client =
3080 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
3081 assert_eq!(
3082 client
3083 .fetch_blob_bytes(&mut transport, "not json")
3084 .unwrap_err(),
3085 (
3086 "client.failed".to_owned(),
3087 "blob ref is not JSON".to_owned()
3088 )
3089 );
3090 }
3091
3092 #[test]
3093 fn visible_blob_references_survive_cache_trimming() {
3094 let schema = json!({"version":1,"tables":[
3095 {"name":"attachments","primaryKey":"id","columns":[
3096 {"name":"id","type":"string","nullable":false},
3097 {"name":"body","type":"blob_ref","nullable":true},
3098 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
3099 {"name":"avatars","primaryKey":"id","columns":[
3100 {"name":"id","type":"string","nullable":false},
3101 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
3102 ]});
3103 let bytes = b"referenced above cap".to_vec();
3104 let blob_id = blob_id_for(&bytes);
3105 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
3106 let mut client = SyncClient::new(
3107 "visible-blob-refs".to_owned(),
3108 &schema,
3109 ClientLimits {
3110 blob_cache_max_bytes: Some(1),
3111 ..Default::default()
3112 },
3113 )
3114 .unwrap();
3115 client.conn.execute(
3116 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
3117 rusqlite::params![reference, reference, reference, reference],
3118 ).unwrap();
3119 client
3120 .conn
3121 .execute(
3122 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
3123 rusqlite::params![reference],
3124 )
3125 .unwrap();
3126 client.conn.execute(
3127 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
3128 [],
3129 ).unwrap();
3130 let mut transport = CountingRealtimeTransport {
3131 blob_download: Some(bytes.clone()),
3132 ..Default::default()
3133 };
3134
3135 assert_eq!(
3136 client
3137 .fetch_blob_bytes(&mut transport, &blob_id)
3138 .unwrap()
3139 .bytes,
3140 bytes
3141 );
3142 let rows = client
3143 .conn
3144 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
3145 .unwrap()
3146 .query_map([], |row| row.get::<_, String>(0))
3147 .unwrap()
3148 .collect::<Result<Vec<_>, _>>()
3149 .unwrap();
3150 assert_eq!(rows, vec![blob_id]);
3151 }
3152
3153 #[test]
3154 fn closed_transport_never_invokes_network_and_reopen_is_host_owned() {
3155 let mut transport = CountingRealtimeTransport::default();
3156 let path = std::env::temp_dir().join(format!(
3157 "syncular-transport-gate-{}.db",
3158 uuid::Uuid::new_v4()
3159 ));
3160 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
3161 {"name":"id","type":"string","nullable":false},
3162 {"name":"project_id","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
3163 let mut client = SyncClient::open_path(
3164 "gate".into(),
3165 &schema,
3166 ClientLimits::default(),
3167 path.to_str().unwrap(),
3168 )
3169 .unwrap();
3170 client.connect_realtime(&mut transport).unwrap();
3171 client.set_transport_enabled(&mut transport, false);
3172 assert_eq!(transport.closes, 1);
3173 client.begin_security_preflight();
3174 client.activate_security(Default::default()).unwrap();
3175 client
3176 .subscribe(
3177 "tasks".into(),
3178 "tasks".into(),
3179 vec![("project_id".into(), vec!["p1".into()])],
3180 None,
3181 )
3182 .unwrap();
3183 for id in ["one", "two"] {
3184 client
3185 .mutate(vec![Mutation::Upsert {
3186 table: "tasks".into(),
3187 values: Map::from_iter([
3188 ("id".into(), json!(id)),
3189 ("project_id".into(), json!("p1")),
3190 ]),
3191 base_version: None,
3192 }])
3193 .unwrap();
3194 }
3195 assert_eq!(
3196 client
3197 .query("SELECT id FROM tasks ORDER BY id", &[])
3198 .unwrap()
3199 .len(),
3200 2
3201 );
3202 assert!(
3203 matches!(client.sync(&mut transport), SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.offline")
3204 );
3205 assert!(client
3206 .connect_realtime(&mut transport)
3207 .unwrap_err()
3208 .starts_with("sync.offline:"));
3209 assert!(client
3210 .set_presence(&mut transport, "project:p1", None)
3211 .unwrap_err()
3212 .starts_with("sync.offline:"));
3213 assert_eq!(transport.sync_calls, 0);
3214 assert_eq!(transport.realtime_sync_calls, 0);
3215 assert_eq!(transport.connects, 1);
3216 assert!(transport.messages.is_empty());
3217 assert!(matches!(
3218 client.drain_sync_intents().as_slice(),
3219 [SyncIntent::None]
3220 ));
3221 let pending = client.pending_commit_ids();
3222 drop(client);
3223 let mut reopened = SyncClient::open_path(
3224 "gate".into(),
3225 &schema,
3226 ClientLimits::default(),
3227 path.to_str().unwrap(),
3228 )
3229 .unwrap();
3230 assert!(reopened.transport_enabled());
3231 reopened.set_transport_enabled(&mut transport, false);
3232 assert_eq!(reopened.pending_commit_ids(), pending);
3233 assert_eq!(
3234 reopened.query("SELECT id FROM tasks", &[]).unwrap().len(),
3235 2
3236 );
3237 reopened.set_transport_enabled(&mut transport, true);
3238 assert!(matches!(
3239 reopened.drain_sync_intents().as_slice(),
3240 [SyncIntent::Interactive]
3241 ));
3242 reopened.set_transport_enabled(&mut transport, true);
3243 assert!(reopened.drain_sync_intents().is_empty());
3244 drop(reopened);
3245 std::fs::remove_file(path).unwrap();
3246 }
3247
3248 #[test]
3249 fn realtime_connection_ownership_is_idempotent() {
3250 let mut client = client();
3251 let mut transport = CountingRealtimeTransport {
3252 connects: 0,
3253 closes: 0,
3254 ..Default::default()
3255 };
3256 client
3257 .connect_realtime(&mut transport)
3258 .expect("first connect");
3259 client
3260 .connect_realtime(&mut transport)
3261 .expect("idempotent connect");
3262 assert_eq!(transport.connects, 1);
3263 client.disconnect_realtime(&mut transport);
3264 client.disconnect_realtime(&mut transport);
3265 assert_eq!(transport.closes, 1);
3266 client
3267 .connect_realtime(&mut transport)
3268 .expect("deliberate reconnect");
3269 assert_eq!(transport.connects, 2);
3270 }
3271
3272 #[test]
3273 fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
3274 let mut client = client();
3275 client
3276 .set_realtime_policy(RealtimePolicy::Required)
3277 .expect("set policy");
3278 let mut transport = CountingRealtimeTransport::default();
3279 match client.sync(&mut transport) {
3280 SyncOutcome::RealtimeUnavailable {
3281 state,
3282 reason_code,
3283 retry_delay_ms,
3284 } => {
3285 assert_eq!(state, RealtimeState::Disconnected);
3286 assert_eq!(reason_code, None);
3287 assert_eq!(retry_delay_ms, 250);
3288 }
3289 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3290 }
3291 assert_eq!(transport.sync_calls, 0);
3292 assert!(matches!(
3293 client.drain_sync_intents().as_slice(),
3294 [SyncIntent::Background { delay_ms: 250 }]
3295 ));
3296 let snapshot = client
3297 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3298 .expect("diagnostics");
3299 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
3300 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
3301 assert_eq!(snapshot.host.realtime_reason_code, None);
3302 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3303 assert_eq!(
3304 snapshot
3305 .last_round
3306 .as_ref()
3307 .and_then(|round| round.error_code.clone()),
3308 Some(REALTIME_UNAVAILABLE_CODE.to_owned())
3309 );
3310 }
3311
3312 #[test]
3313 fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
3314 let mut client = client();
3315 client
3316 .set_realtime_policy(RealtimePolicy::Required)
3317 .expect("set policy");
3318 let mut transport = CountingRealtimeTransport {
3319 refuse_connect: true,
3320 ..Default::default()
3321 };
3322 assert!(client.connect_realtime(&mut transport).is_err());
3323 assert_eq!(client.realtime_state(), RealtimeState::Refused);
3324 let snapshot = client
3325 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3326 .expect("diagnostics");
3327 assert_eq!(
3328 snapshot.host.realtime_reason_code.as_deref(),
3329 Some("transport.failed")
3330 );
3331 match client.sync(&mut transport) {
3332 SyncOutcome::RealtimeUnavailable {
3333 state, reason_code, ..
3334 } => {
3335 assert_eq!(state, RealtimeState::Refused);
3336 assert_eq!(reason_code.as_deref(), Some("transport.failed"));
3337 }
3338 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3339 }
3340 assert_eq!(transport.sync_calls, 0);
3341 assert_eq!(transport.realtime_sync_calls, 0);
3342 }
3343
3344 #[test]
3345 fn an_unsolicited_socket_end_is_lost_with_the_transport_code_and_a_retry_intent() {
3346 let mut client = client();
3347 let mut transport = CountingRealtimeTransport::default();
3348 let ended = TransportError::new("transport.failed", "realtime connection closed");
3349 client.realtime_lost(&mut transport, &ended);
3350 assert_eq!(
3351 client.realtime_state(),
3352 RealtimeState::Disconnected,
3353 "a socket that was never connected has nothing to lose"
3354 );
3355 assert!(client.drain_sync_intents().is_empty());
3356
3357 client.connect_realtime(&mut transport).expect("connect");
3358 client.realtime_lost(&mut transport, &ended);
3359 assert_eq!(client.realtime_state(), RealtimeState::Lost);
3360 assert_eq!(transport.closes, 1);
3361 assert!(matches!(
3362 client.drain_sync_intents().as_slice(),
3363 [SyncIntent::Background { delay_ms: 250 }]
3364 ));
3365 let snapshot = client
3366 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3367 .expect("diagnostics");
3368 assert_eq!(snapshot.host.realtime, RealtimeState::Lost);
3369 assert_eq!(
3370 snapshot.host.realtime_reason_code.as_deref(),
3371 Some("transport.failed")
3372 );
3373 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3374
3375 client.realtime_lost(&mut transport, &ended);
3376 assert!(
3377 client.drain_sync_intents().is_empty(),
3378 "a second report for the same loss schedules nothing"
3379 );
3380 client.connect_realtime(&mut transport).expect("reconnect");
3381 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3382 }
3383
3384 #[test]
3385 fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
3386 let mut client = client();
3387 client
3388 .set_realtime_policy(RealtimePolicy::Required)
3389 .expect("set policy");
3390 let mut transport = CountingRealtimeTransport::default();
3391 client.connect_realtime(&mut transport).expect("connect");
3392 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3393 assert!(
3394 client.set_realtime_policy(RealtimePolicy::Off).is_err(),
3395 "a policy change cannot silently re-route a connected socket"
3396 );
3397 assert!(matches!(
3398 client.sync(&mut transport),
3399 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
3400 ));
3401 assert_eq!(client.realtime_state(), RealtimeState::Lost);
3402 assert_eq!(transport.realtime_sync_calls, 1);
3403 assert_eq!(transport.sync_calls, 0);
3404 let snapshot = client
3405 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3406 .expect("diagnostics");
3407 assert_eq!(
3408 snapshot.host.realtime_reason_code.as_deref(),
3409 Some("sync.transport_failed")
3410 );
3411 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3412 match client.sync(&mut transport) {
3413 SyncOutcome::RealtimeUnavailable {
3414 state, reason_code, ..
3415 } => {
3416 assert_eq!(state, RealtimeState::Lost);
3417 assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
3418 }
3419 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3420 }
3421 assert_eq!(transport.sync_calls, 0);
3422 client.disconnect_realtime(&mut transport);
3423 assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
3424 assert_eq!(transport.closes, 1);
3425 client.connect_realtime(&mut transport).expect("reconnect");
3426 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3427 assert_eq!(transport.connects, 2);
3428 }
3429
3430 #[test]
3431 fn off_policy_uses_http_and_refuses_an_explicit_connect() {
3432 let mut client = client();
3433 client
3434 .set_realtime_policy(RealtimePolicy::Off)
3435 .expect("set policy");
3436 let mut transport = CountingRealtimeTransport::default();
3437 assert!(client.connect_realtime(&mut transport).is_err());
3438 assert_eq!(client.realtime_state(), RealtimeState::Disabled);
3439 assert!(matches!(
3440 client.sync(&mut transport),
3441 SyncOutcome::Failed { .. }
3442 ));
3443 assert_eq!(transport.sync_calls, 1);
3444 assert_eq!(transport.realtime_sync_calls, 0);
3445 let snapshot = client
3446 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3447 .expect("diagnostics");
3448 assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
3449 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
3450 }
3451
3452 #[test]
3453 fn optional_policy_keeps_the_http_round() {
3454 let mut client = client();
3455 let mut transport = CountingRealtimeTransport::default();
3456 assert!(matches!(
3457 client.sync(&mut transport),
3458 SyncOutcome::Failed { .. }
3459 ));
3460 assert_eq!(transport.sync_calls, 1);
3461 assert_eq!(transport.realtime_sync_calls, 0);
3462 let snapshot = client
3463 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3464 .expect("diagnostics");
3465 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
3466 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
3467 }
3468
3469 #[test]
3470 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
3471 let mut client = client();
3472 client
3473 .subscribe(
3474 "repair-tasks".to_owned(),
3475 "tasks".to_owned(),
3476 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3477 None,
3478 )
3479 .expect("subscribe");
3480 {
3481 let sub = client
3482 .subs
3483 .iter_mut()
3484 .find(|sub| sub.id == "repair-tasks")
3485 .expect("subscription");
3486 sub.cursor = 42;
3487 sub.synced_once = true;
3488 let persisted = sub.clone();
3489 client.persist_sub(&persisted).unwrap();
3490 }
3491 for table in ["_syncular_base_tasks", "tasks"] {
3492 client
3493 .conn
3494 .execute(
3495 &format!(
3496 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
3497 ),
3498 rusqlite::params!["server-row", "p1"],
3499 )
3500 .expect("seed server row");
3501 }
3502 let pending = client
3503 .mutate(vec![Mutation::Upsert {
3504 table: "tasks".to_owned(),
3505 values: Map::from_iter([
3506 ("id".to_owned(), Value::from("offline-row")),
3507 ("project_id".to_owned(), Value::from("p1")),
3508 ]),
3509 base_version: None,
3510 }])
3511 .expect("queue offline work");
3512 client.drain_change_batches();
3513 client.drain_sync_intents();
3514
3515 assert_eq!(
3516 client
3517 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3518 rebootstrap_id: "support-case-001".to_owned(),
3519 })
3520 .expect("rebootstrap"),
3521 LocalDataRebootstrapResult {
3522 already_applied: false,
3523 retained_commits: 1,
3524 reset_subscriptions: 1,
3525 }
3526 );
3527 let visible_ids = client
3528 .conn
3529 .prepare("SELECT id FROM tasks ORDER BY id")
3530 .expect("prepare visible ids")
3531 .query_map([], |row| row.get::<_, String>(0))
3532 .expect("query visible ids")
3533 .collect::<Result<Vec<_>, _>>()
3534 .expect("collect visible ids");
3535 assert_eq!(visible_ids, vec!["offline-row"]);
3536 assert_eq!(client.pending_commit_ids(), vec![pending]);
3537 assert_eq!(
3538 client
3539 .subscription_state("repair-tasks")
3540 .expect("subscription")
3541 .cursor,
3542 -1
3543 );
3544 assert!(client.upgrading());
3545 assert!(client.sync_needed());
3546 assert!(matches!(
3547 client.drain_sync_intents().as_slice(),
3548 [SyncIntent::Interactive]
3549 ));
3550 assert_eq!(client.drain_change_batches().len(), 1);
3551
3552 assert_eq!(
3553 client
3554 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3555 rebootstrap_id: "support-case-001".to_owned(),
3556 })
3557 .expect("idempotent retry"),
3558 LocalDataRebootstrapResult {
3559 already_applied: true,
3560 retained_commits: 1,
3561 reset_subscriptions: 1,
3562 }
3563 );
3564 assert!(client.drain_change_batches().is_empty());
3565 }
3566
3567 #[test]
3568 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
3569 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
3570 assert_eq!(
3571 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
3572 (3, 4)
3573 );
3574 assert_eq!(
3575 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
3576 (0, 0)
3577 );
3578 for malformed in [
3579 "",
3580 "{}",
3581 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
3582 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
3583 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
3584 ] {
3585 assert_eq!(
3586 decode_local_rebootstrap_receipt(malformed)
3587 .expect_err("malformed receipt must fail"),
3588 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3589 );
3590 }
3591 }
3592
3593 #[test]
3594 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
3595 let path = std::env::temp_dir().join(format!(
3596 "syncular-rebootstrap-receipt-{}.db",
3597 uuid::Uuid::new_v4()
3598 ));
3599 let schema = json!({
3600 "version": 1,
3601 "tables": [{
3602 "name": "tasks",
3603 "primaryKey": "id",
3604 "columns": [
3605 { "name": "id", "type": "string", "nullable": false },
3606 { "name": "project_id", "type": "string", "nullable": false }
3607 ],
3608 "scopes": [{ "pattern": "project:{project_id}" }]
3609 }]
3610 });
3611 let path_string = path.to_str().expect("UTF-8 temp path");
3612
3613 {
3614 let mut first = SyncClient::open_path(
3615 "repair-restart-client".to_owned(),
3616 &schema,
3617 ClientLimits::default(),
3618 path_string,
3619 )
3620 .expect("first open");
3621 first
3622 .subscribe(
3623 "repair-tasks".to_owned(),
3624 "tasks".to_owned(),
3625 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3626 None,
3627 )
3628 .expect("subscribe");
3629 first
3630 .mutate(vec![Mutation::Upsert {
3631 table: "tasks".to_owned(),
3632 values: Map::from_iter([
3633 ("id".to_owned(), Value::from("offline-row")),
3634 ("project_id".to_owned(), Value::from("p1")),
3635 ]),
3636 base_version: None,
3637 }])
3638 .expect("queue offline work");
3639 assert_eq!(
3640 first
3641 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3642 rebootstrap_id: "restart-receipt".to_owned(),
3643 })
3644 .expect("first rebootstrap"),
3645 LocalDataRebootstrapResult {
3646 already_applied: false,
3647 retained_commits: 1,
3648 reset_subscriptions: 1,
3649 }
3650 );
3651 }
3652
3653 let mut reopened = SyncClient::open_path(
3654 "repair-restart-client".to_owned(),
3655 &schema,
3656 ClientLimits::default(),
3657 path_string,
3658 )
3659 .expect("reopen");
3660 reopened.drain_change_batches();
3661 reopened.drain_sync_intents();
3662 assert_eq!(
3663 reopened
3664 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3665 rebootstrap_id: "restart-receipt".to_owned(),
3666 })
3667 .expect("receipt replay"),
3668 LocalDataRebootstrapResult {
3669 already_applied: true,
3670 retained_commits: 1,
3671 reset_subscriptions: 1,
3672 }
3673 );
3674 assert!(reopened.drain_change_batches().is_empty());
3675 assert!(reopened.drain_sync_intents().is_empty());
3676 drop(reopened);
3677 std::fs::remove_file(path).expect("remove temp database");
3678 }
3679
3680 #[test]
3681 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
3682 let mut malformed = client();
3683 malformed
3684 .subscribe(
3685 "repair-tasks".to_owned(),
3686 "tasks".to_owned(),
3687 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3688 None,
3689 )
3690 .expect("subscribe");
3691 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
3692 malformed.drain_change_batches();
3693 malformed.drain_sync_intents();
3694 let malformed_error = malformed
3695 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3696 rebootstrap_id: "malformed".to_owned(),
3697 })
3698 .expect_err("malformed receipt must fail closed");
3699 assert_eq!(
3700 malformed_error,
3701 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3702 );
3703 assert!(!malformed.upgrading());
3704 assert_eq!(
3705 malformed
3706 .subscription_state("repair-tasks")
3707 .expect("unchanged subscription")
3708 .cursor,
3709 -1
3710 );
3711 assert!(malformed.drain_change_batches().is_empty());
3712 assert!(malformed.drain_sync_intents().is_empty());
3713
3714 let mut unreadable = client();
3715 unreadable
3716 .conn
3717 .execute(
3718 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
3719 rusqlite::params!["server-row", "p1"],
3720 )
3721 .expect("seed visible row");
3722 unreadable
3723 .conn
3724 .execute("DROP TABLE _syncular_meta", [])
3725 .expect("break marker storage");
3726 let unreadable_error = unreadable
3727 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3728 rebootstrap_id: "unreadable".to_owned(),
3729 })
3730 .expect_err("unreadable marker storage must fail closed");
3731 assert_eq!(
3732 unreadable_error,
3733 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
3734 );
3735 let visible_rows = unreadable
3736 .conn
3737 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
3738 .expect("visible projection remains");
3739 assert_eq!(visible_rows, 1);
3740 assert!(!unreadable.upgrading());
3741 assert!(unreadable.drain_change_batches().is_empty());
3742 assert!(unreadable.drain_sync_intents().is_empty());
3743 }
3744
3745 #[test]
3746 fn local_rebootstrap_cannot_bypass_schema_floor() {
3747 let mut client = client();
3748 client.set_schema_floor(Some(SchemaFloor {
3749 required_schema_version: Some(2),
3750 latest_schema_version: Some(2),
3751 }));
3752 let error = client
3753 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3754 rebootstrap_id: "blocked-floor".to_owned(),
3755 })
3756 .expect_err("schema floor must block repair");
3757 assert!(error.contains("cannot bypass an active schema-floor stop"));
3758 }
3759
3760 #[test]
3761 fn diego_applied_sparse_moves_stay_visible_until_pull_image_arrives() {
3762 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3763 "columns":[{"name":"id","type":"string","nullable":false},
3764 {"name":"project_id","type":"string","nullable":false},
3765 {"name":"starts_at_ms","type":"integer","nullable":false},
3766 {"name":"version","type":"integer","nullable":false}],
3767 "scopes":[{"pattern":"project:{project_id}"}]}]});
3768 let mut client =
3769 SyncClient::new("diego-ack-gap".into(), &schema, ClientLimits::default()).unwrap();
3770 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3771 client
3772 .write_base_row(
3773 "tasks",
3774 &vec![
3775 Some(ColumnValue::String("t1".into())),
3776 Some(ColumnValue::String("p1".into())),
3777 Some(ColumnValue::Integer(100)),
3778 Some(ColumnValue::Integer(1)),
3779 ],
3780 1,
3781 )
3782 .unwrap();
3783 client.rebuild_overlay().expect("rebuild overlay");
3784 let first = client
3785 .patch(
3786 "tasks",
3787 "t1",
3788 Map::from_iter([
3789 ("starts_at_ms".into(), json!(200)),
3790 ("version".into(), json!(2)),
3791 ]),
3792 None,
3793 )
3794 .unwrap();
3795 let (_, first_request) = client.build_request(false).unwrap();
3796 let second = client
3798 .patch(
3799 "tasks",
3800 "t1",
3801 Map::from_iter([
3802 ("starts_at_ms".into(), json!(300)),
3803 ("version".into(), json!(3)),
3804 ]),
3805 None,
3806 )
3807 .unwrap();
3808 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3809 for (id, seq, meta) in [
3810 (first, 2, first_request),
3811 (second, 3, client.build_request(false).unwrap().1),
3812 ] {
3813 let response = Message {
3814 wire_version: WIRE_VERSION,
3815 msg_kind: MsgKind::Response,
3816 frames: vec![
3817 Frame::RespHeader {
3818 required_schema_version: None,
3819 latest_schema_version: None,
3820 log_epoch: Some("epoch-1".into()),
3821 reset_required: Some(false),
3822 },
3823 Frame::PushResult {
3824 client_commit_id: id,
3825 status: PushStatus::Applied,
3826 commit_seq: Some(seq),
3827 results: vec![OpResult::Applied { op_index: 0 }],
3828 },
3829 ],
3830 };
3831 assert!(matches!(
3832 client.process_response(&mut transport, response, &meta),
3833 SyncOutcome::Ok(_)
3834 ));
3835 let row = &client
3836 .query("SELECT starts_at_ms, version FROM tasks WHERE id='t1'", &[])
3837 .unwrap()[0];
3838 assert_eq!(
3839 row["starts_at_ms"],
3840 json!(300),
3841 "an applied ACK without a pull image must not expose the old base"
3842 );
3843 assert_eq!(
3844 row["version"],
3845 json!(3),
3846 "the queued domain version must remain monotonic"
3847 );
3848 }
3849 }
3850
3851 #[test]
3852 fn first_epoch_acquisition_preserves_tables_subscriptions_and_local_intent() {
3853 for stored in [None, Some("epoch-old")] {
3854 let mut client = client();
3855 client
3856 .subscribe(
3857 "tasks".into(),
3858 "tasks".into(),
3859 vec![("project_id".into(), vec!["p1".into()])],
3860 None,
3861 )
3862 .unwrap();
3863 client
3864 .conn
3865 .execute_batch(
3866 "CREATE TRIGGER epoch_table_identity AFTER INSERT ON tasks BEGIN SELECT 1; END",
3867 )
3868 .unwrap();
3869 let id = client
3870 .mutate(vec![Mutation::Upsert {
3871 table: "tasks".into(),
3872 values: Map::from_iter([
3873 ("id".into(), json!("offline")),
3874 ("project_id".into(), json!("p1")),
3875 ]),
3876 base_version: None,
3877 }])
3878 .unwrap();
3879 if let Some(epoch) = stored {
3880 client.set_meta(LOG_EPOCH_KEY, epoch);
3881 }
3882 let before = serde_json::to_value(client.subscription_state("tasks")).unwrap();
3883 let (_, meta) = client.build_request(false).unwrap();
3884 client.drain_change_batches();
3885 client.drain_sync_intents();
3886 let response = Message {
3887 wire_version: WIRE_VERSION,
3888 msg_kind: MsgKind::Response,
3889 frames: vec![Frame::RespHeader {
3890 required_schema_version: None,
3891 latest_schema_version: None,
3892 log_epoch: Some("epoch-new".into()),
3893 reset_required: Some(true),
3894 }],
3895 };
3896 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3897 let outcome = client.process_response(&mut transport, response, &meta);
3898 let SyncOutcome::Ok(report) = outcome else {
3899 panic!("epoch round failed: {outcome:?}");
3900 };
3901 assert_eq!(
3902 report.resets,
3903 if stored.is_none() {
3904 Vec::<String>::new()
3905 } else {
3906 vec!["tasks".into()]
3907 }
3908 );
3909 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3910 assert_eq!(client.pending_commit_ids(), vec![id]);
3911 assert_eq!(
3912 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3913 json!("offline")
3914 );
3915 let trigger: i64 = client
3916 .conn
3917 .query_row(
3918 "SELECT count(*) FROM sqlite_master WHERE name='epoch_table_identity'",
3919 [],
3920 |row| row.get(0),
3921 )
3922 .unwrap();
3923 if stored.is_none() {
3924 assert!(!client.upgrading);
3925 assert!(client
3926 .drain_change_batches()
3927 .iter()
3928 .all(|batch| batch.status.as_ref().is_none_or(|status| !status.upgrading)));
3929 assert_eq!(
3930 serde_json::to_value(client.subscription_state("tasks")).unwrap(),
3931 before
3932 );
3933 assert_eq!(trigger, 1, "acquisition must not drop the synced table");
3934 } else {
3935 assert!(client.upgrading);
3936 assert_eq!(trigger, 0, "an epoch change must drop the old table");
3937 }
3938 assert!(matches!(
3939 client.drain_sync_intents().as_slice(),
3940 [SyncIntent::Interactive]
3941 ));
3942 }
3943 }
3944
3945 #[test]
3946 fn epoch_acquisition_storage_failures_preserve_ready_state_and_intent() {
3947 for key in ["logEpoch", "localRevision"] {
3948 let mut client = client();
3949 let id = client
3950 .mutate(vec![Mutation::Upsert {
3951 table: "tasks".into(),
3952 values: Map::from_iter([
3953 ("id".into(), json!("offline")),
3954 ("project_id".into(), json!("p1")),
3955 ]),
3956 base_version: None,
3957 }])
3958 .unwrap();
3959 let revision = client.local_revision();
3960 let needed = client.sync_needed;
3961 client.drain_change_batches();
3962 client.drain_sync_intents();
3963 client.conn.execute_batch(&format!("CREATE TRIGGER fail_acquisition BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected acquisition failure'); END")).unwrap();
3964 assert!(client.run_log_epoch_reset("epoch-new").is_err());
3965 assert!(client.get_meta(LOG_EPOCH_KEY).is_none());
3966 assert!(!client.upgrading);
3967 assert_eq!(client.sync_needed, needed);
3968 assert_eq!(client.local_revision(), revision);
3969 assert_eq!(client.pending_commit_ids(), [id]);
3970 assert!(client.drain_change_batches().is_empty());
3971 assert!(client.drain_sync_intents().is_empty());
3972 }
3973 }
3974
3975 #[test]
3976 fn failed_log_epoch_resets_restore_memory_and_durable_state() {
3977 for key in [
3978 "localSchemaVersion",
3979 "localSchemaDescriptor",
3980 "logEpoch",
3981 "localRevision",
3982 ] {
3983 let mut client = client();
3984 client.set_meta(LOG_EPOCH_KEY, "epoch-old");
3985 let commit = client
3986 .mutate(vec![Mutation::Upsert {
3987 table: "tasks".into(),
3988 values: Map::from_iter([
3989 ("id".into(), json!("offline")),
3990 ("project_id".into(), json!("p1")),
3991 ]),
3992 base_version: None,
3993 }])
3994 .unwrap();
3995 client.active_round = Some(uuid::Uuid::new_v4());
3996 client.overlay_dirty.set(false);
3997 let round = client.active_round;
3998 let needed = client.sync_needed;
3999 let revision = client.local_revision();
4000 let rows = serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap();
4001 client.drain_change_batches();
4002 client.drain_sync_intents();
4003 client.conn.execute_batch(&format!("CREATE TRIGGER fail_reset BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected reset failure'); END")).unwrap();
4004 assert!(client.run_log_epoch_reset("epoch-new").is_err(), "{key}");
4005 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-old"));
4006 assert!(!client.upgrading);
4007 assert_eq!(client.active_round, round);
4008 assert!(!client.overlay_dirty.get());
4009 assert_eq!(client.sync_needed, needed);
4010 assert_eq!(client.local_revision(), revision);
4011 assert_eq!(
4012 client.pending_commit_ids().as_slice(),
4013 std::slice::from_ref(&commit)
4014 );
4015 assert_eq!(
4016 serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap(),
4017 rows
4018 );
4019 assert!(client.drain_change_batches().is_empty());
4020 assert!(client.drain_sync_intents().is_empty());
4021 assert!(client.conn.is_autocommit());
4022 client
4023 .conn
4024 .execute_batch("DROP TRIGGER fail_reset")
4025 .unwrap();
4026 client.run_log_epoch_reset("epoch-new").unwrap();
4027 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
4028 assert_eq!(client.pending_commit_ids(), [commit]);
4029 }
4030 }
4031
4032 #[test]
4033 fn acknowledged_restore_and_replay_ignore_unrelated_subscription_count() {
4034 for subscriptions in [0, 2, 6, 12] {
4035 let mut client = SyncClient::new("ack-scope".into(), &json!({ "version": 1, "tables": [
4036 { "name": "tasks", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] },
4037 { "name": "docs", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] }
4038 ] }), ClientLimits::default()).unwrap();
4039 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4040 client.conn.execute_batch("CREATE TABLE ack_writes(kind TEXT); CREATE TRIGGER count_ack_restore AFTER DELETE ON tasks BEGIN INSERT INTO ack_writes VALUES('restore'); END; CREATE TRIGGER count_ack_replay AFTER INSERT ON tasks BEGIN INSERT INTO ack_writes VALUES('replay'); END;").unwrap();
4041 for index in 0..32 {
4042 client
4043 .mutate(vec![Mutation::Upsert {
4044 table: "tasks".into(),
4045 values: Map::from_iter([
4046 ("id".into(), json!(format!("task-{index}"))),
4047 ("project_id".into(), json!("p1")),
4048 ]),
4049 base_version: None,
4050 }])
4051 .unwrap();
4052 }
4053 let (_, meta) = client.build_request(false).unwrap();
4054 let mut frames = vec![Frame::RespHeader {
4055 required_schema_version: None,
4056 latest_schema_version: None,
4057 log_epoch: Some("epoch-1".into()),
4058 reset_required: Some(false),
4059 }];
4060 frames.extend(meta.pushed_ids.iter().enumerate().map(|(index, id)| {
4061 Frame::PushResult {
4062 client_commit_id: id.clone(),
4063 status: PushStatus::Applied,
4064 commit_seq: Some(index as i64 + 1),
4065 results: vec![OpResult::Applied { op_index: 0 }],
4066 }
4067 }));
4068 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
4069 client.acknowledged_replay_count.set(0);
4070 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
4071 assert!(matches!(
4072 client.process_response(
4073 &mut transport,
4074 Message {
4075 wire_version: WIRE_VERSION,
4076 msg_kind: MsgKind::Response,
4077 frames
4078 },
4079 &meta
4080 ),
4081 SyncOutcome::Ok(_)
4082 ));
4083 assert_eq!(client.acknowledged_replay_count.get(), 32);
4084 assert_eq!(
4085 client
4086 .query(
4087 "SELECT count(*) AS n FROM ack_writes WHERE kind='restore'",
4088 &[]
4089 )
4090 .unwrap()[0]["n"],
4091 json!(32)
4092 );
4093 assert_eq!(
4094 client
4095 .query(
4096 "SELECT count(*) AS n FROM ack_writes WHERE kind='replay'",
4097 &[]
4098 )
4099 .unwrap()[0]["n"],
4100 json!(32)
4101 );
4102 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
4103 client.acknowledged_replay_count.set(0);
4104 for index in 0..subscriptions {
4105 client
4106 .subscribe(
4107 format!("docs-{index}"),
4108 "docs".into(),
4109 vec![("project_id".into(), vec![format!("scope-{index}")])],
4110 None,
4111 )
4112 .unwrap();
4113 let sub = client.subs.len() - 1;
4114 client.subs[sub].effective =
4115 Some(vec![("project_id".into(), vec![format!("scope-{index}")])]);
4116 let segment = RowsSegment {
4117 table: "docs".into(),
4118 schema_version: 1,
4119 columns: client.schema.table("docs").unwrap().wire_columns.clone(),
4120 blocks: if index % 2 == 0 {
4121 vec![vec![ssp2::segment::SegmentRow {
4122 values: vec![
4123 Some(ColumnValue::String(format!("doc-{index}"))),
4124 Some(ColumnValue::String(format!("scope-{index}"))),
4125 ],
4126 server_version: 1,
4127 }]]
4128 } else {
4129 vec![]
4130 },
4131 };
4132 assert!(client.apply_segment(sub, &segment, true).is_ok());
4133 }
4134 assert_eq!(
4135 client
4136 .query("SELECT count(*) AS n FROM ack_writes", &[])
4137 .unwrap()[0]["n"],
4138 json!(0),
4139 "unrelated bootstrap must not restore/replay tasks"
4140 );
4141 assert_eq!(
4142 client.acknowledged_replay_count.get(),
4143 0,
4144 "unrelated bootstrap must not decode task ACKs"
4145 );
4146 assert_eq!(
4147 client
4148 .query("SELECT count(*) AS n FROM tasks", &[])
4149 .unwrap()[0]["n"],
4150 json!(32)
4151 );
4152 assert_eq!(
4153 client
4154 .query("SELECT count(*) AS n FROM _syncular_acked_rows", &[])
4155 .unwrap()[0]["n"],
4156 json!(32)
4157 );
4158 }
4159 }
4160
4161 #[test]
4162 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
4163 let mut client = client();
4164 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4165 const COMMIT_COUNT: usize = 32;
4166
4167 for index in 0..COMMIT_COUNT {
4168 client
4169 .mutate(vec![Mutation::Upsert {
4170 table: "tasks".to_owned(),
4171 values: Map::from_iter([
4172 ("id".to_owned(), Value::from(format!("task-{index}"))),
4173 ("project_id".to_owned(), Value::from("project-1")),
4174 ]),
4175 base_version: None,
4176 }])
4177 .expect("queue commit");
4178 }
4179
4180 let (_, request_meta) = client.build_request(false).unwrap();
4181 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
4182 let mut frames = vec![Frame::RespHeader {
4183 required_schema_version: None,
4184 latest_schema_version: None,
4185 log_epoch: Some("epoch-1".to_owned()),
4186 reset_required: Some(false),
4187 }];
4188 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
4189 |(index, client_commit_id)| Frame::PushResult {
4190 client_commit_id: client_commit_id.clone(),
4191 status: PushStatus::Applied,
4192 commit_seq: Some(index as i64 + 1),
4193 results: vec![OpResult::Applied { op_index: 0 }],
4194 },
4195 ));
4196 let response = Message {
4197 wire_version: WIRE_VERSION,
4198 msg_kind: MsgKind::Response,
4199 frames,
4200 };
4201 let mut transport =
4202 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
4203
4204 client.overlay_rebuild_count.set(0);
4205 client.outcome_prune_count.set(0);
4206 let outcome = client.process_response(&mut transport, response, &request_meta);
4207 assert!(matches!(outcome, SyncOutcome::Ok(_)));
4208 assert!(client.pending_commit_ids().is_empty());
4209 assert_eq!(
4210 client.overlay_rebuild_count.get(),
4211 1,
4212 "one response must reconcile its acknowledged commits with one overlay rebuild"
4213 );
4214 assert_eq!(
4215 client.outcome_prune_count.get(),
4216 1,
4217 "one response must enforce outcome retention once"
4218 );
4219 }
4220
4221 #[test]
4222 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
4223 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
4224 "columns": [
4225 {"name":"id", "type":"string", "nullable":false},
4226 {"name":"project_id", "type":"string", "nullable":false},
4227 {"name":"title", "type":"string", "nullable":false}
4228 ], "scopes": [{"pattern":"project:{project_id}"}],
4229 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
4230 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
4231 }] });
4232 let mut incremental =
4233 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
4234 let mut reference =
4235 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
4236 let tables = vec!["tasks".to_owned()];
4237 for index in 0..32 {
4238 let id = format!("task-{}", index % 8);
4239 let values = Map::from_iter([
4240 ("id".to_owned(), Value::from(id.clone())),
4241 (
4242 "project_id".to_owned(),
4243 Value::from(format!("p{}", index % 2)),
4244 ),
4245 ("title".to_owned(), Value::from(format!("needle{index}"))),
4246 ]);
4247 let payload = encode_row_json(
4248 incremental.schema.table("tasks").unwrap(),
4249 &id,
4250 &values,
4251 &incremental.encryption,
4252 )
4253 .unwrap();
4254 let change = ssp2::model::Change {
4255 table_index: 0,
4256 row_id: id,
4257 op: if index % 5 == 0 {
4258 Op::Delete
4259 } else {
4260 Op::Upsert
4261 },
4262 row_version: Some(index + 1),
4263 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
4264 row: Some(payload),
4265 };
4266 reference.overlay_dirty.set(true);
4267 for instance in [&mut incremental, &mut reference] {
4268 instance.begin_observation("mirror_test").unwrap();
4269 let mut batch = ChangeAccumulator::default();
4270 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
4271 instance
4272 .apply_commit_changes(&tables, std::slice::from_ref(&change))
4273 .unwrap();
4274 instance
4275 .rebuild_overlay_if_dirty()
4276 .expect("rebuild overlay");
4277 instance.finish_observation("mirror_test", batch).unwrap();
4278 }
4279 for sql in [
4280 "SELECT * FROM tasks ORDER BY id",
4281 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
4282 ] {
4283 assert_eq!(
4284 incremental.query(sql, &[]).unwrap(),
4285 reference.query(sql, &[]).unwrap()
4286 );
4287 }
4288 assert_eq!(incremental.local_revision(), reference.local_revision());
4289 assert_eq!(
4290 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4291 serde_json::to_value(reference.drain_change_batches()).unwrap()
4292 );
4293 }
4294 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
4295 assert_eq!(reference.overlay_rebuild_count.get(), 32);
4296 }
4297
4298 #[test]
4299 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
4300 let cases = [
4301 (
4302 "string",
4303 vec![
4304 json!("1"),
4305 json!("01"),
4306 json!(""),
4307 json!("nul\u{0}id"),
4308 json!("é"),
4309 ],
4310 ),
4311 (
4312 "integer",
4313 vec![
4314 json!(i64::MIN),
4315 json!(-1),
4316 json!(0),
4317 json!(1),
4318 json!(i64::MAX),
4319 ],
4320 ),
4321 ("boolean", vec![json!(false), json!(true)]),
4322 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
4323 ];
4324 for (kind, ids) in cases {
4325 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4326 "columns":[{"name":"id","type":kind,"nullable":false},
4327 {"name":"project_id","type":"string","nullable":false}],
4328 "scopes":[{"pattern":"project:{project_id}"}]}]});
4329 let mut client =
4330 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
4331 for (index, id) in ids.iter().enumerate() {
4332 let table = client.schema.table("tasks").unwrap();
4333 client
4334 .write_base_row(
4335 "tasks",
4336 &vec![
4337 if kind == "integer" {
4338 Some(ColumnValue::Integer(id.as_i64().unwrap()))
4341 } else {
4342 json_to_column_value(&table.columns[0], Some(id)).unwrap()
4343 },
4344 Some(ColumnValue::String(format!("p{index}"))),
4345 ],
4346 1,
4347 )
4348 .unwrap();
4349 }
4350 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4351 for base in [true, false] {
4352 let full = if base {
4353 base_table("tasks")
4354 } else {
4355 visible_table("tasks")
4356 };
4357 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
4358 for row_id in [
4359 "1",
4360 "01",
4361 "1.0",
4362 "1.0000000000000002",
4363 "+1",
4364 "1e0",
4365 " 1",
4366 "-1",
4367 "0",
4368 "9223372036854775807",
4369 "-9223372036854775808",
4370 "true",
4371 "false",
4372 "",
4373 "nul\u{0}id",
4374 "é",
4375 "null",
4376 "{\"a\":1}",
4377 ] {
4378 let old_sql =
4379 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
4380 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
4381 let old = client
4382 .conn
4383 .prepare(&old_sql)
4384 .unwrap()
4385 .query_map([row_id], |row| row.get::<_, String>(0))
4386 .unwrap()
4387 .collect::<Result<Vec<_>, _>>()
4388 .unwrap();
4389 let new = client
4390 .conn
4391 .prepare_cached(&new_sql)
4392 .unwrap()
4393 .query_map([row_id], |row| row.get::<_, String>(0))
4394 .unwrap()
4395 .collect::<Result<Vec<_>, _>>()
4396 .unwrap();
4397 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
4398 let mut batch = ChangeAccumulator::default();
4399 assert_eq!(
4400 client.record_row_scopes(&mut batch, "tasks", row_id, base),
4401 !old.is_empty()
4402 );
4403 let expected = old.first().map(|scope| {
4404 BTreeMap::from([(
4405 "tasks".to_owned(),
4406 Some(BTreeSet::from([format!("project:{scope}")])),
4407 )])
4408 });
4409 assert_eq!(batch.tables, expected.unwrap_or_default());
4410 client
4411 .conn
4412 .execute_batch("SAVEPOINT lookup_delete")
4413 .unwrap();
4414 if base {
4415 client.delete_base_row("tasks", row_id).unwrap();
4416 } else {
4417 client
4418 .apply_outbox_ops(&[OutboxOp {
4419 upsert: false,
4420 insert: false,
4421 table: "tasks".into(),
4422 row_id: row_id.into(),
4423 base_version: None,
4424 values: None,
4425 }])
4426 .expect("replay delete");
4427 }
4428 let remaining: i64 = client
4429 .conn
4430 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
4431 row.get(0)
4432 })
4433 .unwrap();
4434 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
4435 client
4436 .conn
4437 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
4438 .unwrap();
4439 client.overlay_dirty.set(false);
4440 }
4441 for (predicate, expected) in [
4442 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
4443 (predicate, "SEARCH"),
4444 ] {
4445 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
4446 let details = client
4447 .conn
4448 .prepare(&sql)
4449 .unwrap()
4450 .query_map(["1"], |row| row.get::<_, String>(3))
4451 .unwrap()
4452 .collect::<Result<Vec<_>, _>>()
4453 .unwrap();
4454 assert!(
4455 details.iter().any(|detail| detail.contains(expected)),
4456 "{kind}: {details:?}"
4457 );
4458 }
4459 }
4460 }
4461 }
4462
4463 #[test]
4464 fn primary_keys_reject_every_ineligible_column_type() {
4465 for ty in ["float", "bytes", "crdt", "blob_ref"] {
4469 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4470 "columns":[{"name":"id","type":ty,"nullable":false},
4471 {"name":"project_id","type":"string","nullable":false}],
4472 "scopes":[{"pattern":"project:{project_id}"}]}]});
4473 let error =
4474 match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
4475 Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
4476 Err(error) => error,
4477 };
4478 assert!(
4479 error.contains("has an ineligible column type (§2.4)"),
4480 "{ty}: {error}"
4481 );
4482 }
4483 for ty in ["string", "integer", "boolean", "json"] {
4484 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4485 "columns":[{"name":"id","type":ty,"nullable":false},
4486 {"name":"project_id","type":"string","nullable":false}],
4487 "scopes":[{"pattern":"project:{project_id}"}]}]});
4488 SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
4489 .unwrap_or_else(|error| panic!("{ty}: {error}"));
4490 }
4491 }
4492
4493 #[test]
4494 fn pending_row_reconciliation_matches_full_fifo_replay() {
4495 for unique in [false, true] {
4496 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
4497 "name": name, "primaryKey": "id", "columns": [
4498 {"name":"id", "type":"string", "nullable":false},
4499 {"name":"project_id", "type":"string", "nullable":false},
4500 {"name":"title", "type":"string", "nullable":false}],
4501 "scopes": [{"pattern":"project:{project_id}"}],
4502 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
4503 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
4504 })))});
4505 let mut incremental =
4506 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
4507 let mut reference =
4508 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
4509 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
4510 for instance in [&mut incremental, &mut reference] {
4511 for table in &tables {
4512 for index in 0..12 {
4513 instance
4514 .write_base_row(
4515 table,
4516 &vec![
4517 Some(ColumnValue::String(format!("row{index}"))),
4518 Some(ColumnValue::String("p0".into())),
4519 Some(ColumnValue::String(format!("base{index}"))),
4520 ],
4521 1,
4522 )
4523 .unwrap();
4524 }
4525 }
4526 instance
4527 .rebuild_overlay_if_dirty()
4528 .expect("rebuild overlay");
4529 }
4530 for index in 0..80 {
4531 let mutations = (0..3)
4532 .map(|offset| {
4533 let table = tables[(index + offset) % 2].clone();
4534 let id = format!("row{}", (index + offset) % 16);
4535 if (index + offset) % 7 == 0 {
4536 Mutation::Delete {
4537 table,
4538 row_id: id,
4539 base_version: None,
4540 }
4541 } else {
4542 Mutation::Upsert {
4543 table,
4544 values: Map::from_iter([
4545 ("id".into(), Value::from(id)),
4546 ("project_id".into(), Value::from(format!("p{}", index % 3))),
4547 (
4548 "title".into(),
4549 Value::from(format!("needle{}", (index + offset) % 16)),
4550 ),
4551 ]),
4552 base_version: None,
4553 }
4554 }
4555 })
4556 .collect::<Vec<_>>();
4557 for instance in [&mut incremental, &mut reference] {
4558 instance.mutate(mutations.clone()).unwrap();
4559 }
4560 let changes = (0..3)
4563 .map(|offset| {
4564 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
4565 let id = format!("row{}", (index + offset) % 16);
4566 let values = Map::from_iter([
4567 ("id".into(), Value::from(id.clone())),
4568 (
4569 "project_id".into(),
4570 Value::from(format!("p{}", (index + 1) % 3)),
4571 ),
4572 (
4573 "title".into(),
4574 Value::from(format!("remote{index}_{offset}")),
4575 ),
4576 ]);
4577 ssp2::model::Change {
4578 table_index: table_index as u16,
4579 row_id: id.clone(),
4580 op: if (index + offset) % 5 == 0 {
4581 Op::Delete
4582 } else {
4583 Op::Upsert
4584 },
4585 row_version: Some(index as i64 + 2),
4586 scopes: vec![],
4587 row: Some(
4588 encode_row_json(
4589 incremental.schema.table(&tables[table_index]).unwrap(),
4590 &id,
4591 &values,
4592 &incremental.encryption,
4593 )
4594 .unwrap(),
4595 ),
4596 }
4597 })
4598 .collect::<Vec<_>>();
4599 reference.overlay_dirty.set(true);
4600 incremental
4601 .apply_commit_frame(&tables, &changes, None)
4602 .unwrap();
4603 reference
4604 .apply_commit_frame(&tables, &changes, None)
4605 .unwrap();
4606 for table in &tables {
4607 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
4608 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
4609 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
4610 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
4611 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
4612 }
4613 }
4614 assert_eq!(incremental.local_revision(), reference.local_revision());
4615 assert_eq!(
4616 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4617 serde_json::to_value(reference.drain_change_batches()).unwrap()
4618 );
4619 }
4620 assert_eq!(
4621 incremental.overlay_rebuild_count.get(),
4622 if unique { 81 } else { 21 }
4623 );
4624 assert_eq!(reference.overlay_rebuild_count.get(), 81);
4625 }
4626 }
4627
4628 #[test]
4629 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
4630 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
4631 "columns":[{"name":"id", "type":"string", "nullable":false},
4632 {"name":"title", "type":"string", "nullable":false}],
4633 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
4634 let mut client =
4635 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
4636 for (id, title) in [("a", "original"), ("b", "occupied")] {
4637 client
4638 .write_base_row(
4639 "tasks",
4640 &vec![
4641 Some(ColumnValue::String(id.into())),
4642 Some(ColumnValue::String(title.into())),
4643 ],
4644 1,
4645 )
4646 .unwrap();
4647 }
4648 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4649 let pending = client
4650 .mutate(vec![Mutation::Upsert {
4651 table: "tasks".into(),
4652 values: Map::from_iter([
4653 ("id".into(), Value::from("a")),
4654 ("title".into(), Value::from("available")),
4655 ]),
4656 base_version: None,
4657 }])
4658 .unwrap();
4659 assert_eq!(
4660 client
4661 .conn
4662 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4663 .get::<_, String>(
4664 0
4665 ))
4666 .unwrap(),
4667 "available"
4668 );
4669 for (title, expected) in [("available", "original"), ("occupied", "available")] {
4670 let change = ssp2::model::Change {
4671 table_index: 0,
4672 row_id: "b".into(),
4673 op: Op::Upsert,
4674 row_version: Some(2),
4675 scopes: vec![],
4676 row: Some(
4677 encode_row_json(
4678 client.schema.table("tasks").unwrap(),
4679 "b",
4680 &Map::from_iter([
4681 ("id".into(), Value::from("b")),
4682 ("title".into(), Value::from(title)),
4683 ]),
4684 &client.encryption,
4685 )
4686 .unwrap(),
4687 ),
4688 };
4689 client
4690 .apply_commit_frame(&["tasks".into()], &[change], None)
4691 .unwrap();
4692 assert_eq!(
4693 client
4694 .conn
4695 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4696 .get::<_, String>(
4697 0
4698 ))
4699 .unwrap(),
4700 expected
4701 );
4702 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
4703 }
4704 assert_eq!(client.overlay_rebuild_count.get(), 3);
4705 }
4706
4707 fn measure_scoped_overlay(unchanged_rows: i64, rounds: i64) -> Vec<u128> {
4708 let schema = json!({ "version": 1, "tables": [
4709 { "name":"tasks", "primaryKey":"id", "columns":[
4710 {"name":"id","type":"string","nullable":false},
4711 {"name":"project_id","type":"string","nullable":false}],
4712 "scopes":[{"pattern":"project:{project_id}"}] },
4713 { "name":"catalogue", "primaryKey":"id", "columns":[
4714 {"name":"id","type":"string","nullable":false},
4715 {"name":"project_id","type":"string","nullable":false},
4716 {"name":"title","type":"string","nullable":false}],
4717 "scopes":[{"pattern":"project:{project_id}"}],
4718 "ftsIndexes":[{"name":"catalogue_fts","columns":["title"],"tokenize":"unicode61"}] }
4719 ] });
4720 let mut client =
4721 SyncClient::new("overlay-scope".into(), &schema, ClientLimits::default()).unwrap();
4722 if unchanged_rows > 0 {
4723 client.conn.execute_batch(&format!("WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i<{unchanged_rows}) INSERT INTO _syncular_base_catalogue SELECT 'c'||i,'p1','needle '||i,1 FROM n")).unwrap();
4724 }
4725 client.rebuild_overlay().expect("rebuild overlay");
4726 client.conn.execute_batch("CREATE TABLE audit(tbl TEXT); CREATE TRIGGER catalogue_changed AFTER DELETE ON catalogue BEGIN INSERT INTO audit VALUES ('catalogue'); END;").unwrap();
4727 let mut elapsed = Vec::new();
4728 for round in 1..=rounds {
4729 let pending = client
4730 .mutate(vec![Mutation::Upsert {
4731 table: "tasks".into(),
4732 values: Map::from_iter([
4733 ("id".into(), json!("t1")),
4734 ("project_id".into(), json!("p1")),
4735 ]),
4736 base_version: None,
4737 }])
4738 .unwrap();
4739 let started = std::time::Instant::now();
4740 client
4741 .write_base_row(
4742 "tasks",
4743 &vec![
4744 Some(ColumnValue::String("t1".into())),
4745 Some(ColumnValue::String("p1".into())),
4746 ],
4747 round,
4748 )
4749 .unwrap();
4750 let mut report = SyncReport::default();
4751 client
4752 .handle_push_results(
4753 &[Frame::PushResult {
4754 client_commit_id: pending.clone(),
4755 status: PushStatus::Applied,
4756 commit_seq: Some(round),
4757 results: vec![OpResult::Applied { op_index: 0 }],
4758 }],
4759 &HashSet::from([pending.as_str()]),
4760 &HashMap::new(),
4761 &mut report,
4762 Some(&pending),
4763 )
4764 .unwrap();
4765 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4766 elapsed.push(started.elapsed().as_nanos());
4767 }
4768 assert_eq!(
4769 client
4770 .conn
4771 .query_row("SELECT count(*) FROM audit", [], |r| r.get::<_, i64>(0))
4772 .unwrap(),
4773 0,
4774 "unchanged catalogue must never be copied"
4775 );
4776 assert_eq!(
4777 client
4778 .conn
4779 .query_row(
4780 "SELECT count(*) FROM catalogue_fts WHERE catalogue_fts MATCH 'needle'",
4781 [],
4782 |r| r.get::<_, i64>(0)
4783 )
4784 .unwrap(),
4785 unchanged_rows
4786 );
4787 assert_eq!(
4788 client.query("SELECT * FROM tasks", &[]).unwrap()[0]["_syncular_version"],
4789 json!(rounds)
4790 );
4791 elapsed
4792 }
4793
4794 #[test]
4795 fn changed_overlay_leaves_large_unchanged_table_and_fts_untouched() {
4796 let elapsed = measure_scoped_overlay(100_000, 1);
4797 eprintln!(
4798 "table-scoped overlay with 100k unchanged rows: {} ns",
4799 elapsed[0]
4800 );
4801 }
4802
4803 #[test]
4804 #[ignore = "isolated performance measurement; run with the shared heavy-check lock"]
4805 fn benchmark_scoped_overlay() {
4806 for rows in [0, 100_000, 200_000] {
4807 let mut elapsed = measure_scoped_overlay(rows, 30);
4808 elapsed.sort_unstable();
4809 eprintln!(
4810 "{}",
4811 json!({"unchangedRows": rows, "rounds": elapsed.len(),
4812 "medianNs": elapsed[elapsed.len()/2], "minNs": elapsed[0],
4813 "maxNs": elapsed[elapsed.len()-1], "unchangedDeletes": 0})
4814 );
4815 }
4816 }
4817
4818 #[test]
4819 fn released_context_discards_in_flight_ack_and_server_rows() {
4820 for barrier in [
4821 "preflight",
4822 "purge",
4823 "rebootstrap",
4824 "subscription",
4825 "failed-purge-replay",
4826 "failed-purge-publish",
4827 "failed-rebootstrap",
4828 ] {
4829 let mut client = client();
4830 client.set_meta(LOG_EPOCH_KEY, "epoch");
4831 client
4832 .subscribe(
4833 "tasks".into(),
4834 "tasks".into(),
4835 vec![("project_id".into(), vec!["p1".into()])],
4836 None,
4837 )
4838 .unwrap();
4839 let commit = client
4840 .mutate(vec![Mutation::Upsert {
4841 table: "tasks".into(),
4842 values: Map::from_iter([
4843 ("id".into(), json!("local")),
4844 ("project_id".into(), json!("p1")),
4845 ]),
4846 base_version: None,
4847 }])
4848 .unwrap();
4849 let prepared = client.prepare_sync_round(false).unwrap();
4850 client.set_transport_enabled(&mut CountingRealtimeTransport::default(), false);
4851 let payload = encode_row_json(
4852 &client.schema.tables[0],
4853 "server",
4854 &Map::from_iter([
4855 ("id".into(), json!("server")),
4856 ("project_id".into(), json!("p1")),
4857 ]),
4858 &client.encryption,
4859 )
4860 .unwrap();
4861 let completed = crate::CompletedSyncRound {
4862 prepared,
4863 exchange: crate::round::ExchangeResult::Reply {
4864 transport_failed: false,
4865 downloads: Default::default(),
4866 response: Ok(Message {
4867 wire_version: WIRE_VERSION,
4868 msg_kind: MsgKind::Response,
4869 frames: vec![
4870 Frame::RespHeader {
4871 required_schema_version: None,
4872 latest_schema_version: None,
4873 log_epoch: Some("epoch".into()),
4874 reset_required: Some(false),
4875 },
4876 Frame::PushResult {
4877 client_commit_id: commit.clone(),
4878 status: PushStatus::Applied,
4879 commit_seq: Some(1),
4880 results: vec![OpResult::Applied { op_index: 0 }],
4881 },
4882 Frame::SubStart {
4883 id: "tasks".into(),
4884 status: SubStatus::Active,
4885 reason_code: String::new(),
4886 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
4887 bootstrap: true,
4888 },
4889 Frame::Commit {
4890 commit_seq: 1,
4891 created_at_ms: 0,
4892 actor_id: "actor".into(),
4893 tables: vec!["tasks".into()],
4894 changes: vec![ssp2::model::Change {
4895 table_index: 0,
4896 row_id: "server".into(),
4897 op: Op::Upsert,
4898 row_version: Some(1),
4899 scopes: vec![("project_id".into(), "p1".into())],
4900 row: Some(payload),
4901 }],
4902 },
4903 Frame::SubEnd {
4904 next_cursor: 1,
4905 bootstrap_state: None,
4906 },
4907 ],
4908 }),
4909 },
4910 };
4911 match barrier {
4912 "preflight" => client.begin_security_preflight(),
4913 "purge" => {
4914 client
4915 .purge_local_data(&LocalDataPurgeInput {
4916 purge_id: "pending-round".into(),
4917 targets: vec![LocalDataPurgeTarget {
4918 table: "tasks".into(),
4919 selectors: BTreeMap::from([(
4920 "project_id".into(),
4921 vec!["p1".into()],
4922 )]),
4923 }],
4924 })
4925 .unwrap();
4926 }
4927 "rebootstrap" => {
4928 client
4929 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4930 rebootstrap_id: "pending-round".into(),
4931 })
4932 .unwrap();
4933 }
4934 "failed-purge-replay" | "failed-purge-publish" | "failed-rebootstrap" => {
4935 let trigger = if barrier == "failed-purge-replay" {
4936 "CREATE TRIGGER fail_local BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4937 } else {
4938 "CREATE TRIGGER fail_local BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4939 };
4940 client.conn.execute_batch(trigger).unwrap();
4941 let revision = client.local_revision();
4942 if barrier == "failed-rebootstrap" {
4943 assert!(client
4944 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4945 rebootstrap_id: "failed-round".into(),
4946 })
4947 .is_err());
4948 } else {
4949 assert!(client
4950 .purge_local_data(&LocalDataPurgeInput {
4951 purge_id: "failed-round".into(),
4952 targets: vec![LocalDataPurgeTarget {
4953 table: "tasks".into(),
4954 selectors: BTreeMap::from([(
4955 "project_id".into(),
4956 vec!["p1".into()]
4957 )]),
4958 }],
4959 })
4960 .is_err());
4961 }
4962 assert_eq!(client.local_revision(), revision);
4963 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
4964 client
4965 .conn
4966 .execute_batch("DROP TRIGGER fail_local")
4967 .unwrap();
4968 let applied = client.apply_sync_round(completed);
4969 assert!(
4970 matches!(
4971 applied,
4972 crate::AppliedSyncRound::Complete {
4973 outcome: SyncOutcome::Ok(_),
4974 ..
4975 }
4976 ),
4977 "{barrier}"
4978 );
4979 assert!(client.pending_commit_ids().is_empty());
4980 assert_eq!(
4981 client
4982 .query("SELECT id FROM tasks WHERE id = 'server'", &[])
4983 .unwrap()
4984 .len(),
4985 1
4986 );
4987 continue;
4988 }
4989 "subscription" => {
4990 client.unsubscribe("tasks");
4991 client
4992 .subscribe(
4993 "tasks".into(),
4994 "tasks".into(),
4995 vec![("project_id".into(), vec!["p2".into()])],
4996 None,
4997 )
4998 .unwrap();
4999 }
5000 _ => unreachable!(),
5001 }
5002 let pending_before_reply = client.pending_commit_ids();
5003 let revision = client.local_revision();
5004 let applied = client.apply_sync_round(completed);
5005 assert!(
5006 matches!(applied, crate::AppliedSyncRound::Complete { outcome: SyncOutcome::Failed { ref error_code, .. }, ref controls, .. } if error_code == "client.round_cancelled" && controls.is_empty())
5007 );
5008 assert_eq!(client.local_revision(), revision);
5009 assert!(client
5010 .conn
5011 .query_row(
5012 "SELECT NOT EXISTS(SELECT 1 FROM tasks WHERE id='server')",
5013 [],
5014 |r| r.get::<_, bool>(0)
5015 )
5016 .unwrap());
5017 assert_eq!(client.pending_commit_ids(), pending_before_reply);
5018 }
5019 }
5020
5021 #[test]
5022 fn sync_round_deadline_is_anchored_once_per_round_and_carried_across_continues() {
5023 struct DeadlineRecording {
5024 budget: std::time::Duration,
5025 seen: Vec<Option<std::time::Instant>>,
5026 }
5027 impl Transport for DeadlineRecording {
5028 fn round_deadline(&self) -> Option<std::time::Duration> {
5029 Some(self.budget)
5030 }
5031 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
5032 self.seen.push(deadline);
5033 }
5034 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
5035 Err(TransportError::new("transport.failed", "injected"))
5036 }
5037 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
5038 self.sync(request)
5039 }
5040 fn download_segment(
5041 &mut self,
5042 _: &SegmentRequest,
5043 _: &mut dyn FnMut(u64),
5044 ) -> Result<Vec<u8>, TransportError> {
5045 Err(TransportError::new("transport.failed", "injected"))
5046 }
5047 fn blob_upload_grant(
5048 &mut self,
5049 _: &str,
5050 _: u64,
5051 _: Option<&str>,
5052 ) -> Result<crate::BlobUploadGrant, TransportError> {
5053 Ok(crate::BlobUploadGrant::Present)
5054 }
5055 fn realtime_connect(&mut self) -> Result<(), TransportError> {
5056 Ok(())
5057 }
5058 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
5059 Ok(())
5060 }
5061 fn realtime_close(&mut self) -> Result<(), TransportError> {
5062 Ok(())
5063 }
5064 }
5065
5066 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5067 {"name":"id","type":"string","nullable":false},
5068 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5069 let mut client =
5070 SyncClient::new("deadline-round".into(), &schema, Default::default()).unwrap();
5071 client.set_meta(LOG_EPOCH_KEY, "epoch");
5072 client
5073 .upload_blob(b"pending upload", Some("text/plain".into()), None)
5074 .unwrap();
5075
5076 let mut transport = DeadlineRecording {
5077 budget: std::time::Duration::from_secs(30),
5078 seen: Vec::new(),
5079 };
5080 let prepared = client.prepare_sync_round(false).unwrap();
5081 assert_eq!(prepared.uploads.len(), 1, "one staged upload");
5082 let prepared = match client.apply_sync_round(prepared.exchange(&mut transport)) {
5084 crate::AppliedSyncRound::Continue(prepared) => prepared,
5085 crate::AppliedSyncRound::Complete { .. } => panic!("the upload must continue"),
5086 };
5087 let completed = prepared.exchange(&mut transport);
5089 let anchor = completed
5090 .prepared
5091 .round_deadline_at
5092 .expect("the upload round anchors a deadline");
5093 assert!(matches!(
5094 client.apply_sync_round(completed),
5095 crate::AppliedSyncRound::Complete { .. }
5096 ));
5097 assert_eq!(transport.seen.len(), 4, "one scope per exchange");
5098 assert_eq!(
5099 transport.seen[0],
5100 Some(anchor),
5101 "the upload anchors the round"
5102 );
5103 assert_eq!(
5104 transport.seen[1], None,
5105 "the upload exchange clears its scope"
5106 );
5107 assert_eq!(
5108 transport.seen[2],
5109 Some(anchor),
5110 "a continuation reuses the carried anchor instead of refreshing it"
5111 );
5112 assert_eq!(transport.seen[3], None, "the continuation clears its scope");
5113
5114 transport.budget = std::time::Duration::from_secs(60);
5117 let prepared = client.prepare_sync_round(false).unwrap();
5118 let completed = prepared.exchange(&mut transport);
5119 let fresh = completed
5120 .prepared
5121 .round_deadline_at
5122 .expect("the next round anchors a deadline");
5123 client.apply_sync_round(completed);
5124 assert!(
5125 fresh >= anchor + std::time::Duration::from_secs(30),
5126 "the next round anchors a fresh deadline from the new budget"
5127 );
5128 assert_eq!(transport.seen[4], Some(fresh));
5129 assert_eq!(
5130 transport.seen[5], None,
5131 "the new round clears its scope too"
5132 );
5133 }
5134
5135 #[derive(Default)]
5138 struct RoundDeadlineProbe {
5139 budget: Option<std::time::Duration>,
5140 budget_queries: std::cell::Cell<usize>,
5141 scoped: Vec<Option<std::time::Instant>>,
5142 }
5143
5144 impl Transport for RoundDeadlineProbe {
5145 fn round_deadline(&self) -> Option<std::time::Duration> {
5146 self.budget_queries.set(self.budget_queries.get() + 1);
5147 self.budget
5148 }
5149 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
5150 self.scoped.push(deadline);
5151 }
5152 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
5153 Err(TransportError::new("transport.failed", "injected"))
5154 }
5155 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
5156 self.sync(request)
5157 }
5158 fn download_segment(
5159 &mut self,
5160 _: &SegmentRequest,
5161 _: &mut dyn FnMut(u64),
5162 ) -> Result<Vec<u8>, TransportError> {
5163 Err(TransportError::new("transport.failed", "injected"))
5164 }
5165 fn blob_upload_grant(
5166 &mut self,
5167 _: &str,
5168 _: u64,
5169 _: Option<&str>,
5170 ) -> Result<crate::BlobUploadGrant, TransportError> {
5171 Ok(crate::BlobUploadGrant::Present)
5172 }
5173 fn realtime_connect(&mut self) -> Result<(), TransportError> {
5174 Ok(())
5175 }
5176 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
5177 Ok(())
5178 }
5179 fn realtime_close(&mut self) -> Result<(), TransportError> {
5180 Ok(())
5181 }
5182 }
5183
5184 #[test]
5185 fn an_unusable_upload_body_does_not_anchor_the_round_deadline() {
5186 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5187 {"name":"id","type":"string","nullable":false},
5188 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5189 let mut client =
5190 SyncClient::new("deadline-unusable".into(), &schema, ClientLimits::default()).unwrap();
5191 client.set_meta(LOG_EPOCH_KEY, "epoch");
5192 client
5193 .upload_blob(b"pending upload", Some("text/plain".into()), None)
5194 .unwrap();
5195 client
5196 .conn
5197 .execute_batch("DELETE FROM _syncular_blobs")
5198 .unwrap();
5199 let prepared = client.prepare_sync_round(false).unwrap();
5200 assert!(
5201 prepared.uploads.front().is_some_and(|u| u.bytes.is_err()),
5202 "the staged body must fail locally"
5203 );
5204 let mut transport = RoundDeadlineProbe {
5205 budget: Some(std::time::Duration::from_secs(30)),
5206 ..Default::default()
5207 };
5208 let completed = prepared.exchange(&mut transport);
5209 assert!(matches!(
5210 completed.exchange,
5211 crate::round::ExchangeResult::Upload { result: Err(_), .. }
5212 ));
5213 assert_eq!(
5214 transport.budget_queries.get(),
5215 0,
5216 "a round with no network work never asks for a budget"
5217 );
5218 assert_eq!(transport.scoped, vec![None]);
5219 }
5220
5221 #[test]
5222 fn a_transport_budget_the_clock_cannot_represent_fails_before_network() {
5223 struct HugeBudget;
5224 impl Transport for HugeBudget {
5225 fn round_deadline(&self) -> Option<std::time::Duration> {
5226 Some(std::time::Duration::MAX)
5227 }
5228 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
5229 panic!("no network for an invalid budget")
5230 }
5231 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
5232 self.sync(request)
5233 }
5234 fn download_segment(
5235 &mut self,
5236 _: &SegmentRequest,
5237 _: &mut dyn FnMut(u64),
5238 ) -> Result<Vec<u8>, TransportError> {
5239 panic!("no network for an invalid budget")
5240 }
5241 fn realtime_connect(&mut self) -> Result<(), TransportError> {
5242 Ok(())
5243 }
5244 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
5245 Ok(())
5246 }
5247 fn realtime_close(&mut self) -> Result<(), TransportError> {
5248 Ok(())
5249 }
5250 }
5251
5252 let mut client = client();
5253 client.set_meta(LOG_EPOCH_KEY, "epoch");
5254 let prepared = client.prepare_sync_round(false).unwrap();
5255 let outcome = match client.apply_sync_round(prepared.exchange(&mut HugeBudget)) {
5256 crate::AppliedSyncRound::Complete { outcome, .. } => outcome,
5257 crate::AppliedSyncRound::Continue(_) => {
5258 panic!("an unrepresentable budget must not continue")
5259 }
5260 };
5261 match outcome {
5262 SyncOutcome::Failed {
5263 error_code,
5264 message,
5265 ..
5266 } => {
5267 assert_eq!(error_code, "sync.invalid_request");
5268 assert_eq!(message, "transport round deadline is out of range");
5269 }
5270 _ => panic!("an unrepresentable budget must fail synchronously"),
5271 }
5272 }
5273
5274 #[test]
5275 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
5276 for failure in ["visible", "revision", "commit"] {
5277 let mut client = client();
5278 let id = client
5279 .mutate(vec![Mutation::Upsert {
5280 table: "tasks".into(),
5281 values: Map::from_iter([
5282 ("id".into(), Value::from("t1")),
5283 ("project_id".into(), Value::from("pending")),
5284 ]),
5285 base_version: None,
5286 }])
5287 .unwrap();
5288 client.drain_change_batches();
5289 let revision = client.local_revision();
5290 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
5291 let trigger = match failure {
5292 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
5293 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
5294 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
5295 };
5296 client.conn.execute_batch(&trigger).unwrap();
5297 let change = ssp2::model::Change {
5298 table_index: 0,
5299 row_id: "t1".into(),
5300 op: Op::Upsert,
5301 row_version: Some(2),
5302 scopes: vec![],
5303 row: Some(
5304 encode_row_json(
5305 client.schema.table("tasks").unwrap(),
5306 "t1",
5307 &Map::from_iter([
5308 ("id".into(), Value::from("t1")),
5309 ("project_id".into(), Value::from("remote")),
5310 ]),
5311 &client.encryption,
5312 )
5313 .unwrap(),
5314 ),
5315 };
5316 assert!(client
5317 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), None)
5318 .is_err());
5319 assert!(client.conn.is_autocommit());
5320 assert!(!client.overlay_dirty.get());
5321 assert_eq!(client.local_revision(), revision);
5322 assert!(client.drain_change_batches().is_empty());
5323 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5324 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
5325 assert!(client
5326 .query("SELECT * FROM _syncular_base_tasks", &[])
5327 .unwrap()
5328 .is_empty());
5329 client
5330 .conn
5331 .execute_batch("DROP TRIGGER fail_pending")
5332 .unwrap();
5333 client
5334 .apply_commit_frame(&["tasks".into()], &[change], None)
5335 .unwrap();
5336 assert_eq!(client.local_revision(), revision + 1);
5337 assert_eq!(client.pending_commit_ids(), vec![id]);
5338 let mut expected = before[0].clone();
5341 expected.insert("_syncular_version".to_owned(), Value::from(2));
5342 assert_eq!(
5343 client.query("SELECT * FROM tasks", &[]).unwrap(),
5344 vec![expected]
5345 );
5346 assert_eq!(
5347 client
5348 .conn
5349 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
5350 .get::<_, String>(0))
5351 .unwrap(),
5352 "remote"
5353 );
5354 }
5355 }
5356
5357 #[test]
5358 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
5359 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
5360 "columns": [
5361 {"name":"id", "type":"string", "nullable":false},
5362 {"name":"project_id", "type":"string", "nullable":false},
5363 {"name":"title", "type":"string", "nullable":false}],
5364 "scopes": [{"pattern":"project:{project_id}"}],
5365 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
5366 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
5367 let mut incremental =
5368 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
5369 let mut reference =
5370 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
5371 for instance in [&mut incremental, &mut reference] {
5372 for index in 0..16 {
5373 instance
5374 .write_base_row(
5375 "tasks",
5376 &vec![
5377 Some(ColumnValue::String(format!("task{index}"))),
5378 Some(ColumnValue::String("p0".to_owned())),
5379 Some(ColumnValue::String(format!("base{index}"))),
5380 ],
5381 1,
5382 )
5383 .unwrap();
5384 }
5385 instance
5386 .rebuild_overlay_if_dirty()
5387 .expect("rebuild overlay");
5388 instance.overlay_rebuild_count.set(0);
5389 }
5390 for index in 0..100 {
5391 let mutations = (0..1 + index % 4)
5392 .map(|offset| {
5393 let id = format!("task{}", (index + offset) % 24);
5394 if (index + offset) % 7 == 0 {
5395 Mutation::Delete {
5396 table: "tasks".to_owned(),
5397 row_id: id,
5398 base_version: None,
5399 }
5400 } else {
5401 Mutation::Upsert {
5402 table: "tasks".to_owned(),
5403 values: Map::from_iter([
5404 ("id".to_owned(), Value::from(id)),
5405 (
5406 "project_id".to_owned(),
5407 Value::from(format!("p{}", index % 2)),
5408 ),
5409 (
5410 "title".to_owned(),
5411 Value::from(format!("needle{}", (index + offset) % 24)),
5412 ),
5413 ]),
5414 base_version: None,
5415 }
5416 }
5417 })
5418 .collect::<Vec<_>>();
5419 reference.overlay_dirty.set(true);
5420 incremental.mutate(mutations.clone()).unwrap();
5421 reference.mutate(mutations).unwrap();
5422 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
5423 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
5424 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
5425 }
5426 assert_eq!(incremental.local_revision(), reference.local_revision());
5427 assert_eq!(
5428 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
5429 serde_json::to_value(reference.drain_change_batches()).unwrap()
5430 );
5431 }
5432 assert_eq!(incremental.outbox.len(), 100);
5433 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
5434 assert_eq!(reference.overlay_rebuild_count.get(), 100);
5435 }
5436
5437 #[test]
5438 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
5439 for fail_at in ["outbox", "revision"] {
5440 let mut client = client();
5441 let values = Map::from_iter([
5442 ("id".to_owned(), Value::from("t1")),
5443 ("project_id".to_owned(), Value::from("p1")),
5444 ]);
5445 let mutation = Mutation::Upsert {
5446 table: "tasks".to_owned(),
5447 values,
5448 base_version: None,
5449 };
5450 let trigger = if fail_at == "outbox" {
5451 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
5452 } else {
5453 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
5454 };
5455 client.conn.execute_batch(&trigger).unwrap();
5456 let revision = client.local_revision();
5457 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
5458 assert_eq!(client.local_revision(), revision);
5459 assert!(client.outbox.is_empty());
5460 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
5461 assert_eq!(
5462 client
5463 .conn
5464 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5465 .get::<_, i64>(0))
5466 .unwrap(),
5467 0
5468 );
5469 assert!(client.drain_change_batches().is_empty());
5470 assert!(!client.overlay_dirty.get());
5471 client
5472 .conn
5473 .execute_batch("DROP TRIGGER fail_append")
5474 .unwrap();
5475 client.mutate(vec![mutation]).unwrap();
5476 assert_eq!(client.outbox.len(), 1);
5477 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
5478 }
5479 }
5480
5481 #[test]
5482 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
5483 let mut client = client();
5484 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
5485 let values = Map::from_iter([
5486 ("id".to_owned(), Value::from("t1")),
5487 ("project_id".to_owned(), Value::from("p1")),
5488 ]);
5489 let payload = encode_row_json(
5490 client.schema.table("tasks").unwrap(),
5491 "t1",
5492 &values,
5493 &client.encryption,
5494 )
5495 .unwrap();
5496 let first = ssp2::model::Change {
5497 table_index: 0,
5498 row_id: "t1".to_owned(),
5499 op: Op::Upsert,
5500 row_version: Some(1),
5501 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
5502 row: Some(payload),
5503 };
5504 let mut invalid = first.clone();
5505 invalid.table_index = 1;
5506 let revision = client.local_revision();
5507 client.begin_observation("failed_mirror").unwrap();
5508 assert!(client
5509 .apply_commit_changes(&tables, &[first, invalid])
5510 .is_err());
5511 client.rollback_observation("failed_mirror");
5512 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
5513 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
5514 assert_eq!(
5515 client
5516 .conn
5517 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
5518 .get::<_, i64>(
5519 0
5520 ))
5521 .unwrap(),
5522 0
5523 );
5524 assert_eq!(client.local_revision(), revision);
5525 assert!(client.drain_change_batches().is_empty());
5526 }
5527
5528 #[test]
5529 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
5530 for failure in ["journal", "revision", "commit", "later-frame"] {
5531 let mut client = client();
5532 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5533 for id in ["first", "second"] {
5534 client
5535 .mutate(vec![Mutation::Upsert {
5536 table: "tasks".to_owned(),
5537 values: Map::from_iter([
5538 ("id".to_owned(), Value::from(id)),
5539 ("project_id".to_owned(), Value::from("p1")),
5540 ]),
5541 base_version: None,
5542 }])
5543 .unwrap();
5544 }
5545 let (_, meta) = client.build_request(false).unwrap();
5546 client
5548 .mutate(vec![Mutation::Upsert {
5549 table: "tasks".to_owned(),
5550 values: Map::from_iter([
5551 ("id".to_owned(), Value::from("later")),
5552 ("project_id".to_owned(), Value::from("p1")),
5553 ]),
5554 base_version: None,
5555 }])
5556 .unwrap();
5557 let ids = client.pending_commit_ids();
5558 let rows = client
5559 .query("SELECT * FROM tasks ORDER BY id", &[])
5560 .unwrap();
5561 let revision = client.local_revision();
5562 client.drain_change_batches();
5563 if failure != "later-frame" {
5564 client.conn.execute_batch(match failure {
5565 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
5566 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
5567 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
5568 }).unwrap();
5569 }
5570 let mut response = Message {
5571 wire_version: WIRE_VERSION,
5572 msg_kind: MsgKind::Response,
5573 frames: vec![Frame::RespHeader {
5574 required_schema_version: None,
5575 latest_schema_version: None,
5576 log_epoch: Some("epoch-1".to_owned()),
5577 reset_required: Some(false),
5578 }],
5579 };
5580 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
5581 response.frames.push(Frame::PushResult {
5582 client_commit_id: id.to_owned(),
5583 status: PushStatus::Applied,
5584 commit_seq: Some(1),
5585 results: vec![OpResult::Applied { op_index: 0 }],
5586 });
5587 }
5588 if failure == "later-frame" {
5589 response.frames.push(Frame::Error {
5590 code: "sync.invalid_request".to_owned(),
5591 message: "later error".to_owned(),
5592 category: "protocol".to_owned(),
5593 retryable: false,
5594 recommended_action: "retry".to_owned(),
5595 details: None,
5596 });
5597 }
5598 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5599 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5600 let expected = match failure {
5603 "later-frame" => "sync.invalid_request",
5604 "commit" => "storage.failed",
5605 _ => "client.outcome_persistence_failed",
5606 };
5607 assert!(
5608 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5609 "{failure}: {outcome:?}"
5610 );
5611 if failure == "later-frame" {
5612 assert_eq!(client.pending_commit_ids(), ids[2..]);
5613 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5614 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
5615 let batches = client.drain_change_batches();
5616 assert_eq!(batches.len(), 1);
5617 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
5618 assert_eq!(client.local_revision(), revision + 1);
5619 response.frames.pop();
5620 } else {
5621 assert_eq!(client.pending_commit_ids(), ids);
5622 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5623 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
5624 assert_eq!(client.local_revision(), revision);
5625 assert!(client.drain_change_batches().is_empty());
5626 assert_eq!(
5627 client
5628 .query("SELECT * FROM tasks ORDER BY id", &[])
5629 .unwrap(),
5630 rows
5631 );
5632 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
5633 }
5634 for frame in &mut response.frames {
5635 if let Frame::PushResult { status, .. } = frame {
5636 *status = PushStatus::Cached;
5637 }
5638 }
5639 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
5640 else {
5641 panic!("retry failed");
5642 };
5643 assert_eq!(
5644 report.applied,
5645 if failure == "later-frame" {
5646 vec![]
5647 } else {
5648 ids[..2].to_vec()
5649 }
5650 );
5651 assert_eq!(client.pending_commit_ids(), ids[2..]);
5652 let batches = client.drain_change_batches();
5653 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
5654 if let Some(batch) = batches.first() {
5655 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
5656 }
5657 assert_eq!(client.local_revision(), revision + 1);
5658 }
5659 }
5660
5661 #[test]
5662 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
5663 for failure in ["revision", "journal", "commit"] {
5664 for status in ["applied", "cached", "rejected", "conflict"] {
5665 let mut client = client();
5666 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5667 for id in ["first", "later"] {
5668 client
5669 .mutate(vec![Mutation::Upsert {
5670 table: "tasks".to_owned(),
5671 values: Map::from_iter([
5672 ("id".to_owned(), Value::from(id)),
5673 ("project_id".to_owned(), Value::from("p1")),
5674 ]),
5675 base_version: None,
5676 }])
5677 .unwrap();
5678 }
5679 let (_, meta) = client.build_request(false).unwrap();
5680 let ids = client.pending_commit_ids();
5681 let rows = client
5682 .query("SELECT * FROM tasks ORDER BY id", &[])
5683 .unwrap();
5684 let revision = client.local_revision();
5685 client.drain_change_batches();
5686 let (push_status, results) = match status {
5687 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
5688 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
5689 "conflict" => (
5690 PushStatus::Rejected,
5691 vec![OpResult::Conflict {
5692 op_index: 0,
5693 code: "sync.version_conflict".to_owned(),
5694 message: "conflict".to_owned(),
5695 server_version: 1,
5696 server_row: encode_row_json(
5697 client.schema.table("tasks").unwrap(),
5698 "first",
5699 &Map::from_iter([
5700 ("id".to_owned(), Value::from("first")),
5701 ("project_id".to_owned(), Value::from("p1")),
5702 ]),
5703 &client.encryption,
5704 )
5705 .unwrap(),
5706 conflict_columns: Vec::new(),
5707 }],
5708 ),
5709 _ => (
5710 PushStatus::Rejected,
5711 vec![OpResult::Error {
5712 op_index: 0,
5713 code: "sync.validation_failed".to_owned(),
5714 message: "rejected".to_owned(),
5715 retryable: false,
5716 }],
5717 ),
5718 };
5719 let response = Message {
5720 wire_version: WIRE_VERSION,
5721 msg_kind: MsgKind::Response,
5722 frames: vec![
5723 Frame::RespHeader {
5724 required_schema_version: None,
5725 latest_schema_version: None,
5726 log_epoch: Some("epoch-1".to_owned()),
5727 reset_required: Some(false),
5728 },
5729 Frame::PushResult {
5730 client_commit_id: ids[0].clone(),
5731 status: push_status,
5732 commit_seq: Some(1),
5733 results,
5734 },
5735 ],
5736 };
5737 client.conn.execute_batch(match failure {
5738 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
5739 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
5740 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
5741 }).unwrap();
5742 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5743 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5744 assert_eq!(
5745 client.pending_commit_ids(),
5746 ids,
5747 "{failure}/{status}: pending identity"
5748 );
5749 assert_eq!(
5750 client.local_revision(),
5751 revision,
5752 "{failure}/{status}: revision"
5753 );
5754 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5755 assert!(client.conflicts().is_empty());
5756 assert!(client.rejections().is_empty());
5757 assert!(client.drain_change_batches().is_empty());
5758 assert_eq!(
5759 client
5760 .query("SELECT * FROM tasks ORDER BY id", &[])
5761 .unwrap(),
5762 rows
5763 );
5764 let expected = if failure == "commit" {
5767 "storage.failed"
5768 } else {
5769 "client.outcome_persistence_failed"
5770 };
5771 assert!(
5772 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5773 "{failure}/{status}: failed round"
5774 );
5775 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
5776 assert!(matches!(
5777 client.process_response(&mut transport, response, &meta),
5778 SyncOutcome::Ok(_)
5779 ));
5780 assert_eq!(client.pending_commit_ids(), ids[1..]);
5781 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5782 assert_eq!(client.drain_change_batches().len(), 1);
5783 }
5784 }
5785 }
5786
5787 #[test]
5788 fn mixed_push_results_reconcile_each_boundary_and_prune_once_per_response() {
5789 let mut client = client();
5790 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5791 for index in 0..4 {
5792 client
5793 .mutate(vec![Mutation::Upsert {
5794 table: "tasks".to_owned(),
5795 values: Map::from_iter([
5796 ("id".to_owned(), Value::from(format!("task-{index}"))),
5797 ("project_id".to_owned(), Value::from("project-1")),
5798 ]),
5799 base_version: None,
5800 }])
5801 .expect("queue commit");
5802 }
5803
5804 let (_, request_meta) = client.build_request(false).unwrap();
5805 let ids = &request_meta.pushed_ids;
5806 assert_eq!(ids.len(), 4);
5807 let response = Message {
5808 wire_version: WIRE_VERSION,
5809 msg_kind: MsgKind::Response,
5810 frames: vec![
5811 Frame::RespHeader {
5812 required_schema_version: None,
5813 latest_schema_version: None,
5814 log_epoch: Some("epoch-1".to_owned()),
5815 reset_required: Some(false),
5816 },
5817 Frame::PushResult {
5818 client_commit_id: ids[0].clone(),
5819 status: PushStatus::Applied,
5820 commit_seq: Some(1),
5821 results: vec![OpResult::Applied { op_index: 0 }],
5822 },
5823 Frame::PushResult {
5824 client_commit_id: ids[1].clone(),
5825 status: PushStatus::Cached,
5826 commit_seq: Some(2),
5827 results: vec![OpResult::Applied { op_index: 0 }],
5828 },
5829 Frame::PushResult {
5830 client_commit_id: ids[2].clone(),
5831 status: PushStatus::Rejected,
5832 commit_seq: None,
5833 results: vec![OpResult::Error {
5834 op_index: 0,
5835 code: "sync.validation_failed".to_owned(),
5836 message: "rejected".to_owned(),
5837 retryable: false,
5838 }],
5839 },
5840 Frame::PushResult {
5841 client_commit_id: ids[3].clone(),
5842 status: PushStatus::Rejected,
5843 commit_seq: None,
5844 results: vec![OpResult::Error {
5845 op_index: 0,
5846 code: "sync.idempotency_cache_miss".to_owned(),
5847 message: "retry".to_owned(),
5848 retryable: true,
5849 }],
5850 },
5851 ],
5852 };
5853 let mut transport =
5854 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
5855
5856 client.overlay_rebuild_count.set(0);
5857 client.outcome_prune_count.set(0);
5858 let outcome = client.process_response(&mut transport, response, &request_meta);
5859 let SyncOutcome::Ok(report) = outcome else {
5860 panic!("mixed push-result response failed");
5861 };
5862
5863 assert_eq!(report.applied, ids[..2]);
5864 assert_eq!(report.rejected, ids[2..3]);
5865 assert_eq!(report.retryable, ids[3..4]);
5866 assert_eq!(client.pending_commit_ids(), ids[3..4]);
5867 assert_eq!(
5868 client
5869 .query("SELECT id FROM tasks ORDER BY id", &[])
5870 .expect("query visible overlay"),
5871 ["task-0", "task-1", "task-3"]
5872 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
5873 .to_vec()
5874 );
5875 assert_eq!(client.overlay_rebuild_count.get(), 2);
5877 assert_eq!(client.outcome_prune_count.get(), 1);
5878 }
5879
5880 fn replay_schema(fts: bool) -> Value {
5881 json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
5882 {"name":"id","type":"string","nullable":false},
5883 {"name":"project_id","type":"string","nullable":false},
5884 {"name":"title","type":"string","nullable":false},
5885 {"name":"done","type":"boolean","nullable":false}
5886 ],"scopes":[{"pattern":"project:{project_id}"}], "ftsIndexes": if fts { json!([{"name":"tasks_fts","columns":["title"],"tokenize":"unicode61"}]) } else { json!([]) }}]})
5887 }
5888
5889 #[test]
5890 fn aborted_schema_and_epoch_resets_restore_memory_and_durable_intent() {
5891 for schema_reset in [false, true] {
5892 let mut schema = replay_schema(false);
5893 let mut client =
5894 SyncClient::new("reset-failure".into(), &schema, ClientLimits::default()).unwrap();
5895 let id = client
5896 .mutate(vec![Mutation::Upsert {
5897 table: "tasks".into(),
5898 values: Map::from_iter([
5899 ("id".into(), json!("queued")),
5900 ("project_id".into(), json!("p1")),
5901 ("title".into(), json!("old")),
5902 ("done".into(), json!(false)),
5903 ]),
5904 base_version: None,
5905 }])
5906 .unwrap();
5907 client.set_meta(LOG_EPOCH_KEY, "old");
5908 let revision = client.local_revision();
5909 let needed = client.sync_needed;
5910 client.drain_change_batches();
5911 client.drain_sync_intents();
5912 client.conn.execute_batch("CREATE TRIGGER refuse_revision BEFORE INSERT ON _syncular_meta WHEN NEW.key='localRevision' BEGIN SELECT RAISE(ABORT,'reset completion failed'); END;").unwrap();
5913 if schema_reset {
5914 schema["version"] = json!(2);
5915 schema["tables"][0]["columns"]
5916 .as_array_mut()
5917 .unwrap()
5918 .retain(|column| column["name"] != "title");
5919 client.schema = parse_schema_json(&schema).unwrap();
5920 assert!(client.run_schema_reset().is_err());
5921 } else {
5922 assert!(client.run_log_epoch_reset("new").is_err());
5923 }
5924 assert!(!client.upgrading);
5925 assert_eq!(client.sync_needed, needed);
5926 assert_eq!(client.local_revision(), revision);
5927 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5928 assert!(client.rejections.is_empty());
5929 assert!(client.commit_outcome(&id).unwrap().is_none());
5930 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("old"));
5931 assert_eq!(
5932 client
5933 .query("SELECT title FROM tasks WHERE id='queued'", &[])
5934 .unwrap()[0]["title"],
5935 "old"
5936 );
5937 assert_eq!(
5938 client
5939 .conn
5940 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5941 .get::<_, i64>(0))
5942 .unwrap(),
5943 1
5944 );
5945 assert!(client.drain_change_batches().is_empty());
5946 assert!(client.drain_sync_intents().is_empty());
5947 assert!(client.conn.is_autocommit());
5948 }
5949 }
5950
5951 #[test]
5952 fn cached_blob_cleanup_propagates_delete_failures() {
5953 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5954 {"name":"id","type":"string","nullable":false},
5955 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5956 let mut client =
5957 SyncClient::new("blob-cleanup".into(), &schema, ClientLimits::default()).unwrap();
5958 client.conn.execute_batch("CREATE TRIGGER refuse_blob_cleanup BEFORE DELETE ON _syncular_blobs BEGIN SELECT RAISE(ABORT,'blob cleanup failed'); END;").unwrap();
5959 client.conn.execute("INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,created_at_ms) VALUES ('test',X'01',1,0)", []).unwrap();
5960 client
5961 .subscribe("attachments".into(), "attachments".into(), vec![], None)
5962 .unwrap();
5963 client.subs[0].effective = Some(vec![]);
5964 client.persist_sub(&client.subs[0]).unwrap();
5965 let revision = client.local_revision();
5966 let (_, meta) = client.build_request(false).unwrap();
5967 let mut transport = CountingRealtimeTransport::default();
5968 let mut report = SyncReport::default();
5969 assert!(client
5970 .process_section(
5971 &mut transport,
5972 "attachments",
5973 SubStatus::Revoked,
5974 "sync.scope_revoked",
5975 vec![],
5976 vec![],
5977 Some((0, None)),
5978 &meta,
5979 &mut report,
5980 &HashSet::new()
5981 )
5982 .is_err());
5983 assert_eq!(client.subs[0].state, SubState::Active);
5984 assert_eq!(client.local_revision(), revision);
5985 assert!(report.revoked.is_empty());
5986 assert!(client.conn.is_autocommit());
5987 client
5988 .conn
5989 .execute_batch("DROP TRIGGER refuse_blob_cleanup")
5990 .unwrap();
5991 assert!(client
5992 .process_section(
5993 &mut transport,
5994 "attachments",
5995 SubStatus::Revoked,
5996 "sync.scope_revoked",
5997 vec![],
5998 vec![],
5999 Some((0, None)),
6000 &meta,
6001 &mut report,
6002 &HashSet::new()
6003 )
6004 .is_ok());
6005 assert_eq!(client.subs[0].state, SubState::Revoked);
6006 assert_eq!(
6007 client
6008 .conn
6009 .query_row("SELECT count(*) FROM _syncular_blobs", [], |row| row
6010 .get::<_, i64>(0))
6011 .unwrap(),
6012 0
6013 );
6014 }
6015
6016 #[test]
6017 fn overlay_replay_trigger_refuses_reopen_and_preserves_durable_intent() {
6018 let path =
6019 std::env::temp_dir().join(format!("syncular-replay-{}.db", uuid::Uuid::new_v4()));
6020 let schema = replay_schema(false);
6021 let mut client = SyncClient::open_path(
6022 "replay".into(),
6023 &schema,
6024 ClientLimits::default(),
6025 path.to_str().unwrap(),
6026 )
6027 .unwrap();
6028 let ids = ["early", "bad"].map(|id| {
6029 client
6030 .mutate(vec![Mutation::Upsert {
6031 table: "tasks".into(),
6032 values: Map::from_iter([
6033 ("id".into(), json!(id)),
6034 ("project_id".into(), json!("p1")),
6035 ("title".into(), json!(id)),
6036 ("done".into(), json!(false)),
6037 ]),
6038 base_version: None,
6039 }])
6040 .unwrap()
6041 });
6042 client.conn.execute_batch("DELETE FROM tasks; CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.id='bad' BEGIN SELECT RAISE(ABORT,'replay failed'); END;").unwrap();
6043 let revision = client.local_revision();
6044 drop(client);
6045 let error = SyncClient::open_path(
6046 "replay".into(),
6047 &schema,
6048 ClientLimits::default(),
6049 path.to_str().unwrap(),
6050 )
6051 .err()
6052 .expect("reopen must fail");
6053 assert!(error.contains("replay failed"));
6054 let conn = Connection::open(&path).unwrap();
6055 assert_eq!(
6056 conn.query_row("SELECT count(*) FROM tasks", [], |row| row.get::<_, i64>(0))
6057 .unwrap(),
6058 0
6059 );
6060 assert_eq!(
6061 conn.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
6062 .get::<_, i64>(0))
6063 .unwrap(),
6064 2
6065 );
6066 assert_eq!(
6067 meta_get(&conn, LOCAL_REVISION_KEY),
6068 Some(revision.to_string())
6069 );
6070 conn.execute_batch("DROP TRIGGER fail_replay").unwrap();
6071 drop(conn);
6072 let reopened = SyncClient::open_path(
6073 "replay".into(),
6074 &schema,
6075 ClientLimits::default(),
6076 path.to_str().unwrap(),
6077 )
6078 .unwrap();
6079 assert_eq!(reopened.pending_commit_ids(), ids);
6080 assert_eq!(
6081 reopened
6082 .query("SELECT id FROM tasks ORDER BY id", &[])
6083 .unwrap()
6084 .len(),
6085 2
6086 );
6087 drop(reopened);
6088 std::fs::remove_file(path).unwrap();
6089 }
6090
6091 #[test]
6092 fn overlay_replay_failure_rolls_back_push_and_pull_apply_boundaries() {
6093 let schema = replay_schema(false);
6094 let mut client =
6095 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
6096 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
6097 client
6098 .write_base_row(
6099 "tasks",
6100 &vec![
6101 Some(ColumnValue::String("queued".into())),
6102 Some(ColumnValue::String("p1".into())),
6103 Some(ColumnValue::String("server".into())),
6104 Some(ColumnValue::Boolean(false)),
6105 ],
6106 1,
6107 )
6108 .unwrap();
6109 client.rebuild_overlay().unwrap();
6110 let id = client
6111 .patch(
6112 "tasks",
6113 "queued",
6114 Map::from_iter([("title".into(), json!("local"))]),
6115 Some(1),
6116 )
6117 .unwrap();
6118 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
6119 let base = client
6120 .query("SELECT * FROM _syncular_base_tasks", &[])
6121 .unwrap();
6122 let revision = client.local_revision();
6123 client.drain_change_batches();
6124 client.conn.execute_batch("CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.title='local' BEGIN SELECT RAISE(ABORT,'replay failed'); END").unwrap();
6125 let remote = vec![
6126 Some(ColumnValue::String("queued".into())),
6127 Some(ColumnValue::String("p1".into())),
6128 Some(ColumnValue::String("remote".into())),
6129 Some(ColumnValue::Boolean(false)),
6130 ];
6131 let bytes = crate::values::encode_row_json(
6132 client.schema.table("tasks").unwrap(),
6133 "queued",
6134 &Map::from_iter(
6135 client
6136 .schema
6137 .table("tasks")
6138 .unwrap()
6139 .columns
6140 .iter()
6141 .zip(&remote)
6142 .map(|(column, value)| (column.name.clone(), column_value_to_json(value))),
6143 ),
6144 &client.encryption,
6145 )
6146 .unwrap();
6147 let change = ssp2::model::Change {
6148 table_index: 0,
6149 op: Op::Upsert,
6150 row_id: "queued".into(),
6151 row_version: Some(2),
6152 scopes: vec![("project_id".into(), "p1".into())],
6153 row: Some(bytes),
6154 };
6155 assert!(client
6156 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), Some(2))
6157 .is_err());
6158 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
6159 assert_eq!(
6160 client
6161 .query("SELECT * FROM _syncular_base_tasks", &[])
6162 .unwrap(),
6163 base
6164 );
6165 assert_eq!(client.local_revision(), revision);
6166 assert!(client.drain_change_batches().is_empty());
6167 let response = Message {
6168 wire_version: 2,
6169 msg_kind: MsgKind::Response,
6170 frames: vec![
6171 Frame::RespHeader {
6172 required_schema_version: None,
6173 latest_schema_version: None,
6174 log_epoch: Some("epoch-1".into()),
6175 reset_required: Some(false),
6176 },
6177 Frame::PushResult {
6178 client_commit_id: id.clone(),
6179 status: PushStatus::Applied,
6180 commit_seq: Some(2),
6181 results: vec![OpResult::Applied { op_index: 0 }],
6182 },
6183 ],
6184 };
6185 let (_, meta) = client.build_request(false).unwrap();
6186 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
6187 assert!(
6188 matches!(client.process_response(&mut transport,response.clone(),&meta),SyncOutcome::Failed { error_code,.. } if error_code=="client.outcome_persistence_failed")
6189 );
6190 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
6191 assert!(client.commit_outcome(&id).unwrap().is_none());
6192 assert_eq!(
6193 client
6194 .conn
6195 .query_row("SELECT count(*) FROM _syncular_acked_rows", [], |row| row
6196 .get::<_, i64>(
6197 0
6198 ))
6199 .unwrap(),
6200 0
6201 );
6202 assert_eq!(
6203 client
6204 .conn
6205 .query_row("SELECT count(*) FROM _syncular_row_deliveries", [], |row| {
6206 row.get::<_, i64>(0)
6207 })
6208 .unwrap(),
6209 0
6210 );
6211 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
6212 assert_eq!(client.local_revision(), revision);
6213 assert!(client.drain_change_batches().is_empty());
6214 client
6215 .conn
6216 .execute_batch("DROP TRIGGER fail_replay")
6217 .unwrap();
6218 assert!(matches!(
6219 client.process_response(&mut transport, response, &meta),
6220 SyncOutcome::Ok(_)
6221 ));
6222 client
6223 .apply_commit_frame(&["tasks".into()], &[change], Some(2))
6224 .unwrap();
6225 assert!(client.pending_commit_ids().is_empty());
6226 assert_eq!(
6227 client.query("SELECT title FROM tasks", &[]).unwrap()[0]["title"],
6228 json!("remote")
6229 );
6230 }
6231
6232 #[test]
6233 fn overlay_replay_distinguishes_absence_from_sql_and_value_decode_errors() {
6234 let client = SyncClient::new(
6235 "replay".into(),
6236 &replay_schema(false),
6237 ClientLimits::default(),
6238 )
6239 .unwrap();
6240 let table = client.schema.table("tasks").unwrap();
6241 let patch = OutboxOp {
6242 table: "tasks".into(),
6243 row_id: "absent".into(),
6244 upsert: true,
6245 insert: false,
6246 base_version: Some(1),
6247 values: Some(Map::from_iter([
6248 ("id".into(), json!("absent")),
6249 ("title".into(), json!("patch")),
6250 ])),
6251 };
6252 assert!(client.visible_row(table, "absent").unwrap().is_none());
6253 client
6254 .apply_outbox_ops([&patch])
6255 .expect("sparse replay over truly absent row");
6256 client
6257 .conn
6258 .execute_batch("INSERT INTO tasks VALUES ('bad','p1','title','invalid',1)")
6259 .unwrap();
6260 assert!(client
6261 .visible_row(table, "bad")
6262 .unwrap_err()
6263 .starts_with("sync.local_corrupt:"));
6264 let mut invalid = patch.clone();
6265 invalid.values = Some(Map::from_iter([
6266 ("id".into(), json!("bad-value")),
6267 ("project_id".into(), json!("p1")),
6268 ("title".into(), json!("title")),
6269 ("done".into(), json!({"bad":true})),
6270 ]));
6271 assert!(client
6272 .apply_outbox_ops([&invalid])
6273 .unwrap_err()
6274 .starts_with("sync.local_corrupt:"));
6275 client.conn.execute_batch("DROP TABLE tasks").unwrap();
6276 assert!(
6277 client.apply_outbox_ops([&patch]).is_err(),
6278 "SQL failure must not count as absence"
6279 );
6280 }
6281
6282 #[test]
6283 fn overlay_replay_keeps_secondary_unique_deferral_and_lazy_base_precedence() {
6284 let mut schema = replay_schema(false);
6285 schema["tables"][0]["indexes"] =
6286 json!([{"name":"tasks_project_title", "columns":["project_id","title"],"unique":true}]);
6287 let mut client =
6288 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
6289 let id = client
6290 .mutate(vec![Mutation::Upsert {
6291 table: "tasks".into(),
6292 values: Map::from_iter([
6293 ("id".into(), json!("queued")),
6294 ("project_id".into(), json!("p1")),
6295 ("title".into(), json!("occupied")),
6296 ("done".into(), json!(false)),
6297 ]),
6298 base_version: None,
6299 }])
6300 .unwrap();
6301 client
6302 .write_base_row(
6303 "tasks",
6304 &vec![
6305 Some(ColumnValue::String("server".into())),
6306 Some(ColumnValue::String("p1".into())),
6307 Some(ColumnValue::String("occupied".into())),
6308 Some(ColumnValue::Boolean(false)),
6309 ],
6310 1,
6311 )
6312 .unwrap();
6313 client.rebuild_overlay_if_dirty().unwrap();
6314 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
6315 assert!(client
6316 .visible_row(client.schema.table("tasks").unwrap(), "queued")
6317 .unwrap()
6318 .is_none());
6319 assert!(client.rejections().is_empty());
6320 client.delete_base_row("tasks", "server").unwrap();
6321 client.rebuild_overlay_if_dirty().unwrap();
6322 assert!(client
6323 .visible_row(client.schema.table("tasks").unwrap(), "queued")
6324 .unwrap()
6325 .is_some());
6326 client
6327 .write_base_row(
6328 "tasks",
6329 &vec![
6330 Some(ColumnValue::String("base".into())),
6331 Some(ColumnValue::String("p1".into())),
6332 Some(ColumnValue::String("base-title".into())),
6333 Some(ColumnValue::Boolean(false)),
6334 ],
6335 1,
6336 )
6337 .unwrap();
6338 client.conn.execute_batch("DROP TABLE tasks").unwrap();
6339 client
6340 .retain_failed_commit(
6341 "retained",
6342 &[OutboxOp {
6343 table: "tasks".into(),
6344 row_id: "base".into(),
6345 upsert: true,
6346 insert: false,
6347 base_version: Some(1),
6348 values: Some(Map::from_iter([
6349 ("id".into(), json!("base")),
6350 ("title".into(), json!("patch")),
6351 ])),
6352 }],
6353 )
6354 .expect("an existing base must not read the unavailable visible table");
6355 }
6356
6357 #[test]
6358 fn overlay_replay_fts_and_savepoint_release_failures_roll_back_all_visible_changes() {
6359 for failure in ["fts", "release"] {
6360 let mut client = SyncClient::new(
6361 "replay".into(),
6362 &replay_schema(true),
6363 ClientLimits::default(),
6364 )
6365 .unwrap();
6366 client
6367 .mutate(vec![Mutation::Upsert {
6368 table: "tasks".into(),
6369 values: Map::from_iter([
6370 ("id".into(), json!("queued")),
6371 ("project_id".into(), json!("p1")),
6372 ("title".into(), json!("needle")),
6373 ("done".into(), json!(false)),
6374 ]),
6375 base_version: None,
6376 }])
6377 .unwrap();
6378 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
6379 let fts_before = client
6380 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
6381 .unwrap();
6382 let triggers_before: Vec<String> = client
6383 .conn
6384 .prepare("SELECT sql FROM sqlite_master WHERE type='trigger' ORDER BY name")
6385 .unwrap()
6386 .query_map([], |row| row.get(0))
6387 .unwrap()
6388 .collect::<Result<_, _>>()
6389 .unwrap();
6390 if failure == "fts" {
6391 client.conn.execute_batch("CREATE TRIGGER fail_fts BEFORE INSERT ON _syncular_fts_tasks_fts BEGIN SELECT RAISE(ABORT,'fts replay failed'); END;").unwrap();
6392 } else {
6393 client.conn.execute_batch("PRAGMA foreign_keys=ON; CREATE TABLE parent(id TEXT PRIMARY KEY); CREATE TABLE deferred(id TEXT REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_release AFTER INSERT ON tasks BEGIN INSERT INTO deferred VALUES ('missing'); END;").unwrap();
6394 }
6395 client.overlay_dirty.set(true);
6396 assert!(client.rebuild_overlay().is_err());
6397 assert!(client.overlay_dirty.get());
6398 assert!(client.conn.is_autocommit());
6399 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
6400 assert_eq!(
6401 client
6402 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
6403 .unwrap(),
6404 fts_before
6405 );
6406 let original_triggers: Vec<String> = client.conn.prepare("SELECT sql FROM sqlite_master WHERE type='trigger' AND name NOT LIKE 'fail_%' ORDER BY name").unwrap().query_map([], |row| row.get(0)).unwrap().collect::<Result<_,_>>().unwrap();
6407 assert_eq!(original_triggers, triggers_before);
6408 assert_eq!(client.pending_commit_ids().len(), 1);
6409 client
6410 .conn
6411 .execute_batch(if failure == "fts" {
6412 "DROP TRIGGER fail_fts"
6413 } else {
6414 "DROP TRIGGER fail_release"
6415 })
6416 .unwrap();
6417 client
6418 .rebuild_overlay()
6419 .expect("retry after removing the fault");
6420 }
6421 }
6422
6423 #[test]
6424 fn secondary_unique_collision_preserves_existing_synced_row() {
6425 let client = SyncClient::new(
6426 "unique-upsert-test".to_owned(),
6427 &json!({
6428 "version": 1,
6429 "tables": [{
6430 "name": "tasks",
6431 "primaryKey": "id",
6432 "columns": [
6433 { "name": "id", "type": "string", "nullable": false },
6434 { "name": "project_id", "type": "string", "nullable": false },
6435 { "name": "title", "type": "string", "nullable": false }
6436 ],
6437 "scopes": [{ "pattern": "project:{project_id}" }],
6438 "indexes": [{
6439 "name": "tasks_by_project_title",
6440 "columns": ["project_id", "title"],
6441 "unique": true
6442 }]
6443 }]
6444 }),
6445 ClientLimits::default(),
6446 )
6447 .expect("test client");
6448 let table = client.schema.table("tasks").expect("tasks table");
6449 let sql = client.insert_row_sql(&base_table("tasks"), table);
6450
6451 client
6452 .conn
6453 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
6454 .expect("insert first row");
6455 client
6456 .conn
6457 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
6458 .expect("update same primary key");
6459 client
6460 .conn
6461 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
6462 .expect("insert second row");
6463 assert!(client
6464 .conn
6465 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
6466 .is_err());
6467
6468 let rows = client
6469 .conn
6470 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
6471 .expect("prepare rows")
6472 .query_map([], |row| {
6473 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
6474 })
6475 .expect("query rows")
6476 .collect::<Result<Vec<_>, _>>()
6477 .expect("collect rows");
6478 assert_eq!(
6479 rows,
6480 vec![
6481 ("t1".to_owned(), "updated".to_owned()),
6482 ("t2".to_owned(), "original".to_owned())
6483 ]
6484 );
6485 }
6486
6487 #[test]
6488 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
6489 let path = std::env::temp_dir().join(format!(
6490 "syncular-startup-intent-{}.db",
6491 uuid::Uuid::new_v4()
6492 ));
6493 let schema = json!({
6494 "version": 1,
6495 "tables": [{
6496 "name": "tasks",
6497 "primaryKey": "id",
6498 "columns": [
6499 { "name": "id", "type": "string", "nullable": false },
6500 { "name": "project_id", "type": "string", "nullable": false }
6501 ],
6502 "scopes": [{ "pattern": "project:{project_id}" }]
6503 }]
6504 });
6505
6506 {
6507 let mut first = SyncClient::open_path_with_identity(
6508 None,
6509 &schema,
6510 ClientLimits::default(),
6511 path.to_str().expect("UTF-8 temp path"),
6512 )
6513 .expect("first open");
6514 first
6515 .set_window(
6516 &WindowBase {
6517 table: "tasks".to_owned(),
6518 variable: "project_id".to_owned(),
6519 fixed_scopes: Vec::new(),
6520 params: None,
6521 },
6522 &["persisted".to_owned()],
6523 )
6524 .expect("persist window");
6525 }
6526
6527 let mut reopened = SyncClient::open_path_with_identity(
6528 None,
6529 &schema,
6530 ClientLimits::default(),
6531 path.to_str().expect("UTF-8 temp path"),
6532 )
6533 .expect("reopen");
6534 assert!(reopened.sync_needed());
6535 assert!(matches!(
6536 reopened.drain_sync_intents().as_slice(),
6537 [SyncIntent::Interactive]
6538 ));
6539 drop(reopened);
6540 std::fs::remove_file(path).expect("remove temp database");
6541 }
6542
6543 #[test]
6544 fn reopening_preserves_immutable_subscription_identity_and_progress() {
6545 let path = std::env::temp_dir().join(format!(
6546 "syncular-subscription-identity-{}.db",
6547 uuid::Uuid::new_v4()
6548 ));
6549 let schema = json!({
6550 "version": 1,
6551 "tables": [
6552 {
6553 "name": "tasks",
6554 "primaryKey": "id",
6555 "columns": [
6556 { "name": "id", "type": "string", "nullable": false },
6557 { "name": "project_id", "type": "string", "nullable": false }
6558 ],
6559 "scopes": [{ "pattern": "project:{project_id}" }]
6560 },
6561 {
6562 "name": "docs",
6563 "primaryKey": "id",
6564 "columns": [
6565 { "name": "id", "type": "string", "nullable": false },
6566 { "name": "org_id", "type": "string", "nullable": false },
6567 { "name": "project_id", "type": "string", "nullable": false }
6568 ],
6569 "scopes": [
6570 { "pattern": "org:{org_id}" },
6571 { "pattern": "project:{projectId}", "column": "project_id" }
6572 ]
6573 }
6574 ]
6575 });
6576
6577 {
6578 let mut first = SyncClient::open_path_with_identity(
6579 None,
6580 &schema,
6581 ClientLimits::default(),
6582 path.to_str().expect("UTF-8 temp path"),
6583 )
6584 .expect("first open");
6585 first
6586 .subscribe(
6587 "stable-subscription".to_owned(),
6588 "tasks".to_owned(),
6589 vec![(
6590 "project_id".to_owned(),
6591 vec!["p2".to_owned(), "p1".to_owned()],
6592 )],
6593 Some(r#"{"view":"v1"}"#.to_owned()),
6594 )
6595 .expect("persist subscription");
6596 let persisted = {
6597 let subscription = first
6598 .subs
6599 .iter_mut()
6600 .find(|subscription| subscription.id == "stable-subscription")
6601 .expect("subscription");
6602 subscription.cursor = 41;
6603 subscription.bootstrap_state = Some("resume-token".to_owned());
6604 subscription.effective = Some(vec![(
6605 "project_id".to_owned(),
6606 vec!["p1".to_owned(), "p2".to_owned()],
6607 )]);
6608 subscription.synced_once = true;
6609 subscription.clone()
6610 };
6611 first.persist_sub(&persisted).unwrap();
6612 }
6613
6614 let mut reopened = SyncClient::open_path_with_identity(
6615 None,
6616 &schema,
6617 ClientLimits::default(),
6618 path.to_str().expect("UTF-8 temp path"),
6619 )
6620 .expect("reopen");
6621 let progress = reopened
6622 .subscription_state("stable-subscription")
6623 .expect("persisted state");
6624 assert_eq!(progress.cursor, 41);
6625 assert!(progress.has_resume_token);
6626
6627 reopened
6628 .subscribe(
6629 "stable-subscription".to_owned(),
6630 "tasks".to_owned(),
6631 vec![(
6632 "project_id".to_owned(),
6633 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
6634 )],
6635 Some(r#"{"view":"v1"}"#.to_owned()),
6636 )
6637 .expect("canonical intent is idempotent");
6638 assert_eq!(
6639 reopened
6640 .subscription_state("stable-subscription")
6641 .expect("unchanged state")
6642 .cursor,
6643 progress.cursor
6644 );
6645
6646 for (table, scopes, params) in [
6647 (
6648 "tasks",
6649 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6650 Some(r#"{"view":"v1"}"#.to_owned()),
6651 ),
6652 (
6653 "tasks",
6654 vec![(
6655 "project_id".to_owned(),
6656 vec!["p2".to_owned(), "p1".to_owned()],
6657 )],
6658 Some(r#"{"view":"v2"}"#.to_owned()),
6659 ),
6660 (
6661 "docs",
6662 vec![
6663 ("org_id".to_owned(), vec!["o1".to_owned()]),
6664 ("projectId".to_owned(), vec!["p1".to_owned()]),
6665 ],
6666 Some(r#"{"view":"v1"}"#.to_owned()),
6667 ),
6668 ] {
6669 let error = reopened
6670 .subscribe(
6671 "stable-subscription".to_owned(),
6672 table.to_owned(),
6673 scopes,
6674 params,
6675 )
6676 .expect_err("identity rebind must fail");
6677 assert!(error.starts_with("client.subscription_intent_mismatch:"));
6678 assert_eq!(
6679 reopened
6680 .subscription_state("stable-subscription")
6681 .expect("unchanged state")
6682 .cursor,
6683 progress.cursor
6684 );
6685 }
6686
6687 drop(reopened);
6688 std::fs::remove_file(path).expect("remove temp database");
6689 }
6690
6691 #[test]
6692 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
6693 let path = std::env::temp_dir().join(format!(
6694 "syncular-satisfied-schema-floor-{}.db",
6695 uuid::Uuid::new_v4()
6696 ));
6697 let schema = json!({
6698 "version": 23,
6699 "tables": [{
6700 "name": "tasks",
6701 "primaryKey": "id",
6702 "columns": [
6703 { "name": "id", "type": "string", "nullable": false },
6704 { "name": "project_id", "type": "string", "nullable": false }
6705 ],
6706 "scopes": [{ "pattern": "project:{project_id}" }]
6707 }]
6708 });
6709
6710 {
6711 let mut first = SyncClient::open_path_with_identity(
6712 None,
6713 &schema,
6714 ClientLimits::default(),
6715 path.to_str().expect("UTF-8 temp path"),
6716 )
6717 .expect("first open");
6718 first
6719 .subscribe(
6720 "tasks".to_owned(),
6721 "tasks".to_owned(),
6722 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6723 None,
6724 )
6725 .expect("persist subscription");
6726 first.set_schema_floor(Some(SchemaFloor {
6727 required_schema_version: Some(22),
6728 latest_schema_version: Some(22),
6729 }));
6730 }
6731
6732 let mut reopened = SyncClient::open_path_with_identity(
6733 None,
6734 &schema,
6735 ClientLimits::default(),
6736 path.to_str().expect("UTF-8 temp path"),
6737 )
6738 .expect("reopen");
6739 assert!(reopened.schema_floor().is_none());
6740 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
6741 assert!(reopened.sync_needed());
6742 assert!(matches!(
6743 reopened.drain_sync_intents().as_slice(),
6744 [SyncIntent::Interactive]
6745 ));
6746 drop(reopened);
6747 std::fs::remove_file(path).expect("remove temp database");
6748 }
6749
6750 #[test]
6751 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
6752 let path = std::env::temp_dir().join(format!(
6753 "syncular-unsatisfied-schema-floor-{}.db",
6754 uuid::Uuid::new_v4()
6755 ));
6756 let schema = json!({
6757 "version": 1,
6758 "tables": [{
6759 "name": "tasks",
6760 "primaryKey": "id",
6761 "columns": [
6762 { "name": "id", "type": "string", "nullable": false },
6763 { "name": "project_id", "type": "string", "nullable": false }
6764 ],
6765 "scopes": [{ "pattern": "project:{project_id}" }]
6766 }]
6767 });
6768
6769 {
6770 let mut first = SyncClient::open_path_with_identity(
6771 None,
6772 &schema,
6773 ClientLimits::default(),
6774 path.to_str().expect("UTF-8 temp path"),
6775 )
6776 .expect("first open");
6777 first
6778 .subscribe(
6779 "tasks".to_owned(),
6780 "tasks".to_owned(),
6781 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6782 None,
6783 )
6784 .expect("persist subscription");
6785 first.set_schema_floor(Some(SchemaFloor {
6786 required_schema_version: Some(2),
6787 latest_schema_version: Some(2),
6788 }));
6789 }
6790
6791 let reopened = SyncClient::open_path_with_identity(
6792 None,
6793 &schema,
6794 ClientLimits::default(),
6795 path.to_str().expect("UTF-8 temp path"),
6796 )
6797 .expect("reopen");
6798 assert_eq!(
6799 reopened.schema_floor(),
6800 Some(&SchemaFloor {
6801 required_schema_version: Some(2),
6802 latest_schema_version: Some(2),
6803 })
6804 );
6805 assert!(!reopened.sync_needed());
6806 drop(reopened);
6807 std::fs::remove_file(path).expect("remove temp database");
6808 }
6809
6810 #[test]
6811 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
6812 let path = std::env::temp_dir().join(format!(
6813 "syncular-indexed-column-bump-{}.db",
6814 uuid::Uuid::new_v4()
6815 ));
6816 let old_schema = json!({
6817 "version": 1,
6818 "tables": [
6819 {
6820 "name": "tasks",
6821 "primaryKey": "id",
6822 "columns": [
6823 { "name": "id", "type": "string", "nullable": false },
6824 { "name": "project_id", "type": "string", "nullable": false }
6825 ],
6826 "scopes": [{ "pattern": "project:{project_id}" }]
6827 },
6828 {
6829 "name": "legacy",
6830 "primaryKey": "id",
6831 "columns": [
6832 { "name": "id", "type": "string", "nullable": false },
6833 { "name": "project_id", "type": "string", "nullable": false }
6834 ],
6835 "scopes": [{ "pattern": "project:{project_id}" }]
6836 }
6837 ]
6838 });
6839 {
6840 let mut first = SyncClient::open_path_with_identity(
6841 None,
6842 &old_schema,
6843 ClientLimits::default(),
6844 path.to_str().expect("UTF-8 temp path"),
6845 )
6846 .expect("open old schema");
6847 first
6848 .subscribe(
6849 "legacy-sub".to_owned(),
6850 "legacy".to_owned(),
6851 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6852 None,
6853 )
6854 .expect("persist legacy subscription");
6855 }
6856
6857 let new_schema = json!({
6858 "version": 2,
6859 "tables": [{
6860 "name": "tasks",
6861 "primaryKey": "id",
6862 "columns": [
6863 { "name": "id", "type": "string", "nullable": false },
6864 { "name": "project_id", "type": "string", "nullable": false },
6865 { "name": "facility_membership_id", "type": "string", "nullable": true }
6866 ],
6867 "scopes": [{ "pattern": "project:{project_id}" }],
6868 "indexes": [{
6869 "name": "tasks_by_membership",
6870 "columns": ["project_id", "facility_membership_id"],
6871 "unique": false
6872 }]
6873 }]
6874 });
6875 let upgraded = SyncClient::open_path_with_identity(
6876 None,
6877 &new_schema,
6878 ClientLimits::default(),
6879 path.to_str().expect("UTF-8 temp path"),
6880 )
6881 .expect("open upgraded schema");
6882 let columns = upgraded
6883 .conn
6884 .prepare("PRAGMA table_info(tasks)")
6885 .expect("prepare columns")
6886 .query_map([], |row| row.get::<_, String>(1))
6887 .expect("query columns")
6888 .collect::<Result<Vec<_>, _>>()
6889 .expect("collect columns");
6890 assert!(columns.contains(&"facility_membership_id".to_owned()));
6891 let index_count: i64 = upgraded
6892 .conn
6893 .query_row(
6894 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
6895 [],
6896 |row| row.get(0),
6897 )
6898 .expect("query index");
6899 assert_eq!(index_count, 1);
6900 assert!(upgraded.subscription_state("legacy-sub").is_none());
6901 drop(upgraded);
6902 std::fs::remove_file(path).expect("remove temp database");
6903 }
6904
6905 #[test]
6906 fn migrates_pre_envelope_outcome_journal_additively() {
6907 let path = std::env::temp_dir().join(format!(
6908 "syncular-outcome-migration-{}.db",
6909 uuid::Uuid::new_v4()
6910 ));
6911 let conn = Connection::open(&path).expect("open legacy database");
6912 conn.execute_batch(
6913 "CREATE TABLE _syncular_commit_outcomes (
6914 seq INTEGER PRIMARY KEY AUTOINCREMENT,
6915 client_commit_id TEXT NOT NULL UNIQUE,
6916 status TEXT NOT NULL,
6917 recorded_at_ms INTEGER NOT NULL,
6918 results_json TEXT NOT NULL,
6919 resolution TEXT NOT NULL DEFAULT 'active',
6920 resolved_at_ms INTEGER,
6921 replacement_client_commit_id TEXT);",
6922 )
6923 .expect("create legacy outcome journal");
6924 drop(conn);
6925 let schema = json!({
6926 "version": 1,
6927 "tables": [{
6928 "name": "tasks",
6929 "primaryKey": "id",
6930 "columns": [
6931 { "name": "id", "type": "string", "nullable": false },
6932 { "name": "project_id", "type": "string", "nullable": false }
6933 ],
6934 "scopes": [{ "pattern": "project:{project_id}" }]
6935 }]
6936 });
6937 let client = SyncClient::open_path(
6938 "migration-native".to_owned(),
6939 &schema,
6940 ClientLimits::default(),
6941 path.to_str().expect("UTF-8 temp path"),
6942 )
6943 .expect("migrate database");
6944 let has_operations = client
6945 .conn
6946 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
6947 .expect("prepare table info")
6948 .query_map([], |row| row.get::<_, String>(1))
6949 .expect("query table info")
6950 .filter_map(Result::ok)
6951 .any(|column| column == "operations_json");
6952 assert!(has_operations);
6953 drop(client);
6954 std::fs::remove_file(path).expect("remove temp database");
6955 }
6956
6957 #[test]
6958 fn retained_rejection_survives_file_reopen_and_security_purge() {
6959 let path =
6960 std::env::temp_dir().join(format!("syncular-retained-{}.db", uuid::Uuid::new_v4()));
6961 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id", "columns": [
6962 { "name": "id", "type": "string", "nullable": false },
6963 { "name": "project_id", "type": "string", "nullable": false },
6964 { "name": "title", "type": "string", "nullable": false }
6965 ], "scopes": [{ "pattern": "project:{project_id}" }] }] });
6966 {
6967 let mut first = SyncClient::open_path(
6968 "retained-native".to_owned(),
6969 &schema,
6970 ClientLimits::default(),
6971 path.to_str().unwrap(),
6972 )
6973 .unwrap();
6974 first
6975 .write_base_row(
6976 "tasks",
6977 &vec![
6978 Some(ColumnValue::String("t1".into())),
6979 Some(ColumnValue::String("p1".into())),
6980 Some(ColumnValue::String("server".into())),
6981 ],
6982 1,
6983 )
6984 .unwrap();
6985 first.rebuild_overlay().expect("rebuild overlay");
6986 let operation = OutboxOp {
6987 table: "tasks".into(),
6988 row_id: "t1".into(),
6989 upsert: true,
6990 insert: false,
6991 base_version: Some(1),
6992 values: Some(Map::from_iter([
6993 ("id".into(), json!("t1")),
6994 ("title".into(), json!("mine")),
6995 ])),
6996 };
6997 first
6998 .retain_failed_commit("retained", std::slice::from_ref(&operation))
6999 .unwrap();
7000 first
7001 .persist_commit_outcome(
7002 "retained",
7003 CommitOutcomeStatus::Rejected,
7004 &[CommitOperationOutcome::Error {
7005 rejection: RejectionRecord {
7006 client_commit_id: "retained".into(),
7007 op_index: 0,
7008 code: "validation.denied".into(),
7009 message: "denied".into(),
7010 retryable: false,
7011 details: None,
7012 operation: None,
7013 },
7014 }],
7015 Some(&[operation]),
7016 )
7017 .unwrap();
7018 first.rebuild_overlay().expect("rebuild overlay");
7019 }
7020 {
7021 let mut reopened = SyncClient::open_path(
7022 "retained-native".to_owned(),
7023 &schema,
7024 ClientLimits::default(),
7025 path.to_str().unwrap(),
7026 )
7027 .unwrap();
7028 let row = reopened
7029 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
7030 .expect("read visible row")
7031 .unwrap();
7032 assert_eq!(row.0[2], Some(ColumnValue::String("mine".into())));
7033 assert_eq!(
7034 reopened
7035 .commit_outcome("retained")
7036 .unwrap()
7037 .unwrap()
7038 .retained_rows
7039 .unwrap()[0]
7040 .server_row
7041 .as_ref()
7042 .unwrap()["title"],
7043 json!("server")
7044 );
7045 reopened
7046 .conn
7047 .execute("DELETE FROM _syncular_base_tasks WHERE id = 't1'", [])
7048 .unwrap();
7049 reopened.rebuild_overlay().expect("rebuild overlay");
7050 assert!(reopened
7051 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
7052 .expect("read visible row")
7053 .is_none());
7054 let retained = reopened.commit_outcome("retained").unwrap().unwrap();
7055 assert!(retained.retained_rows.is_some());
7056 assert_eq!(
7057 retained.operations.unwrap()[0].values.as_ref().unwrap()["title"],
7058 json!("mine")
7059 );
7060 reopened
7061 .purge_local_data(&LocalDataPurgeInput {
7062 purge_id: "protected-purge".into(),
7063 targets: vec![LocalDataPurgeTarget {
7064 table: "tasks".into(),
7065 selectors: BTreeMap::from_iter([("project_id".into(), vec!["p1".into()])]),
7066 }],
7067 })
7068 .unwrap();
7069 assert!(reopened
7070 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
7071 .expect("read visible row")
7072 .is_none());
7073 assert!(reopened
7074 .commit_outcome("retained")
7075 .unwrap()
7076 .unwrap()
7077 .retained_rows
7078 .is_none());
7079 }
7080 std::fs::remove_file(path).unwrap();
7081 }
7082
7083 #[test]
7084 fn durable_conflict_outcome_and_resolution_survive_reopen() {
7085 let path = std::env::temp_dir().join(format!(
7086 "syncular-durable-outcome-{}.db",
7087 uuid::Uuid::new_v4()
7088 ));
7089 let schema = json!({
7090 "version": 1,
7091 "tables": [{
7092 "name": "tasks",
7093 "primaryKey": "id",
7094 "columns": [
7095 { "name": "id", "type": "string", "nullable": false },
7096 { "name": "project_id", "type": "string", "nullable": false }
7097 ],
7098 "scopes": [{ "pattern": "project:{project_id}" }]
7099 }]
7100 });
7101 let conflict = ConflictRecord {
7102 client_commit_id: "losing-commit".to_owned(),
7103 op_index: 0,
7104 table: "tasks".to_owned(),
7105 row_id: "t1".to_owned(),
7106 code: "sync.version_conflict".to_owned(),
7107 message: "stale base version".to_owned(),
7108 server_version: 2,
7109 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
7110 conflict_columns: vec!["project_id".to_owned()],
7111 operation: Some(CommitOperation {
7112 table: "tasks".to_owned(),
7113 row_id: "t1".to_owned(),
7114 op: "upsert".to_owned(),
7115 base_version: Some(1),
7116 values: Some(Map::from_iter([
7117 ("id".into(), json!("t1")),
7118 ("project_id".into(), json!("p1")),
7119 ])),
7120 insert: false,
7121 }),
7122 };
7123 let failed_operations = vec![
7124 OutboxOp {
7125 upsert: true,
7126 insert: false,
7127 table: "tasks".to_owned(),
7128 row_id: "t1".to_owned(),
7129 base_version: Some(1),
7130 values: Some(Map::from_iter([
7131 ("id".into(), json!("t1")),
7132 ("project_id".into(), json!("p1")),
7133 ])),
7134 },
7135 OutboxOp {
7136 upsert: true,
7137 insert: false,
7138 table: "tasks".to_owned(),
7139 row_id: "status-event-1".to_owned(),
7140 base_version: Some(0),
7141 values: Some(Map::from_iter([
7142 ("id".into(), json!("status-event-1")),
7143 ("project_id".into(), json!("p1")),
7144 ])),
7145 },
7146 ];
7147
7148 {
7149 let mut first = SyncClient::open_path(
7150 "durable-native".to_owned(),
7151 &schema,
7152 ClientLimits::default(),
7153 path.to_str().expect("UTF-8 temp path"),
7154 )
7155 .expect("first open");
7156 first
7157 .begin_observation("test_outcome")
7158 .expect("begin outcome");
7159 first
7160 .persist_commit_outcome(
7161 "losing-commit",
7162 CommitOutcomeStatus::Conflict,
7163 &[CommitOperationOutcome::Conflict {
7164 conflict: conflict.clone(),
7165 }],
7166 Some(&failed_operations),
7167 )
7168 .expect("persist outcome");
7169 first.conflicts.push(conflict);
7170 first
7171 .finish_observation(
7172 "test_outcome",
7173 ChangeAccumulator {
7174 conflicts: true,
7175 outcomes: true,
7176 ..ChangeAccumulator::default()
7177 },
7178 )
7179 .expect("commit outcome");
7180 }
7181
7182 {
7183 let mut reopened = SyncClient::open_path(
7184 "durable-native".to_owned(),
7185 &schema,
7186 ClientLimits::default(),
7187 path.to_str().expect("UTF-8 temp path"),
7188 )
7189 .expect("reopen");
7190 assert_eq!(reopened.conflicts().len(), 1);
7191 assert_eq!(
7192 reopened.conflicts()[0].conflict_columns,
7193 vec!["project_id".to_owned()],
7194 "conflictColumns survives the journal round trip"
7195 );
7196 let outcome = reopened
7197 .commit_outcome("losing-commit")
7198 .expect("read outcome")
7199 .expect("outcome");
7200 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
7201 let operations = outcome.operations.expect("aggregate envelope");
7202 assert_eq!(operations.len(), 2);
7203 assert_eq!(operations[1].row_id, "status-event-1");
7204 let resolved = reopened
7205 .resolve_commit_outcome(ResolveCommitOutcomeInput {
7206 client_commit_id: "losing-commit".to_owned(),
7207 resolution: CommitOutcomeResolution::ResolvedKeepServer,
7208 replacement_client_commit_id: None,
7209 })
7210 .expect("resolve");
7211 assert_eq!(
7212 resolved.resolution,
7213 CommitOutcomeResolution::ResolvedKeepServer
7214 );
7215 assert!(reopened.conflicts().is_empty());
7216 }
7217
7218 let reopened = SyncClient::open_path(
7219 "durable-native".to_owned(),
7220 &schema,
7221 ClientLimits::default(),
7222 path.to_str().expect("UTF-8 temp path"),
7223 )
7224 .expect("second reopen");
7225 assert!(reopened.conflicts().is_empty());
7226 assert_eq!(
7227 reopened
7228 .commit_outcome("losing-commit")
7229 .expect("read outcome")
7230 .expect("outcome")
7231 .resolution,
7232 CommitOutcomeResolution::ResolvedKeepServer
7233 );
7234 drop(reopened);
7235 std::fs::remove_file(path).expect("remove temp database");
7236 }
7237
7238 #[test]
7239 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
7240 let path =
7241 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7242 let schema = json!({
7243 "version": 1,
7244 "tables": [{
7245 "name": "tasks",
7246 "primaryKey": "id",
7247 "columns": [
7248 { "name": "id", "type": "string", "nullable": false },
7249 { "name": "project_id", "type": "string", "nullable": false }
7250 ],
7251 "scopes": [{ "pattern": "project:{project_id}" }]
7252 }]
7253 });
7254 let mut client = SyncClient::open_path_with_identity(
7255 Some("sidecar-client".to_owned()),
7256 &schema,
7257 ClientLimits::default(),
7258 path.to_str().expect("UTF-8 temp path"),
7259 )
7260 .expect("open owner");
7261 let base = WindowBase {
7262 table: "tasks".to_owned(),
7263 variable: "project_id".to_owned(),
7264 fixed_scopes: Vec::new(),
7265 params: None,
7266 };
7267 client
7268 .set_window(&base, &["one".to_owned()])
7269 .expect("set window");
7270 client
7271 .mutate(vec![Mutation::Upsert {
7272 table: "tasks".to_owned(),
7273 values: Map::from_iter([
7274 ("id".to_owned(), Value::from("t1")),
7275 ("project_id".to_owned(), Value::from("one")),
7276 ]),
7277 base_version: None,
7278 }])
7279 .expect("local mutate");
7280
7281 let coverage = [WindowCoverage {
7282 base,
7283 units: vec!["one".to_owned(), "missing".to_owned()],
7284 }];
7285 let owner = client
7286 .query_snapshot(
7287 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
7288 &[],
7289 &coverage,
7290 None,
7291 )
7292 .expect("owner snapshot");
7293 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7294 let sidecar = reader
7295 .query_snapshot(
7296 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
7297 &[],
7298 &coverage,
7299 )
7300 .expect("sidecar snapshot");
7301
7302 assert_eq!(sidecar.revision, owner.revision);
7303 assert_eq!(sidecar.rows, owner.rows);
7304 assert_eq!(
7305 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
7306 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
7307 );
7308 assert_eq!(sidecar.revision, "2");
7309 assert_eq!(sidecar.rows[0]["id"], "t1");
7310 assert_eq!(sidecar.rows[0]["server_version"], -1);
7311 assert!(!sidecar.coverage.complete);
7312 assert_eq!(sidecar.coverage.pending.len(), 1);
7313 assert_eq!(sidecar.coverage.missing.len(), 1);
7314
7315 drop(reader);
7316 drop(client);
7317 std::fs::remove_file(path).expect("remove temp database");
7318 }
7319
7320 #[test]
7321 fn file_snapshot_reader_reports_catchup_delivery_and_multi_statement() {
7322 let path =
7323 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7324 let schema = json!({
7325 "version": 1,
7326 "tables": [{
7327 "name": "tasks",
7328 "primaryKey": "id",
7329 "columns": [
7330 { "name": "id", "type": "string", "nullable": false },
7331 { "name": "project_id", "type": "string", "nullable": false }
7332 ],
7333 "scopes": [{ "pattern": "project:{project_id}" }]
7334 }]
7335 });
7336 let mut client = SyncClient::open_path_with_identity(
7337 Some("sidecar-catchup".to_owned()),
7338 &schema,
7339 ClientLimits::default(),
7340 path.to_str().expect("UTF-8 temp path"),
7341 )
7342 .expect("open owner");
7343 client
7344 .subscribe(
7345 "s1".into(),
7346 "tasks".into(),
7347 vec![("project_id".into(), vec!["one".into()])],
7348 None,
7349 )
7350 .expect("subscribe");
7351 let commit_id = client
7352 .mutate(vec![Mutation::Upsert {
7353 table: "tasks".to_owned(),
7354 values: Map::from_iter([
7355 ("id".to_owned(), Value::from("t1")),
7356 ("project_id".to_owned(), Value::from("one")),
7357 ]),
7358 base_version: None,
7359 }])
7360 .expect("mutate");
7361 let revision = client.local_revision().to_string();
7362
7363 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7364 assert_eq!(reader.local_revision().expect("revision"), revision);
7365
7366 let read = reader
7367 .snapshot_read(&SnapshotReadRequest {
7368 statements: vec![
7369 SnapshotStatement {
7370 sql: "SELECT 1 AS n",
7371 params: &[],
7372 },
7373 SnapshotStatement {
7374 sql: "SELECT id FROM tasks ORDER BY id",
7375 params: &[],
7376 },
7377 ],
7378 coverage: &[],
7379 subscriptions: vec!["s1".to_owned(), "absent".to_owned()],
7380 commit_ids: vec![commit_id.clone(), "never-known".to_owned()],
7381 })
7382 .expect("snapshot read");
7383 assert_eq!(read.revision, revision);
7384 assert_eq!(read.queries.len(), 2);
7385 assert_eq!(read.queries[0][0]["n"], 1);
7386 assert_eq!(read.queries[1][0]["id"], "t1");
7387 match &read.subscriptions[0] {
7388 SubscriptionCatchup::Known {
7389 status,
7390 cursor,
7391 bootstrap_complete,
7392 known_pending_pages,
7393 ..
7394 } => {
7395 assert_eq!(status, "active");
7396 assert_eq!(*cursor, -1);
7397 assert!(!bootstrap_complete);
7398 assert!(known_pending_pages);
7399 }
7400 other => panic!("expected a known subscription: {other:?}"),
7401 }
7402 assert!(matches!(
7403 &read.subscriptions[1],
7404 SubscriptionCatchup::Unknown { id } if id.as_str() == "absent"
7405 ));
7406 assert!(matches!(
7407 &read.deliveries[0],
7408 CommitDelivery::Pending { client_commit_id } if client_commit_id == &commit_id
7409 ));
7410 assert!(matches!(
7411 &read.deliveries[1],
7412 CommitDelivery::Unknown { client_commit_id } if client_commit_id.as_str() == "never-known"
7413 ));
7414
7415 assert!(reader
7417 .query_snapshot("SELECT * FROM does_not_exist", &[], &[])
7418 .is_err());
7419 assert_eq!(
7420 reader.local_revision().expect("revision after rollback"),
7421 revision
7422 );
7423 assert!(reader
7425 .query_snapshot("DELETE FROM tasks", &[], &[])
7426 .is_err());
7427
7428 client
7430 .conn
7431 .execute(
7432 "UPDATE _syncular_meta SET value = '+1' WHERE key = 'localRevision'",
7433 [],
7434 )
7435 .expect("corrupt revision");
7436 let failure = reader.local_revision().expect_err("corrupt revision fails");
7437 assert_eq!(failure.code, Some("sync.local_corrupt"));
7438
7439 drop(reader);
7440 drop(client);
7441 std::fs::remove_file(path).expect("remove temp database");
7442 }
7443
7444 #[test]
7445 fn snapshot_delivery_is_the_persisted_journal_view_without_retained_rows() {
7446 let path =
7447 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7448 let schema = json!({
7449 "version": 1,
7450 "tables": [{
7451 "name": "tasks",
7452 "primaryKey": "id",
7453 "columns": [
7454 { "name": "id", "type": "string", "nullable": false },
7455 { "name": "project_id", "type": "string", "nullable": false }
7456 ],
7457 "scopes": [{ "pattern": "project:{project_id}" }]
7458 }]
7459 });
7460 let mut client = SyncClient::open_path_with_identity(
7461 Some("sidecar-delivery".to_owned()),
7462 &schema,
7463 ClientLimits::default(),
7464 path.to_str().expect("UTF-8 temp path"),
7465 )
7466 .expect("open owner");
7467 client.begin_observation("test_delivery").expect("observe");
7468 let operations = [OutboxOp {
7469 upsert: true,
7470 insert: false,
7471 table: "tasks".to_owned(),
7472 row_id: "t1".to_owned(),
7473 base_version: Some(1),
7474 values: Some(Map::from_iter([
7475 ("id".into(), json!("t1")),
7476 ("project_id".into(), json!("p1")),
7477 ])),
7478 }];
7479 client
7480 .persist_commit_outcome(
7481 "conflict-commit",
7482 CommitOutcomeStatus::Conflict,
7483 &[CommitOperationOutcome::Conflict {
7484 conflict: ConflictRecord {
7485 client_commit_id: "conflict-commit".to_owned(),
7486 op_index: 0,
7487 table: "tasks".to_owned(),
7488 row_id: "t1".to_owned(),
7489 code: "sync.version_conflict".to_owned(),
7490 message: "stale base version".to_owned(),
7491 server_version: 2,
7492 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
7493 conflict_columns: vec!["project_id".to_owned()],
7494 operation: None,
7495 },
7496 }],
7497 Some(&operations),
7498 )
7499 .expect("persist conflict");
7500 client
7501 .persist_commit_outcome(
7502 "rejected-commit",
7503 CommitOutcomeStatus::Rejected,
7504 &[CommitOperationOutcome::Error {
7505 rejection: RejectionRecord {
7506 client_commit_id: "rejected-commit".to_owned(),
7507 op_index: 0,
7508 code: "sync.unknown_table".to_owned(),
7509 message: "unknown table".to_owned(),
7510 retryable: false,
7511 details: None,
7512 operation: None,
7513 },
7514 }],
7515 Some(&operations),
7516 )
7517 .expect("persist rejection");
7518 client
7519 .finish_observation(
7520 "test_delivery",
7521 ChangeAccumulator {
7522 conflicts: true,
7523 outcomes: true,
7524 ..ChangeAccumulator::default()
7525 },
7526 )
7527 .expect("commit outcomes");
7528
7529 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7530 match reader.commit_delivery("conflict-commit").expect("conflict") {
7531 CommitDelivery::Known { outcome, .. } => {
7532 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
7533 assert!(matches!(
7534 outcome.results.as_slice(),
7535 [CommitOperationOutcome::Conflict { conflict }]
7536 if conflict.code == "sync.version_conflict"
7537 && conflict.server_version == 2
7538 ));
7539 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
7540 assert!(
7541 outcome.retained_rows.is_none(),
7542 "the sidecar reports the persisted view, not owner-derived retained rows"
7543 );
7544 }
7545 other => panic!("expected a known conflict: {other:?}"),
7546 }
7547 match reader
7548 .commit_delivery("rejected-commit")
7549 .expect("rejection")
7550 {
7551 CommitDelivery::Known { outcome, .. } => {
7552 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
7553 assert!(matches!(
7554 outcome.results.as_slice(),
7555 [CommitOperationOutcome::Error { rejection }]
7556 if rejection.code == "sync.unknown_table"
7557 ));
7558 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
7559 assert!(outcome.retained_rows.is_none());
7560 }
7561 other => panic!("expected a known rejection: {other:?}"),
7562 }
7563 drop(reader);
7564 drop(client);
7565 std::fs::remove_file(path).expect("remove temp database");
7566 }
7567
7568 #[test]
7569 fn snapshot_read_isolates_a_concurrent_writer_commit() {
7570 use std::sync::mpsc::channel;
7571 let path = std::env::temp_dir().join(format!(
7572 "syncular-read-concurrent-{}.db",
7573 uuid::Uuid::new_v4()
7574 ));
7575 let schema = json!({
7576 "version": 1,
7577 "tables": [{
7578 "name": "tasks",
7579 "primaryKey": "id",
7580 "columns": [
7581 { "name": "id", "type": "string", "nullable": false },
7582 { "name": "project_id", "type": "string", "nullable": false }
7583 ],
7584 "scopes": [{ "pattern": "project:{project_id}" }]
7585 }]
7586 });
7587 let mut client = SyncClient::open_path_with_identity(
7588 Some("sidecar-concurrent".to_owned()),
7589 &schema,
7590 ClientLimits::default(),
7591 path.to_str().expect("UTF-8 temp path"),
7592 )
7593 .expect("open owner");
7594 client
7595 .mutate(vec![Mutation::Upsert {
7596 table: "tasks".to_owned(),
7597 values: Map::from_iter([
7598 ("id".to_owned(), Value::from("t1")),
7599 ("project_id".to_owned(), Value::from("one")),
7600 ]),
7601 base_version: None,
7602 }])
7603 .expect("seed row");
7604 let revision_before = client.local_revision().to_string();
7605
7606 let (ready_tx, ready_rx) = channel::<()>();
7609 let (done_tx, done_rx) = channel::<()>();
7610 let writer_path = path.to_string_lossy().into_owned();
7611 let writer_schema = schema.clone();
7612 let writer = std::thread::spawn(move || {
7613 let mut writer = SyncClient::open_path_with_identity(
7614 Some("sidecar-concurrent".to_owned()),
7615 &writer_schema,
7616 ClientLimits::default(),
7617 &writer_path,
7618 )
7619 .expect("open writer");
7620 ready_rx.recv().expect("reader signalled");
7621 writer
7622 .mutate(vec![Mutation::Upsert {
7623 table: "tasks".to_owned(),
7624 values: Map::from_iter([
7625 ("id".to_owned(), Value::from("t2")),
7626 ("project_id".to_owned(), Value::from("one")),
7627 ]),
7628 base_version: None,
7629 }])
7630 .expect("writer commit");
7631 let _ = done_tx.send(());
7632 });
7633 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7634 SNAPSHOT_READ_HOOK.with(|slot| {
7635 *slot.borrow_mut() = Some(Box::new(move || {
7636 ready_tx.send(()).expect("signal writer");
7637 done_rx.recv().expect("writer finished");
7638 }));
7639 });
7640 let read = reader
7641 .snapshot_read(&SnapshotReadRequest {
7642 statements: vec![
7643 SnapshotStatement {
7644 sql: "SELECT count(*) AS n FROM tasks",
7645 params: &[],
7646 },
7647 SnapshotStatement {
7648 sql: "SELECT count(*) AS n FROM tasks",
7649 params: &[],
7650 },
7651 ],
7652 coverage: &[],
7653 subscriptions: Vec::new(),
7654 commit_ids: Vec::new(),
7655 })
7656 .expect("snapshot read");
7657 assert_eq!(
7658 read.queries[0][0]["n"], 1,
7659 "first statement sees the pre-commit row"
7660 );
7661 assert_eq!(
7662 read.queries[1][0]["n"], 1,
7663 "a later statement keeps the one read snapshot"
7664 );
7665 assert_eq!(
7666 read.revision, revision_before,
7667 "the read reports the snapshot revision, not the writer's"
7668 );
7669 writer.join().expect("join writer");
7670
7671 assert_ne!(
7672 reader.local_revision().expect("revision after commit"),
7673 revision_before,
7674 "the next read observes the committed revision"
7675 );
7676 let after = reader
7677 .query_snapshot("SELECT count(*) AS n FROM tasks", &[], &[])
7678 .expect("count after commit");
7679 assert_eq!(
7680 after.rows[0]["n"], 2,
7681 "the committed row is visible afterwards"
7682 );
7683
7684 drop(reader);
7685 drop(client);
7686 std::fs::remove_file(path).expect("remove temp database");
7687 }
7688
7689 #[test]
7690 fn snapshot_read_rejects_every_non_canonical_revision() {
7691 let path = std::env::temp_dir().join(format!(
7692 "syncular-read-revision-{}.db",
7693 uuid::Uuid::new_v4()
7694 ));
7695 let schema = json!({
7696 "version": 1,
7697 "tables": [{
7698 "name": "tasks",
7699 "primaryKey": "id",
7700 "columns": [
7701 { "name": "id", "type": "string", "nullable": false },
7702 { "name": "project_id", "type": "string", "nullable": false }
7703 ],
7704 "scopes": [{ "pattern": "project:{project_id}" }]
7705 }]
7706 });
7707 let client = SyncClient::open_path_with_identity(
7708 Some("sidecar-revision".to_owned()),
7709 &schema,
7710 ClientLimits::default(),
7711 path.to_str().expect("UTF-8 temp path"),
7712 )
7713 .expect("open owner");
7714 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7715 for bad in [
7716 "+1",
7717 "01",
7718 "-1",
7719 " 1",
7720 "1 ",
7721 "18446744073709551616",
7722 "1.0",
7723 "abc",
7724 "",
7725 ] {
7726 client
7727 .conn
7728 .execute(
7729 "UPDATE _syncular_meta SET value = ?1 WHERE key = 'localRevision'",
7730 rusqlite::params![bad],
7731 )
7732 .expect("corrupt revision");
7733 let failure = reader.local_revision().expect_err("corrupt marker fails");
7734 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad:?}");
7735 }
7736 for value in ["X'31'", "CAST(X'FF' AS TEXT)"] {
7737 client
7738 .conn
7739 .execute_batch(&format!(
7740 "UPDATE _syncular_meta SET value={value} WHERE key='localRevision'"
7741 ))
7742 .expect("corrupt stored revision type");
7743 assert_eq!(
7744 reader.local_revision().expect_err("invalid SQL field").code,
7745 Some("sync.local_corrupt")
7746 );
7747 }
7748 client
7750 .conn
7751 .execute("DELETE FROM _syncular_meta WHERE key = 'localRevision'", [])
7752 .expect("delete marker");
7753 assert_eq!(reader.local_revision().expect("legacy zero"), "0");
7754
7755 drop(reader);
7756 drop(client);
7757 std::fs::remove_file(path).expect("remove temp database");
7758 }
7759
7760 #[test]
7761 fn snapshot_read_rejects_corrupt_subscription_state() {
7762 let path = std::env::temp_dir().join(format!(
7763 "syncular-read-substate-{}.db",
7764 uuid::Uuid::new_v4()
7765 ));
7766 let schema = json!({
7767 "version": 1,
7768 "tables": [{
7769 "name": "tasks",
7770 "primaryKey": "id",
7771 "columns": [
7772 { "name": "id", "type": "string", "nullable": false },
7773 { "name": "project_id", "type": "string", "nullable": false }
7774 ],
7775 "scopes": [{ "pattern": "project:{project_id}" }]
7776 }]
7777 });
7778 let mut client = SyncClient::open_path_with_identity(
7779 Some("sidecar-substate".to_owned()),
7780 &schema,
7781 ClientLimits::default(),
7782 path.to_str().expect("UTF-8 temp path"),
7783 )
7784 .expect("open owner");
7785 client
7786 .subscribe(
7787 "s1".into(),
7788 "tasks".into(),
7789 vec![("project_id".into(), vec!["one".into()])],
7790 None,
7791 )
7792 .expect("subscribe");
7793 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7794 for bad in [
7795 "not json".to_owned(),
7796 "{}".to_owned(),
7797 json!({ "status": "bogus", "cursor": 0 }).to_string(),
7798 json!({ "status": "active", "cursor": "x" }).to_string(),
7799 json!({ "status": "active", "cursor": 0, "bootstrapState": 7 }).to_string(),
7800 json!({ "status": "active", "cursor": 0, "reasonCode": 7 }).to_string(),
7801 ] {
7802 client
7803 .conn
7804 .execute(
7805 "UPDATE _syncular_subscriptions SET state_json = ?1 WHERE id = 's1'",
7806 rusqlite::params![bad],
7807 )
7808 .expect("corrupt subscription state");
7809 let failure = reader
7810 .subscription_catchup("s1")
7811 .expect_err("corrupt state fails");
7812 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad}");
7813 }
7814
7815 for assignment in [
7816 "tbl=X'7461736b73'",
7817 "state_json=X'7b7d'",
7818 "state_json=CAST(X'FF' AS TEXT)",
7819 ] {
7820 client.conn.execute_batch(&format!(
7821 "UPDATE _syncular_subscriptions SET tbl='tasks',state_json='{{\"status\":\"active\",\"cursor\":0}}'; UPDATE _syncular_subscriptions SET {assignment} WHERE id='s1'"
7822 )).expect("corrupt stored subscription type");
7823 assert_eq!(
7824 reader
7825 .subscription_catchup("s1")
7826 .expect_err("invalid SQL field")
7827 .code,
7828 Some("sync.local_corrupt")
7829 );
7830 }
7831
7832 drop(reader);
7833 drop(client);
7834 std::fs::remove_file(path).expect("remove temp database");
7835 }
7836
7837 #[test]
7838 fn snapshot_read_rolls_back_a_later_invalid_statement_and_guards_every_write() {
7839 let path = std::env::temp_dir().join(format!(
7840 "syncular-read-rollback-{}.db",
7841 uuid::Uuid::new_v4()
7842 ));
7843 let schema = json!({
7844 "version": 1,
7845 "tables": [{
7846 "name": "tasks",
7847 "primaryKey": "id",
7848 "columns": [
7849 { "name": "id", "type": "string", "nullable": false },
7850 { "name": "project_id", "type": "string", "nullable": false }
7851 ],
7852 "scopes": [{ "pattern": "project:{project_id}" }]
7853 }]
7854 });
7855 let mut client = SyncClient::open_path_with_identity(
7856 Some("sidecar-rollback".to_owned()),
7857 &schema,
7858 ClientLimits::default(),
7859 path.to_str().expect("UTF-8 temp path"),
7860 )
7861 .expect("open owner");
7862 let commit_id = client
7863 .mutate(vec![Mutation::Upsert {
7864 table: "tasks".to_owned(),
7865 values: Map::from_iter([
7866 ("id".to_owned(), Value::from("t1")),
7867 ("project_id".to_owned(), Value::from("one")),
7868 ]),
7869 base_version: None,
7870 }])
7871 .expect("seed row");
7872 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7873 let revision = reader.local_revision().expect("revision");
7874
7875 let failure = reader
7878 .snapshot_read(&SnapshotReadRequest {
7879 statements: vec![
7880 SnapshotStatement {
7881 sql: "SELECT id FROM tasks",
7882 params: &[],
7883 },
7884 SnapshotStatement {
7885 sql: "SELECT * FROM does_not_exist",
7886 params: &[],
7887 },
7888 ],
7889 coverage: &[],
7890 subscriptions: Vec::new(),
7891 commit_ids: vec![commit_id.clone()],
7892 })
7893 .expect_err("second statement fails");
7894 assert_eq!(failure.code, None);
7895 assert_eq!(
7896 reader.local_revision().expect("revision after rollback"),
7897 revision,
7898 "a rolled-back read consumes no revision"
7899 );
7900 assert!(
7901 !matches!(
7902 reader
7903 .commit_delivery(&commit_id)
7904 .expect("delivery after rollback"),
7905 CommitDelivery::Unknown { .. }
7906 ),
7907 "the same reader serves the next read after releasing the connection"
7908 );
7909
7910 let guard = reader
7912 .snapshot_read(&SnapshotReadRequest {
7913 statements: vec![SnapshotStatement {
7914 sql: "DELETE FROM tasks",
7915 params: &[],
7916 }],
7917 coverage: &[],
7918 subscriptions: Vec::new(),
7919 commit_ids: Vec::new(),
7920 })
7921 .expect_err("write guard");
7922 assert!(guard.message.contains("read-only"), "{}", guard.message);
7923
7924 drop(reader);
7925 drop(client);
7926 std::fs::remove_file(path).expect("remove temp database");
7927 }
7928
7929 #[test]
7930 fn owned_query_failures_classify_order_bound_and_clear() {
7931 let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7932 rusqlite::ffi::Error::new(266),
7933 Some("private SQLite prose".to_owned()),
7934 ));
7935 assert_eq!(io.code, Some("client.storage_io"));
7936 assert_eq!(io.sqlite_code, Some(266));
7937 assert_eq!(io.message, "local SQLite storage I/O failed");
7938 let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7939 rusqlite::ffi::Error::new(26),
7940 None,
7941 ));
7942 assert_eq!(corrupt.code, Some("client.storage_corrupt"));
7943 assert!(!io.retryable());
7944 assert!(!corrupt.retryable());
7945
7946 for (raw, extended) in [(5, 5), (5 | (1 << 8), 261), (6, 6), (6 | (1 << 8), 262)] {
7949 let busy = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7950 rusqlite::ffi::Error::new(raw),
7951 Some("database is locked".to_owned()),
7952 ));
7953 assert_eq!(busy.code, Some("client.storage_busy"));
7954 assert_eq!(busy.sqlite_code, Some(extended));
7955 assert_eq!(busy.message, "local SQLite storage is busy");
7956 assert!(busy.retryable());
7957 }
7958 let unrelated = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7959 rusqlite::ffi::Error::new(1),
7960 Some("generic SQLite prose".to_owned()),
7961 ));
7962 assert_eq!(unrelated.code, None);
7963 assert!(!unrelated.retryable());
7964
7965 let mut client = client();
7966 let generic = QueryReadFailure::from("private query prose".to_owned());
7967 client.record_query_read(
7968 QueryOwner {
7969 id: "first",
7970 tables: &["tasks", "tasks"],
7971 },
7972 Some(&generic),
7973 );
7974 client.record_query_read(
7975 QueryOwner {
7976 id: "second",
7977 tables: &["tasks"],
7978 },
7979 Some(&generic),
7980 );
7981 client.record_query_read(
7982 QueryOwner {
7983 id: "first",
7984 tables: &["docs"],
7985 },
7986 Some(&io),
7987 );
7988 let failures = &client
7989 .diagnostics_snapshot(&Default::default())
7990 .expect("diagnostics")
7991 .query_failures;
7992 assert_eq!(
7993 failures
7994 .iter()
7995 .map(|entry| entry.id.as_str())
7996 .collect::<Vec<_>>(),
7997 vec!["second", "first"]
7998 );
7999 assert_eq!(failures[1].tables, vec!["docs"]);
8000 assert_eq!(failures[1].code, "client.storage_io");
8001
8002 for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
8003 client.record_query_read(
8004 QueryOwner {
8005 id: &format!("query-{index}"),
8006 tables: &["tasks"],
8007 },
8008 Some(&generic),
8009 );
8010 }
8011 let failures = &client
8012 .diagnostics_snapshot(&Default::default())
8013 .expect("bounded diagnostics")
8014 .query_failures;
8015 assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
8016 assert_eq!(failures.last().expect("last failure").id, "query-256");
8017
8018 client
8019 .query_snapshot(
8020 "SELECT id FROM tasks",
8021 &[],
8022 &[],
8023 Some(QueryOwner {
8024 id: "query-256",
8025 tables: &["tasks"],
8026 }),
8027 )
8028 .expect("successful owned read");
8029 assert!(client
8030 .diagnostics_snapshot(&Default::default())
8031 .expect("cleared diagnostics")
8032 .query_failures
8033 .iter()
8034 .all(|entry| entry.id != "query-256"));
8035 }
8036
8037 #[test]
8038 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
8039 let mut client = client();
8040 let image = Connection::open_in_memory().unwrap();
8041 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
8042 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
8043 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
8044 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8045 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
8046 let table = client.schema.table("tasks").unwrap().clone();
8047 let scopes = vec![("project_id".into(), vec!["p1".into()])];
8048 assert!(client
8049 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
8050 .is_err());
8051 assert!(client.conn.is_autocommit());
8052 assert_eq!(
8053 client
8054 .query("SELECT count(*) AS n FROM tasks", &[])
8055 .unwrap()[0]["n"],
8056 1024
8057 );
8058 assert_eq!(client.drain_change_batches().len(), 1);
8059 image
8060 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
8061 .unwrap();
8062 assert!(matches!(
8063 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
8064 Ok(2050)
8065 ));
8066 assert_eq!(
8067 client
8068 .query("SELECT count(*) AS n FROM tasks", &[])
8069 .unwrap()[0]["n"],
8070 2050
8071 );
8072 assert_eq!(client.drain_change_batches().len(), 3);
8073 }
8074
8075 #[test]
8076 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
8077 let mut client = client();
8078 let base = WindowBase {
8079 table: "tasks".into(),
8080 variable: "project_id".into(),
8081 fixed_scopes: Vec::new(),
8082 params: None,
8083 };
8084 client
8085 .set_window(&base, &["p1".into(), "p2".into()])
8086 .unwrap();
8087 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
8088 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
8089 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
8090 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
8091 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
8092 client.drain_change_batches();
8093 assert!(client.set_window(&base, &["p2".into()]).is_err());
8094 assert!(client.conn.is_autocommit());
8095 assert_eq!(
8096 client
8097 .query("SELECT count(*) AS n FROM tasks", &[])
8098 .unwrap()[0]["n"],
8099 1027
8100 );
8101 assert_eq!(client.window_state(&base).units, vec!["p2"]);
8102 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
8103 assert_eq!(client.drain_change_batches().len(), 1);
8104 client
8105 .conn
8106 .execute_batch("DROP TRIGGER interrupt_evict")
8107 .unwrap();
8108 client.drain_pending_evictions().unwrap();
8109 assert!(client.load_pending_evictions().unwrap().is_empty());
8110 assert_eq!(
8111 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
8112 "held"
8113 );
8114 assert_eq!(client.drain_change_batches().len(), 2);
8115 }
8116
8117 #[test]
8118 fn eviction_marker_failures_roll_back_window_changes() {
8119 let mut client = client();
8120 let base = WindowBase {
8121 table: "tasks".into(),
8122 variable: "project_id".into(),
8123 fixed_scopes: Vec::new(),
8124 params: None,
8125 };
8126 client.set_window(&base, &["p1".into()]).unwrap();
8127 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
8128 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
8129 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
8130 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
8131 let revision = client.local_revision();
8132 assert!(client.set_window(&base, &[]).is_err());
8133 assert!(client.conn.is_autocommit());
8134 assert_eq!(client.local_revision(), revision);
8135 assert_eq!(
8136 client
8137 .query("SELECT count(*) AS n FROM tasks", &[])
8138 .unwrap()[0]["n"],
8139 2050
8140 );
8141 assert_eq!(client.window_state(&base).units, vec!["p1"]);
8142 client.conn.execute_batch("DROP TRIGGER fail_marker;
8143 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
8144 assert!(client.set_window(&base, &[]).is_err());
8145 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
8146 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
8147 assert!(client.drain_pending_evictions().is_err());
8148 assert!(client.conn.is_autocommit());
8149 assert_eq!(
8150 client
8151 .query("SELECT count(*) AS n FROM tasks", &[])
8152 .unwrap()[0]["n"],
8153 2
8154 );
8155 assert!(client.set_window(&base, &["p1".into()]).is_err());
8156 assert!(client.conn.is_autocommit());
8157 assert!(client.window_state(&base).units.is_empty());
8158 assert!(client.subs.is_empty());
8159 assert_eq!(
8160 client
8161 .query(
8162 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
8163 &[]
8164 )
8165 .unwrap()[0]["n"],
8166 1
8167 );
8168 client
8169 .conn
8170 .execute_batch("DROP TRIGGER fail_marker")
8171 .unwrap();
8172 client.set_window(&base, &["p1".into()]).unwrap();
8173 assert_eq!(client.window_state(&base).units, vec!["p1"]);
8174 }
8175
8176 #[test]
8177 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
8178 let mut client = client();
8179 client
8180 .conn
8181 .execute(
8182 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
8183 [],
8184 )
8185 .unwrap();
8186 assert_eq!(
8187 client.drain_pending_evictions().unwrap_err(),
8188 "sync.local_corrupt: invalid pending eviction scopes"
8189 );
8190 assert!(client.conn.is_autocommit());
8191 assert!(client.enqueue_startup_sync_if_needed().is_err());
8192 }
8193
8194 #[test]
8195 fn image_chunks_reject_null_primary_keys() {
8196 let mut client = client();
8197 let image = Connection::open_in_memory().unwrap();
8198 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8199 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
8200 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
8201 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8202 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
8203 let table = client.schema.table("tasks").unwrap().clone();
8204 assert!(client
8205 .apply_sqlite_image(
8206 &image,
8207 &table,
8208 true,
8209 &[("project_id".into(), vec!["p1".into()])],
8210 1025,
8211 7,
8212 "digest"
8213 )
8214 .is_err());
8215 assert!(client.conn.is_autocommit());
8216 assert_eq!(
8217 client
8218 .query("SELECT count(*) AS n FROM tasks", &[])
8219 .unwrap()[0]["n"],
8220 0
8221 );
8222 }
8223
8224 #[test]
8225 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
8226 for images in [false, true] {
8227 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
8228 "name":name,"primaryKey":"id","columns":[
8229 {"name":"id","type":"string","nullable":false},
8230 {"name":"project_id","type":"string","nullable":false},
8231 {"name":"title","type":"string","nullable":false}],
8232 "scopes":[{"pattern":"project:{project_id}"}],
8233 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
8234 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
8235 })))});
8236 let mut actual =
8237 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
8238 let mut reference =
8239 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
8240 let scopes = vec![("project_id".into(), vec!["p1".into()])];
8241 for client in [&mut actual, &mut reference] {
8242 client
8243 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
8244 .unwrap();
8245 client.subs[0].effective = Some(scopes.clone());
8246 for (table, id, title) in [
8247 ("tasks", "a", "original"),
8248 ("tasks", "b", "occupied"),
8249 ("tasks", "c", "deleted"),
8250 ("notes", "n", "untouched"),
8251 ] {
8252 client
8253 .write_base_row(
8254 table,
8255 &vec![
8256 Some(ColumnValue::String(id.into())),
8257 Some(ColumnValue::String("p1".into())),
8258 Some(ColumnValue::String(title.into())),
8259 ],
8260 1,
8261 )
8262 .unwrap();
8263 }
8264 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
8265 client
8266 .mutate(vec![
8267 Mutation::Upsert {
8268 table: "tasks".into(),
8269 values: Map::from_iter([
8270 ("id".into(), Value::from("a")),
8271 ("project_id".into(), Value::from("p1")),
8272 ("title".into(), Value::from("available")),
8273 ]),
8274 base_version: None,
8275 },
8276 Mutation::Delete {
8277 table: "tasks".into(),
8278 row_id: "c".into(),
8279 base_version: None,
8280 },
8281 Mutation::Upsert {
8282 table: "tasks".into(),
8283 values: Map::from_iter([
8284 ("id".into(), Value::from("local")),
8285 ("project_id".into(), Value::from("p1")),
8286 ("title".into(), Value::from("local value")),
8287 ]),
8288 base_version: None,
8289 },
8290 Mutation::Upsert {
8291 table: "notes".into(),
8292 values: Map::from_iter([
8293 ("id".into(), Value::from("n")),
8294 ("project_id".into(), Value::from("p1")),
8295 ("title".into(), Value::from("pending note")),
8296 ]),
8297 base_version: None,
8298 },
8299 ])
8300 .unwrap();
8301 }
8302 let pending = actual.pending_commit_ids();
8303 actual.overlay_rebuild_count.set(0);
8304 for (id, title) in [
8305 ("b", "available"),
8306 ("b", "occupied"),
8307 ("a", "server original"),
8308 ("d", "local value"),
8309 ("d", "released"),
8310 ("c", "redelivered"),
8311 ] {
8312 let table = actual.schema.table("tasks").unwrap().clone();
8313 let values = vec![
8314 Some(ColumnValue::String(id.into())),
8315 Some(ColumnValue::String("p1".into())),
8316 Some(ColumnValue::String(title.into())),
8317 ];
8318 if images {
8319 let image = Connection::open_in_memory().unwrap();
8320 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8321 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8322 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
8323 image
8324 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
8325 .unwrap();
8326 assert!(matches!(
8327 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
8328 Ok(1)
8329 ));
8330 } else {
8331 let segment = RowsSegment {
8332 table: "tasks".into(),
8333 schema_version: 1,
8334 columns: table.wire_columns.clone(),
8335 blocks: vec![vec![ssp2::segment::SegmentRow {
8336 server_version: 2,
8337 values: values.clone(),
8338 }]],
8339 };
8340 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
8341 }
8342 reference.write_base_row("tasks", &values, 2).unwrap();
8343 reference.rebuild_overlay().expect("rebuild overlay");
8344 for table in ["tasks", "notes"] {
8345 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
8346 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
8347 }
8348 }
8349 assert_eq!(actual.pending_commit_ids(), pending);
8350 }
8351 assert_eq!(
8352 actual.overlay_rebuild_count.get(),
8353 0,
8354 "each chunk must leave unrelated tables alone"
8355 );
8356 let before = actual
8357 .query("SELECT * FROM tasks ORDER BY id", &[])
8358 .unwrap();
8359 let fts_before = actual
8360 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
8361 .unwrap();
8362 let revision = actual.local_revision();
8363 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
8364 let table = actual.schema.table("tasks").unwrap().clone();
8365 let segment = RowsSegment {
8366 table: "tasks".into(),
8367 schema_version: 1,
8368 columns: table.wire_columns.clone(),
8369 blocks: vec![vec![ssp2::segment::SegmentRow {
8370 server_version: 3,
8371 values: vec![
8372 Some(ColumnValue::String("b".into())),
8373 Some(ColumnValue::String("p1".into())),
8374 Some(ColumnValue::String("fresh".into())),
8375 ],
8376 }]],
8377 };
8378 assert!(actual.apply_segment(0, &segment, false).is_err());
8379 assert!(actual.conn.is_autocommit());
8380 assert_eq!(
8381 actual
8382 .query("SELECT * FROM tasks ORDER BY id", &[])
8383 .unwrap(),
8384 before
8385 );
8386 assert_eq!(
8387 actual
8388 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
8389 .unwrap(),
8390 fts_before
8391 );
8392 assert_eq!(actual.local_revision(), revision);
8393 assert_eq!(actual.pending_commit_ids(), pending);
8394 }
8395 }
8396
8397 #[test]
8398 fn import_reconciliation_preserves_typed_pending_primary_keys() {
8399 for (kind, a, b) in [
8400 ("string", json!("a"), json!("b")),
8401 ("integer", json!(1), json!(2)),
8402 ("boolean", json!(true), json!(false)),
8403 ("json", json!("1"), json!("2")),
8404 ] {
8405 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
8406 for images in [false, true] {
8407 let mut client =
8408 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
8409 let table = client.schema.table("tasks").unwrap().clone();
8410 client
8411 .subscribe("tasks".into(), "tasks".into(), vec![], None)
8412 .unwrap();
8413 for (id, title) in [(&a, "original"), (&b, "occupied")] {
8414 client
8415 .write_base_row(
8416 "tasks",
8417 &vec![
8418 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
8419 Some(ColumnValue::String(title.into())),
8420 ],
8421 1,
8422 )
8423 .unwrap();
8424 }
8425 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
8426 client
8427 .mutate(vec![Mutation::Upsert {
8428 table: "tasks".into(),
8429 values: Map::from_iter([
8430 ("id".into(), a.clone()),
8431 ("title".into(), json!("available")),
8432 ]),
8433 base_version: None,
8434 }])
8435 .unwrap();
8436 client.overlay_rebuild_count.set(0);
8437 for (title, expected) in [("available", "original"), ("occupied", "available")] {
8438 let values = vec![
8439 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
8440 Some(ColumnValue::String(title.into())),
8441 ];
8442 if images {
8443 let image = Connection::open_in_memory().unwrap();
8444 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8445 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8446 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
8447 image
8448 .execute(
8449 "INSERT INTO tasks VALUES (?1,?2,2)",
8450 rusqlite::params![RowParam::Cell(&values[0]), title],
8451 )
8452 .unwrap();
8453 assert!(
8454 matches!(
8455 client.apply_sqlite_image(
8456 &image,
8457 &table,
8458 false,
8459 &[],
8460 1,
8461 7,
8462 "digest"
8463 ),
8464 Ok(1)
8465 ),
8466 "{kind}, images={images}"
8467 );
8468 } else {
8469 let segment = RowsSegment {
8470 table: "tasks".into(),
8471 schema_version: 1,
8472 columns: table.wire_columns.clone(),
8473 blocks: vec![vec![ssp2::segment::SegmentRow {
8474 server_version: 2,
8475 values,
8476 }]],
8477 };
8478 assert!(
8479 matches!(client.apply_segment(0, &segment, false), Ok(1)),
8480 "{kind}"
8481 );
8482 }
8483 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
8484 let actual: String = client
8485 .conn
8486 .query_row(
8487 "SELECT title FROM tasks WHERE id=?1",
8488 [RowParam::Cell(&key)],
8489 |row| row.get(0),
8490 )
8491 .unwrap();
8492 assert_eq!(actual, expected, "{kind}, images={images}");
8493 }
8494 assert_eq!(client.overlay_rebuild_count.get(), 0);
8495 }
8496 }
8497 }
8498
8499 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
8500 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
8501 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
8502 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
8503 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
8504 })))});
8505 let mut client =
8506 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
8507 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
8508 client.rebuild_overlay().expect("rebuild overlay");
8509 client
8510 .mutate(vec![Mutation::Upsert {
8511 table: "tasks".into(),
8512 values: Map::from_iter([
8513 ("id".into(), json!("offline")),
8514 ("project_id".into(), json!("p1")),
8515 ("title".into(), json!("pending edit")),
8516 ]),
8517 base_version: None,
8518 }])
8519 .unwrap();
8520 let image = Connection::open_in_memory().unwrap();
8521 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8522 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
8523 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
8524 image
8525 .execute(
8526 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
8527 [rows],
8528 )
8529 .unwrap();
8530 client.overlay_rebuild_count.set(0);
8531 let table = client.schema.table("tasks").unwrap().clone();
8532 (client, image, table)
8533 }
8534
8535 #[test]
8536 fn sqlite_full_image_import_retains_the_first_code_and_recovers() {
8537 let (mut client, image, table) = unique_import_fixture(1);
8538 image
8539 .execute("UPDATE tasks SET title = ?1", ["full ".repeat(32768)])
8540 .unwrap();
8541 client
8542 .conn
8543 .execute_batch("PRAGMA max_page_count = 1")
8544 .unwrap();
8545 let effective = [("project_id".into(), vec!["p1".into()])];
8546 assert!(client
8547 .apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest")
8548 .is_err());
8549 let failure = client.storage_failure.borrow().clone().unwrap();
8550 assert_eq!(failure.code, Some("client.storage_full"));
8551 assert_eq!(failure.sqlite_code, Some(13));
8552 assert!(!failure.message.contains("rollback"));
8553 assert!(client.conn.is_autocommit());
8554 client
8555 .conn
8556 .execute_batch("PRAGMA max_page_count = 1073741823")
8557 .unwrap();
8558 assert!(matches!(
8559 client.apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest"),
8560 Ok(1)
8561 ));
8562 assert!(client.conn.is_autocommit());
8563 }
8564
8565 #[test]
8566 fn authoring_storage_failures_are_structured_and_scoped() {
8567 let path = std::env::temp_dir().join(format!(
8568 "syncular-authoring-failure-{}.db",
8569 uuid::Uuid::new_v4()
8570 ));
8571 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8572 {"name":"id","type":"string","nullable":false},
8573 {"name":"project_id","type":"string","nullable":false},
8574 {"name":"title","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8575 let mut client = SyncClient::open_path(
8576 "authoring".into(),
8577 &schema,
8578 ClientLimits::default(),
8579 path.to_str().unwrap(),
8580 )
8581 .expect("open client");
8582 client.create_synced_tables().expect("create synced tables");
8583 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
8585
8586 let upsert = |id: &str| {
8587 vec![Mutation::Upsert {
8588 table: "tasks".into(),
8589 values: Map::from_iter([
8590 ("id".into(), json!(id)),
8591 ("project_id".into(), json!("p1")),
8592 ("title".into(), json!("first")),
8593 ]),
8594 base_version: None,
8595 }]
8596 };
8597
8598 let locker = Connection::open(&path).expect("second connection");
8599 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
8600 let busy = client.mutate(upsert("t1")).expect_err("busy write");
8601 assert_eq!(busy.code, "client.storage_busy");
8602 assert!(busy.retryable);
8603 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
8604 assert!(client.pending_commit_ids().is_empty());
8605 assert_eq!(client.local_revision(), 0);
8606
8607 let invalid = client.mutate(Vec::new()).expect_err("empty commit");
8609 assert_eq!(invalid.code, "sync.invalid_request");
8610 assert_eq!(invalid.message, "the authoring request is invalid");
8611 assert!(
8612 invalid.details.as_ref().unwrap()["legacyCause"]
8613 .as_str()
8614 .unwrap()
8615 .contains("at least one operation"),
8616 "{invalid:?}"
8617 );
8618
8619 locker.execute_batch("ROLLBACK").unwrap();
8620 let id = client.mutate(upsert("t1")).expect("retry after busy");
8621 assert_eq!(client.pending_commit_ids(), vec![id]);
8622 assert_eq!(client.local_revision(), 1);
8623
8624 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
8626 let busy_again = client.mutate(upsert("t2")).expect_err("busy write again");
8627 assert_eq!(busy_again.code, "client.storage_busy");
8628 assert!(busy_again.retryable);
8629 locker.execute_batch("ROLLBACK").unwrap();
8630
8631 let full = ClientError::from(QueryReadFailure::from(rusqlite::Error::SqliteFailure(
8633 rusqlite::ffi::Error::new(13),
8634 Some("database or disk is full".to_owned()),
8635 )));
8636 assert_eq!(full.code, "client.storage_full");
8637 assert!(!full.retryable);
8638 assert_eq!(full.details.as_ref().unwrap()["sqliteCode"], 13);
8639
8640 drop(locker);
8641 drop(client);
8642 let _ = std::fs::remove_file(&path);
8643 }
8644
8645 #[test]
8646 fn caller_value_failures_are_invalid_request_and_leave_no_outbox_entry() {
8647 let path = std::env::temp_dir().join(format!(
8648 "syncular-caller-values-{}.db",
8649 uuid::Uuid::new_v4()
8650 ));
8651 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8652 {"name":"id","type":"string","nullable":false},
8653 {"name":"project_id","type":"string","nullable":false},
8654 {"name":"title","type":"string","nullable":false},
8655 {"name":"priority","type":"integer","nullable":false},
8656 {"name":"payload","type":"bytes","nullable":false},
8657 {"name":"extra","type":"bytes","nullable":true}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8658 let mut client = SyncClient::open_path(
8659 "caller-values".into(),
8660 &schema,
8661 ClientLimits::default(),
8662 path.to_str().unwrap(),
8663 )
8664 .expect("open client");
8665 client.create_synced_tables().expect("create synced tables");
8666
8667 let values = |id: &str| {
8668 Map::from_iter([
8669 ("id".to_owned(), json!(id)),
8670 ("project_id".to_owned(), json!("p1")),
8671 ("title".to_owned(), json!("first")),
8672 ("priority".to_owned(), json!(1)),
8673 ("payload".to_owned(), json!({"$bytes": "0a"})),
8674 ])
8675 };
8676 let upsert = |values: Map<String, Value>| {
8677 vec![Mutation::Upsert {
8678 table: "tasks".into(),
8679 values,
8680 base_version: None,
8681 }]
8682 };
8683 let seed = client.mutate(upsert(values("t1"))).expect("seed row");
8684 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
8685
8686 for (label, override_value, cause) in [
8691 ("unknown column", ("nope", json!(1)), "unknown column"),
8692 (
8693 "internal sync column",
8694 ("_sync_version", json!(1)),
8695 "internal sync column",
8696 ),
8697 ("wrong value type", ("title", json!(7)), "expected a string"),
8698 (
8699 "malformed bytes envelope",
8700 ("payload", json!({"$bytes": "zz"})),
8701 "expected exactly",
8702 ),
8703 (
8704 "odd-length bytes envelope",
8705 ("payload", json!({"$bytes": "abc"})),
8706 "expected exactly",
8707 ),
8708 (
8709 "plus-sign bytes envelope",
8710 ("payload", json!({"$bytes": "+a"})),
8711 "expected exactly",
8712 ),
8713 (
8714 "bytes envelope with a second key",
8715 ("payload", json!({"$bytes": "0a", "extra": 1})),
8716 "expected exactly",
8717 ),
8718 (
8719 "no bytes envelope",
8720 ("payload", json!({"nested": true})),
8721 "expected exactly",
8722 ),
8723 ] {
8724 let mut full = values("t2");
8725 full.insert(override_value.0.to_owned(), override_value.1);
8726 let error = client.mutate(upsert(full)).expect_err(label);
8727 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8728 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8729 assert!(!error.retryable, "{label}");
8730 let legacy = error.details.as_ref().unwrap()["legacyCause"]
8731 .as_str()
8732 .unwrap();
8733 assert!(
8734 legacy.starts_with("sync.invalid_request: ") && legacy.contains(cause),
8735 "{label}: {legacy}"
8736 );
8737 assert_eq!(client.pending_commit_ids(), vec![seed.clone()], "{label}");
8738 assert_eq!(client.local_revision(), 1, "{label}");
8739 assert_eq!(
8740 client.query("SELECT * FROM tasks", &[]).unwrap(),
8741 rows,
8742 "{label}"
8743 );
8744 }
8745
8746 let mut missing = values("t3");
8749 missing.remove("payload");
8750 let missing_error = client.mutate(upsert(missing)).expect_err("missing column");
8751 assert_eq!(missing_error.code, "sync.invalid_request");
8752 assert!(missing_error.details.as_ref().unwrap()["legacyCause"]
8753 .as_str()
8754 .unwrap()
8755 .contains("is not nullable"));
8756
8757 let mut bad_row_id = values("t4");
8760 bad_row_id.insert("id".to_owned(), json!({"nested": true}));
8761 let row_id_error = client.mutate(upsert(bad_row_id)).expect_err("bad rowId");
8762 assert_eq!(row_id_error.code, "sync.invalid_request");
8763
8764 for (label, partial, cause) in [
8766 (
8767 "patch wrong value type",
8768 ("priority", json!("high")),
8769 "expected an integer",
8770 ),
8771 ("patch unknown column", ("nope", json!(1)), "unknown column"),
8772 ] {
8773 let error = client
8774 .patch(
8775 "tasks",
8776 "t1",
8777 Map::from_iter([(partial.0.to_owned(), partial.1)]),
8778 None,
8779 )
8780 .expect_err(label);
8781 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8782 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8783 assert!(
8784 error.details.as_ref().unwrap()["legacyCause"]
8785 .as_str()
8786 .unwrap()
8787 .contains(cause),
8788 "{label}"
8789 );
8790 }
8791 assert_eq!(client.pending_commit_ids(), vec![seed.clone()]);
8792 assert_eq!(client.local_revision(), 1);
8793 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
8794
8795 for (label, payload) in [
8798 ("explicit null", json!(null)),
8799 ("empty envelope", json!({"$bytes": ""})),
8800 ("uppercase envelope", json!({"$bytes": "0AFF"})),
8801 ] {
8802 let mut accepted = values("t-ok");
8803 accepted.insert("extra".to_owned(), payload);
8804 let encoded = client.mutate(upsert(accepted)).unwrap_or_else(|error| {
8805 panic!("{label} must be accepted: {error:?}");
8806 });
8807 assert!(!encoded.is_empty(), "{label}");
8808 }
8809
8810 let next = client
8812 .mutate(upsert(values("t5")))
8813 .expect("valid after failures");
8814 assert_ne!(next, seed);
8815
8816 drop(client);
8817 let _ = std::fs::remove_file(&path);
8818 }
8819
8820 #[test]
8821 fn a_legacy_outbox_value_the_codec_refuses_is_dropped_instead_of_wedging() {
8822 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8823 {"name":"id","type":"string","nullable":false},
8824 {"name":"project_id","type":"string","nullable":false},
8825 {"name":"payload","type":"bytes","nullable":true}],
8826 "scopes":[{"pattern":"project:{project_id}"}]}]});
8827 let open = |path: &std::path::Path| {
8828 SyncClient::open_path(
8829 "legacy-values".into(),
8830 &schema,
8831 ClientLimits::default(),
8832 path.to_str().unwrap(),
8833 )
8834 .expect("open client")
8835 };
8836 let row = |id: &str| {
8837 Map::from_iter([
8838 ("id".to_owned(), json!(id)),
8839 ("project_id".to_owned(), json!("p1")),
8840 ("payload".to_owned(), json!({"$bytes": "0a"})),
8841 ])
8842 };
8843 let upsert = |values| {
8844 vec![Mutation::Upsert {
8845 table: "tasks".into(),
8846 values,
8847 base_version: None,
8848 }]
8849 };
8850 for (label, stored_cell, representable) in [
8851 ("plus-sign envelope", json!({"$bytes": "+a"}), false),
8852 ("non-hex envelope", json!({"$bytes": "zz"}), false),
8853 ("odd-length envelope", json!({"$bytes": "abc"}), false),
8854 ("no envelope", json!({"nested": true}), false),
8855 ("wrong declared type", json!("high"), true),
8859 ("stored null for a non-nullable column", json!(null), true),
8860 ] {
8861 let path = std::env::temp_dir().join(format!(
8862 "syncular-legacy-values-{}-{}.db",
8863 label.replace(' ', "-"),
8864 uuid::Uuid::new_v4()
8865 ));
8866 let mut client = open(&path);
8867 client.create_synced_tables().expect("create synced tables");
8868 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8869 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8870 drop(client);
8871
8872 let connection = Connection::open(&path).expect("raw connection");
8874 let (column, cell) = if stored_cell.is_null() {
8875 ("project_id", stored_cell.clone())
8876 } else {
8877 ("payload", stored_cell.clone())
8878 };
8879 let mut planted_values = json!({
8880 "id": "t1", "project_id": "p1", "payload": {"$bytes": "0a"}
8881 });
8882 planted_values[column] = cell;
8883 let planted = json!([{
8884 "op": "upsert",
8885 "table": "tasks",
8886 "rowId": "t1",
8887 "baseVersion": null,
8888 "values": planted_values
8889 }]);
8890 connection
8891 .execute(
8892 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8893 rusqlite::params![planted.to_string(), stale],
8894 )
8895 .expect("plant legacy intent");
8896 drop(connection);
8897
8898 let reopened = open(&path);
8901 assert_eq!(
8902 reopened.pending_commit_ids(),
8903 vec![successor.clone()],
8904 "{label}"
8905 );
8906 let rejection = reopened
8907 .rejections()
8908 .iter()
8909 .find(|rejection| rejection.client_commit_id == stale)
8910 .unwrap_or_else(|| panic!("{label}: rejection"));
8911 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8912 assert_eq!(
8913 rejection.message, "the persisted commit carries values the current codec refuses",
8914 "{label}"
8915 );
8916 assert_eq!(rejection.operation.is_some(), representable, "{label}");
8917 assert_eq!(
8918 rejection
8919 .details
8920 .as_ref()
8921 .and_then(|details| details.reason.as_deref()),
8922 Some(INVALID_STORED_VALUES_REASON),
8923 "{label}"
8924 );
8925 let journal = reopened
8927 .commit_outcome(&stale)
8928 .expect("read outcome")
8929 .expect("journal entry");
8930 assert!(matches!(journal.status, CommitOutcomeStatus::Rejected));
8931 drop(reopened);
8932 let mut reopened = open(&path);
8933 let reread = reopened
8934 .commit_outcome(&stale)
8935 .expect("read outcome")
8936 .expect("journal entry");
8937 assert!(matches!(reread.status, CommitOutcomeStatus::Rejected));
8938
8939 assert_eq!(
8942 reopened
8943 .query("SELECT id FROM tasks ORDER BY id", &[])
8944 .unwrap(),
8945 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8946 "{label}"
8947 );
8948 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8949 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8950 assert!(!reopened
8951 .drop_unencodable_outbox(false, None)
8952 .expect("idempotent prepass"));
8953 let mut transport = CountingRealtimeTransport::default();
8954 let outcome = reopened.sync(&mut transport);
8955 assert!(matches!(outcome, SyncOutcome::Failed { .. }), "{outcome:?}");
8956 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8957 let next = reopened.mutate(upsert(row("t3"))).expect("after the drop");
8958 assert_eq!(reopened.pending_commit_ids(), vec![successor, next]);
8959 assert_eq!(reopened.pending_payloads().len(), 2, "{label}");
8960 drop(reopened);
8961 let _ = std::fs::remove_file(&path);
8962 }
8963 }
8964
8965 #[test]
8966 fn two_legacy_values_on_the_same_row_recover_in_fifo_order() {
8967 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8968 {"name":"id","type":"string","nullable":false},
8969 {"name":"project_id","type":"string","nullable":false},
8970 {"name":"payload","type":"bytes","nullable":true}],
8971 "scopes":[{"pattern":"project:{project_id}"}]}]});
8972 for (label, column, planted) in [
8973 ("malformed envelope", "payload", json!({"$bytes": "zz"})),
8974 (
8975 "stored null for a non-nullable column",
8976 "project_id",
8977 json!(null),
8978 ),
8979 ("wrong declared type", "payload", json!(7)),
8980 ] {
8981 let path = std::env::temp_dir().join(format!(
8982 "syncular-shared-row-{}-{}.db",
8983 label.replace(' ', "-"),
8984 uuid::Uuid::new_v4()
8985 ));
8986 let open = |path: &std::path::Path| {
8987 SyncClient::open_path(
8988 "shared-row".into(),
8989 &schema,
8990 ClientLimits::default(),
8991 path.to_str().unwrap(),
8992 )
8993 .expect("open client")
8994 };
8995 let row = |id: &str| {
8996 Map::from_iter([
8997 ("id".to_owned(), json!(id)),
8998 ("project_id".to_owned(), json!("p1")),
8999 ("payload".to_owned(), json!({"$bytes": "0a"})),
9000 ])
9001 };
9002 let upsert = |values| {
9003 vec![Mutation::Upsert {
9004 table: "tasks".into(),
9005 values,
9006 base_version: None,
9007 }]
9008 };
9009 let mut client = open(&path);
9010 client.create_synced_tables().expect("create synced tables");
9011 let older = client.mutate(upsert(row("t1"))).expect("older commit");
9012 let newer = client.mutate(upsert(row("t1"))).expect("newer commit");
9013 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
9014 drop(client);
9015
9016 let connection = Connection::open(&path).expect("raw connection");
9017 for commit_id in [&older, &newer] {
9018 let mut values = row("t1");
9019 values.insert(column.to_owned(), planted.clone());
9020 let planted_ops = json!([{
9021 "op": "upsert",
9022 "table": "tasks",
9023 "rowId": "t1",
9024 "baseVersion": null,
9025 "values": values
9026 }]);
9027 connection
9028 .execute(
9029 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9030 rusqlite::params![planted_ops.to_string(), commit_id],
9031 )
9032 .expect("plant legacy intent");
9033 }
9034 drop(connection);
9035
9036 let mut reopened = open(&path);
9038 assert_eq!(
9039 reopened.pending_commit_ids(),
9040 vec![successor.clone()],
9041 "{label}"
9042 );
9043 assert_eq!(
9044 reopened
9045 .rejections()
9046 .iter()
9047 .map(|rejection| rejection.client_commit_id.clone())
9048 .collect::<Vec<_>>(),
9049 vec![older.clone(), newer.clone()],
9050 "{label}"
9051 );
9052 assert!(
9053 reopened
9054 .rejections()
9055 .iter()
9056 .all(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE
9057 && rejection.details.as_ref().and_then(|d| d.reason.as_deref())
9058 == Some(INVALID_STORED_VALUES_REASON)),
9059 "{label}"
9060 );
9061 for commit_id in [&older, &newer] {
9062 assert!(
9063 matches!(
9064 reopened
9065 .commit_outcome(commit_id)
9066 .expect("outcome")
9067 .map(|o| o.status),
9068 Some(CommitOutcomeStatus::Rejected)
9069 ),
9070 "{label}"
9071 );
9072 }
9073 assert_eq!(
9074 reopened
9075 .query("SELECT id FROM tasks ORDER BY id", &[])
9076 .unwrap(),
9077 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9078 "{label}"
9079 );
9080 let revision = reopened.local_revision();
9081 let batches = reopened.drain_change_batches();
9082 assert!(
9083 batches.iter().any(|batch| batch.rejections_changed
9084 && batch.outcomes_changed
9085 && batch.tables.iter().any(|change| change.table == "tasks")
9086 && batch.revision == revision.to_string()),
9087 "{label}: {batches:?}"
9088 );
9089 drop(reopened);
9090
9091 let mut stable = open(&path);
9094 assert_eq!(stable.local_revision(), revision, "{label}");
9095 assert!(stable.drain_change_batches().is_empty(), "{label}");
9096 assert_eq!(stable.pending_commit_ids(), vec![successor], "{label}");
9097 drop(stable);
9098 let _ = std::fs::remove_file(&path);
9099 }
9100 }
9101
9102 #[test]
9103 fn recovery_omits_a_legacy_operation_the_journal_reader_rejects() {
9104 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9105 {"name":"id","type":"string","nullable":false},
9106 {"name":"project_id","type":"string","nullable":false},
9107 {"name":"payload","type":"bytes","nullable":true}],
9108 "scopes":[{"pattern":"project:{project_id}"}]}]});
9109 for (label, second_op) in [
9110 (
9111 "delete carrying values",
9112 json!({"op": "delete", "table": "tasks", "rowId": "t9", "values": {}}),
9113 ),
9114 (
9115 "baseVersion beyond the safe bound",
9116 json!({"op": "upsert", "table": "tasks", "rowId": "t9",
9117 "baseVersion": MAX_JS_SAFE_INTEGER + 1,
9118 "values": {"id": "t9", "project_id": "p1", "payload": {"$bytes": "0a"}}}),
9119 ),
9120 ] {
9121 let path = std::env::temp_dir().join(format!(
9122 "syncular-journal-shape-{}-{}.db",
9123 label.replace(' ', "-"),
9124 uuid::Uuid::new_v4()
9125 ));
9126 let open = |path: &std::path::Path| {
9127 SyncClient::open_path(
9128 "journal-shape".into(),
9129 &schema,
9130 ClientLimits::default(),
9131 path.to_str().unwrap(),
9132 )
9133 .expect("open client")
9134 };
9135 let values = |id: &str| {
9136 Map::from_iter([
9137 ("id".to_owned(), json!(id)),
9138 ("project_id".to_owned(), json!("p1")),
9139 ("payload".to_owned(), json!({"$bytes": "0a"})),
9140 ])
9141 };
9142 let upsert = |values| {
9143 vec![Mutation::Upsert {
9144 table: "tasks".into(),
9145 values,
9146 base_version: None,
9147 }]
9148 };
9149 let mut client = open(&path);
9150 client.create_synced_tables().expect("create synced tables");
9151 let stale = client.mutate(upsert(values("t1"))).expect("stale commit");
9152 drop(client);
9153
9154 let connection = Connection::open(&path).expect("raw connection");
9155 let planted = json!([
9159 {
9160 "op": "upsert",
9161 "table": "tasks",
9162 "rowId": "t1",
9163 "baseVersion": null,
9164 "values": {"id": "t1", "project_id": null,
9165 "payload": {"$bytes": "0a"}}
9166 },
9167 second_op
9168 ]);
9169 connection
9170 .execute(
9171 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9172 rusqlite::params![planted.to_string(), stale],
9173 )
9174 .expect("plant legacy intent");
9175 drop(connection);
9176
9177 let reopened = open(&path);
9178 let journal = reopened
9179 .commit_outcome(&stale)
9180 .unwrap_or_else(|error| panic!("{label}: journal read: {error}"))
9181 .unwrap_or_else(|| panic!("{label}: journal entry"));
9182 assert!(
9183 matches!(journal.status, CommitOutcomeStatus::Rejected),
9184 "{label}"
9185 );
9186 assert!(journal.operations.is_none(), "{label}");
9187 assert!(
9188 reopened
9189 .commit_outcomes(CommitOutcomeQuery::default())
9190 .is_ok(),
9191 "{label}: outcome list query"
9192 );
9193 assert!(reopened.pending_commit_ids().is_empty(), "{label}");
9194 drop(reopened);
9195 let _ = std::fs::remove_file(&path);
9196 }
9197 }
9198
9199 #[test]
9200 fn legacy_values_recover_at_the_recreate_and_bump_reset_boundaries() {
9201 let schema = |version: i32, extra: bool| {
9202 let mut columns = vec![
9203 json!({"name":"id","type":"string","nullable":false}),
9204 json!({"name":"project_id","type":"string","nullable":false}),
9205 json!({"name":"payload","type":"bytes","nullable":true}),
9206 ];
9207 if extra {
9208 columns.push(json!({"name":"later","type":"string","nullable":true}));
9209 }
9210 json!({"version":version,"tables":[{"name":"tasks","primaryKey":"id","columns":columns,
9211 "scopes":[{"pattern":"project:{project_id}"}]}]})
9212 };
9213 for (label, bump) in [
9214 ("same-version recreation", false),
9215 ("version-bump reset", true),
9216 ("version-only bump reset", true),
9217 ] {
9218 let path = std::env::temp_dir().join(format!(
9219 "syncular-legacy-reset-{}-{}.db",
9220 label.replace(' ', "-"),
9221 uuid::Uuid::new_v4()
9222 ));
9223 let v1 = schema(1, false);
9224 let open = |schema: &Value, path: &std::path::Path| {
9225 SyncClient::open_path(
9226 "legacy-reset".into(),
9227 schema,
9228 ClientLimits::default(),
9229 path.to_str().unwrap(),
9230 )
9231 .expect("open client")
9232 };
9233 let row = |id: &str| {
9234 Map::from_iter([
9235 ("id".to_owned(), json!(id)),
9236 ("project_id".to_owned(), json!("p1")),
9237 ("payload".to_owned(), json!({"$bytes": "0a"})),
9238 ])
9239 };
9240 let upsert = |values| {
9241 vec![Mutation::Upsert {
9242 table: "tasks".into(),
9243 values,
9244 base_version: None,
9245 }]
9246 };
9247 let mut client = open(&v1, &path);
9248 client.create_synced_tables().expect("create synced tables");
9249 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
9250 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
9251 drop(client);
9252
9253 let connection = Connection::open(&path).expect("raw connection");
9254 let planted = json!([{
9255 "op": "upsert",
9256 "table": "tasks",
9257 "rowId": "t1",
9258 "baseVersion": null,
9259 "values": {"id": "t1", "project_id": 7, "payload": {"$bytes": "0a"}}
9260 }]);
9261 connection
9262 .execute(
9263 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9264 rusqlite::params![planted.to_string(), stale],
9265 )
9266 .expect("plant legacy intent");
9267 drop(connection);
9268
9269 let target = if bump {
9270 schema(2, label == "version-bump reset")
9271 } else {
9272 v1.clone()
9273 };
9274 let reopened = open(&target, &path);
9275 let rejection = reopened
9276 .rejections()
9277 .iter()
9278 .find(|rejection| rejection.client_commit_id == stale)
9279 .unwrap_or_else(|| panic!("{label}: rejection"));
9280 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
9281 assert_eq!(
9282 rejection
9283 .details
9284 .as_ref()
9285 .and_then(|details| details.reason.as_deref()),
9286 Some(INVALID_STORED_VALUES_REASON),
9287 "{label}"
9288 );
9289 assert!(rejection.operation.is_some(), "{label}");
9290 assert_eq!(
9291 reopened.pending_commit_ids(),
9292 vec![successor.clone()],
9293 "{label}"
9294 );
9295 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
9299 if label == "version-bump reset" {
9300 assert_eq!(
9301 reopened
9302 .query("SELECT id, later FROM tasks ORDER BY id", &[])
9303 .unwrap(),
9304 vec![Map::from_iter([
9305 ("id".to_owned(), json!("t2")),
9306 ("later".to_owned(), Value::Null),
9307 ])],
9308 "{label}"
9309 );
9310 let mut next = reopened;
9314 let revision = next.local_revision();
9315 let batches = next.drain_change_batches();
9316 assert!(
9317 batches.iter().any(|batch| batch.rejections_changed
9318 && batch.outcomes_changed
9319 && batch.tables.iter().any(|change| change.table == "tasks")
9320 && batch.revision == revision.to_string()),
9321 "{label}: {batches:?}"
9322 );
9323 next.mutate(upsert(Map::from_iter([
9324 ("id".to_owned(), json!("t3")),
9325 ("project_id".to_owned(), json!("p1")),
9326 ("payload".to_owned(), json!({"$bytes": "0b"})),
9327 ("later".to_owned(), json!("v2")),
9328 ])))
9329 .expect("v2 authoring after the reset");
9330 assert_eq!(next.pending_commit_ids().len(), 2, "{label}");
9331 assert_eq!(next.pending_payloads().len(), 2, "{label}");
9332 drop(next);
9333 } else {
9334 assert_eq!(
9335 reopened
9336 .query("SELECT id FROM tasks ORDER BY id", &[])
9337 .unwrap(),
9338 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9339 "{label}"
9340 );
9341 drop(reopened);
9342 }
9343
9344 if !bump {
9347 let mut recreated = open(&v1, &path);
9348 let fresh = recreated
9349 .mutate(upsert(row("t3")))
9350 .expect("fresh commit while open");
9351 recreated.drain_change_batches();
9352 let planted_fresh = json!([{
9353 "op": "upsert",
9354 "table": "tasks",
9355 "rowId": "t3",
9356 "baseVersion": null,
9357 "values": {"id": "t3", "project_id": "p1",
9358 "payload": {"$bytes": "zz"}}
9359 }]);
9360 let connection = Connection::open(&path).expect("raw connection");
9361 let affected = connection
9362 .execute(
9363 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9364 rusqlite::params![planted_fresh.to_string(), fresh],
9365 )
9366 .expect("plant fresh legacy intent");
9367 assert_eq!(
9368 affected, 1,
9369 "{label}: the in-place plant must hit the queued commit"
9370 );
9371 drop(connection);
9372 let revision_before = recreated.local_revision();
9373 recreated
9374 .recreate_with_schema(&v1)
9375 .unwrap_or_else(|error| panic!("{label}: recreation failed: {error}"));
9376 assert_eq!(recreated.pending_commit_ids(), vec![successor], "{label}");
9377 let fresh_rejection = recreated
9378 .rejections()
9379 .iter()
9380 .find(|rejection| rejection.client_commit_id == fresh)
9381 .expect("the in-place recovery rejection");
9382 assert_eq!(fresh_rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
9383 assert_eq!(
9384 fresh_rejection
9385 .details
9386 .as_ref()
9387 .and_then(|details| details.reason.as_deref()),
9388 Some(INVALID_STORED_VALUES_REASON),
9389 "{label}"
9390 );
9391 assert!(
9392 matches!(
9393 recreated
9394 .commit_outcome(&fresh)
9395 .expect("outcome read")
9396 .map(|outcome| outcome.status),
9397 Some(CommitOutcomeStatus::Rejected)
9398 ),
9399 "{label}"
9400 );
9401 assert!(recreated.local_revision() > revision_before, "{label}");
9402 let revision = recreated.local_revision();
9403 let batches = recreated.drain_change_batches();
9404 assert_eq!(batches.len(), 1, "{label}: one recovery batch");
9405 assert!(
9406 batches.iter().any(|batch| batch.rejections_changed
9407 && batch.outcomes_changed
9408 && batch.tables.iter().any(|change| change.table == "tasks")
9409 && batch.revision == revision.to_string()),
9410 "{label}: {batches:?}"
9411 );
9412 assert_eq!(recreated.pending_payloads().len(), 1, "{label}");
9413 assert_eq!(
9414 recreated
9415 .query("SELECT id FROM tasks ORDER BY id", &[])
9416 .unwrap(),
9417 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9418 "{label}: the recovered commit leaves no projection"
9419 );
9420 drop(recreated);
9421 }
9422 let _ = std::fs::remove_file(&path);
9423 }
9424 }
9425
9426 #[test]
9427 fn legacy_values_recover_over_a_synced_authoritative_row() {
9428 let path = std::env::temp_dir().join(format!(
9429 "syncular-authoritative-row-{}.db",
9430 uuid::Uuid::new_v4()
9431 ));
9432 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9433 {"name":"id","type":"string","nullable":false},
9434 {"name":"project_id","type":"string","nullable":false},
9435 {"name":"title","type":"string","nullable":false},
9436 {"name":"payload","type":"bytes","nullable":true}],
9437 "scopes":[{"pattern":"project:{project_id}"}]}]});
9438 let open = |path: &std::path::Path| {
9439 SyncClient::open_path(
9440 "authoritative-row".into(),
9441 &schema,
9442 ClientLimits::default(),
9443 path.to_str().unwrap(),
9444 )
9445 .expect("open client")
9446 };
9447 let row = |title: &str| {
9448 Map::from_iter([
9449 ("id".to_owned(), json!("t1")),
9450 ("project_id".to_owned(), json!("p1")),
9451 ("title".to_owned(), json!(title)),
9452 ("payload".to_owned(), json!({"$bytes": "0a"})),
9453 ])
9454 };
9455 let upsert = |values| {
9456 vec![Mutation::Upsert {
9457 table: "tasks".into(),
9458 values,
9459 base_version: None,
9460 }]
9461 };
9462 let mut client = open(&path);
9463 client.create_synced_tables().expect("create synced tables");
9464 client
9467 .conn
9468 .execute_batch(
9469 "INSERT INTO _syncular_base_tasks VALUES ('t1','p1','authoritative',x'0a',1);
9470 INSERT INTO tasks VALUES ('t1','p1','authoritative',x'0a',1);",
9471 )
9472 .expect("seed authoritative row");
9473 let older = client.mutate(upsert(row("older"))).expect("older");
9474 let newer = client.mutate(upsert(row("newer"))).expect("newer");
9475 let successor = client.mutate(upsert(row("successor"))).expect("successor");
9476 drop(client);
9477
9478 let connection = Connection::open(&path).expect("raw connection");
9479 for commit_id in [&older, &newer] {
9480 let planted = json!([{
9481 "op": "upsert",
9482 "table": "tasks",
9483 "rowId": "t1",
9484 "baseVersion": null,
9485 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
9486 "payload": {"$bytes": "zz"}}
9487 }]);
9488 connection
9489 .execute(
9490 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9491 rusqlite::params![planted.to_string(), commit_id],
9492 )
9493 .expect("plant legacy intent");
9494 }
9495 drop(connection);
9496
9497 let reopened = open(&path);
9498 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
9499 assert_eq!(
9500 reopened
9501 .rejections()
9502 .iter()
9503 .map(|rejection| rejection.client_commit_id.clone())
9504 .collect::<Vec<_>>(),
9505 vec![older.clone(), newer.clone()]
9506 );
9507 assert_eq!(
9510 reopened.query("SELECT id,title,_syncular_version FROM tasks", &[]),
9511 Ok(vec![Map::from_iter([
9512 ("id".to_owned(), json!("t1")),
9513 ("title".to_owned(), json!("successor")),
9514 ("_syncular_version".to_owned(), json!(1)),
9515 ])])
9516 );
9517
9518 drop(reopened);
9521 let connection = Connection::open(&path).expect("raw connection");
9522 connection
9523 .execute(
9524 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9525 rusqlite::params![
9526 json!([{
9527 "op": "upsert",
9528 "table": "tasks",
9529 "rowId": "t1",
9530 "baseVersion": null,
9531 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
9532 "payload": {"$bytes": "zz"}}
9533 }])
9534 .to_string(),
9535 successor
9536 ],
9537 )
9538 .expect("plant successor intent");
9539 drop(connection);
9540 let late = open(&path);
9541 assert!(late.pending_commit_ids().is_empty());
9542 assert_eq!(
9543 late.query("SELECT id,title,_syncular_version FROM tasks", &[]),
9544 Ok(vec![Map::from_iter([
9545 ("id".to_owned(), json!("t1")),
9546 ("title".to_owned(), json!("authoritative")),
9547 ("_syncular_version".to_owned(), json!(1)),
9548 ])])
9549 );
9550 drop(late);
9551 let _ = std::fs::remove_file(&path);
9552 }
9553
9554 #[test]
9555 fn a_failed_value_recovery_commit_restores_every_side_effect() {
9556 let path = std::env::temp_dir().join(format!(
9557 "syncular-value-recovery-commit-{}.db",
9558 uuid::Uuid::new_v4()
9559 ));
9560 let path_str = path.to_str().expect("path");
9561 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9562 {"name":"id","type":"string","nullable":false},
9563 {"name":"project_id","type":"string","nullable":false},
9564 {"name":"payload","type":"bytes","nullable":true}],
9565 "scopes":[{"pattern":"project:{project_id}"}]}]});
9566 let conn = Connection::open(path_str).expect("replica connection");
9569 let mut client = SyncClient::with_connection(
9570 "value-recovery-commit".into(),
9571 &schema,
9572 ClientLimits::default(),
9573 conn,
9574 )
9575 .expect("replica");
9576 assert_eq!(
9577 client
9578 .conn
9579 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
9580 .expect("journal mode"),
9581 "delete"
9582 );
9583 client.create_synced_tables().expect("create synced tables");
9584 let stale = client
9585 .mutate(vec![Mutation::Upsert {
9586 table: "tasks".into(),
9587 values: Map::from_iter([
9588 ("id".to_owned(), json!("t1")),
9589 ("project_id".to_owned(), json!("p1")),
9590 ("payload".to_owned(), json!({"$bytes": "0a"})),
9591 ]),
9592 base_version: None,
9593 }])
9594 .expect("queued write");
9595 client.drain_change_batches();
9596 let affected = client
9597 .conn
9598 .execute(
9599 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9600 rusqlite::params![
9601 json!([{
9602 "op": "upsert",
9603 "table": "tasks",
9604 "rowId": "t1",
9605 "baseVersion": null,
9606 "values": {"id": "t1", "project_id": "p1",
9607 "payload": {"$bytes": "zz"}}
9608 }])
9609 .to_string(),
9610 stale
9611 ],
9612 )
9613 .expect("plant legacy intent");
9614 assert_eq!(affected, 1, "the planted row must exist");
9615 let revision = client.local_revision();
9616 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
9617 let prior_outbox = client.pending_commit_ids();
9618 let prior_rejections = client.rejections().len();
9619 let prior_changes = client.change_queue.len();
9620 let prior_intents = client.sync_intent_queue.len();
9621 let prior_failed = client.failed_commits.len();
9622 let reader = Connection::open(path_str).expect("reader");
9625 reader
9626 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9627 .expect("reader snapshot");
9628 client
9629 .conn
9630 .busy_timeout(std::time::Duration::ZERO)
9631 .expect("busy timeout");
9632 let error = client
9633 .recreate_with_schema(&schema)
9634 .expect_err("a blocked COMMIT must refuse the recovery");
9635 assert!(error.starts_with("client.storage_busy:"), "{error}");
9636 assert!(client.conn.is_autocommit(), "the transaction is released");
9637 assert_eq!(client.local_revision(), revision);
9638 assert_eq!(client.pending_commit_ids(), prior_outbox);
9639 assert_eq!(client.rejections().len(), prior_rejections);
9640 assert_eq!(client.change_queue.len(), prior_changes);
9641 assert_eq!(client.sync_intent_queue.len(), prior_intents);
9642 assert_eq!(client.failed_commits.len(), prior_failed);
9643 assert_eq!(
9644 client.query("SELECT * FROM tasks", &[]).expect("rows"),
9645 rows
9646 );
9647 assert!(
9648 client
9649 .commit_outcome(&stale)
9650 .expect("outcome read")
9651 .is_none(),
9652 "the rolled-back recovery kept no outcome"
9653 );
9654 reader.execute_batch("ROLLBACK").expect("release reader");
9656 client
9657 .recreate_with_schema(&schema)
9658 .expect("retry after the blocked commit");
9659 assert!(client.pending_commit_ids().is_empty());
9660 assert_eq!(client.rejections().len(), prior_rejections + 1);
9661 assert!(client.local_revision() > revision);
9662 let batches = client.drain_change_batches();
9663 assert!(
9664 batches.iter().any(|batch| batch.rejections_changed
9665 && batch.outcomes_changed
9666 && batch.tables.iter().any(|change| change.table == "tasks")),
9667 "{batches:?}"
9668 );
9669 drop(client);
9670 let _ = std::fs::remove_file(&path);
9671 }
9672
9673 #[cfg(feature = "e2ee")]
9674 #[test]
9675 fn a_failed_recovery_restores_retained_failed_commits() {
9676 let path = std::env::temp_dir().join(format!(
9677 "syncular-value-recovery-retention-{}.db",
9678 uuid::Uuid::new_v4()
9679 ));
9680 let path_str = path.to_str().expect("path");
9681 let schema = json!({"version":1,"tables":[{"name":"secrets","primaryKey":"id","columns":[
9682 {"name":"id","type":"string","nullable":false},
9683 {"name":"project_id","type":"string","nullable":false},
9684 {"name":"note","type":"bytes","nullable":true,"encrypted":true,"declaredType":"string"}],
9685 "scopes":[{"pattern":"project:{project_id}"}]}]});
9686 let conn = Connection::open(path_str).expect("replica connection");
9687 let mut client = SyncClient::with_connection(
9688 "value-recovery-retention".into(),
9689 &schema,
9690 ClientLimits::default(),
9691 conn,
9692 )
9693 .expect("replica");
9694 client.create_synced_tables().expect("create synced tables");
9695 client.set_retain_failed_commits(true).expect("retention");
9696 client.set_encryption(crate::values::EncryptionConfig::default());
9698 client
9699 .mutate(vec![Mutation::Upsert {
9700 table: "secrets".into(),
9701 values: Map::from_iter([
9702 ("id".to_owned(), json!("s1")),
9703 ("project_id".to_owned(), json!("p1")),
9704 ("note".to_owned(), json!("secret")),
9705 ]),
9706 base_version: None,
9707 }])
9708 .expect("queued write");
9709 client.drain_change_batches();
9710 let revision = client.local_revision();
9711 let prior_outbox = client.pending_commit_ids();
9712 let prior_rejections = client.rejections().len();
9713 let prior_failed = client.failed_commits.len();
9714 let reader = Connection::open(path_str).expect("reader");
9715 reader
9716 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9717 .expect("reader snapshot");
9718 client
9719 .conn
9720 .busy_timeout(std::time::Duration::ZERO)
9721 .expect("busy timeout");
9722 let error = client
9723 .drop_unencodable_outbox(true, None)
9724 .expect_err("a blocked COMMIT must refuse the recovery");
9725 assert!(error.starts_with("client.storage_busy:"), "{error}");
9726 assert_eq!(client.failed_commits.len(), prior_failed);
9727 assert_eq!(client.pending_commit_ids(), prior_outbox);
9728 assert_eq!(client.rejections().len(), prior_rejections);
9729 assert_eq!(client.local_revision(), revision);
9730 reader.execute_batch("ROLLBACK").expect("release reader");
9732 client
9733 .drop_unencodable_outbox(true, None)
9734 .expect("retry after the blocked commit");
9735 assert_eq!(client.failed_commits.len(), prior_failed + 1);
9736 assert!(client.pending_commit_ids().is_empty());
9737 assert_eq!(client.rejections().len(), prior_rejections + 1);
9738 drop(client);
9739 let _ = std::fs::remove_file(&path);
9740 }
9741
9742 #[test]
9743 fn client_error_maps_legacy_codes_and_static_messages() {
9744 let code_only = ClientError::from("sync.unknown_table".to_owned());
9745 assert_eq!(code_only.code, "sync.unknown_table");
9746 assert_eq!(code_only.message, "the commit targets an unknown table");
9747 assert!(code_only.details.is_none());
9748
9749 let dynamic = ClientError::from(
9750 "sync.invalid_request: table \"tasks\": patch cannot write scope column".to_owned(),
9751 );
9752 assert_eq!(dynamic.code, "sync.invalid_request");
9753 assert_eq!(dynamic.message, "the authoring request is invalid");
9754 assert!(dynamic.details.as_ref().unwrap()["legacyCause"]
9755 .as_str()
9756 .unwrap()
9757 .contains("patch cannot write scope column"));
9758
9759 let unclassified =
9760 ClientError::from("a commit must contain at least one operation".to_owned());
9761 assert_eq!(unclassified.code, "client.failed");
9762 assert_eq!(unclassified.message, "the local authoring operation failed");
9763 assert!(unclassified.details.as_ref().unwrap()["legacyCause"]
9764 .as_str()
9765 .unwrap()
9766 .contains("at least one operation"));
9767
9768 let metadata = ClientError::from(QueryReadFailure {
9770 code: None,
9771 sqlite_code: Some(1),
9772 sqlite_message: Some("generic SQLite prose".to_owned()),
9773 rollback_failure: Some(Box::new(QueryReadFailure::from(
9774 rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(1), None),
9775 ))),
9776 message: "unclassified read".to_owned(),
9777 });
9778 assert_eq!(metadata.code, "client.failed");
9779 assert_eq!(metadata.message, "the local authoring operation failed");
9780 let details = metadata.details.as_ref().unwrap();
9781 assert_eq!(details["sqliteCode"], 1);
9782 assert_eq!(details["sqliteMessage"], "generic SQLite prose");
9783 assert_eq!(details["rollbackFailure"]["sqliteCode"], 1);
9784 assert_eq!(details["legacyCause"], "unclassified read");
9785 }
9786
9787 #[test]
9788 fn schema_marker_read_keeps_storage_classification() {
9789 let busy = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9790 rusqlite::ffi::Error::new(5),
9791 Some("database is locked".to_owned()),
9792 ));
9793 assert!(busy.starts_with("client.storage_busy:"), "{busy}");
9794 let invalid = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9795 rusqlite::ffi::Error::new(1),
9796 Some("unreadable".to_owned()),
9797 ));
9798 assert_eq!(
9799 invalid,
9800 "sync.local_corrupt: persisted local schema marker is unreadable or invalid"
9801 );
9802
9803 let path =
9806 std::env::temp_dir().join(format!("syncular-marker-busy-{}.db", uuid::Uuid::new_v4()));
9807 let setup = Connection::open(&path).expect("setup connection");
9808 setup
9809 .execute_batch(
9810 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value TEXT);\n INSERT INTO _syncular_meta VALUES ('localSchemaVersion','1');",
9811 )
9812 .unwrap();
9813 let locker = Connection::open(&path).expect("locker connection");
9814 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9815 let reader = Connection::open(&path).expect("reader connection");
9816 reader.busy_timeout(std::time::Duration::ZERO).unwrap();
9817 let locked = read_local_schema_version(&reader, 1).expect_err("locked marker read");
9818 assert!(locked.starts_with("client.storage_busy:"), "{locked}");
9819 locker.execute_batch("ROLLBACK").unwrap();
9820 assert_eq!(read_local_schema_version(&reader, 1).unwrap(), Some(1));
9821 let rows = setup
9822 .query_row("SELECT count(*) FROM _syncular_meta", [], |row| {
9823 row.get::<_, i64>(0)
9824 })
9825 .unwrap();
9826 assert_eq!(rows, 1, "the guard performed no writes");
9827 drop(reader);
9828 drop(locker);
9829 drop(setup);
9830 let _ = std::fs::remove_file(&path);
9831 }
9832
9833 #[cfg(feature = "crdt-yjs")]
9834 #[test]
9835 fn crdt_authoring_pre_read_preserves_storage_classification() {
9836 let path =
9837 std::env::temp_dir().join(format!("syncular-crdt-busy-{}.db", uuid::Uuid::new_v4()));
9838 let schema = json!({"version":1,"tables":[{"name":"notes","primaryKey":"id","columns":[
9839 {"name":"id","type":"string","nullable":false},
9840 {"name":"project_id","type":"string","nullable":false},
9841 {"name":"doc","type":"crdt","nullable":true,"crdtType":"yjs-doc"}],
9842 "scopes":[{"pattern":"project:{project_id}"}]}]});
9843 let mut client = SyncClient::open_path(
9844 "crdt-busy".into(),
9845 &schema,
9846 ClientLimits::default(),
9847 path.to_str().unwrap(),
9848 )
9849 .expect("open client");
9850 client.create_synced_tables().expect("create synced tables");
9851 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
9852
9853 let locker = Connection::open(&path).expect("locker connection");
9854 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9855 let busy = client
9856 .crdt_apply_update("notes", "n1", "doc", &[])
9857 .expect_err("busy crdt pre-read");
9858 assert_eq!(busy.code, "client.storage_busy");
9859 assert!(busy.retryable);
9860 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
9861 assert!(client.pending_commit_ids().is_empty());
9862 assert_eq!(client.local_revision(), 0);
9863
9864 locker.execute_batch("ROLLBACK").unwrap();
9865 drop(locker);
9866 drop(client);
9867 let _ = std::fs::remove_file(&path);
9868 }
9869
9870 #[test]
9871 fn large_unique_import_copies_no_unrelated_tables() {
9872 let (mut client, image, table) = unique_import_fixture(10240);
9873 let pending = client.pending_commit_ids();
9874 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
9875 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
9876 assert!(matches!(
9877 client.apply_sqlite_image(
9878 &image,
9879 &table,
9880 true,
9881 &[("project_id".into(), vec!["p1".into()])],
9882 10240,
9883 7,
9884 "digest"
9885 ),
9886 Ok(10240)
9887 ));
9888 assert_eq!(client.overlay_rebuild_count.get(), 0);
9889 assert_eq!(client.pending_commit_ids(), pending);
9890 assert_eq!(
9891 client
9892 .query("SELECT count(*) AS n FROM tasks", &[])
9893 .unwrap()[0]["n"],
9894 10241
9895 );
9896 assert_eq!(
9897 client
9898 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9899 .unwrap()[0]["n"],
9900 10241
9901 );
9902 assert_eq!(
9903 client
9904 .query("SELECT count(*) AS n FROM notes_fts", &[])
9905 .unwrap()[0]["n"],
9906 8192
9907 );
9908 }
9909
9910 #[test]
9911 #[ignore = "manual scaling benchmark; see bench/README.md"]
9912 fn benchmark_unique_image_import() {
9913 let mut samples = Vec::new();
9914 for rows in [10240, 51200, 102400] {
9915 for trial in 0..4 {
9916 let (mut client, image, table) = unique_import_fixture(rows);
9917 let start = std::time::Instant::now();
9918 assert!(
9919 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
9920 );
9921 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
9922 assert_eq!(
9923 client
9924 .query("SELECT count(*) AS n FROM tasks", &[])
9925 .unwrap()[0]["n"],
9926 rows + 1
9927 );
9928 assert_eq!(
9929 client
9930 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9931 .unwrap()[0]["n"],
9932 rows + 1
9933 );
9934 assert_eq!(
9935 client
9936 .query("SELECT count(*) AS n FROM notes_fts", &[])
9937 .unwrap()[0]["n"],
9938 8192
9939 );
9940 assert_eq!(client.pending_commit_ids().len(), 1);
9941 if trial > 0 {
9942 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
9943 }
9944 }
9945 }
9946 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
9947 let output = serde_json::to_string_pretty(&artifact).unwrap();
9948 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
9949 use std::io::Write;
9950 let mut file = std::fs::OpenOptions::new()
9951 .write(true)
9952 .create_new(true)
9953 .open(path)
9954 .unwrap();
9955 file.write_all(output.as_bytes()).unwrap();
9956 } else {
9957 println!("{output}");
9958 }
9959 }
9960
9961 #[test]
9962 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
9963 let schema = json!({
9964 "version": 1,
9965 "tables": [{
9966 "name": "catalogue_codes",
9967 "primaryKey": "id",
9968 "columns": [
9969 { "name": "id", "type": "string", "nullable": false },
9970 { "name": "release_id", "type": "string", "nullable": false },
9971 { "name": "code", "type": "string", "nullable": false },
9972 { "name": "title", "type": "string", "nullable": false }
9973 ],
9974 "scopes": [{ "pattern": "release:{release_id}" }],
9975 "ftsIndexes": [{
9976 "name": "catalogue_codes_fts",
9977 "columns": ["code", "title"],
9978 "tokenize": "unicode61 remove_diacritics 2"
9979 }]
9980 }]
9981 });
9982 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
9983 .expect("FTS5 client");
9984 let insert_trigger: String = client
9985 .conn
9986 .query_row(
9987 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
9988 [],
9989 |row| row.get(0),
9990 )
9991 .expect("insert trigger");
9992 let replace_guard: String = client
9993 .conn
9994 .query_row(
9995 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
9996 [],
9997 |row| row.get(0),
9998 )
9999 .expect("replace guard");
10000 assert!(!insert_trigger.contains("DELETE FROM"));
10001 assert!(replace_guard.contains("BEFORE INSERT"));
10002 assert!(replace_guard.contains("WHEN EXISTS"));
10003 let search = |client: &SyncClient, query: &str| {
10004 client
10005 .query(
10006 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
10007 &[Value::from(query)],
10008 )
10009 .expect("FTS query")
10010 };
10011
10012 client
10013 .mutate(vec![Mutation::Upsert {
10014 table: "catalogue_codes".to_owned(),
10015 values: Map::from_iter([
10016 ("id".to_owned(), Value::from("c1")),
10017 ("release_id".to_owned(), Value::from("r1")),
10018 ("code".to_owned(), Value::from("A01")),
10019 ("title".to_owned(), Value::from("Cholera")),
10020 ]),
10021 base_version: None,
10022 }])
10023 .expect("insert code");
10024 assert_eq!(search(&client, "cholera").len(), 1);
10025 let prior = client
10026 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
10027 .unwrap();
10028 client
10029 .conn
10030 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
10031 .unwrap();
10032 client.create_synced_tables().unwrap();
10033 assert_eq!(
10034 client
10035 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
10036 .unwrap(),
10037 prior
10038 );
10039 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
10040 let rendered = serde_json::to_string(&plan).unwrap();
10041 assert!(rendered.contains("COVERING INDEX"));
10042 assert!(rendered.contains("INDEX 0:="));
10043 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
10044 assert_eq!(search(&client, "cholera").len(), 1);
10045
10046 client
10047 .mutate(vec![Mutation::Upsert {
10048 table: "catalogue_codes".to_owned(),
10049 values: Map::from_iter([
10050 ("id".to_owned(), Value::from("c1")),
10051 ("release_id".to_owned(), Value::from("r1")),
10052 ("code".to_owned(), Value::from("A01")),
10053 ("title".to_owned(), Value::from("Enteric infection")),
10054 ]),
10055 base_version: None,
10056 }])
10057 .expect("update code");
10058 assert!(search(&client, "cholera").is_empty());
10059 assert_eq!(search(&client, "enteric").len(), 1);
10060
10061 client
10062 .mutate(vec![Mutation::Delete {
10063 table: "catalogue_codes".to_owned(),
10064 row_id: "c1".to_owned(),
10065 base_version: None,
10066 }])
10067 .expect("delete code");
10068 assert!(search(&client, "enteric").is_empty());
10069 }
10070
10071 #[test]
10072 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
10073 let schema = json!({
10074 "version": 1,
10075 "tables": [{
10076 "name": "patient_notes",
10077 "primaryKey": "id",
10078 "columns": [
10079 { "name": "id", "type": "string", "nullable": false },
10080 { "name": "practice_id", "type": "string", "nullable": false },
10081 { "name": "encryption_key_id", "type": "string", "nullable": false },
10082 { "name": "title", "type": "string", "nullable": false }
10083 ],
10084 "scopes": [{ "pattern": "practice:{practice_id}" }],
10085 "ftsIndexes": [{
10086 "name": "patient_notes_fts",
10087 "columns": ["title"],
10088 "tokenize": "unicode61 remove_diacritics 2"
10089 }]
10090 }]
10091 });
10092 let mut client = SyncClient::new(
10093 "local-purge-test".to_owned(),
10094 &schema,
10095 ClientLimits::default(),
10096 )
10097 .expect("local purge client");
10098 client
10099 .conn
10100 .execute(
10101 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
10102 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
10103 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
10104 [],
10105 )
10106 .expect("seed base rows");
10107 client.overlay_dirty.set(true);
10108 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
10109
10110 let note = |id: &str, key_id: &str, title: &str| {
10111 Map::from_iter([
10112 ("id".to_owned(), Value::from(id)),
10113 ("practice_id".to_owned(), Value::from("practice-1")),
10114 ("encryption_key_id".to_owned(), Value::from(key_id)),
10115 ("title".to_owned(), Value::from(title)),
10116 ])
10117 };
10118 let doomed = client
10119 .mutate(vec![
10120 Mutation::Upsert {
10121 table: "patient_notes".to_owned(),
10122 values: note("target", "key-revoked", "Target changed"),
10123 base_version: None,
10124 },
10125 Mutation::Upsert {
10126 table: "patient_notes".to_owned(),
10127 values: note("unrelated", "key-held", "Unrelated changed"),
10128 base_version: None,
10129 },
10130 ])
10131 .expect("doomed commit");
10132 let kept = client
10133 .mutate(vec![Mutation::Upsert {
10134 table: "patient_notes".to_owned(),
10135 values: note("kept", "key-held", "Kept optimistic"),
10136 base_version: None,
10137 }])
10138 .expect("kept commit");
10139 client.drain_change_batches();
10140
10141 let input = LocalDataPurgeInput {
10142 purge_id: "purge-001".to_owned(),
10143 targets: vec![LocalDataPurgeTarget {
10144 table: "patient_notes".to_owned(),
10145 selectors: BTreeMap::from([(
10146 "encryption_key_id".to_owned(),
10147 vec!["key-revoked".to_owned()],
10148 )]),
10149 }],
10150 };
10151 assert_eq!(
10152 client.purge_local_data(&input).expect("apply purge"),
10153 LocalDataPurgeResult {
10154 already_applied: false,
10155 purged_rows: 1,
10156 dropped_commits: 1,
10157 }
10158 );
10159 let rows = client
10160 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
10161 .expect("visible rows");
10162 assert_eq!(rows.len(), 2);
10163 assert_eq!(rows[0]["id"], "kept");
10164 assert_eq!(rows[1]["id"], "unrelated");
10165 assert_eq!(rows[1]["title"], "Unrelated original");
10166 let fts = client
10167 .query(
10168 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
10169 &[],
10170 )
10171 .expect("fts query");
10172 assert!(fts.is_empty());
10173 assert_eq!(client.outbox.len(), 1);
10174 assert_eq!(client.outbox[0].client_commit_id, kept);
10175 let outcome = client
10176 .commit_outcome(&doomed)
10177 .expect("read doomed outcome")
10178 .expect("doomed outcome");
10179 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
10180 match &outcome.results[0] {
10181 CommitOperationOutcome::Error { rejection } => {
10182 assert_eq!(rejection.code, "client.local_data_purged");
10183 assert_eq!(rejection.client_commit_id, doomed);
10184 }
10185 other => panic!("expected local purge rejection, got {other:?}"),
10186 }
10187 assert_eq!(client.drain_change_batches().len(), 1);
10188 assert_eq!(
10189 client.purge_local_data(&input).expect("retry purge"),
10190 LocalDataPurgeResult {
10191 already_applied: true,
10192 purged_rows: 0,
10193 dropped_commits: 0,
10194 }
10195 );
10196 let conflicting = LocalDataPurgeInput {
10197 purge_id: input.purge_id.clone(),
10198 targets: vec![LocalDataPurgeTarget {
10199 table: "patient_notes".to_owned(),
10200 selectors: BTreeMap::from([(
10201 "encryption_key_id".to_owned(),
10202 vec!["key-held".to_owned()],
10203 )]),
10204 }],
10205 };
10206 assert!(client
10207 .purge_local_data(&conflicting)
10208 .expect_err("id collision must fail")
10209 .contains("already used with a different plan"));
10210 }
10211
10212 fn sync_split(client: &mut SyncClient, transport: &mut dyn Transport) -> SyncOutcome {
10214 let mut next = match client.prepare_sync_round(transport.supports_url_fetch()) {
10215 Ok(prepared) => prepared,
10216 Err(outcome) => return *outcome,
10217 };
10218 loop {
10219 let completed = crate::round::test_support::drive_round(next, transport);
10220 match client.apply_sync_round(completed) {
10221 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
10222 crate::AppliedSyncRound::Complete {
10223 outcome, controls, ..
10224 } => {
10225 if client.transport_enabled() {
10226 for text in controls {
10227 let _ = transport.realtime_send(&text);
10228 }
10229 } else {
10230 client.disconnect_realtime(transport);
10231 }
10232 return outcome;
10233 }
10234 }
10235 }
10236 }
10237
10238 #[derive(Clone)]
10239 enum RoundReplyScript {
10240 Response(Message),
10241 PushAck,
10243 Error(TransportError),
10244 }
10245
10246 struct SplitParityTransport {
10249 script: RoundReplyScript,
10250 segments: BTreeMap<String, Vec<u8>>,
10251 requests: Vec<Message>,
10252 batches: Vec<Vec<String>>,
10253 uploads: Vec<String>,
10254 sync_calls: usize,
10255 realtime_sync_calls: usize,
10256 }
10257
10258 impl SplitParityTransport {
10259 fn new(script: RoundReplyScript) -> Self {
10260 Self {
10261 script,
10262 segments: BTreeMap::new(),
10263 requests: Vec::new(),
10264 batches: Vec::new(),
10265 uploads: Vec::new(),
10266 sync_calls: 0,
10267 realtime_sync_calls: 0,
10268 }
10269 }
10270
10271 fn with_segments(script: RoundReplyScript, segments: BTreeMap<String, Vec<u8>>) -> Self {
10272 Self {
10273 segments,
10274 ..Self::new(script)
10275 }
10276 }
10277
10278 fn respond(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
10279 let message =
10280 decode_message(request).expect("the client encodes a legal round request");
10281 self.requests.push(message.clone());
10282 match &self.script {
10283 RoundReplyScript::Response(response) => Ok(encode_message(response)),
10284 RoundReplyScript::Error(error) => Err(error.clone()),
10285 RoundReplyScript::PushAck => {
10286 let mut frames = vec![Frame::RespHeader {
10287 required_schema_version: None,
10288 latest_schema_version: None,
10289 log_epoch: Some("epoch-1".into()),
10290 reset_required: Some(false),
10291 }];
10292 let mut commit_seq = 0;
10293 for frame in &message.frames {
10294 if let Frame::PushCommit {
10295 client_commit_id,
10296 operations,
10297 } = frame
10298 {
10299 commit_seq += 1;
10300 frames.push(Frame::PushResult {
10301 client_commit_id: client_commit_id.clone(),
10302 status: PushStatus::Applied,
10303 commit_seq: Some(commit_seq),
10304 results: (0..operations.len())
10305 .map(|op_index| OpResult::Applied {
10306 op_index: op_index as i32,
10307 })
10308 .collect(),
10309 });
10310 }
10311 }
10312 frames.push(Frame::SubStart {
10313 id: "tasks".into(),
10314 status: SubStatus::Active,
10315 reason_code: String::new(),
10316 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
10317 bootstrap: false,
10318 });
10319 frames.push(Frame::SubEnd {
10320 next_cursor: 0,
10321 bootstrap_state: None,
10322 });
10323 Ok(encode_message(&Message {
10324 wire_version: WIRE_VERSION,
10325 msg_kind: MsgKind::Response,
10326 frames,
10327 }))
10328 }
10329 }
10330 }
10331 }
10332
10333 impl Transport for SplitParityTransport {
10334 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
10335 self.sync_calls += 1;
10336 self.respond(request)
10337 }
10338
10339 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
10340 self.realtime_sync_calls += 1;
10341 self.respond(request)
10342 }
10343 fn download_segment(
10344 &mut self,
10345 _request: &SegmentRequest,
10346 _on_progress: &mut dyn FnMut(u64),
10347 ) -> Result<Vec<u8>, TransportError> {
10348 Err(TransportError::new(
10349 "sync.transport_failed",
10350 "per-descriptor fetch",
10351 ))
10352 }
10353
10354 fn download_segments(
10355 &mut self,
10356 fetches: &[crate::SegmentFetch],
10357 on_progress: &mut (dyn FnMut(usize, u64) + Send),
10358 ) -> Vec<Result<Vec<u8>, TransportError>> {
10359 self.batches.push(
10360 fetches
10361 .iter()
10362 .map(|fetch| match fetch {
10363 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
10364 crate::SegmentFetch::Url(url) => url.clone(),
10365 })
10366 .collect(),
10367 );
10368 fetches
10369 .iter()
10370 .enumerate()
10371 .map(|(index, fetch)| {
10372 let crate::SegmentFetch::Direct(request) = fetch else {
10373 panic!("the descriptors are direct-endpoint requests")
10374 };
10375 let bytes =
10376 self.segments
10377 .get(&request.segment_id)
10378 .cloned()
10379 .ok_or_else(|| {
10380 TransportError::new("transport.not_found", "missing segment body")
10381 })?;
10382 on_progress(index, bytes.len() as u64);
10383 Ok(bytes)
10384 })
10385 .collect()
10386 }
10387
10388 fn blob_upload_grant(
10389 &mut self,
10390 _blob_id: &str,
10391 _byte_length: u64,
10392 _media_type: Option<&str>,
10393 ) -> Result<BlobUploadGrant, TransportError> {
10394 Ok(BlobUploadGrant::None)
10395 }
10396
10397 fn blob_upload(
10398 &mut self,
10399 blob_id: &str,
10400 _bytes: &[u8],
10401 _media_type: Option<&str>,
10402 ) -> Result<(), TransportError> {
10403 self.uploads.push(blob_id.to_owned());
10404 Ok(())
10405 }
10406
10407 fn realtime_connect(&mut self) -> Result<(), TransportError> {
10408 Ok(())
10409 }
10410
10411 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
10412 Ok(())
10413 }
10414
10415 fn realtime_close(&mut self) -> Result<(), TransportError> {
10416 Ok(())
10417 }
10418 }
10419
10420 fn parity_clients() -> (SyncClient, SyncClient) {
10421 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
10422 {"name":"id","type":"string","nullable":false},
10423 {"name":"project_id","type":"string","nullable":false}],
10424 "scopes":[{"pattern":"project:{project_id}"}]}]});
10425 let [exchange, split] = ["parity-exchange", "parity-split"].map(|id| {
10426 let mut client =
10427 SyncClient::new(id.to_owned(), &schema, ClientLimits::default()).unwrap();
10428 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
10429 client
10430 .subscribe(
10431 "tasks".into(),
10432 "tasks".into(),
10433 vec![("project_id".into(), vec!["p1".into()])],
10434 None,
10435 )
10436 .unwrap();
10437 client.subs[0].cursor = 0;
10438 client.subs[0].synced_once = true;
10439 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
10440 client.persist_sub(&client.subs[0]).unwrap();
10441 client.drain_change_batches();
10442 client
10443 });
10444 (exchange, split)
10445 }
10446
10447 fn pull_response(client: &SyncClient) -> Message {
10448 let row = encode_row_json(
10449 client.schema.table("tasks").unwrap(),
10450 "kept",
10451 &Map::from_iter([
10452 ("id".into(), json!("kept")),
10453 ("project_id".into(), json!("p1")),
10454 ]),
10455 &client.encryption,
10456 )
10457 .unwrap();
10458 Message {
10459 wire_version: WIRE_VERSION,
10460 msg_kind: MsgKind::Response,
10461 frames: vec![
10462 Frame::RespHeader {
10463 required_schema_version: None,
10464 latest_schema_version: None,
10465 log_epoch: Some("epoch-1".into()),
10466 reset_required: Some(false),
10467 },
10468 Frame::SubStart {
10469 id: "tasks".into(),
10470 status: SubStatus::Active,
10471 reason_code: String::new(),
10472 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
10473 bootstrap: false,
10474 },
10475 Frame::Commit {
10476 commit_seq: 1,
10477 created_at_ms: 1,
10478 actor_id: "writer".into(),
10479 tables: vec!["tasks".into()],
10480 changes: vec![ssp2::model::Change {
10481 table_index: 0,
10482 row_id: "kept".into(),
10483 op: Op::Upsert,
10484 row_version: Some(1),
10485 scopes: vec![("project_id".into(), "p1".into())],
10486 row: Some(row),
10487 }],
10488 },
10489 Frame::SubEnd {
10490 next_cursor: 1,
10491 bootstrap_state: None,
10492 },
10493 ],
10494 }
10495 }
10496
10497 #[test]
10498 fn split_round_matches_exchange_for_a_pull() {
10499 let (mut exchange, mut split) = parity_clients();
10500 let response = pull_response(&exchange);
10501 let mut exchange_transport =
10502 SplitParityTransport::new(RoundReplyScript::Response(response.clone()));
10503 let mut split_transport = SplitParityTransport::new(RoundReplyScript::Response(response));
10504
10505 let exchange_outcome = exchange.sync(&mut exchange_transport);
10506 let split_outcome = sync_split(&mut split, &mut split_transport);
10507 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10508 assert_eq!(
10509 exchange
10510 .query("SELECT id, project_id FROM tasks", &[])
10511 .unwrap(),
10512 split
10513 .query("SELECT id, project_id FROM tasks", &[])
10514 .unwrap()
10515 );
10516 assert_eq!(exchange_transport.requests.len(), 1);
10517 assert_eq!(split_transport.requests.len(), 1);
10518 }
10519
10520 #[test]
10521 fn split_round_matches_exchange_for_a_push_with_an_upload() {
10522 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
10523 {"name":"id","type":"string","nullable":false},
10524 {"name":"project_id","type":"string","nullable":false},
10525 {"name":"file","type":"blob_ref","nullable":true}],
10526 "scopes":[{"pattern":"project:{project_id}"}]}]});
10527 let [mut exchange, mut split] = ["parity-push-exchange", "parity-push-split"].map(|id| {
10528 let mut client =
10529 SyncClient::new(id.to_owned(), &schema, ClientLimits::default()).unwrap();
10530 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
10531 client
10532 .subscribe(
10533 "attachments".into(),
10534 "attachments".into(),
10535 vec![("project_id".into(), vec!["p1".into()])],
10536 None,
10537 )
10538 .unwrap();
10539 client
10540 });
10541 let bytes = b"split upload";
10542 let mut mutations = Vec::new();
10543 for client in [&mut exchange, &mut split] {
10544 client
10545 .upload_blob(bytes, Some("text/plain".into()), None)
10546 .unwrap();
10547 mutations.push(
10548 client
10549 .mutate(vec![Mutation::Upsert {
10550 table: "attachments".into(),
10551 values: Map::from_iter([
10552 ("id".into(), Value::from("a1")),
10553 ("project_id".into(), Value::from("p1")),
10554 (
10555 "file".into(),
10556 Value::from(format!(
10557 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
10558 blob_id_for(bytes),
10559 bytes.len()
10560 )),
10561 ),
10562 ]),
10563 base_version: None,
10564 }])
10565 .unwrap(),
10566 );
10567 }
10568 let mut exchange_transport = SplitParityTransport::new(RoundReplyScript::PushAck);
10569 let mut split_transport = SplitParityTransport::new(RoundReplyScript::PushAck);
10570 let exchange_outcome = exchange.sync(&mut exchange_transport);
10571 let split_outcome = sync_split(&mut split, &mut split_transport);
10572 let SyncOutcome::Ok(exchange_report) = exchange_outcome else {
10573 panic!("the exchange round must succeed: {exchange_outcome:?}")
10574 };
10575 let SyncOutcome::Ok(split_report) = split_outcome else {
10576 panic!("the split round must succeed: {split_outcome:?}")
10577 };
10578 assert_eq!(exchange_report.pushed, 1);
10579 assert_eq!(split_report.pushed, 1);
10580 assert_eq!(
10581 exchange_report.commits_applied,
10582 split_report.commits_applied
10583 );
10584 assert_eq!(
10585 exchange_transport.uploads,
10586 [blob_id_for(bytes)],
10587 "one upload before the metadata request"
10588 );
10589 assert_eq!(split_transport.uploads, exchange_transport.uploads);
10590 assert!(exchange.pending_commit_ids().is_empty());
10591 assert!(split.pending_commit_ids().is_empty());
10592 assert_eq!(
10593 exchange
10594 .query("SELECT id, project_id FROM attachments", &[])
10595 .unwrap(),
10596 split
10597 .query("SELECT id, project_id FROM attachments", &[])
10598 .unwrap()
10599 );
10600 }
10601
10602 #[test]
10603 fn split_round_matches_exchange_for_a_segment_download() {
10604 let (mut exchange, mut split) = parity_clients();
10605 let columns = exchange.schema.table("tasks").unwrap().wire_columns.clone();
10606 let page = ssp2::segment::encode_rows_segment(&RowsSegment {
10607 table: "tasks".into(),
10608 schema_version: 1,
10609 columns,
10610 blocks: vec![vec![ssp2::segment::SegmentRow {
10611 server_version: 1,
10612 values: vec![
10613 Some(ColumnValue::String("page".into())),
10614 Some(ColumnValue::String("p1".into())),
10615 ],
10616 }]],
10617 });
10618 let segment_id = format!("sha256:{}", bytes_to_hex(&Sha256::digest(&page)));
10619 let response = Message {
10620 wire_version: WIRE_VERSION,
10621 msg_kind: MsgKind::Response,
10622 frames: vec![
10623 Frame::RespHeader {
10624 required_schema_version: None,
10625 latest_schema_version: None,
10626 log_epoch: Some("epoch-1".into()),
10627 reset_required: Some(false),
10628 },
10629 Frame::SubStart {
10630 id: "tasks".into(),
10631 status: SubStatus::Active,
10632 reason_code: String::new(),
10633 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
10634 bootstrap: true,
10635 },
10636 Frame::SegmentRef {
10637 segment_id: segment_id.clone(),
10638 media_type: MediaType::Rows,
10639 table: "tasks".into(),
10640 byte_length: page.len() as i64,
10641 row_count: 1,
10642 as_of_commit_seq: 1,
10643 scope_digest: "digest".into(),
10644 row_cursor: None,
10645 next_row_cursor: None,
10646 url: None,
10647 url_expires_at_ms: None,
10648 },
10649 Frame::SubEnd {
10650 next_cursor: 1,
10651 bootstrap_state: None,
10652 },
10653 ],
10654 };
10655 let segments = BTreeMap::from([(segment_id.clone(), page)]);
10656 let mut exchange_transport = SplitParityTransport::with_segments(
10657 RoundReplyScript::Response(response.clone()),
10658 segments.clone(),
10659 );
10660 let mut split_transport =
10661 SplitParityTransport::with_segments(RoundReplyScript::Response(response), segments);
10662 let exchange_outcome = exchange.sync(&mut exchange_transport);
10663 let split_outcome = sync_split(&mut split, &mut split_transport);
10664 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10665 assert_eq!(exchange_transport.batches, [vec![segment_id.clone()]]);
10666 assert_eq!(split_transport.batches, exchange_transport.batches);
10667 assert_eq!(
10668 exchange
10669 .query("SELECT id, project_id FROM tasks", &[])
10670 .unwrap(),
10671 split
10672 .query("SELECT id, project_id FROM tasks", &[])
10673 .unwrap()
10674 );
10675 }
10676
10677 #[test]
10678 fn split_round_matches_exchange_for_a_transport_failure() {
10679 let (mut exchange, mut split) = parity_clients();
10680 let error = TransportError::new("transport.timeout", "injected");
10681 let mut exchange_transport =
10682 SplitParityTransport::new(RoundReplyScript::Error(error.clone()));
10683 let mut split_transport = SplitParityTransport::new(RoundReplyScript::Error(error));
10684 let exchange_outcome = exchange.sync(&mut exchange_transport);
10685 let split_outcome = sync_split(&mut split, &mut split_transport);
10686 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10687 assert!(matches!(
10688 split_outcome,
10689 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.timeout"
10690 ));
10691 assert_eq!(
10692 exchange.query("SELECT id FROM tasks", &[]).unwrap(),
10693 split.query("SELECT id FROM tasks", &[]).unwrap()
10694 );
10695 }
10696
10697 #[test]
10698 fn split_round_matches_exchange_for_a_failed_realtime_round() {
10699 let (mut exchange, mut split) = parity_clients();
10700 let error = TransportError::new("transport.timeout", "socket round failed");
10701 let mut exchange_transport = SplitParityTransport::new(RoundReplyScript::Error(error));
10702 let mut split_transport = SplitParityTransport::new(RoundReplyScript::Error(
10703 TransportError::new("transport.timeout", "socket round failed"),
10704 ));
10705 exchange.connect_realtime(&mut exchange_transport).unwrap();
10706 split.connect_realtime(&mut split_transport).unwrap();
10707 let exchange_outcome = exchange.sync(&mut exchange_transport);
10708 let split_outcome = sync_split(&mut split, &mut split_transport);
10709 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10710 assert_eq!(exchange_transport.realtime_sync_calls, 1);
10711 assert_eq!(exchange_transport.sync_calls, 0);
10712 assert_eq!(split_transport.realtime_sync_calls, 1);
10713 assert_eq!(split_transport.sync_calls, 0);
10714 assert_eq!(exchange.realtime_state(), RealtimeState::Lost);
10715 assert_eq!(split.realtime_state(), RealtimeState::Lost);
10716 }
10717}
10718
10719impl SubState {
10720 fn name(self) -> &'static str {
10721 match self {
10722 SubState::Active => "active",
10723 SubState::Revoked => "revoked",
10724 SubState::Failed => "failed",
10725 }
10726 }
10727
10728 fn parse(value: &str) -> Self {
10729 match value {
10730 "revoked" => Self::Revoked,
10731 "failed" => Self::Failed,
10732 _ => Self::Active,
10733 }
10734 }
10735}
10736
10737#[derive(Debug, Clone)]
10738struct Subscription {
10739 id: String,
10740 table: String,
10741 requested: Vec<(String, Vec<String>)>,
10742 params: Option<String>,
10743 cursor: i64,
10744 bootstrap_state: Option<String>,
10746 state: SubState,
10747 reason_code: Option<String>,
10748 effective: Option<Vec<(String, Vec<String>)>>,
10751 synced_once: bool,
10752}
10753
10754#[doc(hidden)]
10756pub struct CurrentBasePreparation {
10757 client_commit_id: String,
10758 sequence: i64,
10759 subscriptions: Vec<Subscription>,
10760 transport_epoch: u64,
10761 subscription_epoch: u64,
10762 security_epoch: u64,
10763}
10764
10765#[cfg(test)]
10766mod subscription_rearm_tests {
10767 use super::*;
10768 use serde_json::json;
10769
10770 fn client() -> SyncClient {
10771 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
10772 {"name":"id","type":"string","nullable":false},
10773 {"name":"project_id","type":"string","nullable":false}],
10774 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[],"ftsIndexes":[]}]});
10775 SyncClient::new("rearm".into(), &schema, ClientLimits::default()).expect("client")
10776 }
10777
10778 fn scopes() -> Vec<(String, Vec<String>)> {
10779 vec![("project_id".to_owned(), vec!["p1".to_owned()])]
10780 }
10781
10782 fn store_revoked(client: &mut SyncClient) {
10784 client
10785 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10786 .expect("subscribe");
10787 let sub = &mut client.subs[0];
10788 sub.cursor = 41;
10789 sub.bootstrap_state = Some("resume".to_owned());
10790 sub.effective = Some(scopes());
10791 sub.state = SubState::Revoked;
10792 sub.reason_code = Some("sync.scope_revoked".to_owned());
10793 client.persist_sub(&client.subs[0]).expect("persist");
10794 client.sync_needed = false;
10795 client.sync_intent_queue.clear();
10796 }
10797
10798 #[test]
10799 fn resubscribing_the_same_intent_rearms_a_revoked_registration() {
10800 let mut client = client();
10801 store_revoked(&mut client);
10802
10803 client
10804 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10805 .expect("re-arm");
10806
10807 let view = client.subscription_state("sub").expect("state");
10808 assert_eq!(view.status, "active");
10809 assert_eq!(view.cursor, -1);
10810 assert!(!view.has_resume_token);
10811 assert_eq!(view.effective_scopes, None);
10812 assert_eq!(view.reason_code, None);
10813 assert!(client.sync_needed, "a re-arm asks for a pull");
10814 assert!(matches!(
10815 client.drain_sync_intents().as_slice(),
10816 [SyncIntent::Interactive]
10817 ));
10818 let stored: String = client
10819 .conn
10820 .query_row(
10821 "SELECT state_json FROM _syncular_subscriptions WHERE id = 'sub'",
10822 [],
10823 |row| row.get(0),
10824 )
10825 .expect("persisted row");
10826 let stored: Value = serde_json::from_str(&stored).expect("stored json");
10827 assert_eq!(stored["status"], json!("active"));
10828 assert_eq!(stored["cursor"], json!(-1));
10829 assert_eq!(stored["bootstrapState"], Value::Null);
10830 assert_eq!(stored["effectiveScopes"], Value::Null);
10831 assert_eq!(stored["reasonCode"], Value::Null);
10832 }
10833
10834 #[test]
10835 fn a_failed_registration_is_not_rearmed() {
10836 let mut client = client();
10837 client
10838 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10839 .expect("subscribe");
10840 client.subs[0].state = SubState::Failed;
10841 client.subs[0].reason_code = Some("sync.scope_revoked".to_owned());
10842 client.persist_sub(&client.subs[0]).expect("persist");
10843 client.sync_needed = false;
10844 client.sync_intent_queue.clear();
10845
10846 client
10847 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10848 .expect("subscribe");
10849
10850 let view = client.subscription_state("sub").expect("state");
10851 assert_eq!(view.status, "failed");
10852 assert_eq!(view.reason_code.as_deref(), Some("sync.scope_revoked"));
10853 assert!(!client.sync_needed);
10854 assert!(client.drain_sync_intents().is_empty());
10855 }
10856}
10857
10858#[derive(Debug, Clone)]
10859struct OutboxOp {
10860 upsert: bool,
10861 table: String,
10862 row_id: String,
10863 base_version: Option<i64>,
10864 values: Option<Map<String, Value>>,
10868 insert: bool,
10872}
10873
10874impl OutboxOp {
10875 fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
10880 let values = self.values.as_ref()?;
10881 if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
10882 return Some(Cow::Borrowed(values));
10883 }
10884 full_row_values(table, values.clone()).ok().map(Cow::Owned)
10885 }
10886}
10887
10888impl From<&OutboxOp> for CommitOperation {
10889 fn from(operation: &OutboxOp) -> Self {
10890 Self {
10891 table: operation.table.clone(),
10892 row_id: operation.row_id.clone(),
10893 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
10894 base_version: operation.base_version,
10895 values: operation.values.clone(),
10896 insert: operation.insert,
10897 }
10898 }
10899}
10900
10901#[derive(Debug, Clone)]
10902struct OutboxCommit {
10903 client_commit_id: String,
10904 ops: Vec<OutboxOp>,
10905}
10906
10907fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
10911 commit
10912 .ops
10913 .iter()
10914 .filter(|operation| operation.upsert)
10915 .map(|operation| {
10916 (
10917 operation.table.as_str(),
10918 operation
10919 .values
10920 .as_ref()
10921 .map(|values| values.keys().map(String::as_str).collect())
10922 .unwrap_or_default(),
10923 operation.insert,
10924 )
10925 })
10926 .collect()
10927}
10928
10929type FailedCommit = (OutboxCommit, Map<String, Value>);
10930
10931#[derive(Debug, Clone)]
10932struct CompiledLocalDataPurgeTarget {
10933 table: String,
10934 selectors: Vec<(String, Vec<String>)>,
10935}
10936
10937struct StoredCommitOutcomeRow {
10938 sequence: i64,
10939 client_commit_id: String,
10940 status: String,
10941 recorded_at_ms: i64,
10942 results_json: String,
10943 operations_json: Option<String>,
10944 resolution: String,
10945 resolved_at_ms: Option<i64>,
10946 replacement_client_commit_id: Option<String>,
10947}
10948
10949impl StoredCommitOutcomeRow {
10950 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
10951 Ok(Self {
10952 sequence: row.get(0)?,
10953 client_commit_id: row.get(1)?,
10954 status: row.get(2)?,
10955 recorded_at_ms: row.get(3)?,
10956 results_json: row.get(4)?,
10957 operations_json: row.get(5)?,
10958 resolution: row.get(6)?,
10959 resolved_at_ms: row.get(7)?,
10960 replacement_client_commit_id: row.get(8)?,
10961 })
10962 }
10963}
10964
10965enum SectionError {
10968 FailClosed,
10969 Abort(String, String),
10970 Transport(TransportError),
10971}
10972
10973#[derive(Debug)]
10974pub(crate) struct RequestMeta {
10975 pushed_ids: Vec<String>,
10976 fresh: Vec<(String, bool)>,
10979 pub(crate) accept: u8,
10980 deferred_commits: usize,
10984}
10985
10986#[derive(Default)]
10987struct ChangeAccumulator {
10988 tables: BTreeMap<String, Option<BTreeSet<String>>>,
10990 windows: BTreeMap<(String, String), BTreeSet<String>>,
10991 status: bool,
10992 conflicts: bool,
10993 rejections: bool,
10994 outcomes: bool,
10995}
10996
10997impl ChangeAccumulator {
10998 fn table(&mut self, table: &str) {
10999 self.tables.insert(table.to_owned(), None);
11000 }
11001
11002 fn scope(&mut self, table: &str, key: String) {
11003 match self.tables.get_mut(table) {
11004 Some(None) => {}
11005 Some(Some(keys)) => {
11006 keys.insert(key);
11007 }
11008 None => {
11009 self.tables
11010 .insert(table.to_owned(), Some(BTreeSet::from([key])));
11011 }
11012 }
11013 }
11014
11015 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
11016 self.windows
11017 .entry((base_key.to_owned(), table.to_owned()))
11018 .or_default()
11019 .insert(unit.to_owned());
11020 }
11021
11022 fn touched(&self) -> bool {
11023 !self.tables.is_empty()
11024 || !self.windows.is_empty()
11025 || self.status
11026 || self.conflicts
11027 || self.rejections
11028 || self.outcomes
11029 }
11030}
11031
11032const PUSH_OPS_PER_REQUEST: usize = 500;
11038const MAX_LOCAL_PURGE_TARGETS: usize = 64;
11039const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
11040const MAX_LOCAL_PURGE_VALUES: usize = 128;
11041const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
11042pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
11043
11044struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
11046
11047impl OverlayDirty {
11048 fn clean() -> Self {
11049 Self(RefCell::new(Some(BTreeSet::new())))
11050 }
11051 fn get(&self) -> bool {
11052 self.0
11053 .borrow()
11054 .as_ref()
11055 .is_none_or(|tables| !tables.is_empty())
11056 }
11057 fn set(&self, dirty: bool) {
11058 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
11059 }
11060 fn table(&self, table: &str) {
11061 if let Some(tables) = self.0.borrow_mut().as_mut() {
11062 tables.insert(table.to_owned());
11063 }
11064 }
11065 fn snapshot(&self) -> Option<BTreeSet<String>> {
11066 self.0.borrow().clone()
11067 }
11068 fn restore(&self, tables: Option<BTreeSet<String>>) {
11069 *self.0.borrow_mut() = tables;
11070 }
11071}
11072
11073pub struct SyncClient {
11074 #[cfg(feature = "bench-internals")]
11075 benchmark_phases: Recorder,
11076 conn: Connection,
11077 schema: ClientSchema,
11078 client_id: String,
11079 limits: ClientLimits,
11080 subs: Vec<Subscription>,
11081 outbox: Vec<OutboxCommit>,
11082 failed_commits: Vec<FailedCommit>,
11083 retain_failed_commits: bool,
11084 conflicts: Vec<ConflictRecord>,
11085 rejections: Vec<RejectionRecord>,
11086 schema_floor: Option<SchemaFloor>,
11087 lease_state: Option<LeaseState>,
11089 stopped: bool,
11091 upgrading: bool,
11093 sync_needed: bool,
11095 realtime_policy: RealtimePolicy,
11098 realtime_state: RealtimeState,
11099 realtime_reason_code: Option<String>,
11100 realtime_retry_delay_ms: Option<u64>,
11101 presence: HashMap<String, HashMap<String, PresencePeer>>,
11103 now_ms: Option<i64>,
11106 encryption: crate::values::EncryptionConfig,
11110 security_preflight: bool,
11113 security_epoch: u64,
11114 transport_enabled: bool,
11115 transport_epoch: u64,
11116 subscription_epoch: u64,
11117 previous_version: Option<PreviousVersionContextConfig>,
11121 insert_sql: RefCell<HashMap<String, String>>,
11127 overlay_dirty: OverlayDirty,
11130 storage_failure: RefCell<Option<QueryReadFailure>>,
11132 #[cfg(test)]
11135 overlay_rebuild_count: Cell<usize>,
11136 #[cfg(test)]
11137 acknowledged_replay_count: Cell<usize>,
11138 #[cfg(test)]
11141 outcome_prune_count: Cell<usize>,
11142 progress: ProgressObserver,
11144 change_queue: VecDeque<ClientChangeBatch>,
11145 sync_intent_queue: VecDeque<SyncIntent>,
11146 retry_delay_ms: u64,
11148 round_retry_delay_ms: Option<u64>,
11150 round_failure: Option<String>,
11155 active_round: Option<uuid::Uuid>,
11156 last_round: Option<DiagnosticLastRound>,
11157 last_change: Option<DiagnosticLastChange>,
11158 query_failures: Vec<DiagnosticQueryFailure>,
11160 coalesced: Option<CoalescedApply>,
11162 coalesced_failure: Option<String>,
11165}
11166
11167struct CoalescedApply {
11171 rows: usize,
11172 subs: Vec<Subscription>,
11173 outbox: Vec<OutboxCommit>,
11174 failed_commits: Vec<FailedCommit>,
11175 conflicts: Vec<ConflictRecord>,
11176 rejections: Vec<RejectionRecord>,
11177 schema_floor: Option<SchemaFloor>,
11178 lease_state: Option<LeaseState>,
11179 stopped: bool,
11180 upgrading: bool,
11181 sync_needed: bool,
11182 overlay_dirty: Option<BTreeSet<String>>,
11183 change_queue: usize,
11184 sync_intent_queue: usize,
11185 last_change: Option<DiagnosticLastChange>,
11186}
11187
11188const COALESCED_APPLY_ROWS: usize = 1024;
11189
11190pub(crate) fn quote_ident(name: &str) -> String {
11191 format!("\"{}\"", name.replace('"', "\"\""))
11192}
11193pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
11197 conn.query_row(
11198 "SELECT value FROM _syncular_meta WHERE key = ?1",
11199 rusqlite::params![key],
11200 |row| row.get::<_, String>(0),
11201 )
11202 .ok()
11203}
11204
11205pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
11206 let _ = conn.execute(
11207 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
11208 rusqlite::params![key, value],
11209 );
11210}
11211
11212pub(crate) fn meta_delete(conn: &Connection, key: &str) {
11213 let _ = conn.execute(
11214 "DELETE FROM _syncular_meta WHERE key = ?1",
11215 rusqlite::params![key],
11216 );
11217}
11218
11219fn row_id_predicate(table: &TableSchema) -> String {
11221 let key = quote_ident(&table.primary_key);
11222 let text_match = format!("CAST({key} AS TEXT) = ?1");
11223 match table.columns[table.pk_index].ty {
11224 ColumnType::String | ColumnType::Json => {
11225 format!("{key} = ?1 AND {text_match}")
11226 }
11227 ColumnType::Integer | ColumnType::Boolean => {
11231 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
11232 }
11233 _ => text_match,
11236 }
11237}
11238
11239fn is_local_operation_code_like(value: &str) -> bool {
11240 let bytes = value.as_bytes();
11241 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
11242 && bytes
11243 .iter()
11244 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
11245}
11246
11247fn base_table(name: &str) -> String {
11248 quote_ident(&format!("_syncular_base_{name}"))
11249}
11250
11251type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
11256
11257fn window_base_key(base: &WindowBase) -> String {
11258 format!(
11259 "{}\0{}\0{}",
11260 base.table,
11261 base.variable,
11262 canonical_scope_json(&base.fixed_scopes)
11263 )
11264}
11265
11266fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
11268 let mut scopes = base.fixed_scopes.clone();
11269 scopes.retain(|(k, _)| k != &base.variable);
11270 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
11271 scopes
11272}
11273
11274fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
11278 let canonical = canonical_scope_json(&unit_scopes(base, unit));
11279 let digest = Sha256::digest(canonical.as_bytes());
11280 let hex = bytes_to_hex(&digest);
11281 format!("w:{}:{}", base.table, &hex[..16])
11282}
11283
11284fn visible_table(name: &str) -> String {
11285 quote_ident(name)
11286}
11287
11288const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
11289
11290fn is_synced_table_name(name: &str) -> bool {
11293 if name.starts_with("sqlite_") {
11294 return false;
11295 }
11296 if name.starts_with("_syncular_base_") {
11297 return true; }
11299 !name.starts_with("_syncular_")
11301}
11302
11303fn blob_id_for(bytes: &[u8]) -> String {
11305 let digest = Sha256::digest(bytes);
11306 format!("sha256:{}", bytes_to_hex(&digest))
11307}
11308
11309enum RowParam<'a> {
11313 Cell(&'a Option<ColumnValue>),
11314 Version(i64),
11315}
11316
11317impl rusqlite::ToSql for RowParam<'_> {
11318 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
11319 Ok(match self {
11320 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
11321 RowParam::Cell(cell) => match cell {
11322 None => ToSqlOutput::Owned(SqlValue::Null),
11323 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
11324 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
11325 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
11326 Some(ColumnValue::Boolean(b)) => {
11327 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
11328 }
11329 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
11330 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
11331 }
11332 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
11334 ToSqlOutput::Borrowed(ValueRef::Blob(b))
11335 }
11336 },
11337 })
11338 }
11339}
11340
11341fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
11343 match param.to_sql().map_err(|error| error.to_string())? {
11344 ToSqlOutput::Owned(value) => Ok(value),
11345 ToSqlOutput::Borrowed(value) => {
11346 SqlValue::try_from(value).map_err(|error| error.to_string())
11347 }
11348 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
11349 }
11350}
11351
11352fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
11360 use ssp2::segment::ColumnType;
11361 let mismatch = || {
11362 Err(format!(
11363 "image column {:?} holds a value of the wrong type",
11364 column.name
11365 ))
11366 };
11367 match value {
11368 ValueRef::Null => {
11369 if !column.nullable {
11370 return Err(format!(
11371 "image column {:?} is NULL but not nullable",
11372 column.name
11373 ));
11374 }
11375 Ok(ToSqlOutput::Owned(SqlValue::Null))
11376 }
11377 ValueRef::Integer(i) => match column.ty {
11378 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
11379 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
11380 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
11381 _ => mismatch(),
11382 },
11383 ValueRef::Real(_) => match column.ty {
11384 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
11385 _ => mismatch(),
11386 },
11387 ValueRef::Text(t) => {
11388 std::str::from_utf8(t)
11389 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
11390 match column.ty {
11391 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
11392 Ok(ToSqlOutput::Borrowed(value))
11393 }
11394 _ => mismatch(),
11395 }
11396 }
11397 ValueRef::Blob(_) => match column.ty {
11398 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
11400 _ => mismatch(),
11401 },
11402 }
11403}
11404
11405pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
11406 use rusqlite::types::ValueRef;
11407 match value {
11408 ValueRef::Null => Value::Null,
11409 ValueRef::Integer(i) => match column.ty {
11410 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
11411 ssp2::segment::ColumnType::Float => {
11412 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
11413 }
11414 _ => Value::from(i),
11415 },
11416 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
11417 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
11418 ValueRef::Blob(b) => {
11419 let mut map = Map::new();
11420 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
11421 Value::Object(map)
11422 }
11423 }
11424}
11425
11426fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
11430 Ok(match value {
11431 Value::Null => SqlValue::Null,
11432 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
11433 Value::Number(n) => {
11434 if let Some(i) = n.as_i64() {
11435 SqlValue::Integer(i)
11436 } else if let Some(f) = n.as_f64() {
11437 SqlValue::Real(f)
11438 } else {
11439 return Err(format!("query param number {n} is out of range"));
11440 }
11441 }
11442 Value::String(s) => SqlValue::Text(s.clone()),
11443 Value::Object(_) => {
11444 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
11445 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
11446 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
11447 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
11448 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
11449 })?)
11450 } else {
11451 return Err(
11452 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
11453 .to_owned(),
11454 );
11455 }
11456 }
11457 Value::Array(_) => return Err("query array params are not supported".to_owned()),
11458 })
11459}
11460
11461fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
11466 use rusqlite::types::ValueRef;
11467 match value {
11468 ValueRef::Null => Value::Null,
11469 ValueRef::Integer(i) => {
11470 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
11474 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
11475 Value::from(i)
11476 } else {
11477 let mut map = Map::new();
11478 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
11479 Value::Object(map)
11480 }
11481 }
11482 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
11483 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
11484 ValueRef::Blob(b) => {
11485 let mut map = Map::new();
11486 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
11487 Value::Object(map)
11488 }
11489 }
11490}
11491
11492fn query_connection(
11494 conn: &Connection,
11495 sql: &str,
11496 params: &[QueryValue],
11497) -> Result<Vec<QueryRow>, QueryReadFailure> {
11498 crate::query_guard::assert_read_only_query(sql)?;
11499 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
11500 let bound: Vec<SqlValue> = params
11501 .iter()
11502 .map(json_param_to_sql)
11503 .collect::<Result<_, _>>()?;
11504 let mut stmt = conn.prepare(&lowered_sql)?;
11505 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
11506 let bound_refs: Vec<&dyn rusqlite::ToSql> =
11507 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
11508 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
11509 let mut out = Vec::new();
11510 while let Some(row) = sql_rows.next()? {
11511 let mut record = Map::new();
11512 for (i, name) in column_names.iter().enumerate() {
11513 let value = row.get_ref(i)?;
11514 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
11515 }
11516 out.push(record);
11517 }
11518 Ok(out)
11519}
11520
11521fn persisted_window_state(
11522 conn: &Connection,
11523 base: &WindowBase,
11524) -> Result<WindowState, QueryReadFailure> {
11525 let mut stmt = conn.prepare(
11526 "SELECT windows.unit, subscriptions.state_json
11527 FROM _syncular_windows AS windows
11528 JOIN _syncular_subscriptions AS subscriptions
11529 ON subscriptions.id = windows.sub_id
11530 WHERE windows.base = ?1
11531 ORDER BY windows.unit ASC",
11532 )?;
11533 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
11534 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11535 })?;
11536 let mut units = Vec::new();
11537 let mut pending = Vec::new();
11538 for row in rows {
11539 let (unit, raw) = row?;
11540 let state: Value = serde_json::from_str(&raw)
11541 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
11542 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
11543 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
11544 || state
11545 .get("bootstrapState")
11546 .is_some_and(|value| !value.is_null());
11547 if is_pending {
11548 pending.push(unit.clone());
11549 }
11550 units.push(unit);
11551 }
11552 Ok(WindowState { units, pending })
11553}
11554
11555#[derive(Debug, Clone, Copy)]
11557pub struct SnapshotStatement<'a> {
11558 pub sql: &'a str,
11559 pub params: &'a [Value],
11560}
11561
11562#[derive(Debug, Clone, Default)]
11566pub struct SnapshotReadRequest<'a> {
11567 pub statements: Vec<SnapshotStatement<'a>>,
11568 pub coverage: &'a [WindowCoverage],
11569 pub subscriptions: Vec<String>,
11570 pub commit_ids: Vec<String>,
11571}
11572
11573#[cfg(test)]
11574thread_local! {
11575 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
11579 std::cell::RefCell::new(None);
11580}
11581
11582#[cfg(test)]
11583fn run_snapshot_read_hook() {
11584 SNAPSHOT_READ_HOOK.with(|slot| {
11585 if let Some(hook) = slot.borrow_mut().take() {
11586 hook();
11587 }
11588 });
11589}
11590
11591fn snapshot_read_connection(
11596 conn: &Connection,
11597 request: &SnapshotReadRequest<'_>,
11598) -> Result<SnapshotRead, QueryReadFailure> {
11599 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
11600 let corrupt = |message: &str| QueryReadFailure {
11601 code: Some("sync.local_corrupt"),
11602 sqlite_code: None,
11603 sqlite_message: None,
11604 rollback_failure: None,
11605 message: message.to_owned(),
11606 };
11607 let stored_failure = |error: rusqlite::Error, message: &str| match error {
11608 rusqlite::Error::Utf8Error(..)
11609 | rusqlite::Error::InvalidColumnType(..)
11610 | rusqlite::Error::FromSqlConversionFailure(..)
11611 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
11612 _ => QueryReadFailure::from(error),
11613 };
11614 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
11615 let revision: Option<String> = conn
11618 .query_row(
11619 "SELECT value FROM _syncular_meta WHERE key = ?1",
11620 rusqlite::params![LOCAL_REVISION_KEY],
11621 |row| row.get(0),
11622 )
11623 .optional()
11624 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
11625 let revision = match revision {
11626 None => "0".to_owned(),
11627 Some(value)
11628 if value == "0"
11629 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
11630 && value.bytes().all(|b| b.is_ascii_digit())
11631 && value.parse::<u64>().is_ok()) =>
11632 {
11633 value
11634 }
11635 Some(_) => return Err(corrupt("persisted local revision is invalid")),
11636 };
11637 let mut queries = Vec::with_capacity(request.statements.len());
11638 for statement in &request.statements {
11639 queries.push(query_connection(conn, statement.sql, statement.params)?);
11640 #[cfg(test)]
11641 run_snapshot_read_hook();
11642 }
11643 let mut pending = Vec::new();
11644 let mut missing = Vec::new();
11645 for requested in request.coverage {
11646 let base_key = window_base_key(&requested.base);
11647 let state = persisted_window_state(conn, &requested.base)?;
11648 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
11649 let reference = WindowUnitRef {
11650 base_key: base_key.clone(),
11651 unit: unit.clone(),
11652 };
11653 if !state.units.iter().any(|held| held == &unit) {
11654 missing.push(reference);
11655 } else if state.pending.iter().any(|held| held == &unit) {
11656 pending.push(reference);
11657 }
11658 }
11659 }
11660 let mut subscriptions = Vec::with_capacity(request.subscriptions.len());
11661 for id in &request.subscriptions {
11662 let row: Option<(String, String)> = conn
11663 .query_row(
11664 "SELECT tbl, state_json FROM _syncular_subscriptions WHERE id = ?1",
11665 rusqlite::params![id],
11666 |row| Ok((row.get(0)?, row.get(1)?)),
11667 )
11668 .optional()
11669 .map_err(|error| {
11670 stored_failure(error, "persisted subscription state is invalid")
11671 })?;
11672 let Some((table, raw)) = row else {
11673 subscriptions.push(SubscriptionCatchup::Unknown { id: id.clone() });
11674 continue;
11675 };
11676 let state: Value = serde_json::from_str(&raw)
11679 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
11680 let status = match state.get("status").and_then(Value::as_str) {
11681 Some(value @ ("active" | "revoked" | "failed")) => value,
11682 _ => return Err(corrupt("persisted subscription state is invalid")),
11683 };
11684 let Some(cursor) = state.get("cursor").and_then(Value::as_i64) else {
11685 return Err(corrupt("persisted subscription state is invalid"));
11686 };
11687 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
11688 return Err(corrupt("persisted subscription state is invalid"));
11689 }
11690 let effective_scopes = state
11691 .get("effectiveScopes")
11692 .filter(|value| !value.is_null());
11693 if let Some(scopes) = effective_scopes {
11694 json_to_scope_map(scopes)
11695 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
11696 }
11697 let has_resume_token = match state.get("bootstrapState") {
11698 None | Some(Value::Null) => false,
11699 Some(Value::String(_)) => true,
11700 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
11701 };
11702 let reason_code = match state.get("reasonCode") {
11705 None | Some(Value::Null) => None,
11706 Some(Value::String(value)) => Some(value.clone()),
11707 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
11708 };
11709 let active = status == "active";
11710 subscriptions.push(SubscriptionCatchup::Known {
11711 id: id.clone(),
11712 table,
11713 status: status.to_owned(),
11714 cursor,
11715 has_resume_token,
11716 bootstrap_complete: active && cursor >= 0 && !has_resume_token,
11717 known_pending_pages: active && (has_resume_token || cursor < 0),
11718 effective_scopes: effective_scopes.cloned(),
11719 reason_code,
11720 });
11721 }
11722 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
11723 for id in &request.commit_ids {
11724 let pending: bool = conn.query_row(
11725 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
11726 rusqlite::params![id],
11727 |row| row.get(0),
11728 )?;
11729 if pending {
11730 deliveries.push(CommitDelivery::Pending {
11731 client_commit_id: id.clone(),
11732 });
11733 continue;
11734 }
11735 let row = conn
11736 .query_row(
11737 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
11738 resolution, resolved_at_ms, replacement_client_commit_id
11739 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
11740 rusqlite::params![id],
11741 StoredCommitOutcomeRow::from_row,
11742 )
11743 .optional()
11744 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
11745 deliveries.push(match row {
11746 Some(row) => CommitDelivery::Known {
11747 client_commit_id: id.clone(),
11748 outcome: SyncClient::outcome_from_row(row)
11749 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
11750 },
11751 None => CommitDelivery::Unknown {
11752 client_commit_id: id.clone(),
11753 },
11754 });
11755 }
11756 Ok(SnapshotRead {
11757 revision,
11758 queries,
11759 coverage: CoverageSnapshot {
11760 complete: pending.is_empty() && missing.is_empty(),
11761 pending,
11762 missing,
11763 },
11764 subscriptions,
11765 deliveries,
11766 })
11767 })();
11768 match result {
11769 Ok(read) => {
11770 conn.execute_batch("RELEASE syncular_snapshot_read")?;
11771 Ok(read)
11772 }
11773 Err(mut error) => {
11774 if let Err(rollback) = conn
11775 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
11776 {
11777 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
11778 }
11779 Err(error)
11780 }
11781 }
11782}
11783
11784fn snapshot_connection(
11785 conn: &Connection,
11786 sql: &str,
11787 params: &[Value],
11788 coverage: &[WindowCoverage],
11789) -> Result<QuerySnapshot, QueryReadFailure> {
11790 let read = snapshot_read_connection(
11791 conn,
11792 &SnapshotReadRequest {
11793 statements: vec![SnapshotStatement { sql, params }],
11794 coverage,
11795 subscriptions: Vec::new(),
11796 commit_ids: Vec::new(),
11797 },
11798 )?;
11799 Ok(QuerySnapshot {
11800 revision: read.revision,
11801 rows: read
11802 .queries
11803 .into_iter()
11804 .next()
11805 .expect("single-query snapshot returns one result"),
11806 coverage: read.coverage,
11807 })
11808}
11809
11810pub struct FileQuerySnapshotReader {
11815 path: String,
11816 conn: Option<Connection>,
11817}
11818
11819impl FileQuerySnapshotReader {
11820 #[must_use]
11821 pub fn new(path: impl Into<String>) -> Self {
11822 Self {
11823 path: path.into(),
11824 conn: None,
11825 }
11826 }
11827
11828 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
11829 if self.conn.is_none() {
11830 let conn = Connection::open_with_flags(
11831 &self.path,
11832 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
11833 )
11834 .map_err(QueryReadFailure::from)?;
11835 conn.busy_timeout(std::time::Duration::from_millis(250))
11836 .map_err(QueryReadFailure::from)?;
11837 self.conn = Some(conn);
11838 }
11839 self.conn
11840 .as_ref()
11841 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
11842 }
11843
11844 pub fn snapshot_read(
11847 &mut self,
11848 request: &SnapshotReadRequest<'_>,
11849 ) -> Result<SnapshotRead, QueryReadFailure> {
11850 snapshot_read_connection(self.connection()?, request)
11851 }
11852
11853 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
11856 Ok(self
11857 .snapshot_read(&SnapshotReadRequest::default())?
11858 .revision)
11859 }
11860
11861 pub fn subscription_catchup(
11863 &mut self,
11864 id: &str,
11865 ) -> Result<SubscriptionCatchup, QueryReadFailure> {
11866 let read = self.snapshot_read(&SnapshotReadRequest {
11867 subscriptions: vec![id.to_owned()],
11868 ..Default::default()
11869 })?;
11870 Ok(read
11871 .subscriptions
11872 .into_iter()
11873 .next()
11874 .expect("one requested catch-up"))
11875 }
11876
11877 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
11880 let read = self.snapshot_read(&SnapshotReadRequest {
11881 commit_ids: vec![id.to_owned()],
11882 ..Default::default()
11883 })?;
11884 Ok(read
11885 .deliveries
11886 .into_iter()
11887 .next()
11888 .expect("one requested delivery"))
11889 }
11890
11891 pub fn query_snapshot(
11894 &mut self,
11895 sql: &str,
11896 params: &[Value],
11897 coverage: &[WindowCoverage],
11898 ) -> Result<QuerySnapshot, QueryReadFailure> {
11899 snapshot_connection(self.connection()?, sql, params, coverage)
11900 }
11901}
11902
11903fn validate_authority_reads(
11904 schema: &ClientSchema,
11905 declarations: &[crate::api::AuthorityReadDeclaration],
11906) -> Result<(), String> {
11907 let mut seen = BTreeSet::new();
11908 for read in declarations {
11909 let table = schema
11910 .table(&read.table)
11911 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
11912 let plain = |name: &str| {
11913 table.columns.iter().enumerate().any(|(i, c)| {
11914 c.name == name
11915 && name != "authority_version"
11916 && !name.starts_with("_sync")
11917 && !table.encrypted_columns.iter().any(|e| e.index == i)
11918 && matches!(
11919 c.ty,
11920 ssp2::segment::ColumnType::String
11921 | ssp2::segment::ColumnType::Integer
11922 | ssp2::segment::ColumnType::Float
11923 | ssp2::segment::ColumnType::Boolean
11924 | ssp2::segment::ColumnType::Json
11925 )
11926 })
11927 };
11928 if !seen.insert(&read.table)
11929 || read.columns.is_empty()
11930 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
11931 || !read.columns.contains(&table.primary_key)
11932 || !read.columns.iter().all(|c| plain(c))
11933 || read.scopes.is_empty()
11934 || !read.scopes.iter().all(|(v, values)| {
11935 table
11936 .scope_column(v)
11937 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
11938 && !values.is_empty()
11939 && values.iter().all(|v| !v.is_empty() && v != "*")
11940 })
11941 {
11942 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
11943 }
11944 }
11945 Ok(())
11946}
11947
11948fn schema_marker_read_failure(error: rusqlite::Error) -> String {
11953 let failure = QueryReadFailure::from(error);
11954 if failure.code.is_some() {
11955 failure.to_string()
11956 } else {
11957 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
11958 }
11959}
11960
11961fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
11965 if limits.outcome_retention_max_entries == Some(0) {
11966 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
11967 }
11968 if [
11969 limits.max_push_commits_per_request,
11970 limits.max_push_operations_per_request,
11971 limits.max_push_request_bytes,
11972 ]
11973 .into_iter()
11974 .flatten()
11975 .any(|value| value == 0 || value > u32::MAX as usize)
11976 {
11977 return Err(
11978 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
11979 );
11980 }
11981 Ok(())
11982}
11983
11984fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
11985 let corrupt =
11986 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
11987 let table_type: Option<String> = conn
11988 .query_row(
11989 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
11990 [],
11991 |row| row.get(0),
11992 )
11993 .optional()
11994 .map_err(schema_marker_read_failure)?;
11995 let Some(table_type) = table_type else {
11996 return Ok(None);
11997 };
11998 if table_type != "table" {
11999 return Err(corrupt());
12000 }
12001 let mut statement = conn
12005 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
12006 .map_err(schema_marker_read_failure)?;
12007 let mut rows = statement
12008 .query([LOCAL_SCHEMA_VERSION_KEY])
12009 .map_err(schema_marker_read_failure)?;
12010 let value: Option<String> = rows
12011 .next()
12012 .map_err(schema_marker_read_failure)?
12013 .map(|row| row.get(0))
12014 .transpose()
12015 .map_err(schema_marker_read_failure)?;
12016 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
12017 return Err(corrupt());
12018 }
12019 let Some(value) = value else {
12020 let has_descriptor = conn
12021 .query_row(
12022 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
12023 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
12024 |_| Ok(()),
12025 )
12026 .optional()
12027 .map_err(schema_marker_read_failure)?
12028 .is_some();
12029 if has_descriptor {
12030 return Err(corrupt());
12031 }
12032 return Ok(None);
12033 };
12034 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
12035 || !value.bytes().all(|b| b.is_ascii_digit())
12036 {
12037 return Err(corrupt());
12038 }
12039 let version = value.parse::<i32>().map_err(|_| corrupt())?;
12040 if version > requested {
12041 return Err(
12042 "client.schema_downgrade: persisted local schema is newer than the requested schema"
12043 .to_owned(),
12044 );
12045 }
12046 Ok(Some(version))
12047}
12048
12049#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
12053fn prepare_wasm_replica(path: &str) -> Result<(), String> {
12054 if path.is_empty() || path == ":memory:" {
12055 return Ok(());
12056 }
12057 crate::wasm_vfs::disable_foreign_lock_check()
12058}
12059
12060#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
12061fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
12062 Ok(())
12063}
12064
12065impl SyncClient {
12066 pub fn new_with_identity(
12067 client_id: Option<String>,
12068 schema_json: &Value,
12069 limits: ClientLimits,
12070 ) -> Result<Self, String> {
12071 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
12072 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
12073 Self::with_connection(resolved, schema_json, limits, conn)
12074 }
12075
12076 pub fn new(
12077 client_id: String,
12078 schema_json: &Value,
12079 limits: ClientLimits,
12080 ) -> Result<Self, String> {
12081 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
12082 Self::with_connection(client_id, schema_json, limits, conn)
12083 }
12084
12085 pub fn open_path(
12091 client_id: String,
12092 schema_json: &Value,
12093 limits: ClientLimits,
12094 path: &str,
12095 ) -> Result<Self, String> {
12096 validate_client_limits(&limits)?;
12097 parse_schema_json(schema_json)?;
12098 prepare_wasm_replica(path)?;
12099 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
12100 Self::with_connection(client_id, schema_json, limits, conn)
12101 }
12102
12103 pub fn open_path_with_identity(
12107 client_id: Option<String>,
12108 schema_json: &Value,
12109 limits: ClientLimits,
12110 path: &str,
12111 ) -> Result<Self, String> {
12112 Self::open_path_with_identity_journal(
12113 client_id,
12114 schema_json,
12115 limits,
12116 path,
12117 !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
12118 )
12119 }
12120
12121 fn open_path_with_identity_journal(
12122 client_id: Option<String>,
12123 schema_json: &Value,
12124 limits: ClientLimits,
12125 path: &str,
12126 require_wal: bool,
12127 ) -> Result<Self, String> {
12128 validate_client_limits(&limits)?;
12129 parse_schema_json(schema_json)?;
12132 prepare_wasm_replica(path)?;
12133 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
12134 conn.busy_timeout(std::time::Duration::from_millis(250))
12142 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
12143 conn.pragma_update(None, "synchronous", "FULL")
12148 .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
12149 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
12150 if require_wal {
12154 let journal_mode: String = client
12155 .conn
12156 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
12157 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
12158 if journal_mode != "wal" {
12159 return Err(
12160 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
12161 );
12162 }
12163 }
12164 Ok(client)
12165 }
12166
12167 pub fn reset_replica(path: &str) -> Result<(), String> {
12179 if path.is_empty() || path == ":memory:" {
12180 return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
12181 }
12182 let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
12186 if vfs.is_null() {
12187 return Err("client.storage_io: no SQLite VFS is registered".to_owned());
12188 }
12189 let delete = unsafe { (*vfs).xDelete }
12190 .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
12191 for suffix in ["-wal", "-shm", "-journal", ""] {
12192 let file_name = format!("{path}{suffix}");
12193 let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
12194 format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
12195 })?;
12196 let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
12199 if code != rusqlite::ffi::SQLITE_OK {
12200 return Err(format!(
12201 "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
12202 ));
12203 }
12204 }
12205 Ok(())
12206 }
12207
12208 #[must_use]
12209 pub fn client_id(&self) -> &str {
12210 &self.client_id
12211 }
12212
12213 pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> String {
12220 self.disconnect_realtime(transport);
12221 let client_id = uuid::Uuid::new_v4().to_string();
12222 self.client_id.clone_from(&client_id);
12223 self.set_meta(CLIENT_ID_KEY, &client_id);
12224 self.round_failure = None;
12225 client_id
12226 }
12227
12228 pub fn with_connection(
12235 client_id: String,
12236 schema_json: &Value,
12237 limits: ClientLimits,
12238 conn: Connection,
12239 ) -> Result<Self, String> {
12240 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
12241 }
12242
12243 fn with_connection_identity(
12246 client_id: Option<String>,
12247 schema_json: &Value,
12248 limits: ClientLimits,
12249 conn: Connection,
12250 ) -> Result<Self, String> {
12251 validate_client_limits(&limits)?;
12252 let schema = parse_schema_json(schema_json)?;
12253 validate_authority_reads(&schema, &limits.authority_reads)?;
12254 let previous_version = match limits.previous_version_context {
12257 Some(config) => {
12258 config.validate()?;
12259 Some(config)
12260 }
12261 None => None,
12262 };
12263 let mut client = SyncClient {
12264 #[cfg(feature = "bench-internals")]
12265 benchmark_phases: Recorder::default(),
12266 conn,
12267 schema,
12268 client_id: client_id
12269 .clone()
12270 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
12271 limits,
12272 subs: Vec::new(),
12273 outbox: Vec::new(),
12274 failed_commits: Vec::new(),
12275 retain_failed_commits: false,
12276 conflicts: Vec::new(),
12277 rejections: Vec::new(),
12278 schema_floor: None,
12279 lease_state: None,
12280 stopped: false,
12281 upgrading: false,
12282 sync_needed: false,
12283 realtime_policy: RealtimePolicy::Optional,
12284 realtime_state: RealtimeState::Disconnected,
12285 realtime_reason_code: None,
12286 realtime_retry_delay_ms: None,
12287 presence: HashMap::new(),
12288 now_ms: None,
12289 encryption: crate::values::EncryptionConfig::default(),
12290 security_preflight: false,
12291 security_epoch: 0,
12292 transport_enabled: true,
12293 transport_epoch: 0,
12294 subscription_epoch: 0,
12295 previous_version,
12296 insert_sql: RefCell::new(HashMap::new()),
12297 overlay_dirty: OverlayDirty::clean(),
12298 storage_failure: RefCell::new(None),
12299 #[cfg(test)]
12300 overlay_rebuild_count: Cell::new(0),
12301 #[cfg(test)]
12302 acknowledged_replay_count: Cell::new(0),
12303 #[cfg(test)]
12304 outcome_prune_count: Cell::new(0),
12305 progress: ProgressObserver::default(),
12306 change_queue: VecDeque::new(),
12307 sync_intent_queue: VecDeque::new(),
12308 retry_delay_ms: 250,
12309 round_retry_delay_ms: None,
12310 round_failure: None,
12311 active_round: None,
12312 last_round: None,
12313 last_change: None,
12314 query_failures: Vec::new(),
12315 coalesced: None,
12316 coalesced_failure: None,
12317 };
12318 client
12322 .conn
12323 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
12324 client
12330 .conn
12331 .execute_batch("BEGIN IMMEDIATE")
12332 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
12333 let startup = (|| -> Result<(), String> {
12334 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
12337 client.create_bookkeeping_tables()?;
12340 match client.get_meta(CLIENT_ID_KEY) {
12341 Some(existing) => match client_id {
12342 Some(requested) if existing != requested => {
12343 return Err(format!(
12344 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
12345 ))
12346 }
12347 _ => client.client_id = existing,
12348 },
12349 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
12350 }
12351 client.restore_persisted_state()?;
12352 client.failed_commits = client.load_failed_commits()?;
12353 match marker {
12354 None => {
12355 client.create_synced_tables()?;
12356 client.persist_schema_version()?;
12357 client.save_subscription_scope_schema();
12358 }
12359 Some(version) if version == client.schema.version => {
12360 client.create_synced_tables()?;
12361 set_local_schema_descriptor(&client.conn, &client.schema)
12365 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
12366 client.save_subscription_scope_schema();
12367 }
12368 Some(_) => client.run_schema_reset()?,
12369 }
12370 client.reconcile_previous_version_at_boot()?;
12376 client.clear_satisfied_persisted_schema_floor();
12377 client.prune_unknown_subscriptions(false)?;
12378 client.drop_unencodable_outbox(false, None)?;
12387 if marker == Some(client.schema.version)
12388 && (!client.outbox.is_empty()
12389 || !client.failed_commits.is_empty()
12390 || client.has_acknowledged_rows())
12391 {
12392 client.overlay_dirty.set(true);
12396 client.rebuild_overlay()?;
12397 }
12398 client.enqueue_startup_sync_if_needed()?;
12405 Ok(())
12406 })();
12407 match startup {
12408 Ok(()) => match client.conn.execute_batch("COMMIT") {
12409 Ok(()) => Ok(client),
12410 Err(error) => {
12411 let failure = Self::sqlite_failure(&client.storage_failure, error);
12412 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
12413 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
12415 }
12416 Err(failure)
12417 }
12418 },
12419 Err(error) => {
12420 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
12421 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
12423 }
12424 Err(error)
12425 }
12426 }
12427 }
12428
12429 pub fn set_now_ms(&mut self, now_ms: i64) {
12432 self.now_ms = Some(now_ms);
12433 }
12434
12435 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
12439 self.encryption = encryption;
12440 }
12441
12442 #[must_use]
12443 pub fn security_lifecycle(&self) -> &'static str {
12444 if self.security_preflight {
12445 "preflight"
12446 } else {
12447 "active"
12448 }
12449 }
12450
12451 #[must_use]
12452 pub fn security_preflight(&self) -> bool {
12453 self.security_preflight
12454 }
12455
12456 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
12458 if self.transport_enabled != enabled {
12459 self.transport_enabled = enabled;
12460 self.transport_epoch = self.transport_epoch.wrapping_add(1);
12461 self.sync_intent_queue.clear();
12462 self.sync_intent_queue
12463 .push_back(if enabled && !self.security_preflight {
12464 SyncIntent::Interactive
12465 } else {
12466 SyncIntent::None
12467 });
12468 }
12469 if !enabled && self.active_round.is_none() {
12471 self.disconnect_realtime(transport);
12472 }
12473 }
12474
12475 pub fn transport_enabled(&self) -> bool {
12476 self.transport_enabled
12477 }
12478
12479 pub fn begin_security_preflight(&mut self) {
12486 self.seal_security_on_teardown();
12487 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
12490 }
12491
12492 pub fn seal_security_on_teardown(&mut self) {
12502 self.cancel_sync_round();
12503 self.security_preflight = true;
12504 self.security_epoch = self.security_epoch.wrapping_add(1);
12505 self.encryption = crate::values::EncryptionConfig::default();
12506 self.sync_intent_queue.clear();
12507 }
12508
12509 pub fn activate_security(
12511 &mut self,
12512 encryption: crate::values::EncryptionConfig,
12513 ) -> Result<(), String> {
12514 if !self.security_preflight {
12515 return Err(
12516 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
12517 );
12518 }
12519 self.enqueue_startup_sync_if_needed()?;
12520 self.encryption = encryption;
12521 self.security_preflight = false;
12522 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
12523 Ok(())
12524 }
12525
12526 fn clock_now_ms(&self) -> i64 {
12527 self.now_ms.unwrap_or_else(crate::wall_clock_ms)
12528 }
12529
12530 fn create_bookkeeping_tables(&self) -> Result<(), String> {
12531 self.conn
12533 .execute_batch(
12534 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
12535 seq INTEGER PRIMARY KEY AUTOINCREMENT,
12536 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
12537 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
12538 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
12539 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
12540 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
12541 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
12542 PRIMARY KEY(commit_id, idx));
12543 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
12544 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
12545 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
12546 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
12547 seq INTEGER PRIMARY KEY AUTOINCREMENT,
12548 client_commit_id TEXT NOT NULL UNIQUE,
12549 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
12550 recorded_at_ms INTEGER NOT NULL,
12551 results_json TEXT NOT NULL,
12552 operations_json TEXT,
12553 resolution TEXT NOT NULL DEFAULT 'active'
12554 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
12555 resolved_at_ms INTEGER,
12556 replacement_client_commit_id TEXT);
12557 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
12558 ON _syncular_commit_outcomes(resolution, seq);
12559 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
12560 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
12561 CREATE TABLE IF NOT EXISTS _syncular_meta (
12562 key TEXT PRIMARY KEY, value TEXT NOT NULL);
12563 CREATE TABLE IF NOT EXISTS _syncular_windows (
12564 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
12565 PRIMARY KEY (base, unit));
12566 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
12567 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
12568 effective_scopes TEXT NOT NULL);
12569 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
12570 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
12571 PRIMARY KEY(commit_id, blob_id));
12572 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
12573 ON _syncular_blob_commit_refs(blob_id);",
12574 )
12575 .map_err(|e| e.to_string())?;
12576 let _ = self
12579 .conn
12580 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
12581 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
12582 self.set_meta(LOCAL_REVISION_KEY, "0");
12583 }
12584 if self.schema_has_blobs() {
12587 let exists = self
12588 .conn
12589 .query_row(
12590 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
12591 [],
12592 |row| row.get::<_, bool>(0),
12593 )
12594 .map_err(|e| e.to_string())?;
12595 if exists {
12596 let mut statement = self
12597 .conn
12598 .prepare("PRAGMA table_info(_syncular_blobs)")
12599 .map_err(|e| e.to_string())?;
12600 let mut columns = statement
12601 .query_map([], |row| row.get::<_, String>(1))
12602 .map_err(|e| e.to_string())?
12603 .collect::<Result<Vec<_>, _>>()
12604 .map_err(|e| e.to_string())?;
12605 columns.sort();
12606 let mut expected = vec![
12607 "blob_id",
12608 "byte_length",
12609 "bytes",
12610 "created_at_ms",
12611 "media_type",
12612 ];
12613 expected.sort_unstable();
12614 if columns != expected {
12615 return Err(
12616 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
12617 );
12618 }
12619 } else {
12620 self.conn
12621 .execute_batch(
12622 "CREATE TABLE _syncular_blobs(
12623 blob_id TEXT PRIMARY KEY,
12624 bytes BLOB NOT NULL,
12625 byte_length INTEGER NOT NULL,
12626 media_type TEXT,
12627 created_at_ms INTEGER NOT NULL);",
12628 )
12629 .map_err(|e| e.to_string())?;
12630 }
12631 self.conn
12632 .execute_batch(
12633 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
12634 blob_id TEXT PRIMARY KEY,
12635 media_type TEXT,
12636 created_at_ms INTEGER NOT NULL);",
12637 )
12638 .map_err(|e| e.to_string())?;
12639 }
12640 Ok(())
12641 }
12642
12643 fn schema_has_blobs(&self) -> bool {
12645 self.schema
12646 .tables
12647 .iter()
12648 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
12649 }
12650
12651 fn get_meta(&self, key: &str) -> Option<String> {
12654 meta_get(&self.conn, key)
12655 }
12656
12657 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
12658 self.conn
12659 .query_row(
12660 "SELECT value FROM _syncular_meta WHERE key = ?1",
12661 rusqlite::params![key],
12662 |row| row.get::<_, String>(0),
12663 )
12664 .optional()
12665 .map_err(|_| {
12666 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
12667 })
12668 }
12669
12670 fn persist_schema_version(&self) -> Result<(), String> {
12673 self.conn
12674 .execute(
12675 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12676 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
12677 )
12678 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12679 set_local_schema_descriptor(&self.conn, &self.schema)
12680 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12681 }
12682
12683 pub(crate) fn set_meta(&self, key: &str, value: &str) {
12684 meta_set(&self.conn, key, value);
12685 }
12686
12687 fn delete_meta(&self, key: &str) {
12688 meta_delete(&self.conn, key);
12689 }
12690
12691 fn load_outbox(&self) -> Result<Vec<OutboxCommit>, String> {
12693 let mut stmt = self
12694 .conn
12695 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
12696 .map_err(|error| error.to_string())?;
12697 let rows = stmt
12698 .query_map([], |row| {
12699 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12700 })
12701 .map_err(|error| error.to_string())?;
12702 let mut commits = Vec::new();
12703 for row in rows {
12704 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
12705 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
12706 format!("invalid persisted outbox {client_commit_id:?}: {error}")
12707 })?;
12708 let mut ops = Vec::with_capacity(entries.len());
12709 for entry in entries {
12710 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
12711 ops.push(OutboxOp {
12712 upsert: op == "upsert",
12713 table: entry
12714 .get("table")
12715 .and_then(Value::as_str)
12716 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
12717 .to_owned(),
12718 row_id: entry
12719 .get("rowId")
12720 .and_then(Value::as_str)
12721 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
12722 .to_owned(),
12723 base_version: entry.get("baseVersion").and_then(Value::as_i64),
12724 values: entry.get("values").and_then(Value::as_object).cloned(),
12727 insert: entry.get("insert") == Some(&Value::Bool(true)),
12728 });
12729 }
12730 commits.push(OutboxCommit {
12731 client_commit_id,
12732 ops,
12733 });
12734 }
12735 Ok(commits)
12736 }
12737
12738 fn restore_persisted_state(&mut self) -> Result<(), String> {
12739 self.subs = {
12740 let mut stmt = self
12741 .conn
12742 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
12743 .map_err(|error| error.to_string())?;
12744 let rows = stmt
12745 .query_map([], |row| {
12746 Ok((
12747 row.get::<_, String>(0)?,
12748 row.get::<_, String>(1)?,
12749 row.get::<_, String>(2)?,
12750 ))
12751 })
12752 .map_err(|error| error.to_string())?;
12753 let mut subscriptions = Vec::new();
12754 for row in rows {
12755 let (id, table, raw) = row.map_err(|error| error.to_string())?;
12756 let state: Value = serde_json::from_str(&raw)
12757 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
12758 let requested = json_to_scope_map(
12759 state.get("requested").unwrap_or(&Value::Object(Map::new())),
12760 )?;
12761 let effective = state
12762 .get("effectiveScopes")
12763 .filter(|value| !value.is_null())
12764 .map(json_to_scope_map)
12765 .transpose()?;
12766 subscriptions.push(Subscription {
12767 id,
12768 table,
12769 requested,
12770 params: state
12771 .get("params")
12772 .and_then(Value::as_str)
12773 .map(str::to_owned),
12774 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
12775 bootstrap_state: state
12776 .get("bootstrapState")
12777 .and_then(Value::as_str)
12778 .map(str::to_owned),
12779 state: SubState::parse(
12780 state
12781 .get("status")
12782 .and_then(Value::as_str)
12783 .unwrap_or("active"),
12784 ),
12785 reason_code: state
12786 .get("reasonCode")
12787 .and_then(Value::as_str)
12788 .map(str::to_owned),
12789 effective,
12790 synced_once: state
12791 .get("syncedOnce")
12792 .and_then(Value::as_bool)
12793 .unwrap_or(false),
12794 });
12795 }
12796 subscriptions
12797 };
12798
12799 self.outbox = self.load_outbox()?;
12800
12801 self.prune_commit_outcomes()?;
12802 let active = self.commit_outcomes(CommitOutcomeQuery {
12803 active_only: true,
12804 ..CommitOutcomeQuery::default()
12805 })?;
12806 self.conflicts = active
12807 .iter()
12808 .flat_map(|outcome| outcome.results.iter())
12809 .filter_map(|result| match result {
12810 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
12811 _ => None,
12812 })
12813 .collect();
12814 self.rejections = active
12815 .iter()
12816 .flat_map(|outcome| outcome.results.iter())
12817 .filter_map(|result| match result {
12818 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
12819 _ => None,
12820 })
12821 .collect();
12822
12823 self.lease_state = self
12824 .get_meta(LEASE_STATE_KEY)
12825 .map(|raw| serde_json::from_str(&raw))
12826 .transpose()
12827 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
12828 self.schema_floor = self
12829 .get_meta(SCHEMA_FLOOR_KEY)
12830 .map(|raw| serde_json::from_str(&raw))
12831 .transpose()
12832 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
12833 self.stopped = self.schema_floor.is_some();
12834 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
12837 self.security_preflight = true;
12838 }
12839 Ok(())
12840 }
12841
12842 fn save_subscription_scope_schema(&self) {
12843 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
12844 .schema
12845 .tables
12846 .iter()
12847 .map(|table| {
12848 (
12849 table.name.clone(),
12850 table
12851 .scope_variables
12852 .iter()
12853 .map(|scope| {
12854 (
12855 scope.variable.clone(),
12856 [scope.prefix.clone(), scope.column.clone()],
12857 )
12858 })
12859 .collect(),
12860 )
12861 })
12862 .collect();
12863 self.set_meta(
12864 "subscriptionScopeSchema",
12865 &serde_json::to_string(&declarations).expect("scope schema serialization"),
12866 );
12867 }
12868
12869 fn prune_unknown_subscriptions(&mut self, bump: bool) -> Result<(), String> {
12871 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
12872 self.get_meta("subscriptionScopeSchema")
12873 .map(|raw| serde_json::from_str(&raw))
12874 .transpose()
12875 .map_err(|error| format!("invalid persisted scope schema: {error}"))?
12876 } else {
12877 None
12878 };
12879 let compatible = |table_name: &str, requested: &[(String, Vec<String>)]| {
12880 let Some(table) = self.schema.table(table_name) else {
12881 return false;
12882 };
12883 if requested
12884 .iter()
12885 .any(|(variable, _)| table.scope_column(variable).is_none())
12886 {
12887 return false;
12888 }
12889 if !bump {
12890 return true;
12891 }
12892 let Some(old) = previous.as_ref().and_then(|tables| tables.get(table_name)) else {
12893 return false;
12894 };
12895 let variables: BTreeSet<_> = if requested.is_empty() {
12896 old.keys()
12897 .cloned()
12898 .chain(
12899 table
12900 .scope_variables
12901 .iter()
12902 .map(|scope| scope.variable.clone()),
12903 )
12904 .collect()
12905 } else {
12906 requested
12907 .iter()
12908 .map(|(variable, _)| variable.clone())
12909 .collect()
12910 };
12911 variables.iter().all(|variable| {
12912 let current = table
12913 .scope_variables
12914 .iter()
12915 .find(|scope| &scope.variable == variable);
12916 match (old.get(variable), current) {
12917 (Some(old), Some(current)) => {
12918 old == &[current.prefix.clone(), current.column.clone()]
12919 }
12920 _ => false,
12921 }
12922 })
12923 };
12924 let mut stale_ids: BTreeSet<_> = self
12925 .subs
12926 .iter()
12927 .filter(|sub| !compatible(&sub.table, &sub.requested))
12928 .map(|sub| sub.id.clone())
12929 .collect();
12930 for (id, table, effective) in self.load_pending_evictions()? {
12931 if !compatible(&table, &effective) {
12932 stale_ids.insert(id);
12933 }
12934 }
12935 for id in &stale_ids {
12936 for sql in [
12937 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
12938 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
12939 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
12940 ] {
12941 self.conn
12942 .execute(sql, rusqlite::params![id])
12943 .map_err(|error| error.to_string())?;
12944 }
12945 }
12946 self.subs.retain(|sub| !stale_ids.contains(&sub.id));
12947 Ok(())
12948 }
12949
12950 #[must_use]
12951 pub fn local_revision(&self) -> u64 {
12952 self.get_meta(LOCAL_REVISION_KEY)
12953 .and_then(|value| value.parse().ok())
12954 .unwrap_or(0)
12955 }
12956
12957 pub fn snapshot_read(
12963 &mut self,
12964 request: &SnapshotReadRequest<'_>,
12965 owner: Option<QueryOwner<'_>>,
12966 ) -> Result<SnapshotRead, QueryReadFailure> {
12967 let result = snapshot_read_connection(&self.conn, request);
12968 if let Some(owner) = owner {
12969 self.record_query_read(owner, result.as_ref().err());
12970 }
12971 result
12972 }
12973
12974 #[must_use]
12975 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
12976 SyncStatusSnapshot {
12977 current_schema_version: self.schema.version,
12978 outbox: self.outbox.len(),
12979 upgrading: self.upgrading,
12980 lease_state: self.lease_state.clone(),
12981 schema_floor: self.schema_floor.clone(),
12982 sync_needed: self.sync_needed,
12983 previous_version_context: self.previous_version_status(),
12984 }
12985 }
12986
12987 fn previous_version_status(&self) -> PreviousVersionStatus {
12990 let Some(replica_path) = self.previous_version_replica_path() else {
12991 return PreviousVersionStatus {
12992 present: false,
12993 created_at_ms: None,
12994 };
12995 };
12996 match crate::previous_version::read_previous_version_record(&replica_path) {
12997 Ok(Some(record)) => PreviousVersionStatus {
12998 present: true,
12999 created_at_ms: Some(record.created_at_ms),
13000 },
13001 _ => PreviousVersionStatus {
13002 present: std::path::Path::new(
13003 &crate::previous_version::previous_version_container_path(&replica_path),
13004 )
13005 .exists(),
13006 created_at_ms: None,
13007 },
13008 }
13009 }
13010
13011 pub fn previous_version_snapshot(
13017 &self,
13018 spec: &PreviousVersionReadSpec,
13019 ) -> Result<PreviousVersionSnapshot, String> {
13020 let now_ms = self.clock_now_ms();
13021 crate::previous_version::previous_version_snapshot(
13022 &self.conn,
13023 self.previous_version_replica_path().as_deref(),
13024 self.previous_version.as_ref(),
13025 PreviousVersionLifecycle {
13026 lease_inactive: self.previous_version_lease_inactive(now_ms),
13027 scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
13028 coverage_complete: self.previous_version_coverage_complete(),
13029 },
13030 now_ms,
13031 self.schema.version,
13032 spec,
13033 )
13034 }
13035
13036 #[must_use]
13038 pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
13039 crate::previous_version::stored_previous_version_audit(&self.conn)
13040 }
13041
13042 pub fn previous_version_discard(
13046 &self,
13047 ) -> crate::previous_version::PreviousVersionDiscardOutcome {
13048 self.drop_previous_version()
13049 }
13050
13051 fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
13055 let absent = crate::previous_version::PreviousVersionDiscardOutcome {
13056 present: false,
13057 discarded: false,
13058 };
13059 let Some(replica_path) = self.previous_version_replica_path() else {
13060 return absent;
13061 };
13062 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
13063 .unwrap_or(absent)
13064 }
13065
13066 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
13068 let Some(lease) = &self.lease_state else {
13069 return false;
13070 };
13071 if lease.error_code.is_some() {
13072 return true;
13073 }
13074 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
13075 }
13076
13077 fn previous_version_coverage_complete(&self) -> bool {
13081 let active = self
13082 .subs
13083 .iter()
13084 .filter(|sub| sub.state == SubState::Active)
13085 .collect::<Vec<_>>();
13086 if active.is_empty() {
13087 return false;
13088 }
13089 active
13090 .iter()
13091 .all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
13092 }
13093
13094 fn previous_version_lifetime_check(&self) {
13098 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
13099 return;
13100 };
13101 let now_ms = self.clock_now_ms();
13102 if self.previous_version_lease_inactive(now_ms) {
13103 let _ = self.drop_previous_version();
13104 return;
13105 }
13106 let Some(replica_path) = self.previous_version_replica_path() else {
13107 return;
13108 };
13109 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
13110 else {
13111 return;
13112 };
13113 if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
13114 let _ = self.drop_previous_version();
13115 return;
13116 }
13117 if now_ms - record.created_at_ms > config.max_age_ms {
13118 let _ = self.drop_previous_version();
13119 return;
13120 }
13121 if self.previous_version_coverage_complete() {
13122 let _ = self.drop_previous_version();
13123 }
13124 }
13125
13126 pub fn diagnostics_snapshot(
13127 &self,
13128 request: &ClientDiagnosticsRequest,
13129 ) -> Result<ClientDiagnosticsSnapshot, String> {
13130 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
13131 return Err(format!(
13132 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
13133 ));
13134 }
13135 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
13136 for sub in &self.subs {
13137 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
13138 let complete =
13139 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
13140 let state = match sub.state {
13141 SubState::Revoked => "revoked",
13142 SubState::Failed => "failed",
13143 SubState::Active if reset => "reset",
13144 SubState::Active if complete => "complete",
13145 SubState::Active => "bootstrapping",
13146 };
13147 subscriptions.insert(
13148 sub.id.clone(),
13149 DiagnosticSubscription {
13150 id: sub.id.clone(),
13151 table: sub.table.clone(),
13152 state: state.to_owned(),
13153 complete,
13154 cursor: Some(sub.cursor),
13155 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
13156 },
13157 );
13158 }
13159 for expected in &request.expected_subscriptions {
13160 if expected.id.is_empty() || expected.table.is_empty() {
13161 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
13162 }
13163 if subscriptions
13164 .get(&expected.id)
13165 .is_some_and(|registered| registered.table != expected.table)
13166 {
13167 subscriptions.insert(
13168 expected.id.clone(),
13169 DiagnosticSubscription {
13170 id: expected.id.clone(),
13171 table: expected.table.clone(),
13172 state: "failed".to_owned(),
13173 complete: false,
13174 cursor: None,
13175 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
13176 },
13177 );
13178 } else {
13179 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
13180 DiagnosticSubscription {
13181 id: expected.id.clone(),
13182 table: expected.table.clone(),
13183 state: "unregistered".to_owned(),
13184 complete: false,
13185 cursor: None,
13186 reason_code: None,
13187 }
13188 });
13189 }
13190 }
13191 let mut ordered_subscriptions = Vec::new();
13192 let mut included = BTreeSet::new();
13193 for expected in &request.expected_subscriptions {
13194 if included.insert(expected.id.clone()) {
13195 if let Some(subscription) = subscriptions.get(&expected.id) {
13196 ordered_subscriptions.push(subscription.clone());
13197 }
13198 }
13199 }
13200 for (id, subscription) in &subscriptions {
13201 if included.insert(id.clone()) {
13202 ordered_subscriptions.push(subscription.clone());
13203 }
13204 }
13205 let subscriptions_truncated =
13206 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
13207 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
13208 let captured_at_ms = self.clock_now_ms();
13209 let lease = if let Some(error_code) = self
13210 .lease_state
13211 .as_ref()
13212 .and_then(|state| state.error_code.clone())
13213 {
13214 ClientDiagnosticsLease {
13215 state: "stopped".to_owned(),
13216 expires_at_ms: self
13217 .lease_state
13218 .as_ref()
13219 .and_then(|state| state.expires_at_ms),
13220 error_code: Some(Self::diagnostic_code(&error_code)),
13221 }
13222 } else if let Some(expires_at_ms) = self
13223 .lease_state
13224 .as_ref()
13225 .and_then(|state| state.expires_at_ms)
13226 {
13227 ClientDiagnosticsLease {
13228 state: if expires_at_ms <= captured_at_ms {
13229 "expired".to_owned()
13230 } else {
13231 "active".to_owned()
13232 },
13233 expires_at_ms: Some(expires_at_ms),
13234 error_code: None,
13235 }
13236 } else {
13237 ClientDiagnosticsLease {
13238 state: "none".to_owned(),
13239 expires_at_ms: None,
13240 error_code: None,
13241 }
13242 };
13243 let connectivity = match self.last_round.as_ref() {
13244 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
13245 Some(round) if round.status == "succeeded" => "online",
13246 Some(round)
13247 if round.status == "failed"
13248 && matches!(
13249 round.cause_kind,
13250 Some(
13251 crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
13252 )
13253 ) =>
13254 {
13255 "offline"
13256 }
13257 Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
13258 _ => "unknown",
13259 };
13260 Ok(ClientDiagnosticsSnapshot {
13261 version: CLIENT_DIAGNOSTICS_VERSION,
13262 captured_at_ms,
13263 host: ClientDiagnosticsHost {
13264 kind: "direct".to_owned(),
13265 role: "single".to_owned(),
13266 connectivity: connectivity.to_owned(),
13267 realtime: self.realtime_state,
13268 realtime_policy: self.realtime_policy,
13269 realtime_reason_code: self.realtime_reason_code.clone(),
13270 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
13271 },
13272 security_lifecycle: self.security_lifecycle().to_owned(),
13273 schema: ClientDiagnosticsSchema {
13274 current_version: self.schema.version,
13275 upgrading: self.upgrading,
13276 required_version: self
13277 .schema_floor
13278 .as_ref()
13279 .and_then(|floor| floor.required_schema_version),
13280 latest_version: self
13281 .schema_floor
13282 .as_ref()
13283 .and_then(|floor| floor.latest_schema_version),
13284 },
13285 replica: ClientDiagnosticsReplica {
13286 local_revision: self.local_revision().to_string(),
13287 sync_needed: self.sync_needed,
13288 pending_outbox: self.outbox.len(),
13289 },
13290 lease,
13291 subscriptions: ordered_subscriptions,
13292 subscriptions_truncated,
13293 last_round: self.last_round.clone(),
13294 last_change: self.last_change.clone(),
13295 storage: self.diagnostics_storage(),
13296 query_failures: self.query_failures.clone(),
13297 })
13298 }
13299
13300 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
13301 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
13302 let page_count: i64 = self
13303 .conn
13304 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
13305 let page_size: i64 = self
13306 .conn
13307 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
13308 let outbox_bytes: i64 = self
13310 .conn
13311 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
13312 .query_row([], |row| row.get(0))?;
13313 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
13314 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
13315 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
13316 let blob_bytes = if self.schema_has_blobs() {
13317 self.conn
13318 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
13319 .query_row([], |row| row.get(0))?
13320 } else {
13321 0
13322 };
13323 let pressure = self
13324 .limits
13325 .blob_cache_max_bytes
13326 .is_some_and(|limit| blob_bytes > limit);
13327 Ok(ClientDiagnosticsStorage {
13328 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
13329 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
13330 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
13331 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
13332 retained_outcome_entries: Some(outcome_entries.max(0)),
13333 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
13334 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
13335 })
13336 };
13337 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
13338 status: "unreadable".to_owned(),
13339 database_bytes_approx: None,
13340 pending_outbox_bytes_approx: None,
13341 retained_outcome_bytes_approx: None,
13342 retained_outcome_entries: None,
13343 blob_cache_bytes_approx: None,
13344 pressure_reason_code: None,
13345 })
13346 }
13347
13348 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
13349 self.change_queue.drain(..).collect()
13350 }
13351
13352 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
13353 let intents = self.sync_intent_queue.drain(..).collect();
13354 if self.transport_enabled {
13355 intents
13356 } else {
13357 vec![SyncIntent::None]
13358 }
13359 }
13360
13361 fn schedule_background_retry(&mut self) -> u64 {
13362 let delay_ms = self.retry_delay_ms;
13363 self.sync_intent_queue
13364 .push_back(SyncIntent::Background { delay_ms });
13365 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
13366 self.round_retry_delay_ms = Some(delay_ms);
13367 delay_ms
13368 }
13369
13370 fn reset_background_retry(&mut self) {
13371 self.retry_delay_ms = 250;
13372 }
13373
13374 fn retryable_transport_code(code: &str) -> bool {
13375 code == "transport.failed"
13376 || code == "transport.timeout"
13377 || code == "transport.unavailable"
13378 || code == "sync.transport_failed"
13379 }
13380
13381 fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
13386 matches!(
13387 details
13388 .as_ref()
13389 .and_then(|details| details.get("causeKind"))
13390 .and_then(serde_json::Value::as_str),
13391 Some("connect" | "network")
13392 )
13393 }
13394
13395 fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
13400 if ssp2::model::valid_round_failure_code(&error.code) {
13401 self.round_failure = Some(error.code.clone());
13402 }
13403 SectionError::Transport(error)
13404 }
13405
13406 fn retryable_failure_code(code: &str) -> bool {
13411 Self::retryable_transport_code(code)
13412 || matches!(
13413 code,
13414 "sync.auth_required"
13415 | "sync.auth_lease_required"
13416 | "sync.auth_lease_revoked"
13417 | "sync.internal_error"
13418 | "sync.idempotency_cache_miss"
13419 | "sync.schema_not_ready"
13420 | "sync.segment_expired"
13421 | "sync.rate_limited"
13422 | "sync.websocket_connection_limit"
13423 )
13424 }
13425
13426 fn diagnostic_code(code: &str) -> String {
13427 let valid = !code.is_empty()
13428 && code.len() <= 96
13429 && code.contains('.')
13430 && code
13431 .bytes()
13432 .next()
13433 .is_some_and(|byte| byte.is_ascii_lowercase())
13434 && code.bytes().all(|byte| {
13435 byte.is_ascii_lowercase()
13436 || byte.is_ascii_digit()
13437 || matches!(byte, b'.' | b'_' | b'-')
13438 });
13439 if valid {
13440 code.to_owned()
13441 } else {
13442 "client.unknown_failure".to_owned()
13443 }
13444 }
13445
13446 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
13447 if self.sync_needed != value {
13448 if self.begin_observation("syncular_status").is_ok() {
13449 self.sync_needed = value;
13450 let batch = ChangeAccumulator {
13451 status: true,
13452 ..ChangeAccumulator::default()
13453 };
13454 if self.finish_observation("syncular_status", batch).is_err() {
13455 self.rollback_observation("syncular_status");
13456 }
13457 } else {
13458 self.sync_needed = value;
13459 }
13460 }
13461 if value && interactive {
13462 self.sync_intent_queue.push_back(SyncIntent::Interactive);
13463 }
13464 }
13465
13466 fn sqlite_failure(
13467 storage_failure: &RefCell<Option<QueryReadFailure>>,
13468 error: rusqlite::Error,
13469 ) -> String {
13470 let message = error.to_string();
13471 let failure = QueryReadFailure::from(error);
13472 let code = failure.code;
13473 if code.is_some() && storage_failure.borrow().is_none() {
13474 *storage_failure.borrow_mut() = Some(failure);
13475 }
13476 match code {
13477 Some(code) => format!("{code}: {message}"),
13478 None => message,
13479 }
13480 }
13481
13482 fn begin_observation(&self, name: &str) -> Result<(), String> {
13483 if let Some(failure) = &self.coalesced_failure {
13484 return Err(failure.clone());
13485 }
13486 if self.coalesced.is_some() && self.conn.is_autocommit() {
13489 return Err("the coalesced apply transaction was rolled back".to_owned());
13490 }
13491 self.conn
13492 .execute_batch(&format!("SAVEPOINT {name}"))
13493 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13494 }
13495
13496 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
13499 if self.coalesced.is_some() || !self.conn.is_autocommit() {
13500 return Ok(());
13501 }
13502 self.conn
13503 .execute_batch("BEGIN")
13504 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13505 self.coalesced = Some(CoalescedApply {
13506 rows: 0,
13507 subs: self.subs.clone(),
13508 outbox: self.outbox.clone(),
13509 failed_commits: self.failed_commits.clone(),
13510 conflicts: self.conflicts.clone(),
13511 rejections: self.rejections.clone(),
13512 schema_floor: self.schema_floor.clone(),
13513 lease_state: self.lease_state.clone(),
13514 stopped: self.stopped,
13515 upgrading: self.upgrading,
13516 sync_needed: self.sync_needed,
13517 overlay_dirty: self.overlay_dirty.snapshot(),
13518 change_queue: self.change_queue.len(),
13519 sync_intent_queue: self.sync_intent_queue.len(),
13520 last_change: self.last_change.clone(),
13521 });
13522 Ok(())
13523 }
13524
13525 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
13528 let Some(open) = &self.coalesced else {
13529 return Ok(());
13530 };
13531 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
13532 self.commit_coalesced_apply()?;
13533 self.begin_coalesced_apply()?;
13534 }
13535 if let Some(open) = &mut self.coalesced {
13536 open.rows += rows;
13537 }
13538 Ok(())
13539 }
13540
13541 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
13546 let Some(open) = self.coalesced.take() else {
13547 return Ok(());
13548 };
13549 let committed = if self.conn.is_autocommit() {
13550 Err("the coalesced apply transaction was rolled back".to_owned())
13551 } else {
13552 self.conn
13553 .execute_batch("COMMIT")
13554 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13555 };
13556 let Err(error) = committed else {
13557 return Ok(());
13558 };
13559 if !self.conn.is_autocommit() {
13560 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
13561 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
13562 }
13563 }
13564 self.subs = open.subs;
13565 self.outbox = open.outbox;
13566 self.failed_commits = open.failed_commits;
13567 self.conflicts = open.conflicts;
13568 self.rejections = open.rejections;
13569 self.schema_floor = open.schema_floor;
13570 self.lease_state = open.lease_state;
13571 self.stopped = open.stopped;
13572 self.upgrading = open.upgrading;
13573 self.sync_needed = open.sync_needed;
13574 self.overlay_dirty.restore(open.overlay_dirty);
13575 self.change_queue.truncate(open.change_queue);
13576 self.sync_intent_queue.truncate(open.sync_intent_queue);
13577 self.last_change = open.last_change;
13578 self.coalesced_failure = Some(error.clone());
13579 Err(error)
13580 }
13581
13582 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
13584 let committed = self.commit_coalesced_apply();
13585 match self.coalesced_failure.take() {
13586 Some(failure) => Err(failure),
13587 None => committed,
13588 }
13589 }
13590
13591 fn rollback_observation(&self, name: &str) {
13592 if let Err(error) = self
13593 .conn
13594 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
13595 {
13596 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
13597 if failure.rollback_failure.is_none() {
13598 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
13599 }
13600 }
13601 }
13602 }
13603
13604 fn finish_observation(
13605 &mut self,
13606 name: &str,
13607 mut batch: ChangeAccumulator,
13608 ) -> Result<(), String> {
13609 #[cfg(feature = "bench-internals")]
13610 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
13611 if self.failed_commits.iter().any(|(commit, _)| {
13612 commit
13613 .ops
13614 .iter()
13615 .any(|operation| batch.tables.contains_key(&operation.table))
13616 }) {
13617 batch.outcomes = true;
13618 }
13619 if !batch.touched() {
13620 self.conn
13621 .execute_batch(&format!("RELEASE {name}"))
13622 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13623 return Ok(());
13624 }
13625 let revision = self
13626 .local_revision()
13627 .checked_add(1)
13628 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
13629 self.conn
13630 .execute(
13631 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
13632 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
13633 )
13634 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13635 let status = batch.status.then(|| self.status_snapshot());
13636 let event = ClientChangeBatch {
13637 revision: revision.to_string(),
13638 tables: batch
13639 .tables
13640 .into_iter()
13641 .map(|(table, scope_keys)| TableChange {
13642 table,
13643 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
13644 })
13645 .collect(),
13646 windows: batch
13647 .windows
13648 .into_iter()
13649 .map(|((base_key, table), units)| WindowChange {
13650 base_key,
13651 table,
13652 units: units.into_iter().collect(),
13653 })
13654 .collect(),
13655 status,
13656 conflicts_changed: batch.conflicts,
13657 rejections_changed: batch.rejections,
13658 outcomes_changed: batch.outcomes,
13659 };
13660 self.conn
13661 .execute_batch(&format!("RELEASE {name}"))
13662 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13663 let mut diagnostic_tables = event
13664 .tables
13665 .iter()
13666 .map(|entry| entry.table.clone())
13667 .collect::<BTreeSet<_>>()
13668 .into_iter()
13669 .collect::<Vec<_>>();
13670 let mut diagnostic_windows = event
13671 .windows
13672 .iter()
13673 .map(|entry| entry.table.clone())
13674 .collect::<BTreeSet<_>>()
13675 .into_iter()
13676 .collect::<Vec<_>>();
13677 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
13678 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
13679 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
13680 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
13681 self.last_change = Some(DiagnosticLastChange {
13682 revision: event.revision.clone(),
13683 recorded_at_ms: self.clock_now_ms(),
13684 tables: diagnostic_tables,
13685 windows: diagnostic_windows,
13686 domains_truncated,
13687 status_changed: event.status.is_some(),
13688 conflicts_changed: event.conflicts_changed,
13689 rejections_changed: event.rejections_changed,
13690 outcomes_changed: event.outcomes_changed,
13691 });
13692 self.change_queue.push_back(event);
13693 Ok(())
13694 }
13695
13696 fn record_scope_map(
13697 &self,
13698 batch: &mut ChangeAccumulator,
13699 table_name: &str,
13700 scopes: &[(String, Vec<String>)],
13701 ) {
13702 let Some(table) = self.schema.table(table_name) else {
13703 return;
13704 };
13705 for (variable, values) in scopes {
13706 let Some(scope) = table
13707 .scope_variables
13708 .iter()
13709 .find(|scope| &scope.variable == variable)
13710 else {
13711 continue;
13712 };
13713 for value in values {
13714 batch.scope(table_name, format!("{}:{value}", scope.prefix));
13715 }
13716 }
13717 }
13718
13719 fn record_row_scopes(
13721 &self,
13722 batch: &mut ChangeAccumulator,
13723 table_name: &str,
13724 row_id: &str,
13725 base: bool,
13726 ) -> bool {
13727 let Some(table) = self.schema.table(table_name) else {
13728 return false;
13729 };
13730 if table.scope_variables.is_empty() {
13731 return false;
13732 }
13733 let columns = table
13734 .scope_variables
13735 .iter()
13736 .map(|scope| quote_ident(&scope.column))
13737 .collect::<Vec<_>>()
13738 .join(", ");
13739 let full_table = if base {
13740 base_table(table_name)
13741 } else {
13742 visible_table(table_name)
13743 };
13744 let sql = format!(
13745 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
13746 row_id_predicate(table)
13747 );
13748 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
13749 return false;
13750 };
13751 let values = stmt.query_row(rusqlite::params![row_id], |row| {
13752 let mut values = Vec::with_capacity(table.scope_variables.len());
13753 for index in 0..table.scope_variables.len() {
13754 values.push(row.get::<_, Option<String>>(index)?);
13755 }
13756 Ok(values)
13757 });
13758 let Ok(values) = values else {
13759 return false;
13760 };
13761 let mut recorded = false;
13762 for (scope, value) in table.scope_variables.iter().zip(values) {
13763 if let Some(value) = value {
13764 batch.scope(table_name, format!("{}:{value}", scope.prefix));
13765 recorded = true;
13766 }
13767 }
13768 recorded
13769 }
13770
13771 fn record_commit_changes(
13772 &self,
13773 batch: &mut ChangeAccumulator,
13774 tables: &[String],
13775 changes: &[ssp2::model::Change],
13776 ) {
13777 #[cfg(feature = "bench-internals")]
13778 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
13779 for change in changes {
13780 let Some(table_name) = tables.get(change.table_index as usize) else {
13781 continue;
13782 };
13783 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
13784 if let Some(table) = self.schema.table(table_name) {
13785 for (variable, value) in &change.scopes {
13786 if let Some(scope) = table
13787 .scope_variables
13788 .iter()
13789 .find(|scope| &scope.variable == variable)
13790 {
13791 batch.scope(table_name, format!("{}:{value}", scope.prefix));
13792 precise = true;
13793 }
13794 }
13795 }
13796 if !precise {
13797 batch.table(table_name);
13798 }
13799 }
13800 }
13801
13802 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
13803 if effective.is_empty() {
13804 return false;
13805 }
13806 let Some(table) = self.schema.table(table_name) else {
13807 return false;
13808 };
13809 let mut clauses = Vec::new();
13810 let mut params = Vec::new();
13811 for (variable, values) in effective {
13812 let Some(column) = table.scope_column(variable) else {
13813 return false;
13814 };
13815 if values.is_empty() {
13816 return false;
13817 }
13818 let placeholders = values
13819 .iter()
13820 .map(|value| {
13821 params.push(SqlValue::Text(value.clone()));
13822 "?"
13823 })
13824 .collect::<Vec<_>>()
13825 .join(", ");
13826 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
13827 }
13828 let sql = format!(
13829 "SELECT 1 FROM {} WHERE {} LIMIT 1",
13830 base_table(table_name),
13831 clauses.join(" AND ")
13832 );
13833 self.conn
13834 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
13835 .is_ok()
13836 }
13837
13838 pub fn upgrading(&self) -> bool {
13840 self.upgrading
13841 }
13842
13843 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
13851 let new_schema = parse_schema_json(schema_json)?;
13852 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
13853 self.conn
13854 .execute_batch("BEGIN IMMEDIATE")
13855 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13856 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
13857 let prior_subs = self.subs.clone();
13863 let prior_outbox = self.outbox.clone();
13864 let prior_rejections = self.rejections.clone();
13865 let prior_upgrading = self.upgrading;
13866 let prior_stopped = self.stopped;
13867 let prior_schema_floor = self.schema_floor.clone();
13868 let prior_overlay_dirty = self.overlay_dirty.snapshot();
13869 let prior_sync_needed = self.sync_needed;
13870 let prior_active_round = self.active_round;
13871 let prior_changes = self.change_queue.len();
13872 let prior_intents = self.sync_intent_queue.len();
13873 let prior_last_change = self.last_change.clone();
13874 let guarded = (|| -> Result<(), String> {
13875 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
13880 self.create_bookkeeping_tables()?;
13881 match marker {
13882 Some(version) if version < self.schema.version => self.run_schema_reset()?,
13883 None => {
13884 self.create_synced_tables()?;
13885 self.persist_schema_version()?;
13886 }
13887 _ => {}
13888 }
13889 self.outbox = self.load_outbox()?;
13894 self.drop_unencodable_outbox(false, None)?;
13899 if marker == Some(self.schema.version)
13900 && (!self.outbox.is_empty()
13901 || !self.failed_commits.is_empty()
13902 || self.has_acknowledged_rows())
13903 {
13904 self.overlay_dirty.set(true);
13905 self.rebuild_overlay()?;
13906 }
13907 self.prune_unknown_subscriptions(false)?;
13908 self.save_subscription_scope_schema();
13909 self.enqueue_startup_sync_if_needed()?;
13913 Ok(())
13914 })();
13915 let outcome = match guarded {
13916 Ok(()) => self
13917 .conn
13918 .execute_batch("COMMIT")
13919 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
13920 Err(error) => Err(error),
13921 };
13922 if let Err(error) = outcome {
13923 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
13924 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
13926 }
13927 self.schema = prior_schema;
13928 self.subs = prior_subs;
13929 self.outbox = prior_outbox;
13930 self.rejections = prior_rejections;
13931 self.upgrading = prior_upgrading;
13932 self.stopped = prior_stopped;
13933 self.schema_floor = prior_schema_floor;
13934 self.overlay_dirty.restore(prior_overlay_dirty);
13935 self.sync_needed = prior_sync_needed;
13936 self.active_round = prior_active_round;
13937 self.change_queue.truncate(prior_changes);
13938 self.sync_intent_queue.truncate(prior_intents);
13939 self.last_change = prior_last_change;
13940 self.insert_sql.borrow_mut().clear();
13942 return Err(error);
13943 }
13944 Ok(())
13945 }
13946
13947 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
13948 let startup_work = !self.stopped
13949 && (!self.outbox.is_empty()
13950 || self.has_acknowledged_rows()
13951 || !self.load_pending_evictions()?.is_empty()
13952 || self.subs.iter().any(|sub| sub.state == SubState::Active));
13953 if startup_work {
13954 self.sync_needed = true;
13955 self.sync_intent_queue.push_back(SyncIntent::Interactive);
13956 }
13957 Ok(())
13958 }
13959
13960 fn clear_satisfied_persisted_schema_floor(&mut self) {
13967 let satisfied = self
13968 .schema_floor
13969 .as_ref()
13970 .and_then(|floor| floor.required_schema_version)
13971 .is_some_and(|required| self.schema.version >= required);
13972 if !satisfied {
13973 return;
13974 }
13975 self.schema_floor = None;
13976 self.stopped = false;
13977 self.delete_meta(SCHEMA_FLOOR_KEY);
13978 }
13979
13980 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
13985 let Some(replica_path) = self.previous_version_replica_path() else {
13986 return Ok(());
13987 };
13988 let max_age_ms = self
13989 .previous_version
13990 .filter(|config| config.enabled)
13991 .map(|config| config.max_age_ms);
13992 reconcile_previous_version_at_boot(
13993 &self.conn,
13994 &replica_path,
13995 self.schema.version,
13996 max_age_ms,
13997 self.clock_now_ms(),
13998 )
13999 }
14000
14001 fn previous_version_replica_path(&self) -> Option<String> {
14005 let path = self.conn.path()?;
14006 if path.is_empty() || path.starts_with("file:") {
14007 return None;
14008 }
14009 Some(path.to_owned())
14010 }
14011
14012 fn run_schema_reset(&mut self) -> Result<(), String> {
14018 self.begin_observation("syncular_schema_reset")?;
14019 let prior_subs = self.subs.clone();
14020 let prior_outbox = self.outbox.clone();
14021 let prior_rejections = self.rejections.clone();
14022 let prior_upgrading = self.upgrading;
14023 let prior_stopped = self.stopped;
14024 let prior_schema_floor = self.schema_floor.clone();
14025 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14026 let prior_sync_needed = self.sync_needed;
14027 let prior_active_round = self.active_round;
14028 let mut batch = ChangeAccumulator::default();
14029 let result = self
14030 .run_schema_reset_observed(&mut batch, true)
14031 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
14032 if let Err(error) = result {
14033 self.rollback_observation("syncular_schema_reset");
14034 self.subs = prior_subs;
14035 self.outbox = prior_outbox;
14036 self.rejections = prior_rejections;
14037 self.upgrading = prior_upgrading;
14038 self.stopped = prior_stopped;
14039 self.schema_floor = prior_schema_floor;
14040 self.overlay_dirty.restore(prior_overlay_dirty);
14041 self.sync_needed = prior_sync_needed;
14042 self.active_round = prior_active_round;
14043 return Err(error);
14044 }
14045 Ok(())
14046 }
14047
14048 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
14049 if self.get_meta(LOG_EPOCH_KEY).is_none() {
14050 self.begin_observation("syncular_epoch_acquisition")?;
14051 let previous_sync_needed = self.sync_needed;
14052 if let Err(error) = self.conn.execute(
14053 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
14054 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
14055 ) {
14056 self.rollback_observation("syncular_epoch_acquisition");
14057 return Err(Self::sqlite_failure(&self.storage_failure, error));
14058 }
14059 self.sync_needed = true;
14060 let batch = ChangeAccumulator {
14061 status: true,
14062 ..ChangeAccumulator::default()
14063 };
14064 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
14065 self.rollback_observation("syncular_epoch_acquisition");
14066 self.sync_needed = previous_sync_needed;
14067 return Err(error);
14068 }
14069 self.sync_intent_queue.push_back(SyncIntent::Interactive);
14070 return Ok(Vec::new());
14071 }
14072 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
14073 self.begin_observation("syncular_log_epoch_reset")?;
14074 let prior_subs = self.subs.clone();
14075 let prior_outbox = self.outbox.clone();
14076 let prior_rejections = self.rejections.clone();
14077 let prior_upgrading = self.upgrading;
14078 let prior_stopped = self.stopped;
14079 let prior_schema_floor = self.schema_floor.clone();
14080 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14081 let prior_sync_needed = self.sync_needed;
14082 let prior_active_round = self.active_round;
14083 let mut batch = ChangeAccumulator::default();
14084 let result = self
14085 .run_schema_reset_observed(&mut batch, false)
14086 .and_then(|()| {
14087 self.conn
14088 .execute(
14089 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
14090 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
14091 )
14092 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14093 self.sync_needed = true;
14094 batch.status = true;
14095 self.finish_observation("syncular_log_epoch_reset", batch)
14096 });
14097 if let Err(error) = result {
14098 self.rollback_observation("syncular_log_epoch_reset");
14099 self.subs = prior_subs;
14100 self.outbox = prior_outbox;
14101 self.rejections = prior_rejections;
14102 self.upgrading = prior_upgrading;
14103 self.stopped = prior_stopped;
14104 self.schema_floor = prior_schema_floor;
14105 self.overlay_dirty.restore(prior_overlay_dirty);
14106 self.sync_needed = prior_sync_needed;
14107 self.active_round = prior_active_round;
14108 return Err(error);
14109 }
14110 self.sync_intent_queue.push_back(SyncIntent::Interactive);
14111 Ok(resets)
14112 }
14113
14114 fn run_schema_reset_observed(
14115 &mut self,
14116 batch: &mut ChangeAccumulator,
14117 capture: bool,
14118 ) -> Result<(), String> {
14119 read_local_schema_version(&self.conn, self.schema.version)?;
14125 self.conn
14126 .execute_batch(
14127 "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
14128 )
14129 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14130 self.cancel_sync_round();
14131 self.upgrading = true;
14132 batch.status = true;
14133 if self.previous_version.is_some_and(|config| config.enabled) {
14141 let previous_version = self
14142 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
14143 .and_then(|value| value.parse::<i32>().ok())
14144 .unwrap_or(self.schema.version);
14145 let pending = self
14146 .outbox
14147 .iter()
14148 .map(|commit| PendingCommitAudit {
14149 commit_id: &commit.client_commit_id,
14150 operations: commit_audit_operations(commit),
14151 })
14152 .collect::<Vec<_>>();
14153 crate::previous_version::write_previous_version_audit(
14154 &self.conn,
14155 &crate::previous_version::build_previous_version_audit(
14156 &self.schema,
14157 &pending,
14158 previous_version,
14159 self.schema.version,
14160 self.clock_now_ms(),
14161 ),
14162 );
14163 }
14164 if let Some(replica_path) = self.previous_version_replica_path() {
14169 sweep_previous_version_container(&self.conn, &replica_path)?;
14170 if capture {
14171 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
14172 let previous_version = self
14173 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
14174 .and_then(|value| value.parse::<i32>().ok())
14175 .unwrap_or(self.schema.version);
14176 capture_previous_version_from_replica(
14177 &self.conn,
14178 &replica_path,
14179 previous_version,
14180 self.schema.version,
14181 &config,
14182 self.clock_now_ms(),
14183 )?;
14184 }
14185 }
14186 }
14187 for table in &self.schema.tables {
14188 batch.table(&table.name);
14189 }
14190 for (base_key, unit, table) in self.load_registered_window_units() {
14191 batch.window(&base_key, &table, &unit);
14192 }
14193 if capture {
14194 self.prune_unknown_subscriptions(true)?;
14195 }
14196 self.insert_sql.borrow_mut().clear();
14198 self.overlay_dirty.set(true);
14199 let virtual_tables: Vec<String> = {
14207 let mut stmt = self
14208 .conn
14209 .prepare(
14210 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
14211 )
14212 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14213 let rows = stmt
14214 .query_map([], |row| row.get::<_, String>(0))
14215 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14216 rows.filter_map(Result::ok)
14217 .filter(|name| is_synced_table_name(name))
14218 .collect()
14219 };
14220 for name in virtual_tables {
14221 self.conn
14222 .execute_batch(&format!(
14223 "DROP TABLE IF EXISTS {}",
14224 quote_ident(&format!("_syncular_fts_{name}"))
14225 ))
14226 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14227 self.conn
14228 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
14229 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14230 }
14231 let existing: Vec<String> = {
14232 let mut stmt = self
14233 .conn
14234 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
14235 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14236 let rows = stmt
14237 .query_map([], |row| row.get::<_, String>(0))
14238 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14239 rows.filter_map(Result::ok)
14240 .filter(|name| is_synced_table_name(name))
14241 .collect()
14242 };
14243 for name in &existing {
14244 if let Some(table) = name.strip_prefix("_syncular_base_") {
14245 batch.table(table);
14246 } else if !name.starts_with("_syncular_") {
14247 batch.table(name);
14248 }
14249 }
14250 for name in existing {
14251 self.conn
14252 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
14253 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14254 }
14255 self.create_synced_tables()?;
14257 for sub in &mut self.subs {
14259 sub.cursor = -1;
14260 sub.bootstrap_state = None;
14261 sub.effective = None;
14262 sub.state = SubState::Active;
14263 sub.reason_code = None;
14264 sub.synced_once = false;
14265 }
14266 let subs = self.subs.clone();
14267 for sub in &subs {
14268 self.persist_sub(sub)?;
14269 }
14270 self.stopped = false;
14272 self.schema_floor = None;
14273 self.delete_meta(SCHEMA_FLOOR_KEY);
14274 self.persist_schema_version()?;
14276 self.save_subscription_scope_schema();
14277 self.drop_unencodable_outbox(false, Some(batch))?;
14288 self.rebuild_overlay()?;
14290 Ok(())
14291 }
14292
14293 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
14302 let incompatible = self
14303 .outbox
14304 .iter()
14305 .filter(|commit| {
14306 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
14307 })
14308 .cloned()
14309 .collect::<Vec<_>>();
14310 if incompatible.is_empty() {
14311 return Ok(false);
14312 }
14313 let prior_outbox = self.outbox.clone();
14319 let prior_rejections = self.rejections.clone();
14320 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14321 let owns_transaction = self.conn.is_autocommit();
14322 self.begin_observation("syncular_outbox_drop")?;
14323 let mut batch = ChangeAccumulator::default();
14324 let dropped = (|| -> Result<(), String> {
14325 let mut rejections = Vec::new();
14326 for commit in &incompatible {
14327 let results = commit
14328 .ops
14329 .iter()
14330 .enumerate()
14331 .map(|(op_index, operation)| {
14332 let rejection = RejectionRecord {
14333 client_commit_id: commit.client_commit_id.clone(),
14334 op_index: op_index as i32,
14335 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
14336 message: "the persisted commit cannot encode under the current schema"
14337 .to_owned(),
14338 retryable: false,
14339 details: None,
14340 operation: Some(CommitOperation::from(operation)),
14341 };
14342 rejections.push(rejection.clone());
14343 CommitOperationOutcome::Error { rejection }
14344 })
14345 .collect::<Vec<_>>();
14346 self.persist_commit_outcome(
14347 &commit.client_commit_id,
14348 CommitOutcomeStatus::Rejected,
14349 &results,
14350 Some(&commit.ops),
14351 )?;
14352 self.delete_outbox_persisted(&commit.client_commit_id)?;
14353 for operation in &commit.ops {
14356 if self.schema.table(&operation.table).is_some() {
14357 batch.table(&operation.table);
14358 }
14359 }
14360 }
14361 self.prune_commit_outcomes()?;
14362 let incompatible_ids = incompatible
14363 .iter()
14364 .map(|commit| commit.client_commit_id.as_str())
14365 .collect::<BTreeSet<_>>();
14366 self.outbox
14367 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
14368 self.rejections.extend(rejections);
14369 self.overlay_dirty.set(true);
14372 self.rebuild_overlay()?;
14373 batch.status = true;
14374 batch.rejections = true;
14375 batch.outcomes = true;
14376 Ok(())
14377 })();
14378 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
14379 if let Err(error) = result {
14380 if owns_transaction {
14384 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
14385 let mut failure = self.storage_failure.borrow_mut();
14386 let failure =
14387 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
14388 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
14389 }
14390 } else {
14391 self.rollback_observation("syncular_outbox_drop");
14392 }
14393 self.outbox = prior_outbox;
14394 self.rejections = prior_rejections;
14395 self.overlay_dirty.restore(prior_overlay_dirty);
14396 return Err(error);
14397 }
14398 Ok(true)
14399 }
14400
14401 fn create_synced_tables(&self) -> Result<(), String> {
14404 for table in &self.schema.tables {
14405 for (full, index_prefix) in [
14409 (base_table(&table.name), "_syncular_base_"),
14410 (visible_table(&table.name), ""),
14411 ] {
14412 let mut cols: Vec<String> =
14413 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
14414 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
14415 let sql = format!(
14416 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
14417 cols.join(", "),
14418 quote_ident(&table.primary_key)
14419 );
14420 self.conn
14421 .execute(&sql, [])
14422 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14423 for index in &table.indexes {
14427 let unique = if index.unique { "UNIQUE " } else { "" };
14428 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
14429 let cols_sql = index
14430 .columns
14431 .iter()
14432 .map(|c| quote_ident(c))
14433 .collect::<Vec<_>>()
14434 .join(", ");
14435 let index_sql = format!(
14436 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
14437 );
14438 self.conn
14439 .execute(&index_sql, [])
14440 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14441 }
14442 }
14443 }
14444 for table in &self.schema.tables {
14447 for index in &table.fts_indexes {
14448 self.create_fts_projection(table, index)?;
14449 }
14450 }
14451 Ok(())
14452 }
14453
14454 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
14455 let count: i64 = self
14456 .conn
14457 .query_row(
14458 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
14459 rusqlite::params![index.name],
14460 |row| row.get(0),
14461 )
14462 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14463 Ok(count > 0)
14464 }
14465
14466 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
14467 for suffix in ["ai", "ad", "au"] {
14468 self.conn
14469 .execute(
14470 &format!(
14471 "DROP TRIGGER IF EXISTS {}",
14472 quote_ident(&format!("{}_{suffix}", index.name))
14473 ),
14474 [],
14475 )
14476 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14477 }
14478 Ok(())
14479 }
14480
14481 fn create_fts_triggers(
14482 &self,
14483 table: &TableSchema,
14484 index: &FtsIndexSchema,
14485 ) -> Result<(), String> {
14486 let fts = quote_ident(&index.name);
14487 let source = visible_table(&table.name);
14488 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
14489 let pk = quote_ident(&table.primary_key);
14490 let projection_columns = std::iter::once(source_id.clone())
14491 .chain(index.columns.iter().map(|column| quote_ident(column)))
14492 .collect::<Vec<_>>()
14493 .join(", ");
14494 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
14495 .chain(
14496 index
14497 .columns
14498 .iter()
14499 .map(|column| format!("new.{}", quote_ident(column))),
14500 )
14501 .collect::<Vec<_>>()
14502 .join(", ");
14503 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
14504 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
14505 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
14506 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
14507 let bi = quote_ident(&format!("{}_bi", index.name));
14508 let ai = quote_ident(&format!("{}_ai", index.name));
14509 let ad = quote_ident(&format!("{}_ad", index.name));
14510 let au = quote_ident(&format!("{}_au", index.name));
14511 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
14512 let sql = format!(
14513 "DROP TRIGGER IF EXISTS {bi};
14514 DROP TRIGGER IF EXISTS {ai};
14515 DROP TRIGGER IF EXISTS {ad};
14516 DROP TRIGGER IF EXISTS {au};
14517 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
14518 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
14519 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
14520 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
14521 );
14522 self.conn
14523 .execute_batch(&sql)
14524 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14525 }
14526
14527 fn rebuild_fts_projection(
14528 &self,
14529 table: &TableSchema,
14530 index: &FtsIndexSchema,
14531 ) -> Result<(), String> {
14532 let fts = quote_ident(&index.name);
14533 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
14534 let pk = quote_ident(&table.primary_key);
14535 let indexed_columns = index
14536 .columns
14537 .iter()
14538 .map(|column| quote_ident(column))
14539 .collect::<Vec<_>>();
14540 let projection_columns = std::iter::once(source_id.clone())
14541 .chain(indexed_columns.iter().cloned())
14542 .collect::<Vec<_>>()
14543 .join(", ");
14544 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
14545 let sql = format!(
14546 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
14547 columns = indexed_columns.join(", "),
14548 source = visible_table(&table.name),
14549 );
14550 self.conn
14551 .execute_batch(&sql)
14552 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14553 }
14554
14555 fn create_fts_projection(
14556 &self,
14557 table: &TableSchema,
14558 index: &FtsIndexSchema,
14559 ) -> Result<(), String> {
14560 self.begin_observation("syncular_fts_schema")?;
14561 let result = (|| {
14562 let existed = self.fts_projection_exists(index)?;
14563 let tokenizer = index.tokenize.replace('\'', "''");
14564 let columns = index
14565 .columns
14566 .iter()
14567 .map(|column| quote_ident(column))
14568 .collect::<Vec<_>>()
14569 .join(", ");
14570 let sql = format!(
14571 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
14572 fts = quote_ident(&index.name),
14573 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
14574 );
14575 self.conn.execute(&sql, []).map_err(|error| {
14576 format!(
14577 "cannot create local FTS5 projection {:?}: {error}",
14578 index.name
14579 )
14580 })?;
14581 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
14582 let mapped: bool = self
14583 .conn
14584 .query_row(
14585 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
14586 [format!("_syncular_fts_{}", index.name)],
14587 |row| row.get(0),
14588 )
14589 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14590 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14591 if !mapped && existed {
14592 self.conn
14593 .execute_batch(&format!(
14594 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
14595 quote_ident(FTS_SOURCE_ID_COLUMN),
14596 quote_ident(&index.name)
14597 ))
14598 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14599 }
14600 self.create_fts_triggers(table, index)?;
14601 if !existed {
14602 self.rebuild_fts_projection(table, index)?;
14603 }
14604 Ok(())
14605 })();
14606 if let Err(error) = result {
14607 self.rollback_observation("syncular_fts_schema");
14608 return Err(error);
14609 }
14610 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
14611 self.rollback_observation("syncular_fts_schema");
14612 return Err(error.to_string());
14613 }
14614 Ok(())
14615 }
14616
14617 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
14620 let state = serde_json::json!({
14621 "requested": scope_map_to_json(&sub.requested),
14622 "params": sub.params,
14623 "cursor": sub.cursor,
14624 "bootstrapState": sub.bootstrap_state,
14625 "status": sub.state.name(),
14626 "reasonCode": sub.reason_code,
14627 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
14628 "syncedOnce": sub.synced_once,
14629 });
14630 self.conn.execute(
14631 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
14632 rusqlite::params![sub.id, sub.table, state.to_string()],
14633 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14634 }
14635
14636 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
14637 let ops: Vec<Value> = commit
14638 .ops
14639 .iter()
14640 .map(|op| {
14641 let mut entry = serde_json::json!({
14642 "op": if op.upsert { "upsert" } else { "delete" },
14643 "table": op.table,
14644 "rowId": op.row_id,
14645 "baseVersion": op.base_version,
14646 "values": op.values.clone().map(Value::Object),
14647 });
14648 if op.insert {
14649 entry["insert"] = Value::Bool(true);
14650 }
14651 entry
14652 })
14653 .collect();
14654 self.conn
14655 .execute(
14656 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
14657 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
14658 )
14659 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14660 for operation in &commit.ops {
14661 let Some(values) = &operation.values else {
14662 continue;
14663 };
14664 let Some(table) = self.schema.table(&operation.table) else {
14665 continue;
14666 };
14667 for column in table
14668 .columns
14669 .iter()
14670 .filter(|column| column.ty == ColumnType::BlobRef)
14671 {
14672 let Some(Value::String(raw)) = values.get(&column.name) else {
14673 continue;
14674 };
14675 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
14676 continue;
14677 };
14678 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
14679 continue;
14680 };
14681 self.conn
14682 .execute(
14683 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
14684 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
14685 rusqlite::params![commit.client_commit_id, blob_id],
14686 )
14687 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14688 }
14689 }
14690 Ok(())
14691 }
14692
14693 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
14694 if let Some(commit) = self
14695 .outbox
14696 .iter()
14697 .find(|commit| commit.client_commit_id == client_commit_id)
14698 {
14699 for operation in &commit.ops {
14700 self.overlay_dirty.table(&operation.table);
14701 }
14702 }
14703
14704 self.conn
14705 .execute(
14706 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
14707 rusqlite::params![client_commit_id],
14708 )
14709 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14710 self.conn
14711 .execute(
14712 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
14713 rusqlite::params![client_commit_id],
14714 )
14715 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14716 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
14717 .map(|_| ())
14718 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14719 }
14720
14721 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
14722 match status {
14723 CommitOutcomeStatus::Applied => "applied",
14724 CommitOutcomeStatus::Cached => "cached",
14725 CommitOutcomeStatus::Conflict => "conflict",
14726 CommitOutcomeStatus::Rejected => "rejected",
14727 }
14728 }
14729
14730 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
14731 match resolution {
14732 CommitOutcomeResolution::Active => "active",
14733 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
14734 CommitOutcomeResolution::Superseded => "superseded",
14735 CommitOutcomeResolution::Dismissed => "dismissed",
14736 }
14737 }
14738
14739 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
14740 match value {
14741 "applied" => Ok(CommitOutcomeStatus::Applied),
14742 "cached" => Ok(CommitOutcomeStatus::Cached),
14743 "conflict" => Ok(CommitOutcomeStatus::Conflict),
14744 "rejected" => Ok(CommitOutcomeStatus::Rejected),
14745 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
14746 }
14747 }
14748
14749 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
14750 match value {
14751 "active" => Ok(CommitOutcomeResolution::Active),
14752 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
14753 "superseded" => Ok(CommitOutcomeResolution::Superseded),
14754 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
14755 _ => Err(format!(
14756 "invalid persisted commit outcome resolution {value:?}"
14757 )),
14758 }
14759 }
14760
14761 fn persist_commit_outcome(
14762 &self,
14763 client_commit_id: &str,
14764 status: CommitOutcomeStatus,
14765 results: &[CommitOperationOutcome],
14766 operations: Option<&[OutboxOp]>,
14767 ) -> Result<(), String> {
14768 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
14769 let operations_json = operations
14770 .map(|items| {
14771 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
14772 })
14773 .transpose()
14774 .map_err(|error| error.to_string())?;
14775 self.conn
14776 .execute(
14777 "INSERT INTO _syncular_commit_outcomes (
14778 client_commit_id, status, recorded_at_ms, results_json,
14779 operations_json, resolution
14780 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
14781 rusqlite::params![
14782 client_commit_id,
14783 Self::outcome_status_name(status),
14784 self.clock_now_ms(),
14785 results_json,
14786 operations_json
14787 ],
14788 )
14789 .map(|_| ())
14790 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14791 }
14792
14793 fn stored_cell_is_representable(cell: &Value) -> bool {
14796 cell.is_null()
14797 || cell.is_string()
14798 || cell.is_boolean()
14799 || cell.is_number()
14800 || is_canonical_bytes_envelope(cell)
14801 }
14802
14803 fn stored_row_is_representable(value: &Value) -> bool {
14804 value
14805 .as_object()
14806 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
14807 }
14808
14809 fn stored_operation_is_representable(value: &Value) -> bool {
14810 value.get("table").is_some_and(Value::is_string)
14811 && value.get("rowId").is_some_and(Value::is_string)
14812 && value.get("baseVersion").is_none_or(|v| {
14813 v.as_i64()
14814 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
14815 })
14816 && match value.get("op").and_then(Value::as_str) {
14817 Some("upsert") => {
14818 value
14819 .get("values")
14820 .is_some_and(Self::stored_row_is_representable)
14821 && value
14822 .get("insert")
14823 .is_none_or(|insert| *insert == Value::Bool(true))
14824 }
14825 Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
14826 _ => false,
14827 }
14828 }
14829
14830 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
14834 serde_json::to_value(CommitOperation::from(operation))
14835 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
14836 }
14837
14838 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
14839 let StoredCommitOutcomeRow {
14840 sequence,
14841 client_commit_id,
14842 status,
14843 recorded_at_ms,
14844 results_json,
14845 operations_json,
14846 resolution,
14847 resolved_at_ms,
14848 replacement_client_commit_id,
14849 } = row;
14850 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
14851 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
14852 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
14853 {
14854 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
14855 }
14856 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
14857 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
14858 for result in &stored {
14859 let nested = match result.get("status").and_then(Value::as_str) {
14860 Some("conflict") => result.get("conflict"),
14861 Some("error") => result.get("rejection"),
14862 _ => None,
14863 };
14864 if nested
14865 .and_then(|record| record.get("operation"))
14866 .is_some_and(|op| !Self::stored_operation_is_representable(op))
14867 || (result.get("status").and_then(Value::as_str) == Some("conflict")
14868 && nested
14869 .and_then(|record| record.get("serverRow"))
14870 .is_some_and(|row| !Self::stored_row_is_representable(row)))
14871 {
14872 return Err(invalid());
14873 }
14874 }
14875 let results: Vec<CommitOperationOutcome> =
14876 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
14877 let has_conflict = results
14878 .iter()
14879 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
14880 let has_error = results
14881 .iter()
14882 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
14883 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
14884 let redacted = results.is_empty() && operations_json.is_none();
14885 if match status {
14886 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
14887 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
14888 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
14889 } {
14890 return Err(invalid());
14891 }
14892 let operations: Option<Vec<CommitOperation>> = operations_json
14893 .map(|raw| {
14894 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
14895 if !stored.iter().all(Self::stored_operation_is_representable) {
14896 return Err(invalid());
14897 }
14898 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
14899 })
14900 .transpose()?;
14901 for result in &results {
14902 match result {
14903 CommitOperationOutcome::Conflict { conflict }
14904 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
14905 {
14906 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
14907 }
14908 CommitOperationOutcome::Error { rejection } => {
14909 if let Some(details) = &rejection.details {
14910 let encoded = serde_json::to_string(details).map_err(|_| {
14911 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
14912 })?;
14913 RejectionDetails::parse(&encoded).map_err(|_| {
14914 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
14915 })?;
14916 }
14917 }
14918 _ => {}
14919 }
14920 }
14921 Ok(CommitOutcome {
14922 sequence,
14923 client_commit_id,
14924 status,
14925 recorded_at_ms,
14926 results,
14927 operations,
14928 retained_rows: None,
14929 resolution: Self::parse_outcome_resolution(&resolution)?,
14930 resolved_at_ms,
14931 replacement_client_commit_id,
14932 })
14933 }
14934
14935 fn unique_conflicts(
14937 &self,
14938 table: &TableSchema,
14939 values: &Map<String, Value>,
14940 base: bool,
14941 ) -> Result<Vec<RetainedUniqueConflict>, String> {
14942 let source = if base {
14943 base_table(&table.name)
14944 } else {
14945 visible_table(&table.name)
14946 };
14947 let mut conflicts = Vec::new();
14948 for index in table.indexes.iter().filter(|index| index.unique) {
14949 let mut params = vec![json_param_to_sql(
14950 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
14951 )?];
14952 for name in &index.columns {
14953 let column = table
14954 .columns
14955 .iter()
14956 .find(|column| column.name == *name)
14957 .ok_or("sync.local_corrupt")?;
14958 let cell = json_to_column_value(column, values.get(name))?;
14959 params.push(owned_sql_value(RowParam::Cell(&cell))?);
14960 }
14961 let sql = format!(
14962 "SELECT * FROM {source} WHERE {} != ? AND {}",
14963 quote_ident(&table.primary_key),
14964 index
14965 .columns
14966 .iter()
14967 .map(|column| format!("{} = ?", quote_ident(column)))
14968 .collect::<Vec<_>>()
14969 .join(" AND ")
14970 );
14971 let mut statement = self
14972 .conn
14973 .prepare_cached(&sql)
14974 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14975 let records = statement
14976 .query_map(rusqlite::params_from_iter(params), |record| {
14977 let values = table
14978 .columns
14979 .iter()
14980 .enumerate()
14981 .map(|(index, column)| {
14982 let value = sql_ref_to_json(column, record.get_ref(index)?);
14983 json_to_column_value(column, Some(&value)).map_err(|message| {
14984 rusqlite::Error::ToSqlConversionFailure(message.into())
14985 })
14986 })
14987 .collect::<Result<Row, _>>()?;
14988 Ok((values, record.get::<_, i64>(table.columns.len())?))
14989 })
14990 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14991 for record in records {
14992 let (row, version) =
14993 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14994 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
14995 conflicts.push(RetainedUniqueConflict {
14996 index: index.name.clone(),
14997 columns: index.columns.clone(),
14998 row_id,
14999 server_row: table
15000 .columns
15001 .iter()
15002 .zip(&row)
15003 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
15004 .collect(),
15005 server_version: version,
15006 });
15007 }
15008 }
15009 Ok(conflicts)
15010 }
15011
15012 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
15013 if let Some((commit, initial)) = self
15014 .failed_commits
15015 .iter()
15016 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
15017 {
15018 let mut rows = Vec::new();
15019 for operation in &commit.ops {
15020 let table = self
15021 .schema
15022 .table(&operation.table)
15023 .ok_or_else(|| "sync.unknown_table".to_owned())?;
15024 let key =
15025 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
15026 .map_err(|_| "sync.local_corrupt".to_owned())?;
15027 let base = self.stored_row(table, &operation.row_id, true)?;
15028 rows.push(RetainedCommitRow {
15029 table: operation.table.clone(),
15030 row_id: operation.row_id.clone(),
15031 unique_conflicts: if operation.upsert {
15032 self.unique_conflicts(
15033 table,
15034 initial
15035 .get(&key)
15036 .and_then(Value::as_object)
15037 .or(operation.values.as_ref())
15038 .ok_or("sync.local_corrupt")?,
15039 true,
15040 )?
15041 } else {
15042 Vec::new()
15043 },
15044 local_row: if operation.upsert {
15045 initial
15046 .get(&key)
15047 .and_then(Value::as_object)
15048 .cloned()
15049 .or_else(|| operation.values.clone())
15050 } else {
15051 None
15052 },
15053 server_row: base.as_ref().map(|(values, _)| {
15054 table
15055 .columns
15056 .iter()
15057 .zip(values)
15058 .map(|(column, value)| {
15059 (column.name.clone(), column_value_to_json(value))
15060 })
15061 .collect()
15062 }),
15063 server_version: base.map(|(_, version)| version),
15064 });
15065 }
15066 outcome.retained_rows = Some(rows);
15067 }
15068 Ok(outcome)
15069 }
15070
15071 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
15072 let row = self
15073 .conn
15074 .query_row(
15075 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
15076 resolution, resolved_at_ms, replacement_client_commit_id
15077 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
15078 rusqlite::params![client_commit_id],
15079 |row| {
15080 Ok(StoredCommitOutcomeRow {
15081 sequence: row.get(0)?,
15082 client_commit_id: row.get(1)?,
15083 status: row.get(2)?,
15084 recorded_at_ms: row.get(3)?,
15085 results_json: row.get(4)?,
15086 operations_json: row.get(5)?,
15087 resolution: row.get(6)?,
15088 resolved_at_ms: row.get(7)?,
15089 replacement_client_commit_id: row.get(8)?,
15090 })
15091 },
15092 )
15093 .optional()
15094 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15095 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
15096 .transpose()
15097 }
15098
15099 #[doc(hidden)]
15101 pub fn prepare_current_base(
15102 &self,
15103 client_commit_id: &str,
15104 sequence: i64,
15105 ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
15106 let outcome = self
15107 .commit_outcome(client_commit_id)
15108 .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
15109 let valid = outcome.as_ref().is_some_and(|outcome| {
15110 outcome.sequence == sequence
15111 && outcome.resolution == CommitOutcomeResolution::Active
15112 && matches!(
15113 outcome.status,
15114 CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
15115 )
15116 && outcome
15117 .operations
15118 .as_ref()
15119 .zip(outcome.retained_rows.as_ref())
15120 .is_some_and(|(operations, rows)| {
15121 !rows.is_empty() && operations.len() == rows.len()
15122 })
15123 });
15124 if !valid {
15125 return Err(CurrentBaseProof::failure(
15126 CurrentBaseProofReason::OutcomeUnavailable,
15127 ));
15128 }
15129 if !self.transport_enabled {
15130 return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
15131 }
15132 Ok(CurrentBasePreparation {
15133 client_commit_id: client_commit_id.to_owned(),
15134 sequence,
15135 subscriptions: self.subs.clone(),
15136 transport_epoch: self.transport_epoch,
15137 subscription_epoch: self.subscription_epoch,
15138 security_epoch: self.security_epoch,
15139 })
15140 }
15141
15142 #[doc(hidden)]
15144 pub fn finish_current_base(
15145 &mut self,
15146 prepared: CurrentBasePreparation,
15147 drive: SyncOutcome,
15148 ) -> CurrentBaseProof {
15149 if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
15150 return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
15151 }
15152 if self.subscription_epoch != prepared.subscription_epoch {
15153 return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
15154 }
15155 if self.security_preflight || self.security_epoch != prepared.security_epoch {
15156 return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
15157 }
15158 let report = match drive {
15159 SyncOutcome::Ok(report) => report,
15160 SyncOutcome::BudgetExhausted(_) => {
15161 return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
15162 }
15163 _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
15164 };
15165 if report.schema_floor.is_some() {
15166 return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
15167 }
15168 if self
15169 .conn
15170 .execute_batch("SAVEPOINT syncular_base_proof")
15171 .is_err()
15172 {
15173 return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
15174 }
15175 let proof = (|| {
15176 let snapshot = snapshot_read_connection(
15177 &self.conn,
15178 &SnapshotReadRequest {
15179 subscriptions: prepared
15180 .subscriptions
15181 .iter()
15182 .map(|sub| sub.id.clone())
15183 .collect(),
15184 commit_ids: vec![prepared.client_commit_id.clone()],
15185 ..Default::default()
15186 },
15187 )
15188 .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
15189 let current = self
15190 .commit_outcome(&prepared.client_commit_id)
15191 .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
15192 let unavailable = if report.revoked.is_empty() {
15193 CurrentBaseProofReason::OutcomeUnavailable
15194 } else {
15195 CurrentBaseProofReason::Revoked
15196 };
15197 let Some(outcome) = current else {
15198 return Err(unavailable);
15199 };
15200 if !matches!(
15201 snapshot.deliveries.first(),
15202 Some(CommitDelivery::Known { .. })
15203 ) || outcome.sequence != prepared.sequence
15204 || outcome.resolution != CommitOutcomeResolution::Active
15205 || !matches!(
15206 outcome.status,
15207 CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
15208 )
15209 {
15210 return Err(unavailable);
15211 }
15212 let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
15213 return Err(unavailable);
15214 };
15215 if rows.is_empty() || operations.len() != rows.len() {
15216 return Err(unavailable);
15217 }
15218 let after: HashMap<&str, &Subscription> =
15219 self.subs.iter().map(|sub| (sub.id.as_str(), sub)).collect();
15220 let complete: HashSet<&str> = snapshot
15221 .subscriptions
15222 .iter()
15223 .filter_map(|sub| match sub {
15224 SubscriptionCatchup::Known {
15225 id,
15226 bootstrap_complete: true,
15227 ..
15228 } => Some(id.as_str()),
15229 _ => None,
15230 })
15231 .collect();
15232 let candidates: Vec<_> = prepared
15233 .subscriptions
15234 .iter()
15235 .filter_map(|before| {
15236 let after = after.get(before.id.as_str())?;
15237 if before.table != after.table
15238 || before.params != after.params
15239 || canonical_scope_json(&before.requested)
15240 != canonical_scope_json(&after.requested)
15241 || !complete.contains(before.id.as_str())
15242 {
15243 return None;
15244 }
15245 after
15246 .effective
15247 .as_ref()
15248 .map(|scopes| (after.table.as_str(), scopes))
15249 })
15250 .collect();
15251 let covered = |table_name: &str, row: &Map<String, Value>| {
15252 let Some(table) = self.schema.table(table_name) else {
15253 return false;
15254 };
15255 candidates.iter().any(|(candidate_table, scopes)| {
15256 *candidate_table == table_name
15257 && !scopes.is_empty()
15258 && scopes.iter().all(|(variable, allowed)| {
15259 table
15260 .scope_column(variable)
15261 .and_then(|column| match row.get(column) {
15262 Some(Value::String(value)) => Some(value.clone()),
15263 Some(Value::Bool(value)) => Some(value.to_string()),
15264 Some(Value::Number(value)) => value.as_f64().map(|number| {
15265 ryu_js::Buffer::new().format_finite(number).to_owned()
15266 }),
15267 _ => None,
15268 })
15269 .is_some_and(|value| !value.is_empty() && allowed.contains(&value))
15270 })
15271 })
15272 };
15273 for row in &rows {
15274 let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
15275 return Err(if report.revoked.is_empty() {
15276 CurrentBaseProofReason::MissingCoverage
15277 } else {
15278 CurrentBaseProofReason::Revoked
15279 });
15280 };
15281 if !covered(&row.table, base)
15282 || row
15283 .unique_conflicts
15284 .iter()
15285 .any(|conflict| !covered(&row.table, &conflict.server_row))
15286 {
15287 return Err(if report.revoked.is_empty() {
15288 CurrentBaseProofReason::MissingCoverage
15289 } else {
15290 CurrentBaseProofReason::Revoked
15291 });
15292 }
15293 }
15294 Ok(rows)
15295 })();
15296 if self
15297 .conn
15298 .execute_batch("RELEASE syncular_base_proof")
15299 .is_err()
15300 {
15301 return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
15302 }
15303 match proof {
15304 Ok(rows) => CurrentBaseProof {
15305 proven: true,
15306 rows: Some(rows),
15307 reason: None,
15308 },
15309 Err(reason) => CurrentBaseProof::failure(reason),
15310 }
15311 }
15312
15313 pub fn prove_current_base(
15314 &mut self,
15315 transport: &mut dyn Transport,
15316 client_commit_id: &str,
15317 sequence: i64,
15318 max_rounds: Option<u32>,
15319 ) -> CurrentBaseProof {
15320 let prepared = match self.prepare_current_base(client_commit_id, sequence) {
15321 Ok(prepared) => prepared,
15322 Err(proof) => return proof,
15323 };
15324 let drive = self.sync_until_idle(transport, max_rounds);
15325 self.finish_current_base(prepared, drive)
15326 }
15327
15328 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
15329 if query.limit == Some(0) {
15330 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
15331 }
15332 let mut sql = String::from(
15333 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
15334 resolution, resolved_at_ms, replacement_client_commit_id
15335 FROM _syncular_commit_outcomes",
15336 );
15337 if query.active_only {
15338 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
15339 }
15340 sql.push_str(" ORDER BY seq DESC");
15341 if let Some(limit) = query.limit {
15342 sql.push_str(&format!(" LIMIT {limit}"));
15343 }
15344 let mut stmt = self
15345 .conn
15346 .prepare(&sql)
15347 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15348 let rows = stmt
15349 .query_map([], |row| {
15350 Ok(StoredCommitOutcomeRow {
15351 sequence: row.get(0)?,
15352 client_commit_id: row.get(1)?,
15353 status: row.get(2)?,
15354 recorded_at_ms: row.get(3)?,
15355 results_json: row.get(4)?,
15356 operations_json: row.get(5)?,
15357 resolution: row.get(6)?,
15358 resolved_at_ms: row.get(7)?,
15359 replacement_client_commit_id: row.get(8)?,
15360 })
15361 })
15362 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15363 let mut outcomes = Vec::new();
15364 for row in rows {
15365 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
15366 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
15367 )?)?);
15368 }
15369 Ok(outcomes)
15370 }
15371
15372 fn prune_commit_outcomes(&self) -> Result<(), String> {
15373 #[cfg(test)]
15374 self.outcome_prune_count
15375 .set(self.outcome_prune_count.get() + 1);
15376 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
15377 let count = self
15378 .conn
15379 .query_row(
15380 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
15381 [],
15382 |row| row.get::<_, i64>(0),
15383 )
15384 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15385 as usize;
15386 let excess = count.saturating_sub(max_entries);
15387 if excess == 0 {
15388 return Ok(());
15389 }
15390 let mut stmt = self
15391 .conn
15392 .prepare(
15393 "SELECT seq FROM _syncular_commit_outcomes
15394 WHERE status IN ('applied', 'cached') OR resolution != 'active'
15395 ORDER BY seq ASC LIMIT ?1",
15396 )
15397 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15398 let rows = stmt
15399 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
15400 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15401 let sequences = rows
15402 .collect::<Result<Vec<_>, _>>()
15403 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15404 drop(stmt);
15405 for sequence in sequences {
15406 self.conn
15407 .execute(
15408 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
15409 rusqlite::params![sequence],
15410 )
15411 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15412 }
15413 Ok(())
15414 }
15415
15416 pub fn subscribe(
15419 &mut self,
15420 id: String,
15421 table: String,
15422 scopes: Vec<(String, Vec<String>)>,
15423 params: Option<String>,
15424 ) -> Result<(), String> {
15425 if self.schema.table(&table).is_none() {
15426 return Err(format!("unknown table {table:?}"));
15427 }
15428 let registered_at = self
15429 .subs
15430 .iter()
15431 .position(|subscription| subscription.id == id);
15432 if let Some(index) = registered_at {
15433 let existing = &self.subs[index];
15434 let same_intent = existing.table == table
15435 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
15436 && existing.params == params;
15437 if !same_intent {
15438 return Err(
15439 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
15440 .to_owned(),
15441 );
15442 }
15443 if existing.state != SubState::Revoked {
15448 return Ok(());
15449 }
15450 }
15451 let sub = Subscription {
15452 id: id.clone(),
15453 table,
15454 requested: scopes,
15455 params,
15456 cursor: -1,
15457 bootstrap_state: None,
15458 state: SubState::Active,
15459 reason_code: None,
15460 effective: None,
15461 synced_once: false,
15462 };
15463 self.persist_sub(&sub)?;
15464 match registered_at {
15465 Some(index) => self.subs[index] = sub,
15466 None => self.subs.push(sub),
15467 }
15468 self.subscription_epoch = self.subscription_epoch.wrapping_add(1);
15469 if registered_at.is_some() {
15473 self.set_sync_needed(true, true);
15474 }
15475 Ok(())
15476 }
15477
15478 pub fn subscribe_many(&mut self, requests: Vec<SubscriptionRequest>) -> Result<(), String> {
15484 let prior_subs = self.subs.clone();
15485 self.begin_observation("syncular_subscribe")?;
15486 let registered = (|| {
15487 for request in requests {
15488 self.subscribe(request.id, request.table, request.scopes, request.params)?;
15489 }
15490 self.conn
15491 .execute_batch("RELEASE syncular_subscribe")
15492 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15493 })();
15494 if let Err(error) = registered {
15495 self.rollback_observation("syncular_subscribe");
15496 self.subs = prior_subs;
15497 return Err(error);
15498 }
15499 Ok(())
15500 }
15501
15502 pub fn unsubscribe(&mut self, id: &str) {
15503 if self.subs.iter().any(|s| s.id == id) {
15504 self.cancel_sync_round();
15505 self.subscription_epoch = self.subscription_epoch.wrapping_add(1);
15506 }
15507 self.subs.retain(|s| s.id != id);
15508 let _ = self.conn.execute(
15509 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
15510 rusqlite::params![id],
15511 );
15512 }
15513
15514 pub fn set_window(
15522 &mut self,
15523 base: &WindowBase,
15524 units: &[String],
15525 ) -> Result<CommandEffects, String> {
15526 let table = self
15527 .schema
15528 .table(&base.table)
15529 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
15530 if table.scope_column(&base.variable).is_none() {
15531 return Err(format!(
15532 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
15533 base.table, base.variable
15534 ));
15535 }
15536 let base_key = window_base_key(base);
15537 let wanted: std::collections::HashSet<&String> = units.iter().collect();
15538 let live = self.load_window_units(&base_key);
15539 self.begin_observation("syncular_window")?;
15540 let mut batch = ChangeAccumulator::default();
15541 let mut changed = false;
15542
15543 let prior_subs = self.subs.clone();
15544 let widened = (|| {
15545 for unit in units {
15547 if live.iter().any(|(u, _)| u == unit) {
15548 continue;
15549 }
15550 let sub_id = derive_sub_id(base, unit);
15551 self.delete_pending_evict(&sub_id)?;
15552 self.insert_window_unit(&base_key, unit, &sub_id)?;
15553 self.subscribe(
15554 sub_id,
15555 base.table.clone(),
15556 unit_scopes(base, unit),
15557 base.params.clone(),
15558 )?;
15559 batch.window(&base_key, &base.table, unit);
15560 changed = true;
15561 }
15562
15563 self.finish_observation("syncular_window", batch)
15564 })();
15565 if let Err(error) = widened {
15566 self.rollback_observation("syncular_window");
15567 self.subs = prior_subs;
15568 return Err(error);
15569 }
15570 for (unit, sub_id) in live {
15571 if wanted.contains(&unit) {
15572 continue;
15573 }
15574 self.rebuild_overlay_if_dirty()?;
15575 let prior_subs = self.subs.clone();
15576 self.begin_observation("syncular_window")?;
15577 let effective = self
15578 .subs
15579 .iter()
15580 .find(|sub| sub.id == sub_id)
15581 .and_then(|sub| sub.effective.clone())
15582 .unwrap_or_else(|| unit_scopes(base, &unit));
15583 let mut batch = ChangeAccumulator::default();
15584 self.record_scope_map(&mut batch, &base.table, &effective);
15585 batch.window(&base_key, &base.table, &unit);
15586 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
15587 let remaining = match result {
15588 Ok(remaining) => remaining,
15589 Err(error) => {
15590 self.rollback_observation("syncular_window");
15591 self.subs = prior_subs;
15592 return Err(error);
15593 }
15594 };
15595 if let Err(error) = self.finish_observation("syncular_window", batch) {
15596 self.rollback_observation("syncular_window");
15597 self.subs = prior_subs;
15598 return Err(error);
15599 }
15600 if remaining {
15601 self.drain_pending_evictions()?;
15602 }
15603 changed = true;
15604 }
15605 Ok(if changed {
15606 CommandEffects::interactive()
15607 } else {
15608 CommandEffects::none()
15609 })
15610 }
15611
15612 pub fn window_state(&self, base: &WindowBase) -> WindowState {
15617 let mut units = Vec::new();
15618 let mut pending = Vec::new();
15619 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
15620 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
15621 Some(sub) => {
15622 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
15623 }
15624 None => true,
15625 };
15626 if is_pending {
15627 pending.push(unit.clone());
15628 }
15629 units.push(unit);
15630 }
15631 WindowState { units, pending }
15632 }
15633
15634 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
15635 let mut stmt = match self
15636 .conn
15637 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
15638 {
15639 Ok(stmt) => stmt,
15640 Err(_) => return Vec::new(),
15641 };
15642 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
15643 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
15644 });
15645 match rows {
15646 Ok(rows) => rows.filter_map(Result::ok).collect(),
15647 Err(_) => Vec::new(),
15648 }
15649 }
15650
15651 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
15652 let mut stmt = match self.conn.prepare(
15653 "SELECT windows.base, windows.unit, subscriptions.tbl
15654 FROM _syncular_windows AS windows
15655 JOIN _syncular_subscriptions AS subscriptions
15656 ON subscriptions.id = windows.sub_id
15657 ORDER BY windows.base, windows.unit",
15658 ) {
15659 Ok(stmt) => stmt,
15660 Err(_) => return Vec::new(),
15661 };
15662 let rows = stmt.query_map([], |row| {
15663 Ok((
15664 row.get::<_, String>(0)?,
15665 row.get::<_, String>(1)?,
15666 row.get::<_, String>(2)?,
15667 ))
15668 });
15669 match rows {
15670 Ok(rows) => rows.filter_map(Result::ok).collect(),
15671 Err(_) => Vec::new(),
15672 }
15673 }
15674
15675 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
15676 self.conn
15677 .query_row(
15678 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
15679 rusqlite::params![sub_id],
15680 |row| Ok((row.get(0)?, row.get(1)?)),
15681 )
15682 .ok()
15683 }
15684
15685 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
15686 self.conn
15687 .execute(
15688 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
15689 rusqlite::params![base_key, unit, sub_id],
15690 )
15691 .map(|_| ())
15692 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15693 }
15694
15695 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
15696 self.conn
15697 .execute(
15698 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
15699 rusqlite::params![base_key, unit],
15700 )
15701 .map(|_| ())
15702 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15703 }
15704
15705 fn evict_unit(
15712 &mut self,
15713 base_key: &str,
15714 base: &WindowBase,
15715 unit: &str,
15716 sub_id: &str,
15717 ) -> Result<bool, String> {
15718 let effective = self
15719 .subs
15720 .iter()
15721 .find(|s| s.id == sub_id)
15722 .and_then(|s| s.effective.clone())
15723 .unwrap_or_else(|| unit_scopes(base, unit));
15724 let pinned = self.pinned_row_ids(&base.table);
15725 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
15726 self.delete_window_unit(base_key, unit)?;
15727 self.conn
15728 .execute(
15729 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
15730 [sub_id],
15731 )
15732 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15733 self.subs.retain(|sub| sub.id != sub_id);
15734 self.subscription_epoch = self.subscription_epoch.wrapping_add(1);
15735 if deferred {
15736 self.save_pending_evict(sub_id, &base.table, &effective)?;
15737 } else {
15738 self.delete_pending_evict(sub_id)?;
15739 }
15740 Ok(remaining)
15741 }
15742
15743 fn evict_scope_rows(
15747 &self,
15748 table_name: &str,
15749 effective: &[(String, Vec<String>)],
15750 pinned: &std::collections::HashSet<String>,
15751 ) -> Result<(bool, bool), String> {
15752 if effective.is_empty() {
15753 return Ok((false, false));
15754 }
15755 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
15756 let mut clauses = Vec::new();
15757 let mut params: Vec<SqlValue> = Vec::new();
15758 for (variable, values) in effective {
15759 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
15760 if values.is_empty() {
15761 return Ok((false, false));
15762 }
15763 let holes = values
15764 .iter()
15765 .map(|v| {
15766 params.push(SqlValue::Text(v.clone()));
15767 "?"
15768 })
15769 .collect::<Vec<_>>()
15770 .join(", ");
15771 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
15772 }
15773 let scope = clauses.join(" AND ");
15774 let scope_params = params.len();
15775 let pk = quote_ident(&table.primary_key);
15776 if !pinned.is_empty() {
15777 let holes = pinned
15778 .iter()
15779 .map(|id| {
15780 params.push(SqlValue::Text(id.clone()));
15781 "?"
15782 })
15783 .collect::<Vec<_>>()
15784 .join(", ");
15785 clauses.push(format!("{pk} NOT IN ({holes})"));
15786 }
15787 let predicate = clauses.join(" AND ");
15788 let base = base_table(table_name);
15789 let visible = visible_table(table_name);
15790 let mut ids: Vec<SqlValue> = Vec::new();
15791 for source in [&base, &visible] {
15792 let mut stmt = self
15793 .conn
15794 .prepare_cached(&format!(
15795 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
15796 ))
15797 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15798 ids = stmt
15799 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
15800 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15801 .collect::<Result<_, _>>()
15802 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15803 if !ids.is_empty() {
15804 break;
15805 }
15806 }
15807 if !ids.is_empty() {
15808 for id in &ids {
15809 self.conn
15810 .execute(
15811 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
15812 rusqlite::params![table_name, id],
15813 )
15814 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15815 }
15816 let holes = vec!["?"; ids.len()].join(", ");
15817 for source in [&base, &visible] {
15818 self.conn
15819 .execute(
15820 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
15821 rusqlite::params_from_iter(&ids),
15822 )
15823 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15824 }
15825 }
15826 let mut remaining = false;
15827 let mut deferred = false;
15828 for source in [&base, &visible] {
15829 remaining |= self
15830 .conn
15831 .query_row(
15832 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
15833 rusqlite::params_from_iter(¶ms),
15834 |row| row.get::<_, bool>(0),
15835 )
15836 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15837 deferred |= self
15838 .conn
15839 .query_row(
15840 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
15841 rusqlite::params_from_iter(¶ms[..scope_params]),
15842 |row| row.get::<_, bool>(0),
15843 )
15844 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15845 }
15846 Ok((remaining, deferred))
15847 }
15848
15849 fn drain_pending_evictions(&mut self) -> Result<(), String> {
15851 let pending = self.load_pending_evictions()?;
15852 if pending.is_empty() {
15853 return Ok(());
15854 }
15855 self.rebuild_overlay_if_dirty()?;
15856 for (sub_id, table_name, effective) in pending {
15857 if self.schema.table(&table_name).is_none() {
15858 self.delete_pending_evict(&sub_id)?;
15859 continue;
15860 }
15861 loop {
15862 std::thread::yield_now();
15863 self.begin_observation("syncular_evict")?;
15864 let pinned = self.pinned_row_ids(&table_name);
15865 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
15866 let (remaining, deferred) = match result {
15867 Ok(result) => result,
15868 Err(error) => {
15869 self.rollback_observation("syncular_evict");
15870 return Err(error);
15871 }
15872 };
15873 if !deferred {
15874 if let Err(error) = self.delete_pending_evict(&sub_id) {
15875 self.rollback_observation("syncular_evict");
15876 return Err(error);
15877 }
15878 }
15879 let mut batch = ChangeAccumulator::default();
15880 self.record_scope_map(&mut batch, &table_name, &effective);
15881 if let Err(error) = self.finish_observation("syncular_evict", batch) {
15882 self.rollback_observation("syncular_evict");
15883 return Err(error);
15884 }
15885 if !remaining {
15886 break;
15887 }
15888 }
15889 }
15890 Ok(())
15891 }
15892
15893 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
15896 let mut pinned = std::collections::HashSet::new();
15897 for commit in self
15898 .outbox
15899 .iter()
15900 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
15901 {
15902 for op in &commit.ops {
15903 if op.table == table {
15904 pinned.insert(op.row_id.clone());
15905 }
15906 }
15907 }
15908 pinned
15909 }
15910
15911 fn save_pending_evict(
15912 &self,
15913 sub_id: &str,
15914 table: &str,
15915 effective: &[(String, Vec<String>)],
15916 ) -> Result<(), String> {
15917 self.conn.execute(
15918 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
15919 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
15920 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15921 }
15922
15923 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
15924 self.conn
15925 .execute(
15926 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
15927 rusqlite::params![sub_id],
15928 )
15929 .map(|_| ())
15930 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15931 }
15932
15933 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
15934 let mut stmt = self
15935 .conn
15936 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
15937 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15938 let rows = stmt
15939 .query_map([], |row| {
15940 Ok((
15941 row.get::<_, String>(0)?,
15942 row.get::<_, String>(1)?,
15943 row.get::<_, String>(2)?,
15944 ))
15945 })
15946 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15947 rows.map(|row| {
15948 let (sub_id, table, json) =
15949 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15950 let value: Value = serde_json::from_str(&json)
15951 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
15952 let effective = json_to_scope_map(&value)
15953 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
15954 Ok((sub_id, table, effective))
15955 })
15956 .collect()
15957 }
15958
15959 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
15961 self.retain_failed_commits = enabled;
15962 for (commit, _) in &self.failed_commits {
15963 for operation in &commit.ops {
15964 self.overlay_dirty.table(&operation.table);
15965 }
15966 }
15967 self.failed_commits = self.load_failed_commits()?;
15968 for (commit, _) in &self.failed_commits {
15969 for operation in &commit.ops {
15970 self.overlay_dirty.table(&operation.table);
15971 }
15972 }
15973 Ok(())
15974 }
15975
15976 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
15977 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
15978 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15979 let rows = statement
15980 .query_map([], |row| {
15981 Ok((
15982 row.get::<_, String>(0)?,
15983 row.get::<_, String>(1)?,
15984 row.get::<_, String>(2)?,
15985 ))
15986 })
15987 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15988 rows.map(|row| {
15989 let (id, operations, initial) =
15990 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15991 let operations: Vec<CommitOperation> =
15992 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
15993 let initial =
15994 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
15995 Ok((
15996 OutboxCommit {
15997 client_commit_id: id,
15998 ops: operations
15999 .into_iter()
16000 .map(|op| OutboxOp {
16001 upsert: op.op == "upsert",
16002 table: op.table,
16003 row_id: op.row_id,
16004 base_version: op.base_version,
16005 values: op.values,
16006 insert: op.insert,
16007 })
16008 .collect(),
16009 },
16010 initial,
16011 ))
16012 })
16013 .collect()
16014 }
16015
16016 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
16017 let mut initial = Map::new();
16018 for operation in operations {
16019 let table = self
16020 .schema
16021 .table(&operation.table)
16022 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16023 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
16024 Some(row) => Some(row),
16025 None => self.visible_row(table, &operation.row_id)?,
16026 };
16027 if let Some((values, _)) = initial_row {
16028 let mut intended: Map<String, Value> = table
16029 .columns
16030 .iter()
16031 .zip(values.iter())
16032 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
16033 .collect();
16034 if let Some(patch) = &operation.values {
16035 intended.extend(patch.clone());
16036 }
16037 initial.insert(
16038 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
16039 .map_err(|_| "sync.local_corrupt".to_owned())?,
16040 Value::Object(intended),
16041 );
16042 }
16043 }
16044 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
16045 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
16046 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
16047 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16048 self.failed_commits.push((
16049 OutboxCommit {
16050 client_commit_id: id.to_owned(),
16051 ops: operations.to_vec(),
16052 },
16053 initial,
16054 ));
16055 Ok(())
16056 }
16057
16058 fn drop_failed_commits(
16059 &mut self,
16060 matches: impl Fn(&str, &Map<String, Value>) -> bool,
16061 ) -> Result<Vec<OutboxCommit>, String> {
16062 let mut doomed = self.drop_acknowledged_rows(&matches)?;
16063 for (commit, initial) in &self.failed_commits {
16064 for operation in &commit.ops {
16065 let key =
16066 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
16067 .map_err(|_| "sync.local_corrupt".to_owned())?;
16068 if initial
16069 .get(&key)
16070 .and_then(Value::as_object)
16071 .is_some_and(|values| matches(&operation.table, values))
16072 {
16073 doomed.push(commit.clone());
16074 break;
16075 }
16076 }
16077 }
16078 for commit in &doomed {
16079 self.conn
16080 .execute(
16081 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
16082 rusqlite::params![commit.client_commit_id],
16083 )
16084 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16085 }
16086 for commit in &doomed {
16087 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16088 }
16089 self.conflicts.retain(|record| {
16090 !doomed
16091 .iter()
16092 .any(|commit| commit.client_commit_id == record.client_commit_id)
16093 });
16094 self.rejections.retain(|record| {
16095 !doomed
16096 .iter()
16097 .any(|commit| commit.client_commit_id == record.client_commit_id)
16098 });
16099 self.failed_commits.retain(|(commit, _)| {
16100 !doomed
16101 .iter()
16102 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
16103 });
16104 for commit in &doomed {
16105 for operation in &commit.ops {
16106 self.overlay_dirty.table(&operation.table);
16107 }
16108 }
16109 Ok(doomed)
16110 }
16111
16112 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
16116 let prior = self.storage_failure.borrow_mut().take();
16117 let outcome = (|| -> Result<String, String> {
16118 if mutations.is_empty() {
16119 return Err(
16120 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
16121 .to_owned(),
16122 );
16123 }
16124 let mut ops = Vec::with_capacity(mutations.len());
16125 for mutation in mutations {
16126 match mutation {
16127 Mutation::Patch {
16128 table,
16129 values,
16130 base_version,
16131 } => {
16132 let schema_table = self
16133 .schema
16134 .table(&table)
16135 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16136 let values = normalize_values_casing(schema_table, values)?;
16137 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
16138 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
16139 }
16140 Mutation::Upsert {
16141 table,
16142 values,
16143 base_version,
16144 } => {
16145 let schema_table = self.schema.table(&table).ok_or_else(|| {
16146 format!("sync.unknown_table: unknown table {table:?}")
16147 })?;
16148 let values = normalize_values_casing(schema_table, values)?;
16152 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
16153 let values = full_row_values(schema_table, values)?;
16156 self.validate_author_encode(schema_table, &row_id, &values)?;
16162 let insert = !self.visible_row_exists(schema_table, &row_id)?;
16166 ops.push(OutboxOp {
16167 upsert: true,
16168 table,
16169 row_id,
16170 base_version,
16171 values: Some(values),
16172 insert,
16173 });
16174 }
16175 Mutation::Delete {
16176 table,
16177 row_id,
16178 base_version,
16179 } => {
16180 if self.schema.table(&table).is_none() {
16181 return Err(format!("sync.unknown_table: unknown table {table:?}"));
16182 }
16183 ops.push(OutboxOp {
16184 upsert: false,
16185 table,
16186 row_id,
16187 base_version,
16188 values: None,
16189 insert: false,
16190 });
16191 }
16192 }
16193 }
16194 self.record_outbox_commit(ops)
16195 })();
16196 self.finish_authoring(prior, outcome)
16197 }
16198
16199 fn finish_authoring(
16205 &self,
16206 prior: Option<QueryReadFailure>,
16207 outcome: Result<String, String>,
16208 ) -> Result<String, ClientError> {
16209 let fresh = self.storage_failure.borrow_mut().take();
16210 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
16211 match outcome {
16212 Ok(id) => Ok(id),
16213 Err(message) => Err(fresh
16214 .map(ClientError::from)
16215 .unwrap_or_else(|| ClientError::from(message))),
16216 }
16217 }
16218
16219 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
16220 let commit = OutboxCommit {
16221 client_commit_id: uuid::Uuid::new_v4().to_string(),
16222 ops,
16223 };
16224 self.begin_observation("syncular_mutation")?;
16225 let mut batch = ChangeAccumulator::default();
16226 for op in &commit.ops {
16227 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
16228 if let Some(values) = &op.values {
16229 if let Some(table) = self.schema.table(&op.table) {
16230 for scope in &table.scope_variables {
16231 if let Some(Value::String(value)) = values.get(&scope.column) {
16232 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
16233 precise = true;
16234 }
16235 }
16236 }
16237 }
16238 if !precise {
16239 batch.table(&op.table);
16240 }
16241 }
16242 if let Err(error) = self.persist_outbox_insert(&commit) {
16243 self.rollback_observation("syncular_mutation");
16244 return Err(error);
16245 }
16246 let was_dirty = self.overlay_dirty.snapshot();
16247 if let Err(error) = self.rebuild_overlay_if_dirty() {
16252 self.rollback_observation("syncular_mutation");
16253 self.overlay_dirty.restore(was_dirty.clone());
16254 return Err(error);
16255 }
16256 if let Some(error) = commit
16257 .ops
16258 .iter()
16259 .find_map(|op| self.apply_outbox_op(op).err())
16260 {
16261 self.rollback_observation("syncular_mutation");
16262 self.overlay_dirty.restore(was_dirty.clone());
16263 return Err(error);
16264 }
16265 let id = commit.client_commit_id.clone();
16266 self.outbox.push(commit);
16267 batch.status = true;
16268 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
16269 self.rollback_observation("syncular_mutation");
16270 self.outbox.pop();
16271 self.overlay_dirty.restore(was_dirty.clone());
16272 return Err(error);
16273 }
16274 Ok(id)
16275 }
16276
16277 pub fn patch(
16283 &mut self,
16284 table: &str,
16285 row_id: &str,
16286 partial: Map<String, Value>,
16287 base_version: Option<i64>,
16288 ) -> Result<String, ClientError> {
16289 let prior = self.storage_failure.borrow_mut().take();
16290 let outcome = self.perform_patch(table, row_id, partial, base_version);
16291 self.finish_authoring(prior, outcome)
16292 }
16293
16294 fn perform_patch(
16297 &mut self,
16298 table: &str,
16299 row_id: &str,
16300 partial: Map<String, Value>,
16301 base_version: Option<i64>,
16302 ) -> Result<String, String> {
16303 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
16304 self.record_outbox_commit(vec![operation])
16305 }
16306
16307 fn prepare_patch(
16308 &self,
16309 table: &str,
16310 row_id: &str,
16311 partial: Map<String, Value>,
16312 base_version: Option<i64>,
16313 ) -> Result<OutboxOp, String> {
16314 let schema_table = self
16315 .schema
16316 .table(table)
16317 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
16318 let partial = normalize_values_casing(schema_table, partial)?;
16319 if let Some(pk) = partial.get(&schema_table.primary_key) {
16320 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
16321 return Err(format!(
16322 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
16323 ));
16324 }
16325 }
16326 if !self.visible_row_exists(schema_table, row_id)? {
16327 return Err("sync.row_missing: a sparse patch requires a local row".to_owned());
16328 }
16329 for scope in &schema_table.scope_variables {
16339 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
16340 continue;
16341 }
16342 let Some(column) = schema_table
16343 .columns
16344 .iter()
16345 .find(|column| column.name == scope.column)
16346 else {
16347 continue;
16348 };
16349 let stored = self
16350 .conn
16351 .query_row(
16352 &format!(
16353 "SELECT {} FROM {} WHERE {}",
16354 quote_ident(&scope.column),
16355 visible_table(&schema_table.name),
16356 row_id_predicate(schema_table)
16357 ),
16358 rusqlite::params![row_id],
16359 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
16360 )
16361 .optional()
16362 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16363 let incoming = json_to_column_value(column, partial.get(&scope.column))
16368 .ok()
16369 .map(|value| column_value_to_json(&value));
16370 let equal = stored.as_ref().is_some_and(|stored| {
16374 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
16375 });
16376 if !equal {
16377 return Err(format!(
16378 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
16379 scope.column
16380 ));
16381 }
16382 }
16383 let mut values = partial;
16384 for scope in &schema_table.scope_variables {
16387 values.remove(&scope.column);
16388 }
16389 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
16390 ColumnType::Integer => match row_id.parse::<i64>() {
16392 Ok(v)
16393 if v.to_string() == row_id
16394 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
16395 .contains(&v) =>
16396 {
16397 Value::from(v)
16398 }
16399 _ => {
16400 return Err(format!(
16401 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
16402 ))
16403 }
16404 },
16405 ColumnType::Boolean => match row_id {
16406 "true" => Value::Bool(true),
16407 "false" => Value::Bool(false),
16408 _ => {
16409 return Err(format!(
16410 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
16411 ))
16412 }
16413 },
16414 _ => Value::from(row_id.to_owned()),
16415 };
16416 values.insert(schema_table.primary_key.clone(), pk_value);
16417 self.validate_author_encode(schema_table, row_id, &values)?;
16423 Ok(OutboxOp {
16424 upsert: true,
16425 table: table.to_owned(),
16426 row_id: row_id.to_owned(),
16427 base_version,
16428 values: Some(values),
16429 insert: false,
16430 })
16431 }
16432
16433 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
16438 let selector = self.encryption.key_id_columns.get(&table.name)?;
16439 let index = table
16440 .columns
16441 .iter()
16442 .position(|column| &column.name == selector)?;
16443 let sql = format!(
16444 "SELECT {} FROM {} WHERE {}",
16445 quote_ident(selector),
16446 visible_table(&table.name),
16447 row_id_predicate(table)
16448 );
16449 let key: Option<String> = self
16450 .conn
16451 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
16452 .ok()?;
16453 let key = key.filter(|key| !key.is_empty())?;
16454 let mut fallback: Row = vec![None; table.columns.len()];
16455 fallback[index] = Some(ColumnValue::String(key));
16456 Some(fallback)
16457 }
16458
16459 fn validate_author_encode(
16464 &self,
16465 table: &TableSchema,
16466 row_id: &str,
16467 values: &Map<String, Value>,
16468 ) -> Result<(), String> {
16469 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
16470 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
16471 result => result.map(|_| ()),
16472 }
16473 }
16474
16475 fn drop_unencodable_outbox(
16489 &mut self,
16490 with_encryption: bool,
16491 batch: Option<&mut ChangeAccumulator>,
16492 ) -> Result<bool, String> {
16493 let failures = self
16494 .outbox
16495 .iter()
16496 .filter_map(|commit| {
16497 commit
16498 .ops
16499 .iter()
16500 .find_map(|op| {
16501 let table = self.schema.table(&op.table).filter(|_| op.upsert)?;
16502 let values = op.presented_values(table)?;
16505 let fallback = self.stored_key_fallback(table, &op.row_id);
16506 match encode_sparse_row_json(
16507 table,
16508 &op.row_id,
16509 &values,
16510 &self.encryption,
16511 fallback.as_ref(),
16512 ) {
16513 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
16514 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
16515 }
16516 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
16521 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
16522 OUTBOX_INCOMPATIBLE_CODE,
16523 )),
16524 _ => None,
16525 }
16526 })
16527 .map(|(error, code)| (commit.clone(), error, code))
16528 })
16529 .collect::<Vec<_>>();
16530 if failures.is_empty() {
16531 return Ok(false);
16532 }
16533 let prior_outbox = self.outbox.clone();
16539 let prior_rejections = self.rejections.clone();
16540 let prior_failed_commits = self.failed_commits.clone();
16544 let prior_changes = self.change_queue.len();
16545 let prior_intents = self.sync_intent_queue.len();
16546 let prior_last_change = self.last_change.clone();
16547 let prior_overlay_dirty = self.overlay_dirty.snapshot();
16548 let owns_observation = batch.is_none();
16549 let owns_transaction = self.conn.is_autocommit();
16550 let mut domains = ChangeAccumulator::default();
16551 if owns_observation {
16552 self.begin_observation("syncular_value_recovery")?;
16553 }
16554 let dropped = (|| -> Result<(), String> {
16555 for (commit, message, code) in failures {
16556 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
16557 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
16558 }
16559 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
16560 let rejection = RejectionRecord {
16561 client_commit_id: commit.client_commit_id.clone(),
16562 op_index: 0,
16563 code: code.to_owned(),
16564 message,
16565 retryable: false,
16566 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
16567 field_paths: None,
16568 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
16569 required_action: None,
16570 references: None,
16571 }),
16572 operation: representable
16573 .then(|| commit.ops.first())
16574 .flatten()
16575 .map(CommitOperation::from),
16576 };
16577 self.persist_commit_outcome(
16578 &commit.client_commit_id,
16579 CommitOutcomeStatus::Rejected,
16580 &[CommitOperationOutcome::Error {
16581 rejection: rejection.clone(),
16582 }],
16583 representable.then_some(commit.ops.as_slice()),
16584 )?;
16585 self.delete_outbox_persisted(&commit.client_commit_id)?;
16586 self.outbox
16587 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
16588 self.rejections.push(rejection);
16589 for operation in &commit.ops {
16590 domains.table(operation.table.as_str());
16591 }
16592 }
16593 domains.status = true;
16594 domains.rejections = true;
16595 domains.outcomes = true;
16596 self.prune_commit_outcomes()?;
16597 self.overlay_dirty.set(true);
16599 self.rebuild_overlay()?;
16600 Ok(())
16601 })();
16602 let result = match batch {
16603 Some(batch) => {
16604 if dropped.is_ok() {
16605 for table in domains.tables.keys() {
16606 batch.table(table.as_str());
16607 }
16608 batch.status |= domains.status;
16609 batch.rejections |= domains.rejections;
16610 batch.outcomes |= domains.outcomes;
16611 }
16612 dropped
16613 }
16614 None => {
16615 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
16616 }
16617 };
16618 if let Err(error) = result {
16619 if owns_observation {
16620 if owns_transaction {
16621 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
16622 let mut failure = self.storage_failure.borrow_mut();
16623 let failure =
16624 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
16625 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
16626 }
16627 } else {
16628 self.rollback_observation("syncular_value_recovery");
16629 }
16630 }
16631 self.outbox = prior_outbox;
16632 self.rejections = prior_rejections;
16633 self.failed_commits = prior_failed_commits;
16634 self.change_queue.truncate(prior_changes);
16635 self.sync_intent_queue.truncate(prior_intents);
16636 self.last_change = prior_last_change;
16637 self.overlay_dirty.restore(prior_overlay_dirty);
16638 return Err(error);
16639 }
16640 Ok(true)
16641 }
16642
16643 pub fn pending_commit_ids(&self) -> Vec<String> {
16644 self.outbox
16645 .iter()
16646 .map(|c| c.client_commit_id.clone())
16647 .collect()
16648 }
16649
16650 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
16654 let mut payloads = Vec::new();
16655 for commit in &self.outbox {
16656 for op in &commit.ops {
16657 let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
16658 continue;
16659 };
16660 let Some(values) = op.presented_values(table) else {
16661 continue;
16662 };
16663 let fallback = self.stored_key_fallback(table, &op.row_id);
16664 if let Ok(payload) = encode_sparse_row_json(
16665 table,
16666 &op.row_id,
16667 &values,
16668 &self.encryption,
16669 fallback.as_ref(),
16670 ) {
16671 payloads.push(payload);
16672 }
16673 }
16674 }
16675 payloads
16676 }
16677
16678 pub fn conflicts(&self) -> &[ConflictRecord] {
16679 &self.conflicts
16680 }
16681
16682 pub fn rejections(&self) -> &[RejectionRecord] {
16683 &self.rejections
16684 }
16685
16686 pub fn resolve_commit_outcome(
16687 &mut self,
16688 input: ResolveCommitOutcomeInput,
16689 ) -> Result<CommitOutcome, String> {
16690 let current = self
16691 .commit_outcome(&input.client_commit_id)?
16692 .ok_or_else(|| {
16693 format!(
16694 "sync.outcome_not_found: no durable outcome exists for {:?}",
16695 input.client_commit_id
16696 )
16697 })?;
16698 if current.resolution != CommitOutcomeResolution::Active {
16699 return Ok(current);
16700 }
16701 if input.resolution == CommitOutcomeResolution::Active {
16702 return Err("sync.invalid_request: resolution must leave active state".to_owned());
16703 }
16704 match input.resolution {
16705 CommitOutcomeResolution::Superseded => {
16706 let replacement = input
16707 .replacement_client_commit_id
16708 .as_deref()
16709 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
16710 .ok_or_else(|| {
16711 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
16712 .to_owned()
16713 })?;
16714 let _ = replacement;
16715 }
16716 _ if input.replacement_client_commit_id.is_some() => {
16717 return Err(
16718 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
16719 .to_owned(),
16720 );
16721 }
16722 _ => {}
16723 }
16724 let allowed = match current.status {
16725 CommitOutcomeStatus::Conflict => matches!(
16726 input.resolution,
16727 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
16728 ),
16729 CommitOutcomeStatus::Rejected => {
16730 matches!(
16731 input.resolution,
16732 CommitOutcomeResolution::Superseded
16733 | CommitOutcomeResolution::ResolvedKeepServer
16734 )
16735 }
16736 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
16737 input.resolution == CommitOutcomeResolution::Dismissed
16738 }
16739 };
16740 if !allowed {
16741 return Err(format!(
16742 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
16743 input.resolution, current.status
16744 ));
16745 }
16746
16747 self.begin_observation("syncular_outcome_resolution")?;
16748 let previous_failed = self.failed_commits.clone();
16749 let previous_conflicts = self.conflicts.clone();
16750 let previous_rejections = self.rejections.clone();
16751 let result = (|| {
16752 self.conn
16753 .execute(
16754 "UPDATE _syncular_commit_outcomes
16755 SET resolution = ?1, resolved_at_ms = ?2,
16756 replacement_client_commit_id = ?3
16757 WHERE client_commit_id = ?4 AND resolution = 'active'",
16758 rusqlite::params![
16759 Self::outcome_resolution_name(input.resolution),
16760 self.clock_now_ms(),
16761 input.replacement_client_commit_id,
16762 input.client_commit_id
16763 ],
16764 )
16765 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16766 let resolved = self
16767 .commit_outcome(¤t.client_commit_id)?
16768 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
16769 self.conn
16770 .execute(
16771 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
16772 rusqlite::params![input.client_commit_id],
16773 )
16774 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16775 for (commit, _) in &self.failed_commits {
16776 if commit.client_commit_id == input.client_commit_id {
16777 for operation in &commit.ops {
16778 self.overlay_dirty.table(&operation.table);
16779 }
16780 }
16781 }
16782 self.failed_commits
16783 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
16784 self.prune_commit_outcomes()?;
16785 Ok(resolved)
16786 })();
16787 let resolved = match result {
16788 Ok(outcome) => outcome,
16789 Err(error) => {
16790 self.rollback_observation("syncular_outcome_resolution");
16791 self.failed_commits = previous_failed;
16792 self.conflicts = previous_conflicts;
16793 self.rejections = previous_rejections;
16794 return Err(error);
16795 }
16796 };
16797 self.conflicts
16798 .retain(|record| record.client_commit_id != current.client_commit_id);
16799 self.rejections
16800 .retain(|record| record.client_commit_id != current.client_commit_id);
16801 let mut batch = ChangeAccumulator {
16802 conflicts: current.status == CommitOutcomeStatus::Conflict,
16803 rejections: current.status == CommitOutcomeStatus::Rejected,
16804 outcomes: true,
16805 ..ChangeAccumulator::default()
16806 };
16807 if let Some(operations) = ¤t.operations {
16808 for operation in operations {
16809 batch.table(&operation.table);
16810 }
16811 }
16812 if let Err(error) = self
16813 .rebuild_overlay_if_dirty()
16814 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
16815 {
16816 self.rollback_observation("syncular_outcome_resolution");
16817 self.failed_commits = previous_failed;
16818 self.conflicts = previous_conflicts;
16819 self.rejections = previous_rejections;
16820 return Err(error);
16821 }
16822 Ok(resolved)
16823 }
16824
16825 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
16826 self.schema_floor.as_ref()
16827 }
16828
16829 pub fn lease_state(&self) -> Option<&LeaseState> {
16831 self.lease_state.as_ref()
16832 }
16833
16834 fn record_lease_error(&mut self, code: &str) {
16837 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
16838 return;
16839 }
16840 let mut next = self.lease_state.clone().unwrap_or_default();
16841 next.error_code = Some(code.to_owned());
16842 self.set_lease_state(Some(next));
16843 }
16844
16845 fn set_lease_state(&mut self, next: Option<LeaseState>) {
16846 if self.lease_state == next {
16847 return;
16848 }
16849 if self.begin_observation("syncular_lease").is_err() {
16850 return;
16851 }
16852 self.lease_state = next;
16853 if let Some(lease) = &self.lease_state {
16854 if let Ok(json) = serde_json::to_string(lease) {
16855 self.set_meta(LEASE_STATE_KEY, &json);
16856 }
16857 } else {
16858 self.delete_meta(LEASE_STATE_KEY);
16859 }
16860 let batch = ChangeAccumulator {
16861 status: true,
16862 ..ChangeAccumulator::default()
16863 };
16864 if self.finish_observation("syncular_lease", batch).is_err() {
16865 self.rollback_observation("syncular_lease");
16866 }
16867 }
16868
16869 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
16870 if self.schema_floor == next {
16871 return;
16872 }
16873 if self.begin_observation("syncular_schema_floor").is_err() {
16874 return;
16875 }
16876 self.schema_floor = next;
16877 self.stopped = self.schema_floor.is_some();
16878 if let Some(floor) = &self.schema_floor {
16879 if let Ok(json) = serde_json::to_string(floor) {
16880 self.set_meta(SCHEMA_FLOOR_KEY, &json);
16881 }
16882 } else {
16883 self.delete_meta(SCHEMA_FLOOR_KEY);
16884 }
16885 let batch = ChangeAccumulator {
16886 status: true,
16887 ..ChangeAccumulator::default()
16888 };
16889 if self
16890 .finish_observation("syncular_schema_floor", batch)
16891 .is_err()
16892 {
16893 self.rollback_observation("syncular_schema_floor");
16894 }
16895 }
16896
16897 fn set_upgrading(&mut self, value: bool) {
16898 if self.upgrading == value {
16899 return;
16900 }
16901 if self.begin_observation("syncular_upgrading").is_err() {
16902 return;
16903 }
16904 self.upgrading = value;
16905 let batch = ChangeAccumulator {
16906 status: true,
16907 ..ChangeAccumulator::default()
16908 };
16909 if self
16910 .finish_observation("syncular_upgrading", batch)
16911 .is_err()
16912 {
16913 self.rollback_observation("syncular_upgrading");
16914 }
16915 }
16916
16917 pub fn sync_needed(&self) -> bool {
16918 self.sync_needed
16919 }
16920
16921 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
16922 let sub = self.subs.iter().find(|s| s.id == id)?;
16923 Some(SubscriptionStateView {
16924 id: sub.id.clone(),
16925 table: sub.table.clone(),
16926 status: sub.state.name().to_owned(),
16927 cursor: sub.cursor,
16928 has_resume_token: sub.bootstrap_state.is_some(),
16929 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
16930 reason_code: sub.reason_code.clone(),
16931 })
16932 }
16933
16934 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
16935 let schema_table = self
16936 .schema
16937 .table(table)
16938 .ok_or_else(|| format!("unknown table {table:?}"))?;
16939 let sql = format!(
16940 "SELECT * FROM {} ORDER BY {} ASC",
16941 visible_table(table),
16942 quote_ident(&schema_table.primary_key)
16943 );
16944 let mut stmt = self
16945 .conn
16946 .prepare(&sql)
16947 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16948 let mut rows = stmt
16949 .query([])
16950 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16951 let mut out = Vec::new();
16952 while let Some(row) = rows
16953 .next()
16954 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
16955 {
16956 let mut values = Map::new();
16957 for (i, column) in schema_table.columns.iter().enumerate() {
16958 let value = row
16959 .get_ref(i)
16960 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16961 values.insert(column.name.clone(), sql_ref_to_json(column, value));
16962 }
16963 let version: i64 = row
16964 .get(schema_table.columns.len())
16965 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16966 let row_id = match values.get(&schema_table.primary_key) {
16967 Some(Value::String(s)) => s.clone(),
16968 Some(Value::Number(n)) => n.to_string(),
16969 Some(Value::Bool(b)) => b.to_string(),
16970 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
16971 };
16972 out.push(RowState {
16973 row_id,
16974 version,
16975 values,
16976 });
16977 }
16978 Ok(out)
16979 }
16980
16981 #[cfg(feature = "crdt-yjs")]
16997 fn crdt_column_bytes(
16998 &self,
16999 table: &str,
17000 row_id: &str,
17001 column: &str,
17002 ) -> Result<Option<Vec<u8>>, String> {
17003 let schema_table = self
17004 .schema
17005 .table(table)
17006 .ok_or_else(|| format!("unknown table {table:?}"))?;
17007 let col = schema_table
17008 .columns
17009 .iter()
17010 .find(|c| c.name == column)
17011 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
17012 if col.ty != ColumnType::Crdt {
17013 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
17014 }
17015 let sql = format!(
17016 "SELECT {} FROM {} WHERE {}",
17017 quote_ident(column),
17018 visible_table(table),
17019 row_id_predicate(schema_table)
17020 );
17021 let bytes: Option<Vec<u8>> = self
17022 .conn
17023 .prepare_cached(&sql)
17024 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
17025 .query_row(rusqlite::params![row_id], |row| {
17026 row.get::<_, Option<Vec<u8>>>(0)
17027 })
17028 .map_err(|e| match e {
17029 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
17030 other => Self::sqlite_failure(&self.storage_failure, other),
17031 })?;
17032 Ok(bytes)
17033 }
17034
17035 #[cfg(feature = "crdt-yjs")]
17039 pub fn crdt_text(
17040 &self,
17041 table: &str,
17042 row_id: &str,
17043 column: &str,
17044 name: &str,
17045 ) -> Result<String, String> {
17046 let bytes = self
17047 .crdt_column_bytes(table, row_id, column)?
17048 .unwrap_or_default();
17049 crate::crdt::text(&bytes, name)
17050 }
17051
17052 #[cfg(feature = "crdt-yjs")]
17056 pub fn crdt_insert_text(
17057 &mut self,
17058 table: &str,
17059 row_id: &str,
17060 column: &str,
17061 name: &str,
17062 index: u32,
17063 value: &str,
17064 ) -> Result<String, ClientError> {
17065 let prior = self.storage_failure.borrow_mut().take();
17066 let outcome = (|| -> Result<String, String> {
17067 let current = self
17068 .crdt_column_bytes(table, row_id, column)?
17069 .unwrap_or_default();
17070 let update = crate::crdt::insert_text(¤t, name, index, value)?;
17071 self.perform_crdt_push_update(table, row_id, column, &update)
17072 })();
17073 self.finish_authoring(prior, outcome)
17074 }
17075
17076 #[cfg(feature = "crdt-yjs")]
17080 pub fn crdt_delete_text(
17081 &mut self,
17082 table: &str,
17083 row_id: &str,
17084 column: &str,
17085 name: &str,
17086 index: u32,
17087 len: u32,
17088 ) -> Result<String, ClientError> {
17089 let prior = self.storage_failure.borrow_mut().take();
17090 let outcome = (|| -> Result<String, String> {
17091 let current = self
17092 .crdt_column_bytes(table, row_id, column)?
17093 .unwrap_or_default();
17094 let update = crate::crdt::delete_text(¤t, name, index, len)?;
17095 self.perform_crdt_push_update(table, row_id, column, &update)
17096 })();
17097 self.finish_authoring(prior, outcome)
17098 }
17099
17100 #[cfg(feature = "crdt-yjs")]
17105 pub fn crdt_apply_update(
17106 &mut self,
17107 table: &str,
17108 row_id: &str,
17109 column: &str,
17110 update: &[u8],
17111 ) -> Result<String, ClientError> {
17112 let prior = self.storage_failure.borrow_mut().take();
17113 let outcome = (|| -> Result<String, String> {
17114 let current = self
17115 .crdt_column_bytes(table, row_id, column)?
17116 .unwrap_or_default();
17117 let next = crate::crdt::apply_update(¤t, update)?;
17118 self.perform_crdt_push_update(table, row_id, column, &next)
17119 })();
17120 self.finish_authoring(prior, outcome)
17121 }
17122
17123 #[cfg(feature = "crdt-yjs")]
17130 fn perform_crdt_push_update(
17131 &mut self,
17132 table: &str,
17133 row_id: &str,
17134 column: &str,
17135 crdt_bytes: &[u8],
17136 ) -> Result<String, String> {
17137 let mut bytes_obj = Map::new();
17138 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
17139 let mut partial = Map::new();
17140 partial.insert(column.to_owned(), Value::Object(bytes_obj));
17141 self.perform_patch(table, row_id, partial, None)
17142 }
17143
17144 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
17159 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
17160 }
17161
17162 #[cfg(feature = "bench-internals")]
17164 #[doc(hidden)]
17165 pub fn benchmark_connection(&mut self) -> &mut Connection {
17166 &mut self.conn
17167 }
17168
17169 #[cfg(feature = "bench-internals")]
17171 #[doc(hidden)]
17172 pub fn benchmark_phases(
17173 &self,
17174 enabled: Option<bool>,
17175 reset: bool,
17176 ) -> Result<Option<Value>, String> {
17177 self.benchmark_phases.configure(enabled, reset)?;
17178 Ok(self.benchmark_phases.snapshot())
17179 }
17180
17181 pub fn authority_snapshot(&self) -> Result<Value, String> {
17183 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
17184 if self.limits.authority_reads.is_empty() {
17185 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
17186 }
17187 self.conn
17188 .execute_batch("SAVEPOINT authority_snapshot")
17189 .map_err(|e| e.to_string())?;
17190 let result = (|| {
17191 let revision = self
17192 .get_meta(LOCAL_REVISION_KEY)
17193 .ok_or("sync.local_corrupt: authority revision is missing")?;
17194 let parsed_revision = revision
17195 .parse::<u64>()
17196 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
17197 if parsed_revision.to_string() != revision {
17198 return Err("sync.local_corrupt: authority revision is invalid".into());
17199 }
17200 let mut tables = Vec::new();
17201 for read in &self.limits.authority_reads {
17202 let table = self
17203 .schema
17204 .table(&read.table)
17205 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
17206 let scalar = |column: &str, value: &Value| -> Result<String, String> {
17207 let ty = table
17208 .columns
17209 .iter()
17210 .find(|c| c.name == column)
17211 .expect("validated authority column")
17212 .ty;
17213 if ty == ColumnType::Boolean {
17214 if let Some(value) = value.as_bool() {
17215 return Ok(value.to_string());
17216 }
17217 return match value.as_i64() {
17218 Some(0) => Ok("false".into()),
17219 Some(1) => Ok("true".into()),
17220 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
17221 };
17222 }
17223 match value {
17224 Value::String(s) => Ok(s.clone()),
17225 Value::Number(n) => Ok(n.to_string()),
17226 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
17227 .get("$bigint")
17228 .and_then(Value::as_str)
17229 .map(str::to_owned)
17230 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
17231 _ => {
17232 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
17233 }
17234 }
17235 };
17236 let matches = |row: &QueryRow| -> Result<bool, String> {
17237 for (variable, values) in &read.scopes {
17238 let column = table
17239 .scope_column(variable)
17240 .expect("validated authority scope");
17241 let value = row
17242 .get(column)
17243 .ok_or("sync.local_corrupt: authority scope column is missing")?;
17244 if value.is_null() || !values.contains(&scalar(column, value)?) {
17245 return Ok(false);
17246 }
17247 }
17248 Ok(true)
17249 };
17250 let selected = read
17251 .columns
17252 .iter()
17253 .map(|c| quote_ident(c))
17254 .collect::<Vec<_>>()
17255 .join(",");
17256 let mut intent = BTreeSet::new();
17257 let mut relevant = BTreeSet::new();
17258 for row in query_connection(&self.conn,
17259 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
17260 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
17261 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
17262 intent.insert(id.clone());
17263 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
17264 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
17265 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
17266 }
17267 }
17268 let mut rows = BTreeMap::new();
17269 for full in [base_table(&read.table), visible_table(&read.table)] {
17270 for mut values in query_connection(
17271 &self.conn,
17272 &format!(
17273 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
17274 ),
17275 &[],
17276 )
17277 .map_err(|e| e.to_string())?
17278 {
17279 if !matches(&values)? {
17280 continue;
17281 }
17282 let id = scalar(
17283 &table.primary_key,
17284 values
17285 .get(&table.primary_key)
17286 .expect("selected primary key"),
17287 )?;
17288 relevant.insert(id.clone());
17289 if full == base_table(&read.table) {
17290 let version = values
17291 .remove("authority_version")
17292 .expect("selected version");
17293 let version = version
17294 .as_i64()
17295 .ok_or("sync.local_corrupt: authority server version is invalid")?;
17296 if version >= 0 {
17297 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
17298 }
17299 }
17300 }
17301 }
17302 intent.retain(|id| relevant.contains(id));
17303 let rows = rows.into_values().collect::<Vec<_>>();
17304 let mut persisted = Vec::new();
17305 for row in query_connection(
17306 &self.conn,
17307 "SELECT state_json FROM _syncular_subscriptions WHERE tbl=? ORDER BY rowid",
17308 &[Value::String(read.table.clone())],
17309 )
17310 .map_err(|e| e.message)?
17311 {
17312 let state: Value = serde_json::from_str(
17313 row.get("state_json")
17314 .and_then(Value::as_str)
17315 .ok_or("sync.local_corrupt: authority subscription state is missing")?,
17316 )
17317 .map_err(|_| "sync.local_corrupt: authority subscription state is invalid")?;
17318 let effective = state.get("effectiveScopes").and_then(Value::as_object);
17319 let status = state
17320 .get("status")
17321 .and_then(Value::as_str)
17322 .ok_or("sync.local_corrupt: authority subscription status is missing")?;
17323 let cursor = state
17324 .get("cursor")
17325 .and_then(Value::as_i64)
17326 .ok_or("sync.local_corrupt: authority subscription cursor is missing")?;
17327 let complete = effective.is_some_and(|s| s.len() == read.scopes.len())
17328 && status == "active"
17329 && cursor >= 0
17330 && state.get("bootstrapState").is_none_or(Value::is_null)
17331 && state.get("params").is_none_or(Value::is_null);
17332 let exposed = effective.map(|s| {
17333 read.scopes
17334 .keys()
17335 .map(|v| (v.clone(), s.get(v).cloned().unwrap_or(json!([]))))
17336 .collect::<Map<_, _>>()
17337 });
17338 let requested = read
17339 .scopes
17340 .keys()
17341 .map(|v| {
17342 (
17343 v.clone(),
17344 state
17345 .get("requested")
17346 .and_then(|s| s.get(v))
17347 .cloned()
17348 .unwrap_or(json!([])),
17349 )
17350 })
17351 .collect::<Map<_, _>>();
17352 persisted.push(json!({"requestedScopes": requested, "status": status, "cursor": cursor, "effectiveScopes": exposed, "complete": complete}));
17353 }
17354 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
17355 match selectors.split_first() {
17356 None => !candidates.is_empty(),
17357 Some(((variable, values), rest)) => values.iter().all(|value| {
17358 covered(
17359 rest,
17360 &candidates
17361 .iter()
17362 .copied()
17363 .filter(|p| {
17364 p["effectiveScopes"][variable.as_str()]
17365 .as_array()
17366 .is_some_and(|held| {
17367 held.contains(&Value::String(value.clone()))
17368 || held.contains(&Value::String("*".into()))
17369 })
17370 })
17371 .collect::<Vec<_>>(),
17372 )
17373 }),
17374 }
17375 }
17376 let coverage = if covered(
17377 &read.scopes.iter().collect::<Vec<_>>(),
17378 &persisted
17379 .iter()
17380 .filter(|p| p["complete"] == true)
17381 .collect::<Vec<_>>(),
17382 ) {
17383 "complete"
17384 } else if persisted.iter().any(|p| {
17385 p["status"] == "active"
17386 && read.scopes.iter().all(|(v, values)| {
17387 values.iter().any(|value| {
17388 p["requestedScopes"][v].as_array().is_some_and(|held| {
17389 held.contains(&Value::String(value.clone()))
17390 || held.contains(&Value::String("*".into()))
17391 })
17392 })
17393 })
17394 }) {
17395 "pending"
17396 } else {
17397 "missing"
17398 };
17399 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
17400 }
17401 Ok(
17402 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
17403 )
17404 })();
17405 match result {
17406 Ok(snapshot) => {
17407 self.conn
17408 .execute_batch("RELEASE authority_snapshot")
17409 .map_err(|e| e.to_string())?;
17410 Ok(snapshot)
17411 }
17412 Err(error) => {
17413 self.conn
17414 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
17415 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
17416 Err(error)
17417 }
17418 }
17419 }
17420
17421 pub fn query_snapshot(
17425 &mut self,
17426 sql: &str,
17427 params: &[QueryValue],
17428 coverage: &[WindowCoverage],
17429 owner: Option<QueryOwner<'_>>,
17430 ) -> Result<QuerySnapshot, String> {
17431 let result = snapshot_connection(&self.conn, sql, params, coverage);
17432 if let Some(owner) = owner {
17433 self.record_query_read(owner, result.as_ref().err());
17434 }
17435 result.map_err(|failure| failure.to_string())
17436 }
17437
17438 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
17442 let position = self
17443 .query_failures
17444 .iter()
17445 .position(|entry| entry.id == owner.id);
17446 let Some(failure) = failure else {
17447 if let Some(index) = position {
17448 self.query_failures.remove(index);
17449 }
17450 return;
17451 };
17452 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
17453 .into_iter()
17454 .map(str::to_owned)
17455 .collect();
17456 let code = failure.code.unwrap_or("client.query_failed").to_owned();
17457 if position.is_some_and(|index| {
17458 let entry = &self.query_failures[index];
17459 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
17460 }) {
17461 return;
17462 }
17463 if let Some(index) = position {
17464 self.query_failures.remove(index);
17465 }
17466 self.query_failures.push(DiagnosticQueryFailure {
17467 id: owner.id.to_owned(),
17468 tables,
17469 code,
17470 sqlite_code: failure.sqlite_code,
17471 at_ms: self.clock_now_ms(),
17472 });
17473 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
17474 self.query_failures.remove(0);
17475 }
17476 }
17477
17478 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
17481 #[cfg(feature = "bench-internals")]
17482 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
17483 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
17484 let max_commits = self.limits.max_push_commits_per_request;
17485 let max_ops = self
17486 .limits
17487 .max_push_operations_per_request
17488 .unwrap_or(PUSH_OPS_PER_REQUEST);
17489 let byte_cap = self.limits.max_push_request_bytes;
17490 let accept = self.limits.accept.unwrap_or(if url_capable {
17493 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
17494 } else {
17495 DEFAULT_ACCEPT
17496 });
17497 let header = Frame::ReqHeader {
17498 client_id: self.client_id.clone(),
17499 schema_version: self.schema.version,
17500 log_epoch: log_epoch.clone(),
17501 };
17502 let mut lead = vec![header];
17506 lead.extend(
17507 self.round_failure
17508 .iter()
17509 .map(|code| Frame::RoundFailure { code: code.clone() }),
17510 );
17511 let pull = Frame::PullHeader {
17512 limit_commits: self.limits.limit_commits.unwrap_or(0),
17513 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
17514 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
17515 accept,
17516 };
17517 let mut fresh = Vec::new();
17518 let mut sub_frames = Vec::new();
17519 for sub in &self.subs {
17520 if sub.state != SubState::Active {
17521 continue;
17522 }
17523 let mut scopes = sub.requested.clone();
17524 sort_scope_map(&mut scopes);
17525 sub_frames.push(Frame::Subscription {
17526 id: sub.id.clone(),
17527 table: sub.table.clone(),
17528 scopes,
17529 params: sub.params.clone().map(RawJson),
17530 cursor: sub.cursor,
17531 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
17532 });
17533 fresh.push((
17534 sub.id.clone(),
17535 sub.cursor < 0 && sub.bootstrap_state.is_none(),
17536 ));
17537 }
17538 let fixed_bytes = match byte_cap {
17541 Some(_) => {
17542 let mut frames = lead.clone();
17543 frames.push(pull.clone());
17544 frames.extend(sub_frames.iter().cloned());
17545 encode_message(&Message {
17546 wire_version: WIRE_VERSION,
17547 msg_kind: MsgKind::Request,
17548 frames,
17549 })
17550 .len()
17551 }
17552 None => 0,
17553 };
17554 if let Some(cap) = byte_cap {
17555 if fixed_bytes > cap {
17556 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
17557 }
17558 }
17559 let base_bytes = match byte_cap {
17560 Some(_) => {
17561 let mut frames = lead.clone();
17562 frames.push(pull.clone());
17563 encode_message(&Message {
17564 wire_version: WIRE_VERSION,
17565 msg_kind: MsgKind::Request,
17566 frames,
17567 })
17568 .len()
17569 }
17570 None => 0,
17571 };
17572 let total_outbox = if log_epoch.is_some() {
17573 self.outbox.len()
17574 } else {
17575 0
17576 };
17577 let mut push_frames: Vec<Frame> = Vec::new();
17578 let mut pushed_ids = Vec::new();
17579 let mut ops_in_request = 0usize;
17580 let mut bytes_in_request = 0usize;
17581 let mut deferred_commits = 0usize;
17582 #[cfg(feature = "bench-internals")]
17583 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
17584 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
17585 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
17589 deferred_commits = total_outbox - index;
17590 break;
17591 }
17592 if ops_in_request + commit.ops.len() > max_ops {
17593 if push_frames.is_empty() {
17594 return Err(Self::push_capacity_outcome(
17595 "operations",
17596 max_ops,
17597 commit.ops.len(),
17598 Some(commit.client_commit_id.as_str()),
17599 ));
17600 }
17601 deferred_commits = total_outbox - index;
17602 break;
17603 }
17604 let operations = commit
17605 .ops
17606 .iter()
17607 .map(|op| {
17608 let payload = self.schema.table(&op.table).and_then(|table| {
17609 let values = op.presented_values(table)?;
17610 let fallback = self.stored_key_fallback(table, &op.row_id);
17620 encode_sparse_row_json(
17621 table,
17622 &op.row_id,
17623 &values,
17624 &self.encryption,
17625 fallback.as_ref(),
17626 )
17627 .ok()
17628 });
17629 ssp2::model::Operation {
17630 table: op.table.clone(),
17631 row_id: op.row_id.clone(),
17632 op: if op.upsert { Op::Upsert } else { Op::Delete },
17633 base_version: op.base_version,
17634 payload,
17635 }
17636 })
17637 .collect();
17638 let frame = Frame::PushCommit {
17639 client_commit_id: commit.client_commit_id.clone(),
17640 operations,
17641 };
17642 let frame_bytes = match byte_cap {
17643 Some(_) => {
17644 let mut frames = lead.clone();
17645 frames.push(frame.clone());
17646 frames.push(pull.clone());
17647 encode_message(&Message {
17648 wire_version: WIRE_VERSION,
17649 msg_kind: MsgKind::Request,
17650 frames,
17651 })
17652 .len()
17653 - base_bytes
17654 }
17655 None => 0,
17656 };
17657 if let Some(cap) = byte_cap {
17661 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
17662 deferred_commits = total_outbox - index;
17663 break;
17664 }
17665 }
17666 push_frames.push(frame);
17667 pushed_ids.push(commit.client_commit_id.clone());
17668 ops_in_request += commit.ops.len();
17669 bytes_in_request += frame_bytes;
17670 }
17671 #[cfg(feature = "bench-internals")]
17672 drop(outbox_phase);
17673 if let Some(cap) = byte_cap {
17674 if fixed_bytes + bytes_in_request > cap {
17675 let blocked = if push_frames.is_empty() {
17678 None
17679 } else {
17680 self.outbox.first().map(|c| c.client_commit_id.as_str())
17681 };
17682 return Err(Self::push_capacity_outcome(
17683 "bytes",
17684 cap,
17685 fixed_bytes + bytes_in_request,
17686 blocked,
17687 ));
17688 }
17689 }
17690 let mut frames = lead;
17691 frames.extend(push_frames);
17692 frames.push(pull);
17693 frames.extend(sub_frames);
17694 let message = Message {
17695 wire_version: WIRE_VERSION,
17696 msg_kind: MsgKind::Request,
17697 frames,
17698 };
17699 Ok((
17700 message,
17701 RequestMeta {
17702 pushed_ids,
17703 fresh,
17704 accept,
17705 deferred_commits,
17706 },
17707 ))
17708 }
17709
17710 fn push_capacity_outcome(
17712 kind: &str,
17713 limit: usize,
17714 size: usize,
17715 client_commit_id: Option<&str>,
17716 ) -> Box<SyncOutcome> {
17717 let mut details = Map::new();
17718 details.insert("kind".to_owned(), Value::from(kind));
17719 details.insert("limit".to_owned(), Value::from(limit as u64));
17720 details.insert("size".to_owned(), Value::from(size as u64));
17721 if let Some(id) = client_commit_id {
17722 details.insert("clientCommitId".to_owned(), Value::from(id));
17723 }
17724 Box::new(SyncOutcome::Failed {
17725 error_code: "client.push_request_too_large".into(),
17726 message: "the push request exceeds a configured capacity limit".into(),
17727 details: Some(Value::Object(details)),
17728 recommended_action: None,
17729 })
17730 }
17731
17732 pub fn progress(&self) -> ProgressObserver {
17736 self.progress.clone()
17737 }
17738
17739 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
17740 let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
17741 Ok(prepared) => prepared,
17742 Err(outcome) => return *outcome,
17743 };
17744 let mut next = prepared;
17749 loop {
17750 match self.apply_sync_round(next.exchange(transport)) {
17751 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
17752 crate::AppliedSyncRound::Complete {
17753 outcome, controls, ..
17754 } => {
17755 if self.transport_enabled {
17756 for text in controls {
17757 let _ = transport.realtime_send(&text);
17758 }
17759 } else {
17760 self.disconnect_realtime(transport);
17761 }
17762 return outcome;
17763 }
17764 }
17765 }
17766 }
17767
17768 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
17769 if let SyncOutcome::Failed {
17770 error_code,
17771 message,
17772 details,
17773 ..
17774 } = &mut outcome
17775 {
17776 if let Some(failure) = self.storage_failure.borrow().as_ref() {
17777 let code = failure
17778 .code
17779 .expect("retained storage failures are classified");
17780 *error_code = code.to_owned();
17781 *message = match code {
17782 "client.storage_busy" => "local SQLite storage is busy",
17783 "client.storage_full" => "local SQLite storage is full",
17784 "client.storage_io" => "local SQLite storage I/O failed",
17785 "client.storage_corrupt" => "local SQLite storage is corrupt",
17786 _ => unreachable!("unrecognized classified storage failure"),
17787 }
17788 .to_owned();
17789 *details = failure.details();
17790 }
17791 }
17792 let details = match &outcome {
17793 SyncOutcome::Failed { details, .. } => details.as_ref(),
17794 _ => None,
17795 };
17796 let cause_kind = details
17797 .and_then(|details| details.get("causeKind"))
17798 .and_then(Value::as_str)
17799 .and_then(|cause| match cause {
17800 "network" => Some(crate::TransportCauseKind::Network),
17801 "connect" => Some(crate::TransportCauseKind::Connect),
17802 "tls" => Some(crate::TransportCauseKind::Tls),
17803 "io" => Some(crate::TransportCauseKind::Io),
17804 "request" => Some(crate::TransportCauseKind::Request),
17805 "protocol" => Some(crate::TransportCauseKind::Protocol),
17806 "timeout" => Some(crate::TransportCauseKind::Timeout),
17807 "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
17808 "aborted" => Some(crate::TransportCauseKind::Aborted),
17809 "body" => Some(crate::TransportCauseKind::Body),
17810 "status" => Some(crate::TransportCauseKind::Status),
17811 "redirect" => Some(crate::TransportCauseKind::Redirect),
17812 "unknown" => Some(crate::TransportCauseKind::Unknown),
17813 _ => None,
17814 });
17815 let http_status = if matches!(
17816 cause_kind,
17817 Some(
17818 crate::TransportCauseKind::Status
17819 | crate::TransportCauseKind::Body
17820 | crate::TransportCauseKind::Redirect
17821 )
17822 ) {
17823 details
17824 .and_then(|details| details.get("httpStatus"))
17825 .and_then(Value::as_f64)
17826 .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
17827 .map(|status| status as u16)
17828 } else {
17829 None
17830 };
17831 let retry_delay_ms = self.round_retry_delay_ms;
17832 self.progress.update(|p| match &outcome {
17833 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
17834 if report.failed.is_empty() =>
17835 {
17836 p.state = ProgressState::Complete
17837 }
17838 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
17839 p.state = ProgressState::Failed;
17840 p.error_code = Some("sync.scope_revoked".into());
17841 }
17842 SyncOutcome::Failed { error_code, .. } => {
17843 p.state = ProgressState::Failed;
17844 p.error_code = Some(Self::diagnostic_code(error_code));
17845 p.cause_kind = cause_kind;
17846 p.http_status = http_status;
17847 p.retry_delay_ms = retry_delay_ms;
17848 }
17849 SyncOutcome::RealtimeUnavailable { .. } => {
17850 p.state = ProgressState::Failed;
17851 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
17852 p.retry_delay_ms = retry_delay_ms;
17853 }
17854 });
17855 let completed_at_ms = self.clock_now_ms();
17856 self.last_round = Some(match &outcome {
17857 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
17858 status: "succeeded".to_owned(),
17859 started_at_ms,
17860 completed_at_ms,
17861 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
17862 counters: Some(DiagnosticRoundCounters {
17863 pushed: report.pushed,
17864 applied: report.applied.len(),
17865 rejected: report.rejected.len(),
17866 retryable: report.retryable.len(),
17867 conflicts: report.conflicts,
17868 commits_applied: report.commits_applied,
17869 segment_rows_applied: report.segment_rows_applied,
17870 bootstrapping: report.bootstrapping.len(),
17871 resets: report.resets.len(),
17872 revoked: report.revoked.len(),
17873 failed: report.failed.len(),
17874 deferred_commits: report.deferred_commits,
17875 }),
17876 error_code: None,
17877 cause_kind: None,
17878 http_status: None,
17879 },
17880 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
17881 status: "failed".to_owned(),
17882 started_at_ms,
17883 completed_at_ms,
17884 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
17885 counters: None,
17886 error_code: Some(Self::diagnostic_code(error_code)),
17887 cause_kind,
17888 http_status,
17889 },
17890 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
17891 status: "failed".to_owned(),
17892 started_at_ms,
17893 completed_at_ms,
17894 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
17895 counters: None,
17896 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
17897 cause_kind: None,
17898 http_status: None,
17899 },
17900 });
17901 outcome
17902 }
17903
17904 pub fn prepare_sync_round(
17907 &mut self,
17908 url_capable: bool,
17909 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
17910 if !self.transport_enabled {
17911 return Err(Box::new(SyncOutcome::Failed {
17912 error_code: "sync.offline".into(),
17913 message: "transport gate is closed".into(),
17914 details: None,
17915 recommended_action: None,
17916 }));
17917 }
17918 if self.active_round.is_some() {
17919 return Err(Box::new(SyncOutcome::Failed {
17920 error_code: "client.round_in_progress".into(),
17921 message: "a sync round is already in flight".into(),
17922 details: None,
17923 recommended_action: None,
17924 }));
17925 }
17926 self.storage_failure.borrow_mut().take();
17927 self.progress.start();
17928 self.round_retry_delay_ms = None;
17929 let started_at_ms = self.clock_now_ms();
17930 let prepared = (|| {
17931 if self.stopped {
17932 return Err(Box::new(SyncOutcome::Ok(SyncReport {
17933 schema_floor: self.schema_floor.clone(),
17934 ..SyncReport::default()
17935 })));
17936 }
17937 if self.security_preflight {
17938 return Err(Box::new(SyncOutcome::Failed {
17939 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
17940 message: "security preflight must complete before syncing".into(),
17941 details: None,
17942 recommended_action: None,
17943 }));
17944 }
17945 self.set_sync_needed(false, false);
17946 self.drain_pending_evictions()
17947 .and_then(|_| self.drop_incompatible_outbox())
17948 .and_then(|_| self.drop_unencodable_outbox(true, None))
17949 .map_err(|message| {
17950 Box::new(SyncOutcome::Failed {
17951 error_code: "storage.failed".into(),
17952 message,
17953 details: None,
17954 recommended_action: None,
17955 })
17956 })?;
17957 let uploads = if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
17958 self.pending_blob_uploads().map_err(|e| {
17959 Box::new(SyncOutcome::Failed {
17960 error_code: e.code,
17961 message: e.message,
17962 details: e.details,
17963 recommended_action: None,
17964 })
17965 })?
17966 } else {
17967 Vec::new()
17968 };
17969 if self.realtime_state != RealtimeState::Connected
17970 && self.realtime_policy == RealtimePolicy::Required
17971 {
17972 let retry_delay_ms = self.schedule_background_retry();
17973 self.realtime_retry_delay_ms = Some(retry_delay_ms);
17974 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
17975 state: self.realtime_state,
17976 reason_code: self.realtime_reason_code.clone(),
17977 retry_delay_ms,
17978 }));
17979 }
17980 let (message, meta) = self.build_request(url_capable)?;
17981 let id = uuid::Uuid::new_v4();
17982 self.active_round = Some(id);
17983 let phase = if uploads.is_empty() {
17984 crate::round::RoundPhase::Sync
17985 } else {
17986 crate::round::RoundPhase::UploadGrant
17987 };
17988 Ok(crate::PreparedSyncRound {
17989 id,
17990 started_at_ms,
17991 message,
17992 meta,
17993 uploads: uploads.into(),
17994 phase,
17995 encoded: None,
17996 continuation: None,
17997 completed: None,
17998 realtime: self.realtime_state == RealtimeState::Connected,
17999 scopes: self
18000 .subs
18001 .iter()
18002 .map(|s| (s.id.clone(), canonical_scope_json(&s.requested)))
18003 .collect(),
18004 #[cfg(feature = "bench-internals")]
18005 benchmark_phases: self.benchmark_phases.clone(),
18006 progress: self.progress.clone(),
18007 fixed_now: self.now_ms,
18008 round_deadline_at: None,
18009 })
18010 })();
18011 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
18012 }
18013
18014 pub fn cancel_sync_round(&mut self) {
18017 self.active_round = None;
18018 }
18019
18020 pub fn apply_sync_round(
18023 &mut self,
18024 completed: crate::CompletedSyncRound,
18025 ) -> crate::AppliedSyncRound {
18026 let prepared = completed.prepared;
18027 if self.active_round != Some(prepared.id) || self.security_preflight {
18028 return crate::AppliedSyncRound::Complete {
18029 outcome: SyncOutcome::Failed {
18030 error_code: "client.round_cancelled".into(),
18031 message: "sync round belongs to a released client context".into(),
18032 details: None,
18033 recommended_action: None,
18034 },
18035 controls: Vec::new(),
18036 more: false,
18037 bootstrap_advanced: false,
18038 };
18039 }
18040 self.storage_failure.borrow_mut().take();
18041 let (response, mut downloads, transport_failed) = match completed.exchange {
18042 crate::round::ExchangeResult::Upload { id, result } => {
18043 let result = result.and_then(|()| {
18044 self.conn
18045 .execute(
18046 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
18047 rusqlite::params![id],
18048 )
18049 .map(|_| ())
18050 .map_err(|error| {
18051 TransportError::new(
18052 "client.failed",
18053 Self::sqlite_failure(&self.storage_failure, error),
18054 )
18055 })
18056 });
18057 match result {
18058 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
18059 Err(TransportError {
18060 code,
18061 message,
18062 details,
18063 }) => {
18064 self.active_round = None;
18065 if Self::retryable_failure_code(&code)
18066 && !Self::unreachable_transport(&details)
18067 {
18068 self.schedule_background_retry();
18069 }
18070 let recommended_action = recommended_action_of(&details);
18071 let outcome = self.finish_sync_round(
18072 prepared.started_at_ms,
18073 SyncOutcome::Failed {
18074 error_code: code,
18075 message,
18076 details,
18077 recommended_action,
18078 },
18079 );
18080 return crate::AppliedSyncRound::Complete {
18081 outcome,
18082 controls: Vec::new(),
18083 more: false,
18084 bootstrap_advanced: false,
18085 };
18086 }
18087 }
18088 }
18089 crate::round::ExchangeResult::Reply {
18090 response,
18091 downloads,
18092 transport_failed,
18093 } => (response, downloads, transport_failed),
18094 };
18095 self.active_round = None;
18096 let before = self
18097 .subs
18098 .iter()
18099 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
18100 .collect::<Vec<_>>();
18101 let outcome = (|| {
18102 let response = match response {
18103 Ok(response) => {
18104 self.round_failure = None;
18107 response
18108 }
18109 Err(TransportError {
18110 code,
18111 message,
18112 details,
18113 }) => {
18114 if ssp2::model::valid_round_failure_code(&code) {
18119 self.round_failure = Some(code.clone());
18120 }
18121 if transport_failed {
18122 if prepared.realtime {
18123 self.set_realtime_state(RealtimeState::Lost, Some(&code));
18124 }
18125 self.record_lease_error(&code);
18126 }
18127 if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
18128 {
18129 let delay = self.schedule_background_retry();
18130 if prepared.realtime && transport_failed {
18131 self.realtime_retry_delay_ms = Some(delay);
18132 }
18133 }
18134 let recommended_action = recommended_action_of(&details);
18135 return SyncOutcome::Failed {
18136 error_code: code,
18137 message,
18138 details,
18139 recommended_action,
18140 };
18141 }
18142 };
18143 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
18144 if let SyncOutcome::Ok(report) = &mut outcome {
18145 report.deferred_commits = prepared.meta.deferred_commits;
18146 }
18147 match &outcome {
18148 SyncOutcome::Ok(_) => self.reset_background_retry(),
18149 SyncOutcome::Failed {
18150 error_code,
18151 details,
18152 ..
18153 } if Self::retryable_failure_code(error_code)
18154 && !Self::unreachable_transport(details) =>
18155 {
18156 self.schedule_background_retry();
18157 }
18158 _ => {}
18159 }
18160 if prepared.meta.deferred_commits > 0 {
18161 self.set_sync_needed(true, true);
18162 }
18163 self.previous_version_lifetime_check();
18164 outcome
18165 })();
18166 let more = self.transport_enabled
18167 && matches!(&outcome, SyncOutcome::Ok(report)
18168 if report.schema_floor.is_none()
18169 && (!report.bootstrapping.is_empty()
18170 || report.commits_applied > 0
18171 || report.segment_rows_applied > 0
18172 || !report.resets.is_empty()
18173 || report.deferred_commits > 0
18174 || self.sync_needed));
18175 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
18176 && before
18177 != self
18178 .subs
18179 .iter()
18180 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
18181 .collect::<Vec<_>>();
18182 crate::AppliedSyncRound::Complete {
18183 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
18184 controls: if self.transport_enabled {
18185 downloads.controls
18186 } else {
18187 Vec::new()
18188 },
18189 more,
18190 bootstrap_advanced,
18191 }
18192 }
18193
18194 pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
18196
18197 pub fn sync_until_idle(
18198 &mut self,
18199 transport: &mut dyn Transport,
18200 max_rounds: Option<u32>,
18201 ) -> SyncOutcome {
18202 self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
18203 }
18204
18205 pub fn sync_until_idle_with_deadline(
18210 &mut self,
18211 transport: &mut dyn Transport,
18212 max_rounds: Option<u32>,
18213 timeout_ms: Option<u64>,
18214 ) -> SyncOutcome {
18215 if max_rounds == Some(0) {
18218 return SyncOutcome::Failed {
18219 details: None,
18220 error_code: "sync.invalid_request".into(),
18221 message: "maxRounds must be a positive integer".into(),
18222 recommended_action: None,
18223 };
18224 }
18225 if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
18226 return SyncOutcome::Failed {
18227 details: None,
18228 error_code: "sync.invalid_request".into(),
18229 message: "timeoutMs must be a positive integer".into(),
18230 recommended_action: None,
18231 };
18232 }
18233 let started_at_ms = self.clock_now_ms();
18234 let deadline_ms = timeout_ms.map(|ms| started_at_ms.saturating_add(ms as i64));
18235 self.sync_until_idle_bounded(transport, max_rounds, started_at_ms, deadline_ms)
18236 }
18237
18238 fn sync_until_idle_bounded(
18239 &mut self,
18240 transport: &mut dyn Transport,
18241 max_rounds: Option<u32>,
18242 started_at_ms: i64,
18243 deadline_ms: Option<i64>,
18244 ) -> SyncOutcome {
18245 let rounds = max_rounds.unwrap_or(20);
18246 let mut aggregate = SyncReport::default();
18247 let mut spent = 0;
18248 while spent < rounds {
18249 if let Some(deadline_ms) = deadline_ms {
18250 if self.clock_now_ms() >= deadline_ms {
18251 return SyncOutcome::Failed {
18252 error_code: "sync.idle_timeout".into(),
18253 message: "sync_until_idle exceeded its deadline".into(),
18254 details: Some(serde_json::json!({
18255 "timeoutMs": deadline_ms
18256 .saturating_sub(started_at_ms) as u64,
18257 "rounds": spent,
18258 })),
18259 recommended_action: Some("retryLater".into()),
18260 };
18261 }
18262 }
18263 spent += 1;
18264 let before = self
18265 .subs
18266 .iter()
18267 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
18268 .collect::<Vec<_>>();
18269 match self.sync(transport) {
18270 SyncOutcome::Failed {
18271 error_code,
18272 message,
18273 details,
18274 recommended_action,
18275 } => {
18276 return SyncOutcome::Failed {
18277 details,
18278 error_code,
18279 message,
18280 recommended_action,
18281 };
18282 }
18283 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
18284 SyncOutcome::BudgetExhausted(_) => {
18285 unreachable!("sync() never exhausts a round budget")
18286 }
18287 SyncOutcome::Ok(report) => {
18288 if max_rounds.is_none()
18289 && report.segment_rows_applied > 0
18290 && !report.bootstrapping.is_empty()
18291 && before
18292 != self
18293 .subs
18294 .iter()
18295 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
18296 .collect::<Vec<_>>()
18297 {
18298 spent = 0;
18299 }
18300 let stop = !self.transport_enabled() || report.schema_floor.is_some();
18303 aggregate.merge(&report);
18304 if stop {
18305 return SyncOutcome::Ok(aggregate);
18306 }
18307 let more = !report.bootstrapping.is_empty()
18313 || report.commits_applied > 0
18314 || report.segment_rows_applied > 0
18315 || !report.resets.is_empty()
18316 || report.deferred_commits > 0
18317 || self.sync_needed;
18318 if !more {
18319 return SyncOutcome::Ok(aggregate);
18320 }
18321 }
18322 }
18323 }
18324 SyncOutcome::BudgetExhausted(aggregate)
18327 }
18328
18329 fn process_response(
18330 &mut self,
18331 transport: &mut dyn Transport,
18332 response: Message,
18333 meta: &RequestMeta,
18334 ) -> SyncOutcome {
18335 #[cfg(feature = "bench-internals")]
18336 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
18337 let mut report = SyncReport {
18338 pushed: meta.pushed_ids.len() as u32,
18339 ..SyncReport::default()
18340 };
18341 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
18342 HashMap::new();
18343 let pushed_ids = meta
18344 .pushed_ids
18345 .iter()
18346 .map(String::as_str)
18347 .collect::<HashSet<_>>();
18348 let mut incomplete_subscriptions: HashSet<&str> =
18349 meta.fresh.iter().map(|(id, _)| id.as_str()).collect();
18350 let mut last_final_push_result_id: Option<String> = None;
18351 for frame in &response.frames {
18352 match frame {
18353 Frame::PushResultDetails {
18354 client_commit_id,
18355 entries,
18356 } => {
18357 let details = rejection_details_by_commit
18358 .entry(client_commit_id.clone())
18359 .or_default();
18360 for entry in entries {
18361 let parsed = match RejectionDetails::parse(&entry.details.0) {
18362 Ok(value) => value,
18363 Err(message) => {
18364 return SyncOutcome::Failed {
18365 details: None,
18366 error_code: "sync.invalid_request".to_owned(),
18367 message,
18368 recommended_action: None,
18369 };
18370 }
18371 };
18372 details.insert(entry.op_index, parsed);
18373 }
18374 }
18375 Frame::PushResult {
18376 client_commit_id,
18377 status,
18378 results,
18379 ..
18380 } if pushed_ids.contains(client_commit_id.as_str())
18381 && Self::push_result_is_final(*status, results) =>
18382 {
18383 last_final_push_result_id = Some(client_commit_id.clone());
18384 }
18385 _ => {}
18386 }
18387 }
18388 if response.wire_version < 2 {
18389 return SyncOutcome::Failed {
18390 details: None,
18391 error_code: "client.invalid_host_response".to_owned(),
18392 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
18393 recommended_action: None,
18394 };
18395 }
18396 let coalesced_deletes = coalesced_delete_keys(&response.frames, |id, status, _| {
18397 status == SubStatus::Active && self.subs.iter().any(|sub| sub.id == id)
18398 });
18399 let mut frames = response.frames.into_iter().peekable();
18400 match frames.next() {
18401 Some(Frame::RespHeader {
18402 required_schema_version,
18403 latest_schema_version,
18404 log_epoch,
18405 reset_required,
18406 }) => {
18407 let Some(log_epoch) = log_epoch else {
18408 return SyncOutcome::Failed {
18409 details: None,
18410 error_code: "client.invalid_host_response".to_owned(),
18411 message: "response header omits logEpoch".to_owned(),
18412 recommended_action: None,
18413 };
18414 };
18415 let Some(reset_required) = reset_required else {
18416 return SyncOutcome::Failed {
18417 details: None,
18418 error_code: "client.invalid_host_response".to_owned(),
18419 message: "response header omits resetRequired".to_owned(),
18420 recommended_action: None,
18421 };
18422 };
18423 if let Some(required) = required_schema_version {
18424 let floor = SchemaFloor {
18427 required_schema_version: Some(required),
18428 latest_schema_version,
18429 };
18430 self.set_schema_floor(Some(floor.clone()));
18431 report.schema_floor = Some(floor);
18432 return SyncOutcome::Ok(report);
18433 }
18434 if reset_required {
18435 if frames.next().is_some() {
18436 return SyncOutcome::Failed {
18437 details: None,
18438 error_code: "client.invalid_host_response".to_owned(),
18439 message: "log-epoch reset response contains body frames".to_owned(),
18440 recommended_action: None,
18441 };
18442 }
18443 match self.run_log_epoch_reset(&log_epoch) {
18444 Ok(resets) => {
18445 report.resets = resets;
18446 return SyncOutcome::Ok(report);
18447 }
18448 Err(message) => {
18449 return SyncOutcome::Failed {
18450 details: None,
18451 error_code: "sync.local_corrupt".to_owned(),
18452 message,
18453 recommended_action: None,
18454 };
18455 }
18456 }
18457 }
18458 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
18459 return SyncOutcome::Failed {
18460 details: None,
18461 error_code: "client.invalid_host_response".to_owned(),
18462 message: "server changed logEpoch without requiring a reset".to_owned(),
18463 recommended_action: None,
18464 };
18465 }
18466 }
18467 _ => {
18468 return SyncOutcome::Failed {
18469 details: None,
18470 error_code: "sync.invalid_request".to_owned(),
18471 message: "response does not start with RESP_HEADER".to_owned(),
18472 recommended_action: None,
18473 };
18474 }
18475 }
18476
18477 if let Err(message) = self.begin_coalesced_apply() {
18478 return SyncOutcome::Failed {
18479 details: None,
18480 error_code: "storage.failed".into(),
18481 message,
18482 recommended_action: None,
18483 };
18484 }
18485 let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
18486 while let Some(frame) = frames.next() {
18487 match frame {
18488 frame @ Frame::PushResult { .. } => {
18489 let successful = matches!(
18490 &frame,
18491 Frame::PushResult {
18492 status: PushStatus::Applied | PushStatus::Cached,
18493 ..
18494 }
18495 );
18496 let mut results = vec![frame];
18497 if successful {
18498 while matches!(
18499 frames.peek(),
18500 Some(Frame::PushResult {
18501 status: PushStatus::Applied | PushStatus::Cached,
18502 ..
18503 })
18504 ) {
18505 if let Some(next) = frames.next() {
18506 results.push(next);
18507 }
18508 }
18509 }
18510 if self
18511 .handle_push_results(
18512 &results,
18513 &pushed_ids,
18514 &rejection_details_by_commit,
18515 &mut report,
18516 last_final_push_result_id.as_deref(),
18517 )
18518 .is_err()
18519 {
18520 failure = Some((
18521 "client.outcome_persistence_failed".to_owned(),
18522 "local commit outcome could not be persisted".to_owned(),
18523 None,
18524 None,
18525 ));
18526 break;
18527 }
18528 }
18529 Frame::PushResultDetails { .. } => {}
18530 Frame::SubStart {
18531 id,
18532 status,
18533 reason_code,
18534 effective_scopes,
18535 bootstrap: _,
18536 } => {
18537 let mut body = Vec::new();
18538 let mut sub_end: Option<(i64, Option<String>)> = None;
18539 for inner in frames.by_ref() {
18540 match inner {
18541 Frame::SubEnd {
18542 next_cursor,
18543 bootstrap_state,
18544 } => {
18545 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
18546 break;
18547 }
18548 Frame::Unknown { .. } => {}
18549 other => body.push(other),
18550 }
18551 }
18552 if sub_end.is_some() {
18553 incomplete_subscriptions.remove(id.as_str());
18554 }
18555 let section_failure = match self.process_section(
18556 transport,
18557 &id,
18558 status,
18559 &reason_code,
18560 effective_scopes,
18561 body,
18562 sub_end,
18563 meta,
18564 &mut report,
18565 &coalesced_deletes,
18566 ) {
18567 Err(SectionError::Abort(code, message)) => {
18568 Some((code, message, None, None))
18569 }
18570 Err(SectionError::Transport(error)) => {
18571 Some((error.code, error.message, None, error.details))
18572 }
18573 _ => None,
18574 };
18575 if section_failure.is_some() {
18576 failure = section_failure;
18577 break;
18578 }
18579 }
18580 Frame::Lease {
18581 lease_id,
18582 expires_at_ms,
18583 } => {
18584 self.set_lease_state(Some(LeaseState {
18587 lease_id: Some(lease_id),
18588 expires_at_ms: Some(expires_at_ms),
18589 error_code: None,
18590 }));
18591 }
18592 Frame::Error {
18593 code,
18594 message,
18595 recommended_action,
18596 details,
18597 ..
18598 } => {
18599 let recommended_action =
18602 (!recommended_action.is_empty()).then_some(recommended_action);
18603 let details = details
18604 .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
18605 .filter(Value::is_object);
18606 failure = Some((code, message, recommended_action, details));
18607 break;
18608 }
18609 Frame::Unknown { .. } => {}
18610 _ => {
18611 failure = Some((
18612 "sync.invalid_request".to_owned(),
18613 "unexpected frame in response".to_owned(),
18614 None,
18615 None,
18616 ));
18617 break;
18618 }
18619 }
18620 }
18621
18622 let applied = match failure {
18623 Some(failure) => Err(failure),
18624 None => self
18629 .rebuild_overlay_if_dirty()
18630 .and_then(|()| self.drain_pending_evictions())
18631 .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
18632 };
18633 if let Err(message) = self.finish_coalesced_apply() {
18636 return SyncOutcome::Failed {
18637 details: None,
18638 error_code: "storage.failed".into(),
18639 message,
18640 recommended_action: None,
18641 };
18642 }
18643 if let Err((error_code, message, recommended_action, details)) = applied {
18644 return SyncOutcome::Failed {
18645 details,
18646 error_code,
18647 message,
18648 recommended_action,
18649 };
18650 }
18651 if !incomplete_subscriptions.is_empty() {
18652 self.set_sync_needed(true, true);
18653 }
18654 self.ack_after_pull(transport);
18655 if self.upgrading && report.bootstrapping.is_empty() {
18658 self.set_upgrading(false);
18659 }
18660 SyncOutcome::Ok(report)
18661 }
18662
18663 fn handle_push_results(
18666 &mut self,
18667 frames: &[Frame],
18668 pushed_ids: &HashSet<&str>,
18669 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
18670 report: &mut SyncReport,
18671 last_final_push_result_id: Option<&str>,
18672 ) -> Result<(), String> {
18673 self.begin_observation("syncular_push_results")?;
18674 let previous_failed = self.failed_commits.clone();
18675 let previous_sync_needed = self.sync_needed;
18676 let conflict_count = self.conflicts.len();
18677 let rejection_count = self.rejections.len();
18678 let applied_count = report.applied.len();
18679 let rejected_count = report.rejected.len();
18680 let retryable_count = report.retryable.len();
18681 let was_dirty = self.overlay_dirty.snapshot();
18682 let mut removed = Vec::new();
18683 let mut batch = ChangeAccumulator::default();
18684 let persisted = (|| {
18685 for frame in frames {
18686 let Frame::PushResult {
18687 client_commit_id,
18688 status,
18689 results,
18690 commit_seq,
18691 } = frame
18692 else {
18693 return Err("unexpected acknowledgement frame".to_owned());
18694 };
18695 if !pushed_ids.contains(client_commit_id.as_str()) {
18696 continue;
18697 }
18698 let Some(index) = self
18699 .outbox
18700 .iter()
18701 .position(|commit| commit.client_commit_id == *client_commit_id)
18702 else {
18703 continue;
18704 };
18705 let status = *status;
18706 let rejection_details = rejection_details_by_commit.get(client_commit_id);
18707 let operations = self.outbox[index].ops.clone();
18708 match status {
18709 PushStatus::Applied | PushStatus::Cached => {
18710 let sequence =
18713 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
18714 let affected: HashSet<_> = operations
18715 .iter()
18716 .map(|op| (op.table.as_str(), op.row_id.as_str()))
18717 .collect();
18718 let prior = self.acknowledged_operations(Some(&affected), None)?;
18719 for (op_index, operation) in operations.iter().enumerate() {
18720 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
18721 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18722 if delivered {
18723 continue;
18724 }
18725 let table = self
18726 .schema
18727 .table(&operation.table)
18728 .ok_or_else(|| "sync.unknown_table".to_owned())?;
18729 let mut intended: Map<String, Value> = self
18730 .stored_row(table, &operation.row_id, true)?
18731 .map(|(row, _)| {
18732 table
18733 .columns
18734 .iter()
18735 .zip(&row)
18736 .map(|(column, value)| {
18737 (column.name.clone(), column_value_to_json(value))
18738 })
18739 .collect()
18740 })
18741 .unwrap_or_default();
18742 let mut scope_image = intended.clone();
18743 for op in prior
18744 .iter()
18745 .chain(
18746 self.failed_commits
18747 .iter()
18748 .flat_map(|(commit, _)| &commit.ops),
18749 )
18750 .chain(
18751 self.outbox
18752 .iter()
18753 .take(index)
18754 .flat_map(|commit| &commit.ops),
18755 )
18756 .chain(operations.iter().take(op_index + 1))
18757 {
18758 if op.table != operation.table || op.row_id != operation.row_id {
18759 continue;
18760 }
18761 if let Some(values) = &op.values {
18762 intended.extend(values.clone());
18763 } else {
18764 scope_image = intended.clone();
18765 intended.clear();
18766 }
18767 }
18768 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
18769 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
18770 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
18771 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
18772 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18773 }
18774 let journal_results = results
18775 .iter()
18776 .map(|result| {
18777 let op_index = match result {
18778 OpResult::Applied { op_index }
18779 | OpResult::Conflict { op_index, .. }
18780 | OpResult::Error { op_index, .. } => *op_index,
18781 };
18782 CommitOperationOutcome::Applied { op_index }
18783 })
18784 .collect::<Vec<_>>();
18785 let outcome_status = if status == PushStatus::Applied {
18786 CommitOutcomeStatus::Applied
18787 } else {
18788 CommitOutcomeStatus::Cached
18789 };
18790 let persisted = self
18791 .persist_commit_outcome(
18792 client_commit_id,
18793 outcome_status,
18794 &journal_results,
18795 None,
18796 )
18797 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
18798 persisted?;
18799 for operation in &operations {
18800 self.overlay_dirty.table(&operation.table);
18801 }
18802 self.sync_needed = true;
18803 batch.status = true;
18804 batch.outcomes = true;
18805 }
18806 PushStatus::Rejected => {
18807 if results.iter().any(|result| {
18808 matches!(
18809 result,
18810 OpResult::Error {
18811 code,
18812 retryable: true,
18813 ..
18814 } if code == "sync.idempotency_cache_miss"
18815 )
18816 }) {
18817 report.retryable.push(client_commit_id.to_owned());
18820 continue;
18821 }
18822
18823 let mut journal_results = Vec::with_capacity(results.len());
18824 let mut conflicts = Vec::new();
18825 let mut rejections = Vec::new();
18826 for result in results {
18827 match result {
18828 OpResult::Applied { op_index } => {
18829 journal_results.push(CommitOperationOutcome::Applied {
18830 op_index: *op_index,
18831 });
18832 }
18833 OpResult::Conflict {
18834 op_index,
18835 code,
18836 message,
18837 server_version,
18838 server_row,
18839 conflict_columns: conflict_columns_bitmap,
18840 } => {
18841 let operation = operations
18842 .get(*op_index as usize)
18843 .map(CommitOperation::from);
18844 let (table, row_id) = operation
18845 .as_ref()
18846 .map(|op| (op.table.clone(), op.row_id.clone()))
18847 .unwrap_or_default();
18848 let server_row_json = self
18849 .schema
18850 .table(&table)
18851 .and_then(|t| {
18852 decode_row_bytes(t, server_row, &self.encryption)
18853 .ok()
18854 .map(|row| (t, row))
18855 })
18856 .map(|(t, row)| {
18857 let mut map = Map::new();
18858 for (i, column) in t.columns.iter().enumerate() {
18859 map.insert(
18860 column.name.clone(),
18861 column_value_to_json(
18862 row.get(i).unwrap_or(&None),
18863 ),
18864 );
18865 }
18866 map
18867 })
18868 .unwrap_or_default();
18869 let conflict_columns = self
18872 .schema
18873 .table(&table)
18874 .map(|t| {
18875 t.columns
18876 .iter()
18877 .enumerate()
18878 .filter(|(index, _)| {
18879 conflict_columns_bitmap
18880 .get(index / 8)
18881 .is_some_and(|byte| {
18882 byte & (1 << (index % 8)) != 0
18883 })
18884 })
18885 .map(|(_, column)| column.name.clone())
18886 .collect::<Vec<_>>()
18887 })
18888 .unwrap_or_default();
18889 let conflict = ConflictRecord {
18890 client_commit_id: client_commit_id.to_owned(),
18891 op_index: *op_index,
18892 table,
18893 row_id,
18894 code: code.clone(),
18895 message: message.clone(),
18896 server_version: *server_version,
18897 server_row: server_row_json,
18898 conflict_columns,
18899 operation,
18900 };
18901 journal_results.push(CommitOperationOutcome::Conflict {
18902 conflict: conflict.clone(),
18903 });
18904 conflicts.push(conflict);
18905 }
18906 OpResult::Error {
18907 op_index,
18908 code,
18909 message,
18910 retryable,
18911 } => {
18912 let rejection = RejectionRecord {
18913 client_commit_id: client_commit_id.to_owned(),
18914 op_index: *op_index,
18915 code: code.clone(),
18916 message: message.clone(),
18917 retryable: *retryable,
18918 details: rejection_details
18919 .and_then(|details| details.get(op_index))
18920 .cloned(),
18921 operation: operations
18922 .get(*op_index as usize)
18923 .map(CommitOperation::from),
18924 };
18925 journal_results.push(CommitOperationOutcome::Error {
18926 rejection: rejection.clone(),
18927 });
18928 rejections.push(rejection);
18929 }
18930 }
18931 }
18932 if self.retain_failed_commits {
18933 self.retain_failed_commit(client_commit_id, &operations)?;
18934 }
18935 let outcome_status = if conflicts.is_empty() {
18936 CommitOutcomeStatus::Rejected
18937 } else {
18938 CommitOutcomeStatus::Conflict
18939 };
18940 let persisted = self
18941 .persist_commit_outcome(
18942 client_commit_id,
18943 outcome_status,
18944 &journal_results,
18945 Some(&operations),
18946 )
18947 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
18948 persisted?;
18949 batch.conflicts = !conflicts.is_empty();
18950 batch.rejections = !rejections.is_empty();
18951 batch.status = true;
18952 batch.outcomes = true;
18953 self.conflicts.extend(conflicts);
18954 self.rejections.extend(rejections);
18955 for operation in &operations {
18956 self.overlay_dirty.table(&operation.table);
18957 }
18958 }
18959 }
18960 removed.push((index, self.outbox.remove(index)));
18961 if batch.status {
18962 for operation in &operations {
18963 if !self.record_row_scopes(
18964 &mut batch,
18965 &operation.table,
18966 &operation.row_id,
18967 false,
18968 ) {
18969 batch.table(&operation.table);
18970 }
18971 }
18972 }
18973 match status {
18974 PushStatus::Applied | PushStatus::Cached => {
18975 report.applied.push(client_commit_id.to_owned())
18976 }
18977 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
18978 }
18979 }
18980 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
18981 self.prune_commit_outcomes()?;
18982 }
18983 self.rebuild_overlay_if_dirty()?;
18984 self.finish_observation("syncular_push_results", batch)
18985 })();
18986 if let Err(error) = persisted {
18987 self.rollback_observation("syncular_push_results");
18988 self.failed_commits = previous_failed;
18989 self.sync_needed = previous_sync_needed;
18990 for (index, commit) in removed.into_iter().rev() {
18991 self.outbox.insert(index, commit);
18992 }
18993 self.conflicts.truncate(conflict_count);
18994 self.rejections.truncate(rejection_count);
18995 self.overlay_dirty.restore(was_dirty.clone());
18996 report.applied.truncate(applied_count);
18997 report.rejected.truncate(rejected_count);
18998 report.retryable.truncate(retryable_count);
18999 return Err(error);
19000 }
19001 if !previous_sync_needed && self.sync_needed {
19002 self.sync_intent_queue.push_back(SyncIntent::Interactive);
19003 }
19004 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
19005 Ok(())
19006 }
19007
19008 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
19009 status != PushStatus::Rejected
19010 || !results.iter().any(|result| {
19011 matches!(
19012 result,
19013 OpResult::Error {
19014 code,
19015 retryable: true,
19016 ..
19017 } if code == "sync.idempotency_cache_miss"
19018 )
19019 })
19020 }
19021
19022 #[allow(clippy::too_many_arguments)]
19025 fn process_section(
19026 &mut self,
19027 transport: &mut dyn Transport,
19028 id: &str,
19029 status: SubStatus,
19030 reason_code: &str,
19031 effective_scopes: Vec<(String, Vec<String>)>,
19032 body: Vec<Frame>,
19033 sub_end: Option<(i64, Option<String>)>,
19034 meta: &RequestMeta,
19035 report: &mut SyncReport,
19036 coalesced_deletes: &HashSet<(i64, String, String)>,
19037 ) -> Result<(), SectionError> {
19038 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
19039 return Ok(()); };
19041 if status != SubStatus::Active && sub_end.is_none() {
19042 return Err(SectionError::Abort(
19043 "sync.invalid_request".into(),
19044 "subscription section without SUB_END".into(),
19045 ));
19046 }
19047 match status {
19048 SubStatus::Revoked => {
19049 self.begin_observation("syncular_revocation")
19050 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
19051 let previous = self.subs[sub_index].clone();
19052 let was_dirty = self.overlay_dirty.snapshot();
19053 let previous_outbox = self.outbox.clone();
19054 let previous_failed = self.failed_commits.clone();
19055 let previous_rejections = self.rejections.clone();
19056 let previous_conflicts = self.conflicts.clone();
19057 let revoked_count = report.revoked.len();
19058 let failed_count = report.failed.len();
19059 let result = (|| {
19060 let mut batch = ChangeAccumulator::default();
19061 let registered = self.window_unit_by_sub_id(id);
19062 let (table, effective) = {
19064 let sub = &self.subs[sub_index];
19065 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
19066 };
19067 let scoped_table = self.schema.table(&table).cloned().ok_or_else(|| {
19068 SectionError::Abort(
19069 "sync.unknown_table".to_owned(),
19070 "revoked scope references an unknown table".to_owned(),
19071 )
19072 })?;
19073 let dropped = self
19074 .drop_failed_commits(|name, values| {
19075 name == table
19076 && !effective.is_empty()
19077 && effective.iter().all(|(variable, allowed)| {
19078 scoped_table
19079 .scope_column(variable)
19080 .and_then(|column| values.get(column))
19081 .and_then(Value::as_str)
19082 .is_some_and(|value| {
19083 allowed.iter().any(|allowed| allowed == value)
19084 })
19085 })
19086 })
19087 .map_err(|message| {
19088 SectionError::Abort("storage.failed".to_owned(), message)
19089 })?;
19090 for commit in dropped {
19091 batch.conflicts = true;
19092 batch.rejections = true;
19093 batch.outcomes = true;
19094 for operation in commit.ops {
19095 batch.table(&operation.table);
19096 }
19097 }
19098 let purged = self.purge_scope_rows(&table, &effective);
19099 match purged {
19100 Ok(()) => {
19101 self.record_scope_map(&mut batch, &table, &effective);
19102 let sub = &mut self.subs[sub_index];
19103 sub.state = SubState::Revoked;
19104 sub.reason_code = Some(if reason_code.is_empty() {
19105 "sync.scope_revoked".to_owned()
19106 } else {
19107 reason_code.to_owned()
19108 });
19109 report.revoked.push(id.to_owned());
19110 let doomed_effective = effective;
19111 let sub_table = table;
19112 self.persist_sub(&self.subs[sub_index].clone())
19113 .map_err(|message| {
19114 SectionError::Abort("storage.failed".into(), message)
19115 })?;
19116 let dropped = self
19117 .drop_doomed_outbox(&sub_table, &doomed_effective)
19118 .map_err(|message| {
19119 SectionError::Abort("storage.failed".to_owned(), message)
19120 })?;
19121 if dropped {
19122 batch.status = true;
19123 batch.rejections = true;
19124 batch.outcomes = true;
19125 }
19126 self.delete_unreferenced_cached_blobs().map_err(|message| {
19129 SectionError::Abort("storage.failed".into(), message)
19130 })?;
19131 }
19132 Err(()) => {
19133 let sub = &mut self.subs[sub_index];
19136 sub.state = SubState::Failed;
19137 sub.reason_code = Some("sync.scope_revoked".to_owned());
19138 report.failed.push(id.to_owned());
19139 self.persist_sub(&self.subs[sub_index].clone())
19140 .map_err(|message| {
19141 SectionError::Abort("storage.failed".into(), message)
19142 })?;
19143 }
19144 }
19145 if let Some((base_key, unit)) = registered {
19146 batch.window(&base_key, &self.subs[sub_index].table, &unit);
19147 }
19148 self.rebuild_overlay_if_dirty()
19149 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19150 self.finish_observation("syncular_revocation", batch)
19151 .map_err(|message| {
19152 SectionError::Abort("storage.failed".to_owned(), message)
19153 })?;
19154 Ok(())
19155 })();
19156 if result.is_err() {
19157 self.rollback_observation("syncular_revocation");
19158 self.subs[sub_index] = previous;
19159 self.overlay_dirty.restore(was_dirty.clone());
19160 self.outbox = previous_outbox;
19161 self.failed_commits = previous_failed;
19162 self.rejections = previous_rejections;
19163 self.conflicts = previous_conflicts;
19164 report.revoked.truncate(revoked_count);
19165 report.failed.truncate(failed_count);
19166 }
19167 result
19168 }
19169 SubStatus::Reset => {
19170 self.begin_observation("syncular_reset")
19171 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
19172 let previous = self.subs[sub_index].clone();
19173 let was_dirty = self.overlay_dirty.snapshot();
19174 let reset_count = report.resets.len();
19175 let result = (|| {
19176 let mut batch = ChangeAccumulator::default();
19177 let registered = self.window_unit_by_sub_id(id);
19178 let sub = &mut self.subs[sub_index];
19181 sub.cursor = -1;
19182 sub.bootstrap_state = None;
19183 sub.reason_code = Some(if reason_code.is_empty() {
19184 "sync.cursor_expired".to_owned()
19185 } else {
19186 reason_code.to_owned()
19187 });
19188 report.resets.push(id.to_owned());
19189 self.persist_sub(&self.subs[sub_index].clone())
19190 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19191 if let Some((base_key, unit)) = registered {
19192 batch.window(&base_key, &self.subs[sub_index].table, &unit);
19193 }
19194 self.finish_observation("syncular_reset", batch)
19195 .map_err(|message| {
19196 SectionError::Abort("storage.failed".to_owned(), message)
19197 })?;
19198 Ok(())
19199 })();
19200 if result.is_err() {
19201 self.rollback_observation("syncular_reset");
19202 self.subs[sub_index] = previous;
19203 self.overlay_dirty.restore(was_dirty.clone());
19204 report.resets.truncate(reset_count);
19205 }
19206 result
19207 }
19208 SubStatus::Active => {
19209 let fresh = meta
19210 .fresh
19211 .iter()
19212 .find(|(fid, _)| fid == id)
19213 .map(|(_, fresh)| *fresh)
19214 .unwrap_or(false);
19215 let previous = self.subs[sub_index].clone();
19216 self.subs[sub_index].effective = Some(effective_scopes);
19217 let outcome = self
19218 .apply_section_body(
19219 transport,
19220 sub_index,
19221 body,
19222 fresh,
19223 meta,
19224 report,
19225 coalesced_deletes,
19226 )
19227 .and_then(|()| {
19228 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
19229 SectionError::Abort(
19230 "sync.invalid_request".into(),
19231 "subscription section without SUB_END".into(),
19232 )
19233 })?;
19234 self.apply_sub_end(sub_index, cursor, bootstrap_state)
19235 .map_err(|message| {
19236 SectionError::Abort("storage.failed".into(), message)
19237 })
19238 });
19239 match outcome {
19240 Ok(()) => {
19241 if self.subs[sub_index].bootstrap_state.is_some() {
19242 report.bootstrapping.push(id.to_owned());
19243 }
19244 Ok(())
19245 }
19246 Err(SectionError::FailClosed) => {
19247 self.subs[sub_index] = previous.clone();
19248 self.begin_observation("syncular_section_failure")
19249 .map_err(|message| {
19250 SectionError::Abort("storage.failed".into(), message)
19251 })?;
19252 let persisted = (|| {
19253 let mut batch = ChangeAccumulator::default();
19254 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
19255 batch.window(&base_key, &self.subs[sub_index].table, &unit);
19256 }
19257 self.subs[sub_index].state = SubState::Failed;
19258 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
19259 self.persist_sub(&self.subs[sub_index])?;
19260 self.finish_observation("syncular_section_failure", batch)
19261 })();
19262 if let Err(message) = persisted {
19263 self.rollback_observation("syncular_section_failure");
19264 self.subs[sub_index] = previous;
19265 return Err(SectionError::Abort("storage.failed".into(), message));
19266 }
19267 report.failed.push(id.to_owned());
19268 Ok(())
19269 }
19270 Err(error) => {
19271 self.subs[sub_index] = previous;
19274 Err(error)
19275 }
19276 }
19277 }
19278 }
19279 }
19280
19281 fn apply_sub_end(
19282 &mut self,
19283 sub_index: usize,
19284 cursor: i64,
19285 bootstrap_state: Option<String>,
19286 ) -> Result<(), String> {
19287 #[cfg(feature = "bench-internals")]
19288 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
19289 let previous = self.subs[sub_index].clone();
19290 self.begin_observation("syncular_sub_end")?;
19291 let persisted = (|| {
19292 let mut batch = ChangeAccumulator::default();
19293 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
19294 && cursor >= 0
19295 && bootstrap_state.is_none();
19296 if completed {
19297 let table = self
19298 .schema
19299 .table(&previous.table)
19300 .ok_or_else(|| "sync.unknown_table".to_owned())?;
19301 let effective = previous.effective.as_deref().unwrap_or(&[]);
19302 let mut statement = self.conn.prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
19303 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19304 let rows = statement
19305 .query_map(rusqlite::params![previous.table, cursor], |row| {
19306 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
19307 })
19308 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19309 let mut covered = HashSet::new();
19310 for row in rows {
19311 let (id, intent) =
19312 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19313 let values: Map<String, Value> = serde_json::from_str(&intent)
19314 .map_err(|_| "sync.local_corrupt".to_owned())?;
19315 if !effective.is_empty()
19316 && effective.iter().all(|(variable, allowed)| {
19317 table
19318 .scope_column(variable)
19319 .and_then(|column| values.get(column))
19320 .is_some_and(|value| {
19321 allowed.contains(
19322 &value
19323 .as_str()
19324 .map_or_else(|| value.to_string(), str::to_owned),
19325 )
19326 })
19327 })
19328 {
19329 covered.insert(id);
19330 }
19331 }
19332 drop(statement);
19333 for id in covered {
19334 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![previous.table, id, cursor])
19335 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19336 self.overlay_dirty.table(&previous.table);
19337 batch.table(&previous.table);
19338 }
19339 self.rebuild_overlay_if_dirty()?;
19340 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
19341 batch.window(&base_key, &previous.table, &unit);
19342 }
19343 }
19344 let sub = &mut self.subs[sub_index];
19345 sub.cursor = cursor;
19346 sub.bootstrap_state = bootstrap_state;
19347 sub.reason_code = None;
19350 sub.synced_once = true;
19351 self.persist_sub(&self.subs[sub_index])?;
19352 self.finish_observation("syncular_sub_end", batch)
19353 })();
19354 if persisted.is_err() {
19355 self.rollback_observation("syncular_sub_end");
19356 self.subs[sub_index] = previous;
19357 }
19358 persisted
19359 }
19360
19361 #[allow(clippy::too_many_arguments)]
19362 fn apply_section_body(
19363 &mut self,
19364 transport: &mut dyn Transport,
19365 sub_index: usize,
19366 body: Vec<Frame>,
19367 fresh: bool,
19368 meta: &RequestMeta,
19369 report: &mut SyncReport,
19370 coalesced_deletes: &HashSet<(i64, String, String)>,
19371 ) -> Result<(), SectionError> {
19372 let mut saw_segment = false;
19373 for frame in body {
19374 match frame {
19375 Frame::Commit {
19376 tables,
19377 changes,
19378 commit_seq,
19379 ..
19380 } => {
19381 if self
19382 .apply_commit_frame_coalesced(
19383 &tables,
19384 changes,
19385 commit_seq,
19386 coalesced_deletes,
19387 )
19388 .map_err(|(c, m)| SectionError::Abort(c, m))?
19389 {
19390 report.commits_applied += 1;
19391 }
19392 }
19393 Frame::SegmentInline { payload } => {
19394 let segment = decode_rows_segment(&payload)
19395 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
19396 self.progress.update(|p| {
19397 p.phase = ProgressPhase::Import;
19398 p.subscription_id = Some(self.subs[sub_index].id.clone());
19399 p.table = Some(self.subs[sub_index].table.clone());
19400 p.segment_id = None;
19401 p.bytes_received = payload.len() as u64;
19402 p.bytes_total = Some(payload.len() as u64);
19403 p.rows_processed = 0;
19404 p.rows_total =
19405 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
19406 });
19407 let first = !saw_segment;
19408 saw_segment = true;
19409 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
19410 report.segment_rows_applied += applied;
19411 }
19412 Frame::SegmentRef {
19413 segment_id,
19414 byte_length,
19415 media_type,
19416 table,
19417 row_count,
19418 as_of_commit_seq,
19419 scope_digest,
19420 row_cursor,
19421 next_row_cursor,
19422 url,
19423 url_expires_at_ms,
19424 ..
19425 } => {
19426 let advertised = match media_type {
19429 MediaType::Rows => {
19430 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
19431 || meta.accept & ACCEPT_INLINE_ROWS != 0
19432 }
19433 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
19434 };
19435 if !advertised {
19436 return Err(SectionError::Abort(
19437 "sync.invalid_request".to_owned(),
19438 format!(
19439 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
19440 media_type.name()
19441 ),
19442 ));
19443 }
19444 self.progress.update(|p| {
19445 p.phase = ProgressPhase::Download;
19446 p.subscription_id = Some(self.subs[sub_index].id.clone());
19447 p.table = Some(table.clone());
19448 p.segment_id = Some(segment_id.clone());
19449 p.bytes_received = 0;
19450 p.bytes_total = u64::try_from(byte_length).ok();
19451 p.rows_processed = 0;
19452 p.rows_total = u64::try_from(row_count).ok();
19453 });
19454 let observer = self.progress.clone();
19455 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
19456 let bytes = if let Some(url) = url {
19457 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
19462 return Err(SectionError::Abort(
19463 "sync.invalid_request".to_owned(),
19464 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
19465 .to_owned(),
19466 ));
19467 }
19468 if url_expires_at_ms.is_some_and(|exp| {
19470 exp <= transport
19471 .segment_fetch_started_at(&url)
19472 .unwrap_or_else(|| self.clock_now_ms())
19473 }) {
19474 return Err(SectionError::Abort(
19475 "sync.segment_expired".to_owned(),
19476 format!(
19477 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
19478 ),
19479 ));
19480 }
19481 transport
19482 .fetch_url(&url, &mut on_progress)
19483 .map_err(|e| self.segment_fetch_failed(e))?
19484 } else {
19485 let requested_scopes_json =
19486 canonical_scope_json(&self.subs[sub_index].requested);
19487 transport
19488 .download_segment(
19489 &SegmentRequest {
19490 segment_id: segment_id.clone(),
19491 table,
19492 requested_scopes_json,
19493 },
19494 &mut on_progress,
19495 )
19496 .map_err(|e| self.segment_fetch_failed(e))?
19497 };
19498 on_progress(bytes.len() as u64);
19500 let digest = Sha256::digest(&bytes);
19501 let expected = segment_id
19502 .strip_prefix("sha256:")
19503 .unwrap_or(segment_id.as_str());
19504 if bytes_to_hex(&digest) != expected {
19505 self.schedule_background_retry();
19507 return Err(SectionError::Abort(
19508 "sync.invalid_request".to_owned(),
19509 "segment bytes do not match the content address (§5.1)".to_owned(),
19510 ));
19511 }
19512 self.progress.update(|p| p.phase = ProgressPhase::Import);
19513 if media_type == MediaType::Sqlite {
19514 if row_cursor.is_some() || next_row_cursor.is_some() {
19517 return Err(SectionError::Abort(
19518 "sync.invalid_request".to_owned(),
19519 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
19520 .to_owned(),
19521 ));
19522 }
19523 let first = !saw_segment;
19524 saw_segment = true;
19525 let applied = self.apply_sqlite_segment(
19526 sub_index,
19527 &bytes,
19528 fresh && first,
19529 row_count,
19530 as_of_commit_seq,
19531 &scope_digest,
19532 )?;
19533 report.segment_rows_applied += applied;
19534 } else {
19535 let segment = decode_rows_segment(&bytes).map_err(|e| {
19536 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
19537 })?;
19538 let first = row_cursor.is_none();
19539 saw_segment = true;
19540 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
19541 report.segment_rows_applied += applied;
19542 }
19543 }
19544 Frame::Unknown { .. } => {}
19545 Frame::Error { code, message, .. } => {
19546 return Err(SectionError::Abort(code, message))
19547 }
19548 _ => {
19549 return Err(SectionError::Abort(
19550 "sync.invalid_request".to_owned(),
19551 "unexpected frame inside a subscription section".to_owned(),
19552 ));
19553 }
19554 }
19555 }
19556 Ok(())
19557 }
19558
19559 fn apply_commit_frame_coalesced(
19563 &mut self,
19564 tables: &[String],
19565 changes: Vec<ssp2::model::Change>,
19566 commit_seq: i64,
19567 coalesced_deletes: &HashSet<(i64, String, String)>,
19568 ) -> Result<bool, (String, String)> {
19569 let changes: Vec<ssp2::model::Change> = changes
19570 .into_iter()
19571 .filter(|change| {
19572 if change.op != Op::Delete {
19573 return true;
19574 }
19575 let Some(table) = tables.get(change.table_index as usize) else {
19576 return true;
19577 };
19578 !coalesced_deletes.contains(&(commit_seq, table.clone(), change.row_id.clone()))
19579 })
19580 .collect();
19581 if changes.is_empty() {
19582 return Ok(false);
19583 }
19584 self.apply_commit_frame(tables, &changes, Some(commit_seq))?;
19585 Ok(true)
19586 }
19587
19588 fn apply_commit_frame(
19590 &mut self,
19591 tables: &[String],
19592 changes: &[ssp2::model::Change],
19593 commit_seq: Option<i64>,
19594 ) -> Result<(), (String, String)> {
19595 #[cfg(feature = "bench-internals")]
19596 let _phase = self.benchmark_phases.start(Phase::CommitApply);
19597 let was_dirty = self.overlay_dirty.snapshot();
19598 self.coalesce_unit(changes.len())
19599 .and_then(|()| self.begin_observation("syncular_commit"))
19600 .map_err(|message| ("storage.failed".into(), message))?;
19601 let applied = (|| {
19602 let mut batch = ChangeAccumulator::default();
19603 self.record_commit_changes(&mut batch, tables, changes);
19604 if let Some(sequence) = commit_seq {
19605 for change in changes {
19606 let table = tables.get(change.table_index as usize).ok_or_else(|| {
19607 (
19608 "sync.invalid_request".to_owned(),
19609 "change tableIndex out of range".to_owned(),
19610 )
19611 })?;
19612 if self
19613 .outbox
19614 .iter()
19615 .flat_map(|commit| &commit.ops)
19616 .any(|op| op.table == *table && op.row_id == change.row_id)
19617 {
19618 self.conn.execute("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)", rusqlite::params![table, change.row_id, sequence])
19619 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
19620 }
19621 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![table, change.row_id, sequence])
19622 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
19623 }
19624 }
19625 self.apply_commit_changes(tables, changes)?;
19626 self.rebuild_overlay_if_dirty()
19627 .map_err(|message| ("storage.failed".into(), message))?;
19628 self.finish_observation("syncular_commit", batch)
19629 .map_err(|message| ("storage.failed".into(), message))
19630 })();
19631 if applied.is_err() {
19632 self.rollback_observation("syncular_commit");
19633 self.overlay_dirty.restore(was_dirty.clone());
19634 }
19635 applied
19636 }
19637
19638 fn apply_commit_changes(
19639 &mut self,
19640 tables: &[String],
19641 changes: &[ssp2::model::Change],
19642 ) -> Result<(), (String, String)> {
19643 let mirror_visible = !self.overlay_dirty.get()
19648 && self.failed_commits.is_empty()
19649 && ((self.outbox.is_empty() && !self.has_acknowledged_rows())
19650 || changes.iter().all(|change| {
19651 tables
19652 .get(change.table_index as usize)
19653 .and_then(|name| self.schema.table(name))
19654 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
19655 }));
19656 for change in changes {
19657 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
19658 (
19659 "sync.invalid_request".to_owned(),
19660 "change tableIndex out of range".to_owned(),
19661 )
19662 })?;
19663 let table = self.schema.table(table_name).ok_or_else(|| {
19664 (
19665 "sync.schema_mismatch".to_owned(),
19666 format!("change targets unknown table {table_name:?}"),
19667 )
19668 })?;
19669 match change.op {
19670 Op::Upsert => {
19671 let payload = change.row.as_ref().ok_or_else(|| {
19672 (
19673 "sync.invalid_request".to_owned(),
19674 "upsert change without row payload".to_owned(),
19675 )
19676 })?;
19677 let row = {
19679 #[cfg(feature = "bench-internals")]
19680 let _phase = self.benchmark_phases.start(Phase::RowDecode);
19681 decode_row_bytes(table, payload, &self.encryption)
19682 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
19683 };
19684 let version = change.row_version.unwrap_or(0);
19685 let table_name = table.name.clone();
19686 self.write_base_row(&table_name, &row, version)
19687 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
19688 if mirror_visible {
19689 self.write_row(&visible_table(&table_name), &table_name, &row, version)
19690 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
19691 }
19692 }
19693 Op::Delete => {
19694 self.delete_base_row(table_name, &change.row_id)
19695 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
19696 if mirror_visible {
19697 let sql = format!(
19698 "DELETE FROM {} WHERE {}",
19699 visible_table(table_name),
19700 row_id_predicate(table)
19701 );
19702 self.conn
19703 .prepare_cached(&sql)
19704 .and_then(|mut statement| {
19705 statement.execute(rusqlite::params![change.row_id])
19706 })
19707 .map_err(|error| {
19708 ("sync.invalid_request".to_owned(), error.to_string())
19709 })?;
19710 }
19711 }
19712 }
19713 }
19714 if mirror_visible {
19715 {
19716 let changed_rows: HashSet<(&str, &str)> = changes
19717 .iter()
19718 .map(|change| {
19719 (
19720 tables[change.table_index as usize].as_str(),
19721 change.row_id.as_str(),
19722 )
19723 })
19724 .collect();
19725 let acknowledged = self
19726 .acknowledged_operations(Some(&changed_rows), None)
19727 .map_err(|error| ("storage.failed".to_owned(), error))?;
19728 self.apply_outbox_ops(&acknowledged)
19729 .map_err(|message| ("storage.failed".into(), message))?;
19730 self.apply_outbox_ops(
19731 self.outbox
19732 .iter()
19733 .flat_map(|commit| &commit.ops)
19734 .filter(|op| {
19735 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
19736 }),
19737 )
19738 .map_err(|message| ("storage.failed".into(), message))?;
19739 }
19740 self.overlay_dirty.set(false);
19741 }
19742 Ok(())
19743 }
19744
19745 fn apply_segment(
19750 &mut self,
19751 sub_index: usize,
19752 segment: &RowsSegment,
19753 first_fresh_page: bool,
19754 ) -> Result<u32, SectionError> {
19755 let (sub_table, effective) = {
19756 let sub = &self.subs[sub_index];
19757 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
19758 };
19759 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
19760 SectionError::Abort(
19761 "sync.schema_mismatch".to_owned(),
19762 format!("subscription table {sub_table:?} is not in the client schema"),
19763 )
19764 })?;
19765 let matches = segment.table == table.name
19770 && segment.schema_version == self.schema.version
19771 && segment.columns.len() == table.wire_columns.len()
19772 && segment
19773 .columns
19774 .iter()
19775 .zip(table.wire_columns.iter())
19776 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
19777 if !matches {
19778 return Err(SectionError::Abort(
19779 "sync.schema_mismatch".to_owned(),
19780 "segment column table does not match the generated schema (§5.2)".to_owned(),
19781 ));
19782 }
19783 let mut applied = 0u32;
19784 let empty = Vec::new();
19785 let blocks = if segment.blocks.is_empty() {
19786 std::slice::from_ref(&empty)
19787 } else {
19788 &segment.blocks
19789 };
19790 for (index, block) in blocks.iter().enumerate() {
19791 let clear = first_fresh_page && index == 0;
19792 if !clear && block.is_empty() {
19793 continue;
19794 }
19795 let was_dirty = self.overlay_dirty.snapshot();
19796 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
19797 && self
19798 .outbox
19799 .iter()
19800 .flat_map(|commit| &commit.ops)
19801 .any(|op| op.table == table.name);
19802 let mirror_visible = !reconcile_pending
19803 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
19804 self.coalesce_unit(block.len())
19805 .and_then(|()| self.begin_observation("syncular_segment_block"))
19806 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19807 let outcome = (|| {
19808 self.rebuild_overlay_if_dirty()
19809 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19810 let mut changed_keys = Vec::new();
19811 let mut batch = ChangeAccumulator::default();
19812 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
19813 batch.table(&table.name);
19814 }
19815 if clear {
19816 self.purge_scope_rows(&table.name, &effective)
19817 .map_err(|()| SectionError::FailClosed)?;
19818 }
19819 for (row_index, row) in block.iter().enumerate() {
19820 let decrypted;
19821 let values = if table.has_encrypted_columns() {
19822 let mut values = row.values.clone();
19823 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
19824 .map_err(|message| {
19825 SectionError::Abort("client.decrypt_failed".into(), message)
19826 })?;
19827 decrypted = values;
19828 &decrypted
19829 } else {
19830 &row.values
19831 };
19832 self.write_base_row(&table.name, values, row.server_version)
19833 .map_err(|message| {
19834 SectionError::Abort("sync.invalid_request".into(), message)
19835 })?;
19836 if reconcile_pending {
19837 changed_keys.push(
19838 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
19839 |message| SectionError::Abort("storage.failed".into(), message),
19840 )?,
19841 );
19842 }
19843 if mirror_visible {
19844 self.write_row(
19845 &visible_table(&table.name),
19846 &table.name,
19847 values,
19848 row.server_version,
19849 )
19850 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19851 }
19852 let processed = u64::from(applied) + row_index as u64 + 1;
19853 if processed.is_multiple_of(1024) {
19854 self.progress.update(|p| p.rows_processed = processed);
19855 }
19856 }
19857 if reconcile_pending {
19858 self.reconcile_imported_rows(&table, &mut changed_keys)
19859 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19860 }
19861 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
19862 self.apply_outbox_ops(
19863 self.outbox
19864 .iter()
19865 .flat_map(|commit| &commit.ops)
19866 .filter(|op| op.table == table.name),
19867 )
19868 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19869 self.overlay_dirty.set(false);
19870 } else {
19871 self.rebuild_overlay_if_dirty()
19872 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19873 }
19874 self.finish_observation("syncular_segment_block", batch)
19875 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
19876 })();
19877 if let Err(error) = outcome {
19878 self.rollback_observation("syncular_segment_block");
19879 self.overlay_dirty.restore(was_dirty.clone());
19880 return Err(error);
19881 }
19882 applied += block.len() as u32;
19883 self.progress
19884 .update(|p| p.rows_processed = u64::from(applied));
19885 std::thread::yield_now();
19886 }
19887 self.progress
19888 .update(|p| p.rows_processed = u64::from(applied));
19889 Ok(applied)
19890 }
19891
19892 fn apply_sqlite_segment(
19900 &mut self,
19901 sub_index: usize,
19902 bytes: &[u8],
19903 first_fresh_page: bool,
19904 row_count: i64,
19905 as_of_commit_seq: i64,
19906 scope_digest: &str,
19907 ) -> Result<u32, SectionError> {
19908 let invalid = |detail: &str| {
19909 SectionError::Abort(
19910 "sync.invalid_request".to_owned(),
19911 format!("sqlite segment rejected: {detail} (§5.3)"),
19912 )
19913 };
19914 let (sub_table, effective) = {
19915 let sub = &self.subs[sub_index];
19916 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
19917 };
19918 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
19919 SectionError::Abort(
19920 "sync.schema_mismatch".to_owned(),
19921 format!("subscription table {sub_table:?} is not in the client schema"),
19922 )
19923 })?;
19924
19925 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
19926 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
19927 let img = match rusqlite::Connection::open_with_flags(
19928 &path,
19929 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
19930 ) {
19931 Ok(conn) => conn,
19932 Err(_) => {
19933 let _ = std::fs::remove_file(&path);
19934 return Err(invalid("bytes do not open as a SQLite database"));
19935 }
19936 };
19937 let outcome = self.apply_sqlite_image(
19938 &img,
19939 &table,
19940 first_fresh_page,
19941 &effective,
19942 row_count,
19943 as_of_commit_seq,
19944 scope_digest,
19945 );
19946 drop(img);
19947 let _ = std::fs::remove_file(&path);
19948 outcome
19949 }
19950
19951 #[allow(clippy::too_many_arguments)]
19952 fn apply_sqlite_image(
19953 &mut self,
19954 img: &rusqlite::Connection,
19955 table: &crate::schema::TableSchema,
19956 first_fresh_page: bool,
19957 effective: &[(String, Vec<String>)],
19958 row_count: i64,
19959 as_of_commit_seq: i64,
19960 scope_digest: &str,
19961 ) -> Result<u32, SectionError> {
19962 let invalid = |detail: String| {
19963 SectionError::Abort(
19964 "sync.invalid_request".to_owned(),
19965 format!("sqlite segment rejected: {detail} (§5.3)"),
19966 )
19967 };
19968
19969 type MetaRow = (i64, String, i64, i64, String, i64, i64);
19972 let meta: MetaRow = img
19973 .query_row(
19974 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
19975 \"scopeDigest\", \"rowCount\",
19976 (SELECT count(*) FROM _syncular_segment)
19977 FROM _syncular_segment",
19978 [],
19979 |row| {
19980 Ok((
19981 row.get(0)?,
19982 row.get(1)?,
19983 row.get(2)?,
19984 row.get(3)?,
19985 row.get(4)?,
19986 row.get(5)?,
19987 row.get(6)?,
19988 ))
19989 },
19990 )
19991 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
19992 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
19993 if meta_count != 1 {
19994 return Err(invalid(format!(
19995 "_syncular_segment must contain exactly one row, found {meta_count}"
19996 )));
19997 }
19998 if format != 1 {
19999 return Err(invalid(format!("format {format}")));
20000 }
20001 if meta_table != table.name {
20002 return Err(invalid(format!("image table {meta_table:?}")));
20003 }
20004 if schema_version != i64::from(self.schema.version) {
20005 return Err(invalid(format!("schemaVersion {schema_version}")));
20006 }
20007 if pin != as_of_commit_seq {
20008 return Err(invalid(format!("asOfCommitSeq {pin}")));
20009 }
20010 if digest != scope_digest {
20011 return Err(invalid("scopeDigest mismatch".to_owned()));
20012 }
20013 if meta_rows != row_count {
20014 return Err(invalid(format!("rowCount {meta_rows}")));
20015 }
20016
20017 let mut names: Vec<String> = Vec::new();
20019 let mut primary_keys: Vec<String> = Vec::new();
20020 {
20021 let mut stmt = img
20022 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
20023 .map_err(|_| invalid("image data table missing".to_owned()))?;
20024 let mut rows = stmt
20025 .query([])
20026 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
20027 while let Some(row) = rows
20028 .next()
20029 .map_err(|_| invalid("image data table unreadable".to_owned()))?
20030 {
20031 if row
20032 .get::<_, i64>(5)
20033 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
20034 > 0
20035 {
20036 primary_keys.push(row.get(1).map_err(|error| {
20037 invalid(Self::sqlite_failure(&self.storage_failure, error))
20038 })?);
20039 }
20040 names.push(
20041 row.get::<_, String>(1)
20042 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
20043 );
20044 }
20045 }
20046 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
20047 expected.push("_syncular_version");
20048 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
20049 return Err(SectionError::Abort(
20050 "sync.schema_mismatch".to_owned(),
20051 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
20052 ));
20053 }
20054
20055 if primary_keys != vec![table.primary_key.clone()] {
20056 return Err(invalid(
20057 "image primary key does not match generated schema".into(),
20058 ));
20059 }
20060 let actual_count: i64 = img
20061 .query_row(
20062 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
20063 [],
20064 |row| row.get(0),
20065 )
20066 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
20067 if actual_count != row_count {
20068 return Err(invalid("image row count does not match descriptor".into()));
20069 }
20070 let insert = self.insert_row_sql(&base_table(&table.name), table);
20071 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
20072 let column_list = names
20073 .iter()
20074 .map(|n| quote_ident(n))
20075 .collect::<Vec<_>>()
20076 .join(", ");
20077 let mut stmt = img
20078 .prepare(&format!(
20079 "SELECT {column_list} FROM {} ORDER BY {}",
20080 quote_ident(&table.name),
20081 quote_ident(&table.primary_key)
20082 ))
20083 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
20084 let mut rows = stmt
20085 .query([])
20086 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
20087 let mut applied = 0u32;
20088 loop {
20089 let clear = applied == 0 && first_fresh_page;
20090 let was_dirty = self.overlay_dirty.snapshot();
20091 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
20092 && self
20093 .outbox
20094 .iter()
20095 .flat_map(|commit| &commit.ops)
20096 .any(|op| op.table == table.name);
20097 let mirror_visible = !reconcile_pending
20098 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
20099 self.coalesce_unit(1024)
20101 .and_then(|()| self.begin_observation("syncular_image_chunk"))
20102 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20103 let result = (|| {
20104 self.rebuild_overlay_if_dirty()
20105 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20106 let mut changed_keys = Vec::new();
20107 let mut batch = ChangeAccumulator::default();
20108 if clear {
20109 if self.scoped_rows_exist(&table.name, effective) {
20110 batch.table(&table.name);
20111 }
20112 self.purge_scope_rows(&table.name, effective)
20113 .map_err(|()| SectionError::FailClosed)?;
20114 }
20115 let mut processed = 0u32;
20116 {
20117 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
20118 invalid(Self::sqlite_failure(&self.storage_failure, error))
20119 })?;
20120 let mut mirror = if mirror_visible {
20121 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
20122 invalid(Self::sqlite_failure(&self.storage_failure, error))
20123 })?)
20124 } else {
20125 None
20126 };
20127 for _ in 0..1024 {
20128 let Some(row) = rows.next().map_err(|error| {
20129 invalid(Self::sqlite_failure(&self.storage_failure, error))
20130 })?
20131 else {
20132 break;
20133 };
20134 for (i, column) in table.columns.iter().enumerate() {
20135 let cell = row.get_ref(i).map_err(|error| {
20136 invalid(Self::sqlite_failure(&self.storage_failure, error))
20137 })?;
20138 let param = image_cell_param(column, cell).map_err(invalid)?;
20139 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
20140 invalid(Self::sqlite_failure(&self.storage_failure, error))
20141 })?;
20142 if let Some(mirror) = &mut mirror {
20143 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
20144 invalid(Self::sqlite_failure(&self.storage_failure, error))
20145 })?;
20146 }
20147 }
20148 let version_index = table.columns.len();
20149 let version: i64 = row.get(version_index).map_err(|error| {
20150 invalid(Self::sqlite_failure(&self.storage_failure, error))
20151 })?;
20152 if version < 1 {
20153 return Err(invalid("image row version must be positive".into()));
20154 }
20155 ins.raw_bind_parameter(version_index + 1, version)
20156 .map_err(|error| {
20157 invalid(Self::sqlite_failure(&self.storage_failure, error))
20158 })?;
20159 ins.raw_execute().map_err(|error| {
20160 invalid(Self::sqlite_failure(&self.storage_failure, error))
20161 })?;
20162 if let Some(mirror) = &mut mirror {
20163 mirror
20164 .raw_bind_parameter(version_index + 1, version)
20165 .map_err(|error| {
20166 invalid(Self::sqlite_failure(&self.storage_failure, error))
20167 })?;
20168 mirror.raw_execute().map_err(|error| {
20169 invalid(Self::sqlite_failure(&self.storage_failure, error))
20170 })?;
20171 }
20172 if reconcile_pending {
20173 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
20174 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
20175 )?);
20176 }
20177 processed += 1;
20178 }
20179 }
20180 if processed > 0 {
20181 batch.table(&table.name);
20182 self.overlay_dirty.table(&table.name);
20183 }
20184 if reconcile_pending {
20185 self.reconcile_imported_rows(table, &mut changed_keys)
20186 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20187 }
20188 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
20189 self.apply_outbox_ops(
20190 self.outbox
20191 .iter()
20192 .flat_map(|commit| &commit.ops)
20193 .filter(|op| op.table == table.name),
20194 )
20195 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20196 self.overlay_dirty.set(false);
20197 } else {
20198 self.rebuild_overlay_if_dirty()
20199 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20200 }
20201 self.finish_observation("syncular_image_chunk", batch)
20202 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20203 Ok(processed)
20204 })();
20205 let processed = match result {
20206 Ok(processed) => processed,
20207 Err(error) => {
20208 self.rollback_observation("syncular_image_chunk");
20209 self.overlay_dirty.restore(was_dirty.clone());
20210 return Err(error);
20211 }
20212 };
20213 applied += processed;
20214 self.progress
20215 .update(|p| p.rows_processed = u64::from(applied));
20216 std::thread::yield_now();
20217 if i64::from(applied) == actual_count {
20218 break;
20219 }
20220 }
20221 Ok(applied)
20222 }
20223
20224 fn reconcile_imported_rows(
20228 &self,
20229 table: &TableSchema,
20230 keys: &mut Vec<SqlValue>,
20231 ) -> Result<(), String> {
20232 let visible = visible_table(&table.name);
20233 let base = base_table(&table.name);
20234 let pk = quote_ident(&table.primary_key);
20235 let predicate = row_id_predicate(table);
20236 let mut lookup = self.conn.prepare_cached(&format!(
20237 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
20238 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20239 let mut deleted = HashSet::new();
20240 for op in self
20241 .outbox
20242 .iter()
20243 .flat_map(|commit| &commit.ops)
20244 .filter(|op| op.table == table.name)
20245 {
20246 if op.upsert {
20247 if let Some(values) = &op.values {
20248 let key = json_to_column_value(
20249 &table.columns[table.pk_index],
20250 values.get(&table.primary_key),
20251 )?;
20252 keys.push(owned_sql_value(RowParam::Cell(&key))?);
20253 }
20254 } else if deleted.insert(&op.row_id) {
20255 keys.extend(
20256 lookup
20257 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
20258 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
20259 .collect::<Result<Vec<_>, _>>()
20260 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
20261 );
20262 }
20263 }
20264 for chunk in keys.chunks(400) {
20267 let holes = vec!["?"; chunk.len()].join(",");
20268 self.conn
20269 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
20270 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
20271 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20272 }
20273 let updates = table
20274 .columns
20275 .iter()
20276 .map(|column| column.name.as_str())
20277 .chain(std::iter::once("_syncular_version"))
20278 .filter(|name| *name != table.primary_key)
20279 .map(|name| {
20280 let name = quote_ident(name);
20281 format!("{name}=excluded.{name}")
20282 })
20283 .collect::<Vec<_>>()
20284 .join(",");
20285 for chunk in keys.chunks(400) {
20286 let holes = vec!["?"; chunk.len()].join(",");
20287 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
20288 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
20289 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20290 }
20291 Ok(())
20292 }
20293
20294 fn compile_local_data_purge(
20297 &self,
20298 input: &LocalDataPurgeInput,
20299 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
20300 let invalid = |message: String| format!("sync.invalid_request: {message}");
20301 if input.purge_id.is_empty()
20302 || input.purge_id.len() > 128
20303 || !is_local_operation_code_like(&input.purge_id)
20304 {
20305 return Err(invalid(
20306 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
20307 ));
20308 }
20309 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
20310 return Err(invalid(format!(
20311 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
20312 )));
20313 }
20314
20315 let mut deduplicated: BTreeMap<
20316 String,
20317 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
20318 > = BTreeMap::new();
20319 for target in &input.targets {
20320 let table = self.schema.table(&target.table).ok_or_else(|| {
20321 invalid(format!(
20322 "local purge names unknown table {:?}",
20323 target.table
20324 ))
20325 })?;
20326 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
20327 return Err(invalid(format!(
20328 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
20329 target.table
20330 )));
20331 }
20332 let mut selectors = Vec::with_capacity(target.selectors.len());
20333 let mut canonical_selectors = BTreeMap::new();
20334 for (column_name, raw_values) in &target.selectors {
20335 let Some((column_index, column)) = table
20336 .columns
20337 .iter()
20338 .enumerate()
20339 .find(|(_, column)| column.name == *column_name)
20340 else {
20341 return Err(invalid(format!(
20342 "local purge target {:?} names unknown column {:?}",
20343 target.table, column_name
20344 )));
20345 };
20346 let encrypted = table
20347 .encrypted_columns
20348 .iter()
20349 .any(|candidate| candidate.index == column_index);
20350 if column.ty != ColumnType::String || encrypted {
20351 return Err(invalid(format!(
20352 "local purge selector {:?}.{:?} must be a plaintext string column",
20353 target.table, column_name
20354 )));
20355 }
20356 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
20357 return Err(invalid(format!(
20358 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
20359 target.table, column_name
20360 )));
20361 }
20362 let mut values = raw_values.clone();
20363 values.sort();
20364 values.dedup();
20365 if values.iter().any(|value| {
20366 value.is_empty()
20367 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
20368 || !is_local_operation_code_like(value)
20369 }) {
20370 return Err(invalid(format!(
20371 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
20372 )));
20373 }
20374 selectors.push((column_name.clone(), values.clone()));
20375 canonical_selectors.insert(column_name.clone(), values);
20376 }
20377 let canonical = LocalDataPurgeTarget {
20378 table: target.table.clone(),
20379 selectors: canonical_selectors,
20380 };
20381 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
20382 deduplicated.insert(
20383 key,
20384 (
20385 CompiledLocalDataPurgeTarget {
20386 table: target.table.clone(),
20387 selectors,
20388 },
20389 canonical,
20390 ),
20391 );
20392 }
20393 let targets = deduplicated
20394 .values()
20395 .map(|(compiled, _)| compiled.clone())
20396 .collect::<Vec<_>>();
20397 let canonical = deduplicated
20398 .values()
20399 .map(|(_, target)| target.clone())
20400 .collect::<Vec<_>>();
20401 let canonical_plan =
20402 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
20403 Ok((targets, canonical_plan))
20404 }
20405
20406 fn local_purge_base_row_ids(
20407 &self,
20408 targets: &[CompiledLocalDataPurgeTarget],
20409 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
20410 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
20411 for target in targets {
20412 let table = self
20413 .schema
20414 .table(&target.table)
20415 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
20416 let mut clauses = Vec::with_capacity(target.selectors.len());
20417 let mut params = Vec::new();
20418 for (column, values) in &target.selectors {
20419 clauses.push(format!(
20420 "{} IN ({})",
20421 quote_ident(column),
20422 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
20423 ));
20424 params.extend(values.iter().cloned().map(SqlValue::Text));
20425 }
20426 let sql = format!(
20427 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
20428 quote_ident(&table.primary_key),
20429 base_table(&target.table),
20430 clauses.join(" AND ")
20431 );
20432 let mut statement = self
20433 .conn
20434 .prepare(&sql)
20435 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20436 let rows = statement
20437 .query_map(rusqlite::params_from_iter(params), |row| {
20438 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
20439 render_row_id_json(Some(&value))
20440 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
20441 })
20442 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20443 let ids = by_table.entry(target.table.clone()).or_default();
20444 for row in rows {
20445 ids.insert(
20446 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
20447 );
20448 }
20449 }
20450 Ok(by_table)
20451 }
20452
20453 fn local_purge_values_match(
20454 target: &CompiledLocalDataPurgeTarget,
20455 values: &Map<String, Value>,
20456 ) -> bool {
20457 target.selectors.iter().all(|(column, allowed)| {
20458 values
20459 .get(column)
20460 .and_then(Value::as_str)
20461 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
20462 })
20463 }
20464
20465 pub fn purge_local_data(
20469 &mut self,
20470 input: &LocalDataPurgeInput,
20471 ) -> Result<LocalDataPurgeResult, String> {
20472 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
20473 let meta_key = format!("localPurge:{}", input.purge_id);
20474 let applied_plan = self
20475 .conn
20476 .query_row(
20477 "SELECT value FROM _syncular_meta WHERE key = ?1",
20478 rusqlite::params![meta_key],
20479 |row| row.get::<_, String>(0),
20480 )
20481 .optional()
20482 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20483 if let Some(applied_plan) = applied_plan {
20484 if applied_plan != canonical_plan {
20485 return Err(format!(
20486 "sync.invalid_request: local purge id {:?} was already used with a different plan",
20487 input.purge_id
20488 ));
20489 }
20490 return Ok(LocalDataPurgeResult {
20491 already_applied: true,
20492 purged_rows: 0,
20493 dropped_commits: 0,
20494 });
20495 }
20496
20497 let _ = self.drop_previous_version();
20501
20502 let prior_outbox = self.outbox.clone();
20503 let prior_failed = self.failed_commits.clone();
20504 let prior_rejections = self.rejections.clone();
20505 let prior_conflicts = self.conflicts.clone();
20506 let prior_overlay_dirty = self.overlay_dirty.snapshot();
20507 self.begin_observation("syncular_local_purge")?;
20508 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
20509 let row_ids = self.local_purge_base_row_ids(&targets)?;
20510 let retained_dropped = self.drop_failed_commits(|table, values| {
20511 targets.iter().any(|target| {
20512 target.table == table && Self::local_purge_values_match(target, values)
20513 })
20514 })?;
20515 let doomed = self
20516 .outbox
20517 .iter()
20518 .filter(|commit| {
20519 commit.ops.iter().any(|operation| {
20520 let matching_targets = targets
20521 .iter()
20522 .filter(|target| target.table == operation.table)
20523 .collect::<Vec<_>>();
20524 if matching_targets.is_empty() {
20525 return false;
20526 }
20527 if row_ids
20528 .get(&operation.table)
20529 .is_some_and(|ids| ids.contains(&operation.row_id))
20530 {
20531 return true;
20532 }
20533 operation.values.as_ref().is_some_and(|values| {
20534 matching_targets
20535 .iter()
20536 .any(|target| Self::local_purge_values_match(target, values))
20537 })
20538 })
20539 })
20540 .cloned()
20541 .collect::<Vec<_>>();
20542 let doomed_ids = doomed
20543 .iter()
20544 .map(|commit| commit.client_commit_id.clone())
20545 .collect::<BTreeSet<_>>();
20546 let mut batch = ChangeAccumulator::default();
20547 for commit in &retained_dropped {
20548 batch.conflicts = true;
20549 batch.rejections = true;
20550 batch.outcomes = true;
20551 for operation in &commit.ops {
20552 batch.table(&operation.table);
20553 }
20554 }
20555 let mut rejections = Vec::new();
20556 for commit in &doomed {
20557 for operation in &commit.ops {
20558 batch.table(&operation.table);
20559 }
20560 let results = commit
20561 .ops
20562 .iter()
20563 .enumerate()
20564 .map(|(op_index, operation)| {
20565 let rejection = RejectionRecord {
20566 client_commit_id: commit.client_commit_id.clone(),
20567 op_index: op_index as i32,
20568 code: "client.local_data_purged".to_owned(),
20569 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
20570 retryable: false,
20571 details: None,
20572 operation: Some(CommitOperation::from(operation)),
20573 };
20574 rejections.push(rejection.clone());
20575 CommitOperationOutcome::Error { rejection }
20576 })
20577 .collect::<Vec<_>>();
20578 self.persist_commit_outcome(
20579 &commit.client_commit_id,
20580 CommitOutcomeStatus::Rejected,
20581 &results,
20582 Some(&commit.ops),
20583 )?;
20584 self.delete_outbox_persisted(&commit.client_commit_id)?;
20585 }
20586 if !doomed.is_empty() {
20587 self.outbox
20588 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
20589 self.rejections.extend(rejections);
20590 self.prune_commit_outcomes()?;
20591 batch.status = true;
20592 batch.rejections = true;
20593 batch.outcomes = true;
20594 }
20595
20596 let mut purged_rows = 0usize;
20597 for (table, ids) in &row_ids {
20598 if ids.is_empty() {
20599 continue;
20600 }
20601 batch.table(table);
20602 for row_id in ids {
20603 self.delete_base_row(table, row_id)?;
20604 purged_rows += 1;
20605 }
20606 }
20607 self.rebuild_overlay_if_dirty()?;
20608 self.delete_unreferenced_cached_blobs()?;
20609 self.conn
20610 .execute(
20611 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
20612 rusqlite::params![meta_key, canonical_plan],
20613 )
20614 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20615 Ok((
20616 batch,
20617 LocalDataPurgeResult {
20618 already_applied: false,
20619 purged_rows,
20620 dropped_commits: doomed.len(),
20621 },
20622 ))
20623 })();
20624 let (batch, result) = match applied {
20625 Ok(value) => value,
20626 Err(error) => {
20627 self.rollback_observation("syncular_local_purge");
20628 self.outbox = prior_outbox;
20629 self.failed_commits = prior_failed;
20630 self.rejections = prior_rejections;
20631 self.conflicts = prior_conflicts;
20632 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20633 return Err(error);
20634 }
20635 };
20636 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
20637 self.rollback_observation("syncular_local_purge");
20638 self.outbox = prior_outbox;
20639 self.failed_commits = prior_failed;
20640 self.rejections = prior_rejections;
20641 self.conflicts = prior_conflicts;
20642 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20643 return Err(error);
20644 }
20645 self.cancel_sync_round();
20646 Ok(result)
20647 }
20648
20649 pub fn rebootstrap_local_data(
20655 &mut self,
20656 input: &LocalDataRebootstrapInput,
20657 ) -> Result<LocalDataRebootstrapResult, String> {
20658 if input.rebootstrap_id.is_empty()
20659 || input.rebootstrap_id.len() > 128
20660 || !is_local_operation_code_like(&input.rebootstrap_id)
20661 {
20662 return Err(
20663 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
20664 .to_owned(),
20665 );
20666 }
20667 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
20668 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
20669 let (retained_commits, reset_subscriptions) =
20670 decode_local_rebootstrap_receipt(&persisted)?;
20671 return Ok(LocalDataRebootstrapResult {
20672 already_applied: true,
20673 retained_commits,
20674 reset_subscriptions,
20675 });
20676 }
20677 if self.stopped || self.schema_floor.is_some() {
20678 return Err(
20679 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
20680 .to_owned(),
20681 );
20682 }
20683
20684 let retained_commits = self.outbox.len();
20685 let reset_subscriptions = self.subs.len();
20686 let prior_subs = self.subs.clone();
20687 let prior_upgrading = self.upgrading;
20688 let prior_stopped = self.stopped;
20689 let prior_schema_floor = self.schema_floor.clone();
20690 let prior_overlay_dirty = self.overlay_dirty.snapshot();
20691 let prior_sync_needed = self.sync_needed;
20692 let prior_sync_intents = self.sync_intent_queue.clone();
20693 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
20694
20695 let prior_active_round = self.active_round;
20696 self.begin_observation("syncular_local_rebootstrap")?;
20697 let mut batch = ChangeAccumulator::default();
20698 let applied = (|| -> Result<(), String> {
20699 self.run_schema_reset_observed(&mut batch, false)?;
20700 self.conn
20701 .execute(
20702 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
20703 rusqlite::params![meta_key, receipt],
20704 )
20705 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20706 self.sync_needed = true;
20707 self.sync_intent_queue.push_back(SyncIntent::Interactive);
20708 batch.status = true;
20709 Ok(())
20710 })();
20711 if let Err(error) = applied {
20712 self.rollback_observation("syncular_local_rebootstrap");
20713 self.active_round = prior_active_round;
20714 self.subs = prior_subs;
20715 self.upgrading = prior_upgrading;
20716 self.stopped = prior_stopped;
20717 self.schema_floor = prior_schema_floor;
20718 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20719 self.sync_needed = prior_sync_needed;
20720 self.sync_intent_queue = prior_sync_intents;
20721 return Err(error);
20722 }
20723 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
20724 self.rollback_observation("syncular_local_rebootstrap");
20725 self.active_round = prior_active_round;
20726 self.subs = prior_subs;
20727 self.upgrading = prior_upgrading;
20728 self.stopped = prior_stopped;
20729 self.schema_floor = prior_schema_floor;
20730 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20731 self.sync_needed = prior_sync_needed;
20732 self.sync_intent_queue = prior_sync_intents;
20733 return Err(error);
20734 }
20735
20736 Ok(LocalDataRebootstrapResult {
20737 already_applied: false,
20738 retained_commits,
20739 reset_subscriptions,
20740 })
20741 }
20742
20743 fn purge_scope_rows(
20748 &mut self,
20749 table_name: &str,
20750 effective: &[(String, Vec<String>)],
20751 ) -> Result<(), ()> {
20752 if effective.is_empty() {
20753 return Ok(());
20754 }
20755 let table = self.schema.table(table_name).ok_or(())?.clone();
20756 let mut clauses = Vec::new();
20757 let mut params: Vec<SqlValue> = Vec::new();
20758 for (variable, values) in effective {
20759 let column = table.scope_column(variable).ok_or(())?;
20760 let placeholders: Vec<String> = values
20761 .iter()
20762 .map(|v| {
20763 params.push(SqlValue::Text(v.clone()));
20764 "?".to_owned()
20765 })
20766 .collect();
20767 clauses.push(format!(
20768 "{} IN ({})",
20769 quote_ident(column),
20770 placeholders.join(", ")
20771 ));
20772 }
20773 let sql = format!(
20774 "DELETE FROM {} WHERE {}",
20775 base_table(table_name),
20776 clauses.join(" AND ")
20777 );
20778 self.overlay_dirty.table(table_name);
20779 self.conn
20780 .execute(&sql, rusqlite::params_from_iter(¶ms))
20781 .map_err(|_| ())?;
20782 self.conn
20783 .execute(
20784 &format!(
20785 "DELETE FROM {} WHERE {}",
20786 visible_table(table_name),
20787 clauses.join(" AND ")
20788 ),
20789 rusqlite::params_from_iter(¶ms),
20790 )
20791 .map_err(|_| ())?;
20792 Ok(())
20793 }
20794
20795 fn drop_doomed_outbox(
20798 &mut self,
20799 table_name: &str,
20800 effective: &[(String, Vec<String>)],
20801 ) -> Result<bool, String> {
20802 if effective.is_empty() {
20803 return Ok(false);
20804 }
20805 let Some(table) = self.schema.table(table_name).cloned() else {
20806 return Ok(false);
20807 };
20808 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
20809 for (variable, values) in effective {
20810 match table.scope_column(variable) {
20811 Some(column) => mappings.push((column, values)),
20812 None => return Ok(false), }
20814 }
20815 let doomed: Vec<OutboxCommit> = self
20816 .outbox
20817 .iter()
20818 .filter(|commit| {
20819 commit.ops.iter().any(|op| {
20820 op.upsert
20821 && op.table == table_name
20822 && op.values.as_ref().is_some_and(|values| {
20823 mappings
20824 .iter()
20825 .all(|(column, allowed)| match values.get(*column) {
20826 Some(Value::String(s)) => allowed.contains(s),
20827 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
20828 Some(Value::Bool(b)) => allowed.contains(&b.to_string()),
20829 _ => false,
20830 })
20831 })
20832 })
20833 })
20834 .cloned()
20835 .collect();
20836 if doomed.is_empty() {
20837 return Ok(false);
20838 }
20839 let mut rejections = Vec::new();
20840 for commit in &doomed {
20841 let results = commit
20842 .ops
20843 .iter()
20844 .enumerate()
20845 .map(|(op_index, operation)| {
20846 let rejection = RejectionRecord {
20847 client_commit_id: commit.client_commit_id.clone(),
20848 op_index: op_index as i32,
20849 code: "sync.scope_revoked".to_owned(),
20850 message: "the commit was dropped because its effective scope was revoked"
20851 .to_owned(),
20852 retryable: false,
20853 details: None,
20854 operation: Some(CommitOperation::from(operation)),
20855 };
20856 rejections.push(rejection.clone());
20857 CommitOperationOutcome::Error { rejection }
20858 })
20859 .collect::<Vec<_>>();
20860 self.persist_commit_outcome(
20861 &commit.client_commit_id,
20862 CommitOutcomeStatus::Rejected,
20863 &results,
20864 Some(&commit.ops),
20865 )?;
20866 self.delete_outbox_persisted(&commit.client_commit_id)?;
20867 }
20868 self.prune_commit_outcomes()?;
20869 let doomed_ids = doomed
20870 .iter()
20871 .map(|commit| commit.client_commit_id.as_str())
20872 .collect::<BTreeSet<_>>();
20873 self.outbox
20874 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
20875 self.rejections.extend(rejections);
20876 Ok(true)
20877 }
20878
20879 pub fn upload_blob(
20886 &mut self,
20887 bytes: &[u8],
20888 media_type: Option<String>,
20889 name: Option<String>,
20890 ) -> Result<Value, String> {
20891 let blob_id = blob_id_for(bytes);
20892 let now = self.clock_now_ms();
20893 let tx = self
20894 .conn
20895 .transaction()
20896 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20897 tx.execute(
20898 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
20899 ON CONFLICT(blob_id) DO NOTHING",
20900 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
20901 )
20902 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20903 tx.execute(
20904 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
20905 rusqlite::params![blob_id, media_type, now],
20906 )
20907 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20908 tx.commit()
20909 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20910 self.enforce_blob_cache_cap();
20911 let mut obj = Map::new();
20912 obj.insert("blobId".to_owned(), Value::from(blob_id));
20913 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
20914 if let Some(mt) = media_type {
20915 obj.insert("mediaType".to_owned(), Value::from(mt));
20916 }
20917 if let Some(n) = name {
20918 obj.insert("name".to_owned(), Value::from(n));
20919 }
20920 Ok(Value::Object(obj))
20921 }
20922
20923 pub fn fetch_blob_bytes(
20927 &mut self,
20928 transport: &mut dyn Transport,
20929 blob_id_or_ref: &str,
20930 ) -> Result<FetchedBlob, (String, String)> {
20931 let simple = |m: String| ("client.failed".to_owned(), m);
20932 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
20933 blob_id_or_ref.to_owned()
20934 } else {
20935 let value: Value = serde_json::from_str(blob_id_or_ref)
20936 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
20937 value
20938 .get("blobId")
20939 .and_then(Value::as_str)
20940 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
20941 .to_owned()
20942 };
20943 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
20944 return Ok(cached);
20945 }
20946 if !self.transport_enabled {
20947 return Err(("sync.offline".into(), "transport gate is closed".into()));
20948 }
20949 #[cfg(feature = "bench-internals")]
20955 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
20956 let bytes = match transport
20957 .blob_download(&blob_id)
20958 .map_err(|e| (e.code, e.message))?
20959 {
20960 BlobDownload::Bytes(bytes) => bytes,
20961 BlobDownload::Url {
20962 url,
20963 url_expires_at_ms,
20964 } => {
20965 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
20967 return Err((
20968 "sync.segment_expired".to_owned(),
20969 format!(
20970 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
20971 ),
20972 ));
20973 }
20974 transport
20975 .fetch_blob_url(&url)
20976 .map_err(|e| (e.code, e.message))?
20977 }
20978 };
20979 #[cfg(feature = "bench-internals")]
20980 drop(download_phase);
20981 #[cfg(feature = "bench-internals")]
20982 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
20983 if blob_id_for(&bytes) != blob_id {
20985 return Err(simple(format!(
20986 "blob content address mismatch for {blob_id}"
20987 )));
20988 }
20989 #[cfg(feature = "bench-internals")]
20990 drop(validate_phase);
20991 #[cfg(feature = "bench-internals")]
20992 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
20993 let now = self.clock_now_ms();
20994 self.conn
20995 .execute(
20996 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
20997 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
20998 )
20999 .map_err(|e| simple(e.to_string()))?;
21000 self.conn
21001 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
21002 .map_err(|e| simple(e.to_string()))?;
21003 self.enforce_blob_cache_cap();
21004 #[cfg(feature = "bench-internals")]
21005 drop(insert_phase);
21006 self.get_cached_blob_bytes(&blob_id)
21007 .map_err(simple)?
21008 .ok_or_else(|| simple("blob cache write failed".to_owned()))
21009 }
21010
21011 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
21012 #[cfg(feature = "bench-internals")]
21013 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
21014 let mut stmt = self
21015 .conn
21016 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
21017 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21018 let mut rows = stmt
21019 .query(rusqlite::params![blob_id])
21020 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21021 if let Some(row) = rows
21022 .next()
21023 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
21024 {
21025 let bytes: Vec<u8> = row
21026 .get(0)
21027 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21028 let byte_length: i64 = row
21029 .get(1)
21030 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21031 let media_type: Option<String> = row
21032 .get(2)
21033 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21034 return Ok(Some(FetchedBlob {
21035 blob_id: blob_id.to_owned(),
21036 byte_length,
21037 bytes,
21038 media_type,
21039 }));
21040 }
21041 Ok(None)
21042 }
21043
21044 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
21046 let pending: Vec<(String, Option<String>)> = {
21047 let mut stmt = self
21048 .conn
21049 .prepare(
21050 "SELECT blob_id, media_type FROM (
21051 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
21052 UNION ALL
21053 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
21054 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
21055 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
21056 ) ORDER BY created_at_ms, blob_id",
21057 )
21058 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
21059 let rows = stmt
21060 .query_map([], |row| {
21061 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
21062 })
21063 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
21064 rows.collect::<Result<_, _>>()
21065 .map_err(|error| match error {
21066 rusqlite::Error::InvalidColumnType(..)
21067 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
21068 "sync.local_corrupt",
21069 "Pending blob upload metadata is invalid",
21070 ),
21071 _ => TransportError::new("client.failed", error.to_string()),
21072 })?
21073 };
21074 let mut uploads = Vec::new();
21075 for (blob_id, media_type) in pending {
21076 let bytes = (|| {
21077 let stored: Option<(SqlValue, SqlValue)> = self
21078 .conn
21079 .query_row(
21080 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
21081 rusqlite::params![blob_id],
21082 |row| Ok((row.get(0)?, row.get(1)?)),
21083 )
21084 .optional()
21085 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
21086 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
21087 return Err(TransportError::new(
21088 "sync.local_corrupt",
21089 "Pending blob upload body is missing or corrupt",
21090 ));
21091 };
21092 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
21093 return Err(TransportError::new(
21094 "sync.local_corrupt",
21095 "Pending blob upload body is missing or corrupt",
21096 ));
21097 }
21098 Ok(bytes)
21099 })();
21100 let failed = bytes.is_err();
21101 uploads.push(crate::round::PendingUpload {
21102 id: blob_id,
21103 bytes,
21104 media_type,
21105 });
21106 if failed {
21107 break;
21108 }
21109 }
21110 Ok(uploads)
21111 }
21112
21113 fn visible_blob_ids_sql(&self) -> String {
21114 let mut selects = Vec::new();
21115 for table in &self.schema.tables {
21116 for column in table
21117 .columns
21118 .iter()
21119 .filter(|column| column.ty == ColumnType::BlobRef)
21120 {
21121 let identifier = quote_ident(&column.name);
21122 selects.push(format!(
21123 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
21124 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
21125 AND json_type({identifier}, '$.blobId') = 'text'",
21126 quote_ident(&table.name),
21127 ));
21128 }
21129 }
21130 if selects.is_empty() {
21131 "SELECT NULL AS blob_id WHERE 0".to_owned()
21132 } else {
21133 selects.join(" UNION ")
21134 }
21135 }
21136
21137 fn enforce_blob_cache_cap(&self) {
21140 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
21141 return;
21142 };
21143 let mut total: i64 = self
21144 .conn
21145 .query_row(
21146 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
21147 [],
21148 |row| row.get(0),
21149 )
21150 .unwrap_or(0);
21151 if total <= max_bytes {
21152 return;
21153 }
21154 let query = format!(
21155 "SELECT blob_id, byte_length FROM _syncular_blobs
21156 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
21157 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
21158 AND blob_id NOT IN ({})
21159 ORDER BY created_at_ms ASC, blob_id ASC",
21160 self.visible_blob_ids_sql(),
21161 );
21162 let candidates: Vec<(String, i64)> = {
21163 let Ok(mut stmt) = self.conn.prepare(&query) else {
21164 return;
21165 };
21166 let Ok(rows) = stmt.query_map([], |row| {
21167 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
21168 }) else {
21169 return;
21170 };
21171 rows.filter_map(Result::ok).collect()
21172 };
21173 for (blob_id, byte_length) in candidates {
21174 if total <= max_bytes {
21175 break;
21176 }
21177 let _ = self.conn.execute(
21178 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
21179 rusqlite::params![blob_id],
21180 );
21181 total -= byte_length;
21182 }
21183 }
21184
21185 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
21186 #[cfg(feature = "bench-internals")]
21187 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
21188 if !self.schema_has_blobs() {
21189 return Ok(());
21190 }
21191 self.rebuild_overlay_if_dirty()?;
21192 self.conn
21193 .execute(
21194 &format!(
21195 "DELETE FROM _syncular_blobs
21196 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
21197 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
21198 AND blob_id NOT IN ({})",
21199 self.visible_blob_ids_sql(),
21200 ),
21201 [],
21202 )
21203 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21204 Ok(())
21205 }
21206
21207 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
21212 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
21213 return sql.clone();
21214 }
21215 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
21216 columns.push(quote_ident("_syncular_version"));
21217 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
21218 let primary_key = quote_ident(&table.primary_key);
21219 let updates = columns
21220 .iter()
21221 .filter(|column| **column != primary_key)
21222 .map(|column| format!("{column}=excluded.{column}"))
21223 .collect::<Vec<_>>()
21224 .join(", ");
21225 let sql = format!(
21226 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
21227 columns.join(", "),
21228 placeholders.join(", ")
21229 );
21230 self.insert_sql
21231 .borrow_mut()
21232 .insert(full_table.to_owned(), sql.clone());
21233 sql
21234 }
21235
21236 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
21237 self.overlay_dirty.table(table_name);
21238 self.write_row(&base_table(table_name), table_name, row, version)
21239 }
21240
21241 fn write_row(
21242 &self,
21243 full_table: &str,
21244 table_name: &str,
21245 row: &Row,
21246 version: i64,
21247 ) -> Result<(), String> {
21248 #[cfg(feature = "bench-internals")]
21249 let _phase = self.benchmark_phases.start(Phase::RowWrite);
21250 let table = self
21251 .schema
21252 .table(table_name)
21253 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
21254 let sql = self.insert_row_sql(full_table, table);
21255 let mut stmt = self
21256 .conn
21257 .prepare_cached(&sql)
21258 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21259 let params = row
21260 .iter()
21261 .map(RowParam::Cell)
21262 .chain(std::iter::once(RowParam::Version(version)));
21263 stmt.execute(rusqlite::params_from_iter(params))
21264 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21265 Ok(())
21266 }
21267
21268 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
21269 let table = self
21270 .schema
21271 .table(table_name)
21272 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
21273 self.overlay_dirty.table(table_name);
21274 let sql = format!(
21275 "DELETE FROM {} WHERE {}",
21276 base_table(table_name),
21277 row_id_predicate(table)
21278 );
21279 let mut stmt = self
21280 .conn
21281 .prepare_cached(&sql)
21282 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21283 stmt.execute(rusqlite::params![row_id])
21284 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21285 Ok(())
21286 }
21287
21288 fn has_acknowledged_rows(&self) -> bool {
21289 self.conn
21290 .query_row(
21291 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
21292 [],
21293 |row| row.get(0),
21294 )
21295 .expect("read acknowledged intent")
21296 }
21297
21298 fn acknowledged_operations(
21299 &self,
21300 rows: Option<&HashSet<(&str, &str)>>,
21301 tables: Option<&BTreeSet<&str>>,
21302 ) -> Result<Vec<OutboxOp>, String> {
21303 let keys = rows.map(|rows| {
21304 serde_json::to_string(
21305 &rows
21306 .iter()
21307 .map(|(table, id)| serde_json::to_string(&(table, id)).expect("row key"))
21308 .collect::<Vec<_>>(),
21309 )
21310 .expect("row keys")
21311 });
21312 let mut statement = self
21313 .conn
21314 .prepare("SELECT op_json,intent_json FROM _syncular_acked_rows WHERE (?1 IS NULL OR json_array(tbl,id) IN(SELECT value FROM json_each(?1))) AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx")
21315 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21316 let rows = statement
21317 .query_map(
21318 rusqlite::params![
21319 keys,
21320 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
21321 ],
21322 |row| {
21323 #[cfg(test)]
21324 self.acknowledged_replay_count
21325 .set(self.acknowledged_replay_count.get() + 1);
21326 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
21327 },
21328 )
21329 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21330 rows.map(|row| {
21331 let (op, intent) =
21332 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21333 let op: CommitOperation =
21334 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
21335 let table = self
21336 .schema
21337 .table(&op.table)
21338 .ok_or_else(|| "sync.unknown_table".to_owned())?;
21339 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
21340 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
21341 } else {
21342 op.values
21343 };
21344 Ok(OutboxOp {
21345 table: op.table,
21346 row_id: op.row_id,
21347 upsert: op.op == "upsert",
21348 base_version: op.base_version,
21349 values,
21350 insert: op.insert,
21351 })
21352 })
21353 .collect()
21354 }
21355
21356 fn drop_acknowledged_rows(
21357 &self,
21358 matches: impl Fn(&str, &Map<String, Value>) -> bool,
21359 ) -> Result<Vec<OutboxCommit>, String> {
21360 let mut statement = self
21361 .conn
21362 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
21363 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21364 let rows = statement
21365 .query_map([], |row| {
21366 Ok((
21367 row.get::<_, String>(0)?,
21368 row.get::<_, String>(1)?,
21369 row.get::<_, String>(2)?,
21370 row.get::<_, String>(3)?,
21371 ))
21372 })
21373 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21374 let mut doomed = HashSet::new();
21375 for row in rows {
21376 let (id, table, intent, row_id) =
21377 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21378 let values =
21379 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
21380 let base_matches = self
21381 .schema
21382 .table(&table)
21383 .map(|schema| self.stored_row(schema, &row_id, true))
21384 .transpose()?
21385 .flatten()
21386 .is_some_and(|(base, _)| {
21387 let schema = self.schema.table(&table).expect("acknowledged table");
21388 let values = schema
21389 .columns
21390 .iter()
21391 .zip(base.iter())
21392 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
21393 .collect();
21394 matches(&table, &values)
21395 });
21396 if matches(&table, &values) || base_matches {
21397 doomed.insert(id);
21398 }
21399 }
21400 let mut dropped = Vec::new();
21401 for id in doomed {
21402 let mut statement = self
21403 .conn
21404 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
21405 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21406 let operations = statement
21407 .query_map([&id], |row| row.get::<_, String>(0))
21408 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21409 let mut ops = Vec::new();
21410 for operation in operations {
21411 let json = operation
21412 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21413 let op: CommitOperation =
21414 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
21415 self.overlay_dirty.table(&op.table);
21416 ops.push(OutboxOp {
21417 table: op.table,
21418 row_id: op.row_id,
21419 upsert: op.op == "upsert",
21420 base_version: op.base_version,
21421 values: op.values,
21422 insert: op.insert,
21423 });
21424 }
21425 dropped.push(OutboxCommit {
21426 client_commit_id: id.clone(),
21427 ops,
21428 });
21429 self.conn
21430 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
21431 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21432 }
21433 Ok(dropped)
21434 }
21435
21436 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
21439 if self.overlay_dirty.get() {
21440 self.rebuild_overlay()?;
21441 }
21442 Ok(())
21443 }
21444
21445 fn rebuild_overlay(&mut self) -> Result<(), String> {
21449 #[cfg(feature = "bench-internals")]
21450 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
21451 #[cfg(test)]
21452 self.overlay_rebuild_count
21453 .set(self.overlay_rebuild_count.get() + 1);
21454 let dirty = self.overlay_dirty.snapshot();
21455 let tables: Vec<_> = self
21456 .schema
21457 .tables
21458 .iter()
21459 .filter(|table| {
21460 dirty
21461 .as_ref()
21462 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
21463 })
21464 .cloned()
21465 .collect();
21466 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
21467 self.begin_observation("syncular_overlay")?;
21468 let rebuilt = (|| {
21469 for table in &tables {
21470 for index in &table.fts_indexes {
21471 self.drop_fts_triggers(index)?;
21472 }
21473 let visible = visible_table(&table.name);
21474 let base = base_table(&table.name);
21475 self.conn
21476 .execute_batch(&format!(
21477 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
21478 ))
21479 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21480 }
21481 let acknowledged = self.acknowledged_operations(None, Some(&names))?;
21482 self.apply_outbox_ops(
21483 acknowledged
21484 .iter()
21485 .filter(|op| names.contains(op.table.as_str())),
21486 )?;
21487 for (commit, initial) in &self.failed_commits {
21488 for operation in &commit.ops {
21489 if !names.contains(operation.table.as_str()) || !operation.upsert {
21490 continue;
21491 }
21492 if let Some(table) = self.schema.table(&operation.table) {
21493 if self.visible_row(table, &operation.row_id)?.is_none()
21494 && operation.presented_values(table).is_some_and(|values| {
21495 table
21496 .columns
21497 .iter()
21498 .all(|column| values.contains_key(&column.name))
21499 })
21500 {
21501 let key = serde_json::to_string(&(
21502 operation.table.clone(),
21503 operation.row_id.clone(),
21504 ))
21505 .expect("row key serialization");
21506 if let Some(Value::Object(values)) = initial.get(&key) {
21507 self.apply_outbox_ops(&[OutboxOp {
21508 values: Some(values.clone()),
21509 ..operation.clone()
21510 }])?;
21511 }
21512 }
21513 }
21514 }
21515 self.apply_outbox_ops(
21516 commit
21517 .ops
21518 .iter()
21519 .filter(|op| names.contains(op.table.as_str())),
21520 )?;
21521 }
21522 for commit in &self.outbox {
21523 self.apply_outbox_ops(
21524 commit
21525 .ops
21526 .iter()
21527 .filter(|op| names.contains(op.table.as_str())),
21528 )?;
21529 }
21530 for table in &tables {
21531 for index in &table.fts_indexes {
21532 self.rebuild_fts_projection(table, index)?;
21533 self.create_fts_triggers(table, index)?;
21534 }
21535 }
21536 self.conn
21537 .execute_batch("RELEASE syncular_overlay")
21538 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21539 Ok(())
21540 })();
21541 if rebuilt.is_err() {
21542 self.conn
21543 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
21544 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21545 } else {
21546 self.overlay_dirty.set(false);
21547 }
21548 rebuilt
21549 }
21550
21551 fn apply_outbox_ops<'a>(
21556 &self,
21557 ops: impl IntoIterator<Item = &'a OutboxOp>,
21558 ) -> Result<(), String> {
21559 #[cfg(feature = "bench-internals")]
21560 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
21561 for op in ops {
21562 #[cfg(feature = "bench-internals")]
21563 if let Some(phase) = &mut phase {
21564 phase.unit();
21565 }
21566 if op.upsert {
21567 if let Some((table, values)) = self
21568 .schema
21569 .table(&op.table)
21570 .filter(|table| table.indexes.iter().any(|index| index.unique))
21571 .and_then(|table| Some((table, op.presented_values(table)?)))
21572 {
21573 let mut intended: Map<String, Value> = self
21574 .visible_row(table, &op.row_id)?
21575 .map(|(row, _)| {
21576 table
21577 .columns
21578 .iter()
21579 .zip(&row)
21580 .map(|(column, value)| {
21581 (column.name.clone(), column_value_to_json(value))
21582 })
21583 .collect()
21584 })
21585 .unwrap_or_default();
21586 intended.extend(values.into_owned());
21587 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
21588 continue;
21589 }
21590 }
21591 }
21592 self.apply_outbox_op(op)?;
21593 }
21594 Ok(())
21595 }
21596
21597 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
21598 let Some(table) = self.schema.table(&op.table) else {
21599 return Ok(());
21600 };
21601 let visible = visible_table(&table.name);
21602 if !op.upsert {
21603 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
21604 self.conn
21605 .prepare_cached(&sql)
21606 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
21607 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21608 return Ok(());
21609 }
21610 let Some(values) = op.presented_values(table) else {
21614 return Ok(());
21615 };
21616 let full = table
21622 .columns
21623 .iter()
21624 .all(|column| values.contains_key(&column.name));
21625 let local = self.visible_row(table, &op.row_id)?;
21626 if local.is_none() && !full {
21627 return Ok(());
21628 }
21629 let mut row: Row = Vec::with_capacity(table.columns.len());
21630 for (index, column) in table.columns.iter().enumerate() {
21631 let incoming = match values.get(&column.name) {
21632 Some(raw) => json_to_column_value(
21633 if raw.get("$bytes").is_some()
21634 && table
21635 .encrypted_columns
21636 .iter()
21637 .any(|encrypted| encrypted.index == index)
21638 {
21639 &table.wire_columns[index]
21640 } else {
21641 column
21642 },
21643 Some(raw),
21644 ),
21645 None => Ok(local
21646 .as_ref()
21647 .and_then(|(values, _)| values.get(index).cloned())
21648 .flatten()),
21649 };
21650 row.push(incoming.map_err(|_| {
21651 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
21652 })?);
21653 }
21654 let version = local.as_ref().map_or(-1, |(_, version)| *version);
21655 self.write_row(&visible, &table.name, &row, version)
21656 .map_err(|error| {
21657 if error.starts_with("UNIQUE constraint failed:") {
21658 "sync.constraint_violation: local write violates a unique constraint".to_owned()
21659 } else {
21660 error
21661 }
21662 })
21663 }
21664
21665 fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
21669 self.conn
21670 .query_row(
21671 &format!(
21672 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
21673 visible_table(&table.name),
21674 row_id_predicate(table)
21675 ),
21676 rusqlite::params![row_id],
21677 |row| row.get(0),
21678 )
21679 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
21680 }
21681
21682 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
21685 self.stored_row(table, row_id, false)
21686 }
21687
21688 fn stored_row(
21689 &self,
21690 table: &TableSchema,
21691 row_id: &str,
21692 base: bool,
21693 ) -> Result<Option<(Row, i64)>, String> {
21694 let sql = format!(
21695 "SELECT * FROM {} WHERE {}",
21696 if base {
21697 base_table(&table.name)
21698 } else {
21699 visible_table(&table.name)
21700 },
21701 row_id_predicate(table)
21702 );
21703 let mut stmt = self
21704 .conn
21705 .prepare_cached(&sql)
21706 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21707 let mut rows = stmt
21708 .query(rusqlite::params![row_id])
21709 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21710 let Some(row) = rows
21711 .next()
21712 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
21713 else {
21714 return Ok(None);
21715 };
21716 let mut values = Vec::with_capacity(table.columns.len());
21717 for (index, column) in table.columns.iter().enumerate() {
21718 let raw = row
21719 .get_ref(index)
21720 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21721 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
21722 if table
21723 .encrypted_columns
21724 .iter()
21725 .any(|encrypted| encrypted.index == index)
21726 {
21727 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
21728 continue;
21729 }
21730 }
21731 let json = sql_ref_to_json(column, raw);
21732 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
21733 "sync.local_corrupt: persisted local row value is invalid".to_owned()
21734 })?);
21735 }
21736 let version: i64 = row
21737 .get(table.columns.len())
21738 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21739 Ok(Some((values, version)))
21740 }
21741
21742 #[must_use]
21746 pub fn realtime_policy(&self) -> RealtimePolicy {
21747 self.realtime_policy
21748 }
21749
21750 #[must_use]
21752 pub fn realtime_state(&self) -> RealtimeState {
21753 self.realtime_state
21754 }
21755
21756 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
21759 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
21760 return Err(
21761 "sync.invalid_request: disconnect realtime before changing the realtime policy"
21762 .to_owned(),
21763 );
21764 }
21765 self.realtime_policy = policy;
21766 match policy {
21767 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
21768 RealtimePolicy::Required | RealtimePolicy::Optional
21769 if self.realtime_state == RealtimeState::Disabled =>
21770 {
21771 self.set_realtime_state(RealtimeState::Disconnected, None);
21772 }
21773 RealtimePolicy::Required | RealtimePolicy::Optional => {}
21774 }
21775 Ok(())
21776 }
21777
21778 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
21781 self.realtime_state = state;
21782 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
21783 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
21784 self.realtime_reason_code = None;
21785 self.realtime_retry_delay_ms = None;
21786 }
21787 }
21788
21789 #[must_use]
21790 fn realtime_connected(&self) -> bool {
21791 self.realtime_state == RealtimeState::Connected
21792 }
21793
21794 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
21795 if !self.transport_enabled {
21796 return Err("sync.offline: transport gate is closed".into());
21797 }
21798 if self.realtime_policy == RealtimePolicy::Off {
21799 return Err(
21800 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
21801 );
21802 }
21803 if self.realtime_connected() {
21804 return Ok(());
21805 }
21806 self.realtime_state = RealtimeState::Connecting;
21807 match transport.realtime_connect_for_client(&self.client_id) {
21808 Ok(()) => {
21809 self.set_realtime_state(RealtimeState::Connected, None);
21810 Ok(())
21811 }
21812 Err(error) => {
21813 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
21814 Err(format!("{}: {}", error.code, error.message))
21815 }
21816 }
21817 }
21818
21819 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
21820 self.cancel_sync_round();
21821 if self.realtime_state == RealtimeState::Disconnected
21824 || self.realtime_state == RealtimeState::Disabled
21825 {
21826 return;
21827 }
21828 let _ = transport.realtime_close();
21829 self.set_realtime_state(
21830 if self.realtime_policy == RealtimePolicy::Off {
21831 RealtimeState::Disabled
21832 } else {
21833 RealtimeState::Disconnected
21834 },
21835 None,
21836 );
21837 self.presence.clear(); }
21839
21840 pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
21846 if self.realtime_state != RealtimeState::Connected {
21847 return;
21848 }
21849 self.cancel_sync_round();
21850 let _ = transport.realtime_close();
21851 self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
21852 self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
21853 self.presence.clear();
21854 }
21855
21856 pub fn set_presence(
21862 &mut self,
21863 transport: &mut dyn Transport,
21864 scope_key: &str,
21865 doc: Option<&Value>,
21866 ) -> Result<(), String> {
21867 if !self.transport_enabled {
21868 return Err("sync.offline: transport gate is closed".into());
21869 }
21870 if !self.realtime_connected() {
21871 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
21872 }
21873 let text = encode_presence_publish(scope_key, doc);
21874 transport
21875 .realtime_send(&text)
21876 .map_err(|e| format!("{}: {}", e.code, e.message))
21877 }
21878
21879 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
21881 self.presence
21882 .get(scope_key)
21883 .map(|peers| peers.values().cloned().collect())
21884 .unwrap_or_default()
21885 }
21886
21887 fn apply_presence(
21889 &mut self,
21890 scope_key: String,
21891 kind: Option<PresenceKind>,
21892 actor_id: Option<String>,
21893 client_id: Option<String>,
21894 doc: Option<Value>,
21895 error: Option<String>,
21896 ) {
21897 if error.is_some() {
21900 return;
21901 }
21902 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
21903 return;
21904 };
21905 let peer_key = format!("{actor_id} {client_id}");
21906 match kind {
21907 PresenceKind::Leave => {
21908 if let Some(peers) = self.presence.get_mut(&scope_key) {
21909 peers.remove(&peer_key);
21910 if peers.is_empty() {
21911 self.presence.remove(&scope_key);
21912 }
21913 }
21914 }
21915 _ => {
21916 let doc = match doc {
21917 Some(Value::Object(_)) => doc.unwrap(),
21918 _ => return,
21919 };
21920 self.presence.entry(scope_key).or_default().insert(
21921 peer_key,
21922 PresencePeer {
21923 actor_id,
21924 client_id,
21925 doc,
21926 },
21927 );
21928 }
21929 }
21930 }
21931
21932 pub fn on_realtime_text(&mut self, text: &str) {
21934 if self.stopped || !self.transport_enabled || self.security_preflight {
21935 return;
21936 }
21937 match parse_control(text) {
21938 Ok(ControlMessage::Hello { requires_sync, .. }) => {
21939 if requires_sync {
21940 self.set_sync_needed(true, true);
21942 }
21943 }
21944 Ok(ControlMessage::Presence {
21945 scope_key,
21946 kind,
21947 actor_id,
21948 client_id,
21949 doc,
21950 error,
21951 ..
21952 }) => {
21953 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
21954 }
21955 Ok(ControlMessage::Wake { .. }) => {
21956 self.set_sync_needed(true, true);
21958 }
21959 _ => {}
21960 }
21961 }
21962
21963 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
21967 let mut transport = crate::round::DownloadResults::default();
21968 self.on_realtime_binary(&mut transport, bytes);
21969 transport.controls
21970 }
21971
21972 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
21973 if self.stopped || !self.transport_enabled || self.security_preflight {
21974 return;
21975 }
21976 #[cfg(feature = "bench-internals")]
21977 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
21978 let message = match decode_message(bytes) {
21979 Ok(m) if m.msg_kind == MsgKind::Response => m,
21980 _ => {
21981 self.set_sync_needed(true, true);
21982 return;
21983 }
21984 };
21985 #[cfg(feature = "bench-internals")]
21986 drop(decode_phase);
21987 let coalesced_deletes =
21988 coalesced_delete_keys(&message.frames, |id, status, next_cursor| {
21989 if status != SubStatus::Active {
21990 return false;
21991 }
21992 let Some(sub) = self.subs.iter().find(|sub| sub.id == id) else {
21993 return false;
21994 };
21995 sub.state == SubState::Active
21996 && sub.bootstrap_state.is_none()
21997 && sub.synced_once
21998 && next_cursor.is_some_and(|cursor| cursor > sub.cursor)
21999 });
22000 let mut frames = message.frames.into_iter();
22001 let mut applied_cursor: Option<i64> = None;
22002 let mut any_covered = false;
22003 let mut dropped = self.begin_coalesced_apply().is_err();
22004 while let Some(frame) = frames.next() {
22005 let Frame::SubStart {
22006 id,
22007 status,
22008 effective_scopes,
22009 ..
22010 } = frame
22011 else {
22012 continue;
22013 };
22014 let mut body = Vec::new();
22015 let mut next_cursor: Option<i64> = None;
22016 for inner in frames.by_ref() {
22017 match inner {
22018 Frame::SubEnd {
22019 next_cursor: nc, ..
22020 } => {
22021 next_cursor = Some(nc);
22022 break;
22023 }
22024 Frame::Unknown { .. } => {}
22025 other => body.push(other),
22026 }
22027 }
22028 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
22029 dropped = true;
22030 continue;
22031 };
22032 let sub = &self.subs[sub_index];
22033 if status != SubStatus::Active
22036 || sub.state != SubState::Active
22037 || sub.bootstrap_state.is_some()
22038 || !sub.synced_once
22039 {
22040 dropped = true;
22041 continue;
22042 }
22043 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
22044 any_covered = true;
22046 continue;
22047 }
22048 let previous_effective = self.subs[sub_index].effective.clone();
22049 let previous_cursor = self.subs[sub_index].cursor;
22050 self.subs[sub_index].effective = Some(effective_scopes);
22051 let mut failed = false;
22052 for inner in body {
22053 match inner {
22054 Frame::Commit {
22055 tables,
22056 changes,
22057 commit_seq,
22058 ..
22059 } => {
22060 if self
22061 .apply_commit_frame_coalesced(
22062 &tables,
22063 changes,
22064 commit_seq,
22065 &coalesced_deletes,
22066 )
22067 .is_err()
22068 {
22069 failed = true;
22070 break;
22071 }
22072 }
22073 _ => {
22074 failed = true;
22075 break;
22076 }
22077 }
22078 }
22079 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
22080 self.apply_sub_end(sub_index, cursor, None)
22081 .ok()
22082 .map(|()| cursor)
22083 });
22084 let Some(next_cursor) = committed else {
22085 self.subs[sub_index].effective = previous_effective;
22086 self.subs[sub_index].cursor = previous_cursor;
22087 dropped = true;
22088 continue;
22089 };
22090 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
22091 }
22092 if self.finish_coalesced_apply().is_err() {
22095 applied_cursor = None;
22096 dropped = true;
22097 }
22098 if let Some(cursor) = applied_cursor {
22099 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
22101 let _ = transport.realtime_send(&ack);
22102 } else if !any_covered || dropped {
22103 self.set_sync_needed(true, true);
22105 }
22106 }
22107
22108 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
22112 if !self.realtime_connected() {
22113 return;
22114 }
22115 let floor = self
22116 .subs
22117 .iter()
22118 .filter(|s| {
22119 s.state == SubState::Active
22120 && s.bootstrap_state.is_none()
22121 && s.synced_once
22122 && s.cursor >= 0
22123 })
22124 .map(|s| s.cursor)
22125 .min();
22126 if let Some(cursor) = floor {
22127 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
22128 let _ = transport.realtime_send(&ack);
22129 }
22130 }
22131}
22132
22133#[cfg(test)]
22134mod previous_version_wiring_tests {
22135 use super::*;
22136 use serde_json::{json, Value};
22137
22138 fn previous_version_schema(version: i32) -> Value {
22139 json!({
22140 "version": version,
22141 "tables": [{
22142 "name": "tasks",
22143 "primaryKey": "id",
22144 "columns": [
22145 { "name": "id", "type": "string", "nullable": false },
22146 { "name": "note", "type": "string", "nullable": true }
22147 ],
22148 "scopes": []
22149 }]
22150 })
22151 }
22152
22153 fn previous_version_limits() -> crate::api::ClientLimits {
22154 crate::api::ClientLimits {
22155 previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
22156 enabled: true,
22157 ..Default::default()
22158 }),
22159 ..Default::default()
22160 }
22161 }
22162
22163 fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
22165 previous_version_limits()
22166 .previous_version_context
22167 .expect("configured")
22168 }
22169
22170 fn recapture_previous_version(client: &SyncClient) {
22173 let replica_path = client
22174 .previous_version_replica_path()
22175 .expect("file-backed replica");
22176 let outcome = crate::previous_version::capture_previous_version_from_replica(
22177 &client.conn,
22178 &replica_path,
22179 2,
22180 2,
22181 &previous_version_config(),
22182 1,
22183 )
22184 .expect("capture");
22185 assert!(matches!(
22186 outcome,
22187 crate::previous_version::CaptureOutcome::Captured(_)
22188 ));
22189 }
22190
22191 fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
22192 Subscription {
22193 id: "wiring-sub".to_owned(),
22194 table: "tasks".to_owned(),
22195 requested: Vec::new(),
22196 params: None,
22197 cursor,
22198 bootstrap_state: bootstrap_state.map(str::to_owned),
22199 state,
22200 reason_code: None,
22201 effective: None,
22202 synced_once: false,
22203 }
22204 }
22205
22206 #[test]
22207 fn schema_downgrade_refuses_all_opens_and_recreate_without_mutation() {
22208 let path =
22209 std::env::temp_dir().join(format!("syncular-downgrade-{}.db", uuid::Uuid::new_v4()));
22210 let path_str = path.to_str().expect("temp path");
22211 let schema = previous_version_schema(3);
22212 let mut older = previous_version_schema(2);
22213 older["tables"][0]["columns"] =
22214 json!([{ "name": "id", "type": "string", "nullable": false }]);
22215 let mut client = SyncClient::open_path(
22216 "downgrade".into(),
22217 &previous_version_schema(1),
22218 previous_version_limits(),
22219 path_str,
22220 )
22221 .expect("fresh");
22222 client
22223 .mutate(vec![Mutation::Upsert {
22224 table: "tasks".into(),
22225 values: Map::from_iter([
22226 ("id".into(), json!("accepted")),
22227 ("note".into(), json!("old")),
22228 ]),
22229 base_version: None,
22230 }])
22231 .expect("v1 write");
22232 client
22233 .recreate_with_schema(&schema)
22234 .expect("upgrade captures context");
22235 let commit = client
22236 .mutate(vec![Mutation::Upsert {
22237 table: "tasks".into(),
22238 values: Map::from_iter([
22239 ("id".into(), json!("queued")),
22240 ("note".into(), json!("v3-only")),
22241 ]),
22242 base_version: None,
22243 }])
22244 .expect("queued v3 write");
22245 let changes = client.conn.total_changes();
22246 let outbox = client.pending_commit_ids();
22247 assert!(outbox.contains(&commit));
22248 let descriptor = client.get_meta("localSchemaDescriptor");
22249 let container = format!(
22250 "{path_str}{}",
22251 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
22252 );
22253 let context = std::fs::read(&container).expect("captured context");
22254 assert!(client
22255 .recreate_with_schema(&older)
22256 .expect_err("refuse recreate")
22257 .starts_with("client.schema_downgrade:"));
22258 assert_eq!(client.schema.version, 3);
22259 assert_eq!(client.conn.total_changes(), changes);
22260 assert_eq!(client.pending_commit_ids(), outbox);
22261 assert_eq!(client.get_meta("localSchemaDescriptor"), descriptor);
22262 assert_eq!(
22263 client
22264 .conn
22265 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
22266 row.get::<_, String>(0)
22267 })
22268 .expect("queued row"),
22269 "v3-only"
22270 );
22271 assert_eq!(std::fs::read(&container).expect("context remains"), context);
22272 drop(client);
22273 let before = std::fs::read(&path).expect("replica bytes");
22274 for entry in 0..3 {
22277 let error = match entry {
22278 0 => SyncClient::open_path(
22279 "downgrade".into(),
22280 &older,
22281 previous_version_limits(),
22282 path_str,
22283 ),
22284 1 => SyncClient::open_path_with_identity(
22285 None,
22286 &older,
22287 previous_version_limits(),
22288 path_str,
22289 ),
22290 _ => SyncClient::with_connection(
22291 "downgrade".into(),
22292 &older,
22293 previous_version_limits(),
22294 Connection::open(&path).expect("connection"),
22295 ),
22296 }
22297 .err()
22298 .expect("refuse open");
22299 assert!(error.starts_with("client.schema_downgrade:"));
22300 let conn = Connection::open(&path).expect("inspect");
22301 assert_eq!(
22302 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
22303 .expect("journal mode"),
22304 "delete"
22305 );
22306 drop(conn);
22307 assert_eq!(std::fs::read(&path).expect("replica unchanged"), before);
22308 assert_eq!(
22309 std::fs::read(&container).expect("context unchanged"),
22310 context
22311 );
22312 }
22313 let reopened = SyncClient::open_path(
22314 "downgrade".into(),
22315 &schema,
22316 previous_version_limits(),
22317 path_str,
22318 )
22319 .expect("compatible reopen");
22320 assert_eq!(reopened.pending_commit_ids(), outbox);
22321 assert_eq!(
22322 reopened
22323 .conn
22324 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
22325 row.get::<_, String>(0)
22326 })
22327 .expect("queued row remains"),
22328 "v3-only"
22329 );
22330 reopened.delete_meta(LOCAL_SCHEMA_VERSION_KEY);
22331 reopened.delete_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
22332 drop(reopened);
22333 let legacy = SyncClient::open_path(
22334 "downgrade".into(),
22335 &schema,
22336 previous_version_limits(),
22337 path_str,
22338 )
22339 .expect("legacy no-marker reopen");
22340 assert_eq!(legacy.pending_commit_ids(), outbox);
22341 assert!(!legacy.upgrading());
22342 assert_eq!(
22343 legacy
22344 .conn
22345 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
22346 row.get::<_, String>(0)
22347 })
22348 .expect("legacy row remains"),
22349 "v3-only"
22350 );
22351 drop(legacy);
22352 std::fs::remove_file(path).expect("remove replica");
22353 let _ = std::fs::remove_file(container);
22354 }
22355
22356 #[test]
22357 fn schema_marker_errors_refuse_open_and_recreate_without_mutation() {
22358 for marker in [
22359 "''",
22360 "'bad'",
22361 "'0'",
22362 "'-1'",
22363 "'2.5'",
22364 "'03'",
22365 "'3\n'",
22366 "' 3'",
22367 "'+3'",
22368 "'3e0'",
22369 "'2147483648'",
22370 "NULL",
22371 "X'33'",
22372 ] {
22373 let conn = Connection::open_in_memory().expect("connection");
22374 conn.execute_batch(&format!("CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value); INSERT INTO _syncular_meta VALUES ('localSchemaVersion', {marker});")).expect("invalid marker fixture");
22375 let error = SyncClient::with_connection(
22376 "marker".into(),
22377 &previous_version_schema(3),
22378 ClientLimits::default(),
22379 conn,
22380 )
22381 .err()
22382 .expect("refuse marker");
22383 assert!(
22384 error.starts_with("sync.local_corrupt:"),
22385 "{marker}: {error}"
22386 );
22387 }
22388 for fault in [
22389 "UPDATE _syncular_meta SET value = 'bad' WHERE key = 'localSchemaVersion'",
22390 "ALTER TABLE _syncular_meta RENAME COLUMN value TO unreadable_value",
22391 ] {
22392 let mut client = SyncClient::new(
22393 "marker".into(),
22394 &previous_version_schema(3),
22395 ClientLimits::default(),
22396 )
22397 .expect("fresh");
22398 let commit = client
22399 .mutate(vec![Mutation::Upsert {
22400 table: "tasks".into(),
22401 values: Map::from_iter([("id".into(), json!("queued"))]),
22402 base_version: None,
22403 }])
22404 .expect("queued write");
22405 client.conn.execute_batch(fault).expect("marker fault");
22406 let changes = client.conn.total_changes();
22407 let error = client
22408 .recreate_with_schema(&previous_version_schema(4))
22409 .expect_err("refuse corrupt marker");
22410 assert!(error.starts_with("sync.local_corrupt:"));
22411 assert_eq!(client.conn.total_changes(), changes);
22412 assert_eq!(client.schema.version, 3);
22413 assert_eq!(client.pending_commit_ids(), vec![commit]);
22414 }
22415 for setup in [
22416 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY)",
22417 "CREATE VIEW _syncular_meta AS SELECT 'localSchemaVersion' AS key, '3' AS value",
22418 ] {
22419 let path =
22420 std::env::temp_dir().join(format!("syncular-marker-{}.db", uuid::Uuid::new_v4()));
22421 let conn = Connection::open(&path).expect("connection");
22422 conn.execute_batch(setup)
22423 .expect("unreadable metadata fixture");
22424 drop(conn);
22425 let before = std::fs::read(&path).expect("before");
22426 let error = SyncClient::open_path_with_identity(
22427 None,
22428 &previous_version_schema(3),
22429 ClientLimits::default(),
22430 path.to_str().expect("path"),
22431 )
22432 .err()
22433 .expect("refuse unreadable marker");
22434 assert!(error.starts_with("sync.local_corrupt:"));
22435 assert_eq!(std::fs::read(&path).expect("after"), before);
22436 std::fs::remove_file(path).expect("remove fixture");
22437 }
22438 }
22439
22440 std::thread_local! {
22441 static CONCURRENT_UPGRADE: RefCell<Option<Connection>> =
22444 const { RefCell::new(None) };
22445 }
22446
22447 fn commit_concurrent_upgrade(attempts: i32) -> bool {
22448 CONCURRENT_UPGRADE.with(|slot| {
22449 if let Some(banner) = slot.borrow_mut().take() {
22450 banner
22451 .execute_batch(
22452 "UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion'; COMMIT",
22453 )
22454 .expect("commit the concurrent upgrade");
22455 }
22456 });
22457 attempts < 10
22458 }
22459
22460 fn journal_mode(conn: &Connection) -> String {
22461 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
22462 .expect("journal mode")
22463 }
22464
22465 fn temp_replica(label: &str) -> std::path::PathBuf {
22466 std::env::temp_dir().join(format!("syncular-{label}-{}.db", uuid::Uuid::new_v4()))
22467 }
22468
22469 #[test]
22470 fn file_identity_constructor_requires_the_returned_wal_mode() {
22471 for path in [":memory:", ""] {
22472 let error = SyncClient::open_path_with_identity(
22473 None,
22474 &previous_version_schema(1),
22475 ClientLimits::default(),
22476 path,
22477 )
22478 .err()
22479 .expect("temporary SQLite databases cannot enter WAL mode");
22480 assert_eq!(
22481 error,
22482 "sync.invalid_request: file-backed client requires WAL journal mode"
22483 );
22484 }
22485 let path = temp_replica("confirmed-wal");
22486 let client = SyncClient::open_path_with_identity(
22487 None,
22488 &previous_version_schema(1),
22489 ClientLimits::default(),
22490 path.to_str().unwrap(),
22491 )
22492 .unwrap();
22493 assert_eq!(journal_mode(&client.conn), "wal");
22494 drop(client);
22495 std::fs::remove_file(path).unwrap();
22496 }
22497
22498 #[test]
22499 fn non_wal_identity_open_is_accepted_when_wal_is_not_required() {
22500 for path in [":memory:", ""] {
22501 let client = SyncClient::open_path_with_identity_journal(
22502 None,
22503 &previous_version_schema(1),
22504 ClientLimits::default(),
22505 path,
22506 false,
22507 )
22508 .expect("a host without WAL can still open the identity constructor");
22509 assert_ne!(journal_mode(&client.conn), "wal");
22510 assert_eq!(
22511 client
22512 .conn
22513 .query_row("PRAGMA synchronous", [], |row| row.get::<_, i64>(0))
22514 .expect("synchronous"),
22515 2,
22516 "every identity open pins FULL durability"
22517 );
22518 }
22519 }
22520
22521 #[test]
22522 fn reset_replica_deletes_the_database_and_its_sidecars_through_the_vfs() {
22523 let path = temp_replica("reset");
22524 let path_str = path.to_str().expect("path");
22525 let client = SyncClient::open_path(
22526 "reset-replica".into(),
22527 &previous_version_schema(1),
22528 ClientLimits::default(),
22529 path_str,
22530 )
22531 .expect("replica");
22532 let original_id = client.client_id().to_owned();
22533 drop(client);
22534 for suffix in ["-journal", "-wal", "-shm"] {
22535 std::fs::write(format!("{path_str}{suffix}"), b"stale sidecar").expect("sidecar");
22536 }
22537
22538 SyncClient::reset_replica(path_str).expect("reset through the VFS");
22539
22540 for suffix in ["", "-journal", "-wal", "-shm"] {
22541 assert!(
22542 !std::path::Path::new(&format!("{path_str}{suffix}")).exists(),
22543 "reset removes the replica file {suffix:?}"
22544 );
22545 }
22546 let reopened = SyncClient::open_path_with_identity(
22547 None,
22548 &previous_version_schema(1),
22549 ClientLimits::default(),
22550 path_str,
22551 )
22552 .expect("fresh replica");
22553 assert_ne!(reopened.client_id(), original_id);
22554 assert!(reopened
22555 .query("SELECT id FROM tasks", &[])
22556 .expect("empty")
22557 .is_empty());
22558 drop(reopened);
22559 assert_eq!(
22560 SyncClient::reset_replica(":memory:").expect_err("memory has nothing to reset"),
22561 "sync.invalid_request: a memory replica has nothing to reset"
22562 );
22563 }
22564
22565 #[test]
22566 fn non_positive_generated_schema_versions_are_refused_before_storage() {
22567 for version in [json!(0), json!(-1), json!(i32::MIN), json!(2147483648i64)] {
22568 let mut schema = previous_version_schema(1);
22569 schema["version"] = version.clone();
22570 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
22571 .err()
22572 .expect("refuse an out-of-range generated version");
22573 assert_eq!(
22574 error,
22575 "sync.invalid_request: generated schema version must be an integer in 1..=2147483647"
22576 );
22577 let path = temp_replica("bad-version");
22579 let path_str = path.to_str().expect("path");
22580 for identity in [false, true] {
22581 let error = if identity {
22582 SyncClient::open_path_with_identity(
22583 None,
22584 &schema,
22585 ClientLimits::default(),
22586 path_str,
22587 )
22588 } else {
22589 SyncClient::open_path(
22590 "bad-version".into(),
22591 &schema,
22592 ClientLimits::default(),
22593 path_str,
22594 )
22595 }
22596 .err()
22597 .expect("refuse before storage");
22598 assert!(
22599 error.starts_with("sync.invalid_request:"),
22600 "{version}: {error}"
22601 );
22602 assert!(
22603 !path.exists(),
22604 "{version}: a refused constructor creates no replica file"
22605 );
22606 }
22607 }
22608 let mut schema = previous_version_schema(1);
22610 schema
22611 .as_object_mut()
22612 .expect("schema object")
22613 .remove("version");
22614 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
22615 .err()
22616 .expect("refuse a missing version");
22617 assert!(error.starts_with("sync.invalid_request:"), "{error}");
22618 }
22619
22620 #[test]
22621 fn a_duplicated_schema_marker_is_refused_without_a_reset() {
22622 let path = temp_replica("duplicate-marker");
22623 let path_str = path.to_str().expect("path");
22624 {
22625 let conn = Connection::open(path_str).expect("fixture");
22628 conn.execute_batch(
22629 "CREATE TABLE _syncular_meta(key TEXT, value);
22630 INSERT INTO _syncular_meta VALUES
22631 ('localSchemaVersion', '1'),
22632 ('localSchemaVersion', '2'),
22633 ('clientId', 'duplicate');",
22634 )
22635 .expect("duplicate marker fixture");
22636 }
22637 let error = SyncClient::open_path_with_identity(
22638 None,
22639 &previous_version_schema(3),
22640 ClientLimits::default(),
22641 path_str,
22642 )
22643 .err()
22644 .expect("refuse a duplicated marker");
22645 assert!(error.starts_with("sync.local_corrupt:"), "{error}");
22646 let probe = Connection::open(path_str).expect("probe");
22647 assert_eq!(
22648 probe
22649 .query_row(
22650 "SELECT COUNT(*) FROM _syncular_meta WHERE key = 'localSchemaVersion'",
22651 [],
22652 |row| row.get::<_, i64>(0),
22653 )
22654 .expect("marker rows"),
22655 2,
22656 "the duplicated rows are left for the operator, not collapsed"
22657 );
22658 assert_eq!(
22659 probe
22660 .query_row(
22661 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_syncular_%' AND name NOT LIKE 'sqlite_%'",
22662 [],
22663 |row| row.get::<_, i64>(0),
22664 )
22665 .expect("synced tables"),
22666 0,
22667 "a duplicated marker never drives the reset that recreates v3 tables"
22668 );
22669 assert_eq!(
22670 journal_mode(&probe),
22671 "delete",
22672 "the refusal leaves the journal mode alone"
22673 );
22674 drop(probe);
22675 std::fs::remove_file(path).expect("remove replica");
22676 }
22677
22678 #[test]
22679 fn a_refused_open_does_not_persist_wal_or_mutate_the_replica() {
22680 let path = temp_replica("refused-wal");
22681 let path_str = path.to_str().expect("path");
22682 drop(
22685 SyncClient::open_path(
22686 "owner".into(),
22687 &previous_version_schema(2),
22688 ClientLimits::default(),
22689 path_str,
22690 )
22691 .expect("v2 replica"),
22692 );
22693 let before = std::fs::read(&path).expect("before");
22694 let error = SyncClient::open_path_with_identity(
22695 Some("intruder".into()),
22696 &previous_version_schema(2),
22697 ClientLimits::default(),
22698 path_str,
22699 )
22700 .err()
22701 .expect("refuse the identity mismatch");
22702 assert!(error.starts_with("client.identity_mismatch:"), "{error}");
22703 let probe = Connection::open(path_str).expect("probe");
22704 assert_eq!(
22705 journal_mode(&probe),
22706 "delete",
22707 "a refused open must not persist WAL"
22708 );
22709 assert_eq!(std::fs::read(&path).expect("after"), before);
22710 assert_eq!(
22711 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
22712 Some("2")
22713 );
22714 drop(probe);
22715 std::fs::remove_file(path).expect("remove replica");
22716 }
22717
22718 #[test]
22719 fn a_concurrent_marker_upgrade_refuses_a_stale_open() {
22720 let path = temp_replica("marker-race");
22721 let path_str = path.to_str().expect("path");
22722 drop(
22723 SyncClient::open_path_with_identity(
22724 Some("stale".into()),
22725 &previous_version_schema(1),
22726 ClientLimits::default(),
22727 path_str,
22728 )
22729 .expect("v1 replica"),
22730 );
22731 let banner = Connection::open(path_str).expect("banner");
22735 banner
22736 .execute_batch(
22737 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
22738 )
22739 .expect("pending upgrade");
22740 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
22741 let conn = Connection::open(path_str).expect("stale connection");
22742 conn.busy_handler(Some(commit_concurrent_upgrade))
22743 .expect("busy handler");
22744 let error = SyncClient::with_connection(
22745 "stale".into(),
22746 &previous_version_schema(1),
22747 ClientLimits::default(),
22748 conn,
22749 )
22750 .err()
22751 .expect("a concurrent upgrade must refuse the stale open");
22752 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
22753 let probe = Connection::open(path_str).expect("probe");
22754 assert_eq!(
22755 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
22756 Some("2"),
22757 "the concurrent upgrade survives"
22758 );
22759 drop(probe);
22760 std::fs::remove_file(path).expect("remove replica");
22761 }
22762
22763 #[test]
22764 fn a_concurrent_marker_upgrade_refuses_a_stale_recreate() {
22765 let path = temp_replica("recreate-race");
22766 let path_str = path.to_str().expect("path");
22767 let mut client = SyncClient::open_path_with_identity(
22768 Some("stale".into()),
22769 &previous_version_schema(1),
22770 ClientLimits::default(),
22771 path_str,
22772 )
22773 .expect("v1 replica");
22774 client
22775 .mutate(vec![Mutation::Upsert {
22776 table: "tasks".into(),
22777 values: Map::from_iter([("id".into(), json!("queued"))]),
22778 base_version: None,
22779 }])
22780 .expect("queued write");
22781 let commit = client.pending_commit_ids();
22782 let banner = Connection::open(path_str).expect("banner");
22783 banner
22784 .execute_batch(
22785 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
22786 )
22787 .expect("pending upgrade");
22788 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
22789 client
22790 .conn
22791 .busy_handler(Some(commit_concurrent_upgrade))
22792 .expect("busy handler");
22793 let error = client
22794 .recreate_with_schema(&previous_version_schema(1))
22795 .expect_err("a concurrent upgrade must refuse the stale recreate");
22796 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
22797 assert_eq!(client.schema.version, 1, "the compiled schema is restored");
22798 assert_eq!(
22799 client.pending_commit_ids(),
22800 commit,
22801 "the outbox is preserved"
22802 );
22803 let probe = Connection::open(path_str).expect("probe");
22804 assert_eq!(
22805 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
22806 Some("2")
22807 );
22808 drop(probe);
22809 drop(client);
22810 std::fs::remove_file(path).expect("remove replica");
22811 }
22812
22813 #[test]
22814 fn failed_schema_record_writes_roll_back_recreation() {
22815 for key in [
22816 LOCAL_SCHEMA_VERSION_KEY,
22817 crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY,
22818 ] {
22819 let mut client = SyncClient::new(
22820 "schema-record".into(),
22821 &previous_version_schema(1),
22822 ClientLimits::default(),
22823 )
22824 .expect("v1");
22825 let marker = client.get_meta(LOCAL_SCHEMA_VERSION_KEY);
22826 let descriptor = client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
22827 client.conn.execute_batch(&format!("CREATE TRIGGER refuse_schema_record BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{key}' BEGIN SELECT RAISE(ABORT, 'refuse schema record'); END")).expect("fault");
22828 client
22829 .recreate_with_schema(&previous_version_schema(2))
22830 .expect_err("both record writes must propagate failure");
22831 assert_eq!(client.schema.version, 1);
22832 assert_eq!(client.get_meta(LOCAL_SCHEMA_VERSION_KEY), marker);
22833 assert_eq!(
22834 client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY),
22835 descriptor
22836 );
22837 client
22838 .conn
22839 .execute_batch("DROP TRIGGER refuse_schema_record")
22840 .expect("remove fault");
22841 client
22842 .recreate_with_schema(&previous_version_schema(2))
22843 .expect("retry");
22844 assert_eq!(client.schema.version, 2);
22845 }
22846 }
22847
22848 #[test]
22849 fn a_failed_outer_commit_restores_every_reset_side_effect() {
22850 let path = temp_replica("outer-commit");
22851 let path_str = path.to_str().expect("path");
22852 let conn = Connection::open(path_str).expect("replica connection");
22855 let mut client = SyncClient::with_connection(
22856 "outer-commit".into(),
22857 &previous_version_schema(1),
22858 previous_version_limits(),
22859 conn,
22860 )
22861 .expect("v1 replica");
22862 assert_eq!(journal_mode(&client.conn), "delete");
22863 let commit = client
22864 .mutate(vec![Mutation::Upsert {
22865 table: "tasks".into(),
22866 values: Map::from_iter([
22867 ("id".into(), json!("queued")),
22868 ("note".into(), json!("dropped-by-v2")),
22869 ]),
22870 base_version: None,
22871 }])
22872 .expect("queued write");
22873 client.drain_change_batches();
22874 let mut dropped = previous_version_schema(2);
22876 dropped["tables"][0]["columns"] =
22877 json!([{ "name": "id", "type": "string", "nullable": false }]);
22878 let marker = meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY);
22881 let revision = client.local_revision();
22882 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
22883 let columns: i64 = client
22884 .conn
22885 .query_row(
22886 "SELECT count(*) FROM pragma_table_info('tasks')",
22887 [],
22888 |row| row.get(0),
22889 )
22890 .expect("column count");
22891 let rebuilds = client.overlay_rebuild_count.get();
22892 let prior_subs = client.subs.len();
22893 let prior_changes = client.change_queue.len();
22894 let prior_intents = client.sync_intent_queue.len();
22895 let prior_last_change = client.last_change.clone();
22896 let prior_overlay_dirty = client.overlay_dirty.get();
22897 let prior_sync_needed = client.sync_needed;
22898 let prior_upgrading = client.upgrading;
22899 let prior_stopped = client.stopped;
22900 let prior_schema_floor = client.schema_floor.clone();
22901 let prior_active_round = client.active_round;
22902 let reader = Connection::open(path_str).expect("reader");
22905 reader
22906 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
22907 .expect("reader snapshot");
22908 client
22909 .conn
22910 .busy_timeout(std::time::Duration::ZERO)
22911 .expect("busy timeout");
22912 let error = client
22913 .recreate_with_schema(&dropped)
22914 .expect_err("a blocked COMMIT must refuse the recreation");
22915 assert!(error.starts_with("client.storage_busy:"), "{error}");
22916 assert!(
22917 client.overlay_rebuild_count.get() > rebuilds,
22918 "the reset ran to its overlay rebuild inside the transaction"
22919 );
22920 assert!(client.conn.is_autocommit(), "the transaction is released");
22921 assert_eq!(meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY), marker);
22923 assert_eq!(client.local_revision(), revision);
22924 assert_eq!(
22925 client.query("SELECT * FROM tasks", &[]).expect("rows"),
22926 rows
22927 );
22928 assert_eq!(
22929 client
22930 .conn
22931 .query_row(
22932 "SELECT count(*) FROM pragma_table_info('tasks')",
22933 [],
22934 |row| { row.get::<_, i64>(0) }
22935 )
22936 .expect("column count"),
22937 columns,
22938 "the previous table layout survives the rollback"
22939 );
22940 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
22941 assert_eq!(client.schema.version, 1, "compiled schema");
22944 assert_eq!(client.subs.len(), prior_subs);
22945 assert_eq!(client.change_queue.len(), prior_changes);
22946 assert_eq!(client.sync_intent_queue.len(), prior_intents);
22947 assert_eq!(client.last_change, prior_last_change);
22948 assert_eq!(client.overlay_dirty.get(), prior_overlay_dirty);
22949 assert_eq!(client.sync_needed, prior_sync_needed);
22950 assert_eq!(client.upgrading, prior_upgrading);
22951 assert_eq!(client.stopped, prior_stopped);
22952 assert_eq!(client.schema_floor, prior_schema_floor);
22953 assert_eq!(client.active_round, prior_active_round);
22954 assert!(
22955 client.rejections.is_empty(),
22956 "the dropped-column rejection is rolled back too"
22957 );
22958 let container = crate::previous_version::previous_version_container_path(path_str);
22961 assert!(std::path::Path::new(&container).exists());
22962 drop(reader);
22965 let snapshot = client
22966 .previous_version_snapshot(&PreviousVersionReadSpec {
22967 table: "tasks".into(),
22968 ..Default::default()
22969 })
22970 .expect("refuse stale capture");
22971 assert!(!snapshot.available);
22972 assert_eq!(snapshot.current_version, 1);
22973 assert!(!std::path::Path::new(&container).exists());
22974 client
22979 .conn
22980 .busy_timeout(std::time::Duration::from_secs(1))
22981 .expect("busy timeout");
22982 client
22983 .recreate_with_schema(&dropped)
22984 .expect("the recreation succeeds once the reader releases");
22985 assert_eq!(client.schema.version, 2);
22986 assert_eq!(
22987 client.pending_commit_ids(),
22988 vec![commit.clone()],
22989 "the reset kept the incompatible commit pending"
22990 );
22991 assert!(
22992 client.rejections.is_empty(),
22993 "no rejection exists before the send-time prepass"
22994 );
22995 client
22997 .prepare_sync_round(false)
22998 .expect("the send prepass runs during round preparation");
22999 assert!(client.pending_commit_ids().is_empty());
23000 assert!(
23001 client
23002 .rejections
23003 .iter()
23004 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE),
23005 "the dropped-column commit surfaced as a rejection at send time"
23006 );
23007 client.previous_version_discard();
23008 drop(client);
23009 std::fs::remove_file(path).expect("remove replica");
23010 }
23011
23012 #[test]
23013 fn the_send_time_incompatible_drop_is_atomic_and_publishes_once() {
23014 let v1 = previous_version_schema(1);
23015 let mut v2 = previous_version_schema(2);
23016 v2["tables"][0]["columns"] = json!([{ "name": "id", "type": "string", "nullable": false }]);
23018 let mut client = SyncClient::new("drop".to_string(), &v1, ClientLimits::default()).unwrap();
23019 let dropped = client
23020 .mutate(vec![Mutation::Upsert {
23021 table: "tasks".into(),
23022 values: Map::from_iter([
23023 ("id".into(), json!("dropped")),
23024 ("note".into(), json!("gone")),
23025 ]),
23026 base_version: None,
23027 }])
23028 .unwrap();
23029 let survivor = client
23033 .mutate(vec![Mutation::Delete {
23034 table: "tasks".into(),
23035 row_id: "gone".into(),
23036 base_version: None,
23037 }])
23038 .unwrap();
23039 client
23040 .recreate_with_schema(&v2)
23041 .expect("the reset preserves the outbox");
23042 assert_eq!(
23043 client.pending_commit_ids(),
23044 vec![dropped.clone(), survivor.clone()],
23045 "the reset preserves both commits"
23046 );
23047 assert!(client.rejections.is_empty(), "the reset classifies nothing");
23048 client.drain_change_batches();
23049 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
23050 assert_eq!(rows.len(), 1, "only the dropped upsert has a visible row");
23051
23052 for fault in [
23058 "outcome-insert",
23059 "outbox-delete",
23060 "overlay-rebuild",
23061 "publish-revision",
23062 ] {
23063 client.set_sync_needed(false, false);
23064 client.drain_change_batches();
23065 let revision = client.local_revision();
23066 let trigger = match fault {
23067 "outcome-insert" => {
23068 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23069 }
23070 "outbox-delete" => {
23071 "CREATE TRIGGER fail_drop BEFORE DELETE ON _syncular_outbox BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23072 }
23073 "publish-revision" => {
23078 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23079 }
23080 _ => {
23081 "CREATE TRIGGER fail_drop BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23082 }
23083 };
23084 client.conn.execute_batch(trigger).unwrap();
23085 let outcome = client
23086 .prepare_sync_round(false)
23087 .err()
23088 .unwrap_or_else(|| panic!("{fault}: a fault in the drop must fail the round"));
23089 assert!(
23090 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "storage.failed"),
23091 "{fault}: {outcome:?}"
23092 );
23093 client.conn.execute_batch("DROP TRIGGER fail_drop").unwrap();
23094 assert!(client.conn.is_autocommit(), "{fault}: no leaked savepoint");
23095 assert_eq!(
23096 client.pending_commit_ids(),
23097 vec![dropped.clone(), survivor.clone()],
23098 "{fault}: the outbox is restored"
23099 );
23100 assert!(
23101 client.rejections.is_empty(),
23102 "{fault}: no in-memory rejection survives"
23103 );
23104 assert!(
23105 client.commit_outcome(&dropped).unwrap().is_none(),
23106 "{fault}: the durable outcome rolled back"
23107 );
23108 assert_eq!(client.local_revision(), revision, "{fault}: revision");
23109 assert_eq!(
23110 client.query("SELECT * FROM tasks", &[]).unwrap(),
23111 rows,
23112 "{fault}: the visible projection is unchanged"
23113 );
23114 assert!(
23115 client.drain_change_batches().is_empty(),
23116 "{fault}: nothing is published"
23117 );
23118 }
23119
23120 client
23123 .conn
23124 .execute_batch(
23125 "BEGIN; INSERT INTO _syncular_meta(key,value) VALUES ('outerProbe','kept');
23126 CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes
23127 BEGIN SELECT RAISE(ABORT,'nested drop fault'); END;",
23128 )
23129 .unwrap();
23130 assert!(client.drop_incompatible_outbox().is_err());
23131 assert!(!client.conn.is_autocommit());
23132 assert_eq!(client.get_meta("outerProbe").as_deref(), Some("kept"));
23133 assert_eq!(
23134 client.pending_commit_ids(),
23135 vec![dropped.clone(), survivor.clone()]
23136 );
23137 assert!(client.commit_outcome(&dropped).unwrap().is_none());
23138 client
23139 .conn
23140 .execute_batch("DROP TRIGGER fail_drop; COMMIT")
23141 .unwrap();
23142
23143 client.set_meta(LOG_EPOCH_KEY, "epoch");
23148 client.set_sync_needed(false, false);
23149 client.drain_change_batches();
23150 let revision = client.local_revision();
23151 let prepared = client.prepare_sync_round(false).expect("the drop succeeds");
23152 assert_eq!(client.pending_commit_ids(), vec![survivor.clone()]);
23153 assert_eq!(
23154 prepared.meta.pushed_ids,
23155 vec![survivor.clone()],
23156 "the incompatible commit is classified before any operation is sent"
23157 );
23158 let rejection = client
23159 .rejections
23160 .iter()
23161 .find(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE)
23162 .expect("a sync.outbox_incompatible rejection");
23163 assert_eq!(rejection.client_commit_id, dropped);
23164 assert!(
23165 !rejection.retryable,
23166 "a schema-incompatible commit is final"
23167 );
23168 assert_eq!(
23169 client
23170 .commit_outcome(&dropped)
23171 .unwrap()
23172 .expect("a durable rejected outcome")
23173 .status,
23174 CommitOutcomeStatus::Rejected
23175 );
23176 assert_eq!(client.local_revision(), revision + 1, "one revision");
23177 let batches = client.drain_change_batches();
23178 assert_eq!(batches.len(), 1, "one change batch");
23179 assert!(
23180 batches[0].rejections_changed
23181 && batches[0].outcomes_changed
23182 && batches[0].status.is_some(),
23183 "{:?}",
23184 batches[0]
23185 );
23186 assert!(
23187 batches[0].tables.iter().any(|entry| entry.table == "tasks"),
23188 "{:?}",
23189 batches[0]
23190 );
23191 let visible = client.query("SELECT * FROM tasks", &[]).unwrap();
23192 assert!(
23193 visible.is_empty(),
23194 "the dropped projection is undone: {visible:?}"
23195 );
23196
23197 let path = temp_replica("outbox-drop-commit");
23201 let path_str = path.to_str().expect("path");
23202 let conn = Connection::open(path_str).expect("replica connection");
23203 let mut client =
23204 SyncClient::with_connection("drop-commit".into(), &v1, ClientLimits::default(), conn)
23205 .expect("v1 replica");
23206 let dropped = client
23207 .mutate(vec![Mutation::Upsert {
23208 table: "tasks".into(),
23209 values: Map::from_iter([
23210 ("id".into(), json!("dropped")),
23211 ("note".into(), json!("gone")),
23212 ]),
23213 base_version: None,
23214 }])
23215 .unwrap();
23216 client
23217 .recreate_with_schema(&v2)
23218 .expect("the reset preserves the outbox");
23219 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
23220 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
23221 assert_eq!(rows.len(), 1, "the optimistic row is visible");
23222 client.set_sync_needed(false, false);
23223 client.drain_change_batches();
23224 let revision = client.local_revision();
23225 let reader = Connection::open(path_str).expect("reader");
23226 reader
23227 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
23228 .expect("reader snapshot");
23229 client
23230 .conn
23231 .busy_timeout(std::time::Duration::ZERO)
23232 .expect("busy timeout");
23233 let outcome = client
23234 .prepare_sync_round(false)
23235 .err()
23236 .expect("a blocked COMMIT must fail the round");
23237 assert!(
23238 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "client.storage_busy"),
23239 "{outcome:?}"
23240 );
23241 assert!(client.conn.is_autocommit(), "the savepoint is released");
23242 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
23243 assert!(client.rejections.is_empty());
23244 assert!(client.commit_outcome(&dropped).unwrap().is_none());
23245 assert_eq!(client.local_revision(), revision);
23246 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
23247 assert!(client.drain_change_batches().is_empty());
23248 drop(reader);
23250 client
23251 .conn
23252 .busy_timeout(std::time::Duration::from_secs(1))
23253 .expect("busy timeout");
23254 client
23255 .prepare_sync_round(false)
23256 .expect("the drop succeeds after the reader releases");
23257 assert!(client.pending_commit_ids().is_empty());
23258 assert!(client
23259 .rejections
23260 .iter()
23261 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE));
23262 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
23263 drop(client);
23264 std::fs::remove_file(path).expect("remove replica");
23265 }
23266
23267 #[test]
23268 fn previous_version_read_reasons_come_from_live_client_state() {
23269 use crate::previous_version::{
23270 PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
23271 };
23272 let path = std::env::temp_dir()
23273 .join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
23274 .to_string_lossy()
23275 .into_owned();
23276 let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
23277 {
23278 let mut setup = SyncClient::open_path(
23279 "wiring".to_owned(),
23280 &previous_version_schema(1),
23281 previous_version_limits(),
23282 &path,
23283 )
23284 .expect("v1 install");
23285 setup
23286 .mutate(vec![Mutation::Upsert {
23287 table: "tasks".to_owned(),
23288 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
23289 base_version: None,
23290 }])
23291 .expect("seed a row");
23292 }
23293 let mut client = SyncClient::open_path(
23294 "wiring".to_owned(),
23295 &previous_version_schema(2),
23296 previous_version_limits(),
23297 &path,
23298 )
23299 .expect("v2 bump captures");
23300 assert!(std::path::Path::new(&container).exists());
23301 let spec = PreviousVersionReadSpec {
23302 table: "tasks".to_owned(),
23303 ..Default::default()
23304 };
23305
23306 recapture_previous_version(&client);
23308 client.lease_state = Some(LeaseState {
23309 lease_id: None,
23310 expires_at_ms: None,
23311 error_code: Some("lease.expired".to_owned()),
23312 });
23313 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23314 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
23315 assert!(!snapshot.available);
23316 assert!(!std::path::Path::new(&container).exists());
23317
23318 recapture_previous_version(&client);
23320 client.set_now_ms(1_000);
23321 client.lease_state = Some(LeaseState {
23322 lease_id: Some("lease".to_owned()),
23323 expires_at_ms: Some(1_000),
23324 error_code: None,
23325 });
23326 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23327 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
23328 assert!(!std::path::Path::new(&container).exists());
23329 client.lease_state = None;
23330
23331 recapture_previous_version(&client);
23333 client.subs.push(subscription(SubState::Revoked, 0, None));
23334 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23335 assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
23336 assert!(!std::path::Path::new(&container).exists());
23337 client.subs.clear();
23338
23339 recapture_previous_version(&client);
23342 client
23343 .subs
23344 .push(subscription(SubState::Active, 0, Some("resume")));
23345 assert!(!client.previous_version_coverage_complete());
23346 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23347 assert!(snapshot.available);
23348 assert_eq!(snapshot.previous_version, Some(2));
23349 assert_eq!(snapshot.rows.len(), 1);
23350
23351 recapture_previous_version(&client);
23352 client.subs[0].bootstrap_state = None;
23353 assert!(client.previous_version_coverage_complete());
23354 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23355 assert_eq!(
23356 snapshot.reason,
23357 Some(PreviousVersionReason::CoverageComplete)
23358 );
23359 assert!(!std::path::Path::new(&container).exists());
23360 client.subs.clear();
23361
23362 assert!(!client.previous_version_coverage_complete());
23364
23365 drop(client);
23366 let _ = std::fs::remove_file(&path);
23367 }
23368
23369 #[test]
23376 fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
23377 let path = std::env::temp_dir()
23378 .join(format!(
23379 "syncular-prev-lifetime-{}.db",
23380 uuid::Uuid::new_v4()
23381 ))
23382 .to_string_lossy()
23383 .into_owned();
23384 let container = format!(
23385 "{path}{}",
23386 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
23387 );
23388 let exists = || std::path::Path::new(&container).exists();
23389 {
23390 let mut setup = SyncClient::open_path(
23391 "lifetime".to_owned(),
23392 &previous_version_schema(1),
23393 previous_version_limits(),
23394 &path,
23395 )
23396 .expect("v1 install");
23397 setup
23398 .mutate(vec![Mutation::Upsert {
23399 table: "tasks".to_owned(),
23400 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
23401 base_version: None,
23402 }])
23403 .expect("seed a row");
23404 }
23405 let mut client = SyncClient::open_path(
23406 "lifetime".to_owned(),
23407 &previous_version_schema(2),
23408 previous_version_limits(),
23409 &path,
23410 )
23411 .expect("v2 bump captures");
23412 client.set_now_ms(1_000);
23415 assert!(exists());
23416
23417 client.lease_state = Some(LeaseState {
23419 lease_id: None,
23420 expires_at_ms: None,
23421 error_code: Some("lease.expired".to_owned()),
23422 });
23423 client.previous_version_lifetime_check();
23424 assert!(!exists());
23425 client.lease_state = None;
23426
23427 recapture_previous_version(&client);
23430 client.subs.push(subscription(SubState::Active, 0, None));
23431 client.previous_version_lifetime_check();
23432 assert!(!exists());
23433 client.subs.clear();
23434
23435 recapture_previous_version(&client);
23437 client.subs.push(subscription(SubState::Revoked, 0, None));
23438 client.previous_version_lifetime_check();
23439 assert!(!exists());
23440 client.subs.clear();
23441
23442 recapture_previous_version(&client);
23445 client
23446 .subs
23447 .push(subscription(SubState::Active, 0, Some("resume")));
23448 client.previous_version_lifetime_check();
23449 assert!(exists());
23450 client.subs.clear();
23451
23452 drop(client);
23453 let _ = std::fs::remove_file(&path);
23454 let _ = std::fs::remove_file(&container);
23455 let _ = std::fs::remove_file(format!("{container}-wal"));
23456 let _ = std::fs::remove_file(format!("{container}-shm"));
23457 }
23458}
23459
23460#[cfg(test)]
23461mod scope_move_delivery_tests {
23462 use super::*;
23463 use crate::native_transport::HostTransport;
23464 use crate::values::encode_row_json;
23465 use serde_json::json;
23466
23467 fn client() -> SyncClient {
23468 SyncClient::new(
23469 "scope-move".to_owned(),
23470 &json!({
23471 "version": 1,
23472 "tables": [{
23473 "name": "tasks",
23474 "primaryKey": "id",
23475 "columns": [
23476 { "name": "id", "type": "string", "nullable": false },
23477 { "name": "project_id", "type": "string", "nullable": false }
23478 ],
23479 "scopes": [{ "pattern": "project:{project_id}" }]
23480 }]
23481 }),
23482 ClientLimits::default(),
23483 )
23484 .expect("test client")
23485 }
23486
23487 fn upsert(client: &SyncClient, project: &str) -> ssp2::model::Change {
23488 ssp2::model::Change {
23489 table_index: 0,
23490 row_id: "t1".to_owned(),
23491 op: Op::Upsert,
23492 row_version: Some(1),
23493 scopes: vec![("project_id".to_owned(), project.to_owned())],
23494 row: Some(
23495 encode_row_json(
23496 client.schema.table("tasks").unwrap(),
23497 "t1",
23498 &Map::from_iter([
23499 ("id".to_owned(), Value::from("t1")),
23500 ("project_id".to_owned(), Value::from(project)),
23501 ]),
23502 &client.encryption,
23503 )
23504 .unwrap(),
23505 ),
23506 }
23507 }
23508
23509 fn delete(project: &str) -> ssp2::model::Change {
23510 ssp2::model::Change {
23511 table_index: 0,
23512 row_id: "t1".to_owned(),
23513 op: Op::Delete,
23514 row_version: None,
23515 scopes: vec![("project_id".to_owned(), project.to_owned())],
23516 row: None,
23517 }
23518 }
23519
23520 #[test]
23521 fn a_scope_move_keeps_the_row_in_either_section_order() {
23522 for order in [["old", "new"], ["new", "old"]] {
23523 let mut client = client();
23524 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
23525 client
23526 .subscribe(
23527 "old".into(),
23528 "tasks".into(),
23529 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
23530 None,
23531 )
23532 .unwrap();
23533 client
23534 .subscribe(
23535 "new".into(),
23536 "tasks".into(),
23537 vec![("project_id".to_owned(), vec!["p2".to_owned()])],
23538 None,
23539 )
23540 .unwrap();
23541 let seed = upsert(&client, "p1");
23542 client
23543 .apply_commit_frame(&["tasks".into()], &[seed], Some(1))
23544 .unwrap();
23545 assert_eq!(
23546 client.query("SELECT project_id FROM tasks", &[]).unwrap()[0]["project_id"],
23547 json!("p1")
23548 );
23549
23550 let mut frames = vec![Frame::RespHeader {
23551 required_schema_version: None,
23552 latest_schema_version: None,
23553 log_epoch: Some("epoch-1".to_owned()),
23554 reset_required: Some(false),
23555 }];
23556 for id in order {
23557 let project = if id == "old" { "p1" } else { "p2" };
23558 frames.push(Frame::SubStart {
23559 id: id.to_owned(),
23560 status: SubStatus::Active,
23561 reason_code: String::new(),
23562 effective_scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
23563 bootstrap: false,
23564 });
23565 frames.push(Frame::Commit {
23566 commit_seq: 2,
23567 created_at_ms: 0,
23568 actor_id: "actor".to_owned(),
23569 tables: vec!["tasks".to_owned()],
23570 changes: vec![if id == "old" {
23571 delete("p1")
23572 } else {
23573 upsert(&client, "p2")
23574 }],
23575 });
23576 frames.push(Frame::SubEnd {
23577 next_cursor: 2,
23578 bootstrap_state: None,
23579 });
23580 }
23581 let (_, meta) = client.build_request(false).unwrap();
23582 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
23583 let outcome = client.process_response(
23584 &mut transport,
23585 Message {
23586 wire_version: WIRE_VERSION,
23587 msg_kind: MsgKind::Response,
23588 frames,
23589 },
23590 &meta,
23591 );
23592 assert!(
23593 matches!(outcome, SyncOutcome::Ok(_)),
23594 "{order:?}: {outcome:?}"
23595 );
23596 let rows = client
23597 .query("SELECT id, project_id FROM tasks", &[])
23598 .unwrap();
23599 assert_eq!(rows.len(), 1, "{order:?}: the row was lost");
23600 assert_eq!(rows[0]["project_id"], json!("p2"), "{order:?}");
23601 }
23602 }
23603
23604 struct RefusalTransport;
23605
23606 impl Transport for RefusalTransport {
23607 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
23608 let mut error = TransportError::new(
23609 "sync.invalid_client_id",
23610 "clientId is bound to a different actor in this partition",
23611 );
23612 error.details = Some(json!({
23613 "causeKind": "status",
23614 "httpStatus": 400,
23615 "category": "invalid-request",
23616 "retryable": false,
23617 "recommendedAction": "resetClientId"
23618 }));
23619 Err(error)
23620 }
23621
23622 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
23623 self.sync(request)
23624 }
23625
23626 fn download_segment(
23627 &mut self,
23628 _request: &SegmentRequest,
23629 _on_progress: &mut dyn FnMut(u64),
23630 ) -> Result<Vec<u8>, TransportError> {
23631 Err(TransportError::new("sync.transport_failed", "offline"))
23632 }
23633
23634 fn realtime_connect(&mut self) -> Result<(), TransportError> {
23635 Ok(())
23636 }
23637
23638 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
23639 Ok(())
23640 }
23641
23642 fn realtime_close(&mut self) -> Result<(), TransportError> {
23643 Ok(())
23644 }
23645 }
23646
23647 #[test]
23648 fn an_http_refusal_carries_its_code_and_recommended_action() {
23649 let mut client = client();
23650 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
23651 let outcome = client.sync(&mut RefusalTransport);
23652 let SyncOutcome::Failed {
23653 error_code,
23654 recommended_action,
23655 details,
23656 ..
23657 } = outcome
23658 else {
23659 panic!("expected a failed round");
23660 };
23661 assert_eq!(error_code, "sync.invalid_client_id");
23662 assert_eq!(recommended_action.as_deref(), Some("resetClientId"));
23663 assert_eq!(
23664 details
23665 .as_ref()
23666 .and_then(|details| details.get("httpStatus")),
23667 Some(&json!(400))
23668 );
23669 }
23670
23671 #[test]
23672 fn reset_client_id_persists_and_changes_the_request_identity() {
23673 let mut client = client();
23674 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
23675 let previous = client.client_id().to_owned();
23676 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
23677 let next = client.reset_client_id(&mut transport);
23678 assert_ne!(next, previous);
23679 assert_eq!(client.client_id(), next);
23680 assert_eq!(client.get_meta(CLIENT_ID_KEY), Some(next.clone()));
23681
23682 client
23683 .subscribe(
23684 "tasks".into(),
23685 "tasks".into(),
23686 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
23687 None,
23688 )
23689 .unwrap();
23690 let (message, _) = client.build_request(false).unwrap();
23691 assert!(matches!(
23692 message.frames.first(),
23693 Some(Frame::ReqHeader { client_id, .. }) if client_id == &next
23694 ));
23695 }
23696}
23697
23698#[cfg(test)]
23699mod insert_intent_tests {
23700 use super::*;
23701 use serde_json::{json, Value};
23702
23703 fn tasks_schema(version: i32, extra: Value) -> Value {
23705 let mut columns = vec![
23706 json!({ "name": "id", "type": "string", "nullable": false }),
23707 json!({ "name": "title", "type": "string", "nullable": true }),
23708 ];
23709 columns.extend(extra.as_array().cloned().unwrap_or_default());
23710 json!({
23711 "version": version,
23712 "tables": [{ "name": "tasks", "primaryKey": "id", "columns": columns, "scopes": [] }]
23713 })
23714 }
23715
23716 fn with_note() -> Value {
23717 tasks_schema(
23718 2,
23719 json!([{ "name": "note", "type": "string", "nullable": true }]),
23720 )
23721 }
23722
23723 fn upsert(id: &str, title: &str) -> Mutation {
23724 Mutation::Upsert {
23725 table: "tasks".into(),
23726 values: Map::from_iter([("id".into(), json!(id)), ("title".into(), json!(title))]),
23727 base_version: None,
23728 }
23729 }
23730
23731 fn persisted_markers(client: &SyncClient) -> Vec<Vec<Option<Value>>> {
23732 let mut statement = client
23733 .conn
23734 .prepare("SELECT ops_json FROM _syncular_outbox ORDER BY seq")
23735 .unwrap();
23736 statement
23737 .query_map([], |row| row.get::<_, String>(0))
23738 .unwrap()
23739 .map(|raw| {
23740 serde_json::from_str::<Vec<Value>>(&raw.unwrap())
23741 .unwrap()
23742 .iter()
23743 .map(|entry| entry.get("insert").cloned())
23744 .collect()
23745 })
23746 .collect()
23747 }
23748
23749 fn queued_insert_after_bump(next: &Value, strip_marker: bool) -> SyncClient {
23753 let mut client = SyncClient::new(
23754 "insert-intent".into(),
23755 &tasks_schema(1, json!([])),
23756 ClientLimits::default(),
23757 )
23758 .unwrap();
23759 client.mutate(vec![upsert("q", "new")]).unwrap();
23760 if strip_marker {
23761 client
23762 .conn
23763 .execute_batch(
23764 "UPDATE _syncular_outbox SET ops_json = json_remove(ops_json, '$[0].insert')",
23765 )
23766 .unwrap();
23767 client.outbox[0].ops[0].insert = false;
23768 }
23769 client.recreate_with_schema(next).unwrap();
23770 client
23771 }
23772
23773 fn seed_base_row(client: &mut SyncClient, row: Row) {
23776 client
23777 .write_row(&base_table("tasks"), "tasks", &row, 1)
23778 .unwrap();
23779 client.overlay_dirty.set(true);
23780 client.rebuild_overlay().unwrap();
23781 }
23782
23783 fn sparse_payload(client: &SyncClient, values: Value) -> Vec<u8> {
23784 let table = client.schema.table("tasks").unwrap();
23785 encode_sparse_row_json(
23786 table,
23787 "q",
23788 values.as_object().unwrap(),
23789 &client.encryption,
23790 None,
23791 )
23792 .unwrap()
23793 }
23794
23795 #[test]
23796 fn mutate_records_insert_intent_only_for_rows_absent_at_call_time() {
23797 let mut client = SyncClient::new(
23798 "insert-intent".into(),
23799 &tasks_schema(1, json!([])),
23800 ClientLimits::default(),
23801 )
23802 .unwrap();
23803 client
23805 .mutate(vec![upsert("a", "first"), upsert("a", "second")])
23806 .unwrap();
23807 client.mutate(vec![upsert("a", "visible")]).unwrap();
23808 client
23809 .patch(
23810 "tasks",
23811 "a",
23812 Map::from_iter([("title".into(), json!("patched"))]),
23813 None,
23814 )
23815 .unwrap();
23816 client
23817 .mutate(vec![Mutation::Delete {
23818 table: "tasks".into(),
23819 row_id: "a".into(),
23820 base_version: None,
23821 }])
23822 .unwrap();
23823
23824 let marker = Some(Value::Bool(true));
23825 assert_eq!(
23826 persisted_markers(&client),
23827 vec![
23828 vec![marker.clone(), marker],
23829 vec![None],
23830 vec![None],
23831 vec![None]
23832 ]
23833 );
23834 let flags = |commits: &[OutboxCommit]| {
23835 commits
23836 .iter()
23837 .map(|commit| commit.ops.iter().map(|op| op.insert).collect::<Vec<_>>())
23838 .collect::<Vec<_>>()
23839 };
23840 let expected = vec![vec![true, true], vec![false], vec![false], vec![false]];
23841 assert_eq!(flags(&client.outbox), expected);
23842 assert_eq!(
23843 flags(&client.load_outbox().unwrap()),
23844 expected,
23845 "reload keeps it"
23846 );
23847
23848 let insert = serde_json::to_value(CommitOperation::from(&client.outbox[0].ops[0])).unwrap();
23850 assert_eq!(insert["insert"], json!(true));
23851 assert!(
23852 serde_json::from_value::<CommitOperation>(insert)
23853 .unwrap()
23854 .insert
23855 );
23856 let update = serde_json::to_value(CommitOperation::from(&client.outbox[1].ops[0])).unwrap();
23857 assert!(update.get("insert").is_none());
23858 }
23859
23860 #[test]
23861 fn insert_intent_replays_and_encodes_an_appended_nullable_column_as_null() {
23862 let mut client = queued_insert_after_bump(&with_note(), false);
23863 assert_eq!(
23864 client
23865 .query("SELECT id, title, note FROM tasks", &[])
23866 .unwrap(),
23867 vec![Map::from_iter([
23868 ("id".into(), json!("q")),
23869 ("title".into(), json!("new")),
23870 ("note".into(), Value::Null),
23871 ])],
23872 "the replayed insert is visible with the appended column NULL"
23873 );
23874 assert_eq!(
23875 client.pending_payloads(),
23876 vec![sparse_payload(
23877 &client,
23878 json!({ "id": "q", "title": "new", "note": null })
23879 )],
23880 "the insert presents the appended column as NULL"
23881 );
23882
23883 seed_base_row(
23885 &mut client,
23886 vec![
23887 Some(ColumnValue::String("q".into())),
23888 Some(ColumnValue::String("base".into())),
23889 Some(ColumnValue::String("server".into())),
23890 ],
23891 );
23892 assert_eq!(
23893 client.query("SELECT title, note FROM tasks", &[]).unwrap(),
23894 vec![Map::from_iter([
23895 ("title".into(), json!("new")),
23896 ("note".into(), Value::Null),
23897 ])]
23898 );
23899 }
23900
23901 #[test]
23902 fn an_operation_without_the_marker_keeps_its_presence_set() {
23903 let mut client = queued_insert_after_bump(&with_note(), true);
23904 assert!(
23905 client
23906 .query("SELECT id FROM tasks", &[])
23907 .unwrap()
23908 .is_empty(),
23909 "a partial operation over an absent row stays invisible"
23910 );
23911 assert_eq!(
23912 client.pending_payloads(),
23913 vec![sparse_payload(
23914 &client,
23915 json!({ "id": "q", "title": "new" })
23916 )],
23917 "the appended column stays absent"
23918 );
23919
23920 seed_base_row(
23921 &mut client,
23922 vec![
23923 Some(ColumnValue::String("q".into())),
23924 Some(ColumnValue::String("base".into())),
23925 Some(ColumnValue::String("server".into())),
23926 ],
23927 );
23928 assert_eq!(
23929 client.query("SELECT title, note FROM tasks", &[]).unwrap(),
23930 vec![Map::from_iter([
23931 ("title".into(), json!("new")),
23932 ("note".into(), json!("server")),
23933 ])],
23934 "the stored appended value survives"
23935 );
23936 }
23937
23938 #[test]
23939 fn insert_intent_missing_a_non_nullable_column_is_outbox_incompatible() {
23940 let with_rank = tasks_schema(
23941 2,
23942 json!([{ "name": "rank", "type": "integer", "nullable": false }]),
23943 );
23944 let mut client = SyncClient::new(
23945 "insert-intent".into(),
23946 &tasks_schema(1, json!([])),
23947 ClientLimits::default(),
23948 )
23949 .unwrap();
23950 let insert = client.mutate(vec![upsert("q", "new")]).unwrap();
23951 let update = client.mutate(vec![upsert("q", "edited")]).unwrap();
23954 client.recreate_with_schema(&with_rank).unwrap();
23955 assert_eq!(
23956 client.pending_commit_ids(),
23957 vec![insert.clone(), update.clone()],
23958 "the reset classifies nothing"
23959 );
23960 assert!(client.rejections.is_empty());
23961 assert!(
23962 client
23963 .query("SELECT id FROM tasks", &[])
23964 .unwrap()
23965 .is_empty(),
23966 "the unencodable insert stays invisible"
23967 );
23968
23969 assert!(client.drop_incompatible_outbox().unwrap());
23970 assert_eq!(client.pending_commit_ids(), vec![update]);
23971 assert_eq!(client.rejections.len(), 1);
23972 let rejection = &client.rejections[0];
23973 assert_eq!(rejection.client_commit_id, insert);
23974 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE);
23975 assert!(rejection.operation.as_ref().is_some_and(|op| op.insert));
23976 assert!(
23977 client.commit_outcome(&insert).unwrap().is_some(),
23978 "the journal accepts the marked operation"
23979 );
23980 }
23981
23982 #[test]
23983 fn the_stored_operation_validator_accepts_the_marker_only_on_upserts() {
23984 let upsert =
23985 json!({ "table": "tasks", "rowId": "q", "op": "upsert", "values": { "id": "q" } });
23986 let delete = json!({ "table": "tasks", "rowId": "q", "op": "delete" });
23987 let with = |base: &Value, insert: Value| {
23988 let mut value = base.clone();
23989 value["insert"] = insert;
23990 value
23991 };
23992 assert!(SyncClient::stored_operation_is_representable(&upsert));
23993 assert!(SyncClient::stored_operation_is_representable(&with(
23994 &upsert,
23995 json!(true)
23996 )));
23997 assert!(!SyncClient::stored_operation_is_representable(&with(
23998 &upsert,
23999 json!(false)
24000 )));
24001 assert!(!SyncClient::stored_operation_is_representable(&with(
24002 &upsert,
24003 json!("yes")
24004 )));
24005 assert!(SyncClient::stored_operation_is_representable(&delete));
24006 assert!(!SyncClient::stored_operation_is_representable(&with(
24007 &delete,
24008 json!(true)
24009 )));
24010 }
24011}