1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17
18use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
19use rusqlite::{Connection, OpenFlags, OptionalExtension};
20use serde::{Deserialize, Serialize};
21use serde_json::{json, Map, Value};
22use sha2::{Digest, Sha256};
23use ssp2::decode::WIRE_VERSION;
24use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
25use ssp2::primitives::RawJson;
26use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
27use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
28
29use crate::api::{
30 AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
31 ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsRequest,
32 ClientDiagnosticsSchema, ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError,
33 ClientLimits, CommandEffects, CommitDelivery, CommitOperation, CommitOperationOutcome,
34 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
35 ConflictRecord, CoverageSnapshot, CurrentBaseProof, CurrentBaseProofReason,
36 DiagnosticLastChange, DiagnosticLastRound, DiagnosticQueryFailure, DiagnosticRoundCounters,
37 DiagnosticSubscription, FetchedBlob, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
38 LocalDataPurgeTarget, LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation,
39 PresencePeer, PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot,
40 QueryValue, RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
41 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SubscriptionCatchup,
42 SubscriptionRequest, SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport,
43 SyncStatusSnapshot, TableChange, WindowBase, WindowChange, WindowCoverage, WindowState,
44 WindowUnitRef, CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
45 MAX_DIAGNOSTIC_QUERY_FAILURES,
46};
47use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
48#[cfg(feature = "bench-internals")]
49use crate::bench::{Phase, Recorder};
50use crate::previous_version::{
51 capture_previous_version_from_replica, first_incompatibility,
52 reconcile_previous_version_at_boot, set_local_schema_descriptor,
53 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
54 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
55};
56use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
57use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
58use crate::values::{
59 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
60 encode_sparse_row_json, full_row_values, is_canonical_bytes_envelope, json_to_column_value,
61 json_to_scope_map, normalize_values_casing, render_row_id_json, scope_map_to_json,
62 sort_scope_map,
63};
64
65const DEFAULT_ACCEPT: u8 = 0b0111;
69const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
70const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
71const ACCEPT_SQLITE: u8 = 1 << 2;
72const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
73const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
74
75fn recommended_action_of(details: &Option<Value>) -> Option<String> {
77 details
78 .as_ref()?
79 .get("recommendedAction")?
80 .as_str()
81 .map(str::to_owned)
82}
83
84fn coalesced_delete_keys(
92 frames: &[Frame],
93 mut applies: impl FnMut(&str, SubStatus, Option<i64>) -> bool,
94) -> HashSet<(i64, String, String)> {
95 let mut sections = Vec::new();
96 let mut open: Option<(usize, String, SubStatus)> = None;
97 for (index, frame) in frames.iter().enumerate() {
98 match frame {
99 Frame::SubStart { id, status, .. } => {
100 open = Some((index, id.clone(), *status));
101 }
102 Frame::SubEnd { next_cursor, .. } => {
103 if let Some((start, id, status)) = open.take() {
104 sections.push((start, index, id, status, Some(*next_cursor)));
105 }
106 }
107 _ => {}
108 }
109 }
110 type Key = (i64, String, String);
111 let mut ops: HashMap<Key, (HashSet<usize>, HashSet<usize>)> = HashMap::new();
112 let mut both_in_frame: HashSet<Key> = HashSet::new();
113 let mut frame_ordinal = 0usize;
114 for (start, end, id, status, next_cursor) in sections {
115 if !applies(&id, status, next_cursor) {
116 continue;
117 }
118 for frame in &frames[start + 1..end] {
119 let Frame::Commit {
120 commit_seq,
121 tables,
122 changes,
123 ..
124 } = frame
125 else {
126 continue;
127 };
128 frame_ordinal += 1;
129 let mut local: HashMap<(String, String), (bool, bool)> = HashMap::new();
130 for change in changes {
131 let Some(table) = tables.get(change.table_index as usize) else {
132 continue;
133 };
134 let key = (*commit_seq, table.clone(), change.row_id.clone());
135 let entry = ops.entry(key).or_default();
136 let local_entry = local
137 .entry((table.clone(), change.row_id.clone()))
138 .or_default();
139 match change.op {
140 Op::Upsert => {
141 entry.0.insert(frame_ordinal);
142 local_entry.0 = true;
143 }
144 Op::Delete => {
145 entry.1.insert(frame_ordinal);
146 local_entry.1 = true;
147 }
148 }
149 }
150 for ((table, row_id), (has_upsert, has_delete)) in local {
151 if has_upsert && has_delete {
152 both_in_frame.insert((*commit_seq, table, row_id));
153 }
154 }
155 }
156 }
157 ops.into_iter()
158 .filter_map(|(key, (upserts, deletes))| {
159 (!both_in_frame.contains(&key) && !upserts.is_empty() && !deletes.is_empty())
160 .then_some(key)
161 })
162 .collect()
163}
164
165pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
167const LOCAL_REVISION_KEY: &str = "localRevision";
168const CLIENT_ID_KEY: &str = "clientId";
169const LEASE_STATE_KEY: &str = "leaseState";
170const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
171pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
172const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
177const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
178const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
179const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
182const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
185const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
186const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
188 "the persisted commit carries values the current codec refuses";
189const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
191
192#[derive(Debug, Serialize, Deserialize)]
193#[serde(rename_all = "camelCase", deny_unknown_fields)]
194struct PersistedLocalDataRebootstrapReceipt {
195 version: u8,
196 retained_commits: u64,
197 reset_subscriptions: u64,
198}
199
200fn invalid_local_rebootstrap_receipt() -> String {
201 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
202}
203
204fn encode_local_rebootstrap_receipt(
205 retained_commits: usize,
206 reset_subscriptions: usize,
207) -> Result<String, String> {
208 let retained_commits =
209 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
210 let reset_subscriptions =
211 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
212 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
213 return Err(invalid_local_rebootstrap_receipt());
214 }
215 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
216 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
217 retained_commits,
218 reset_subscriptions,
219 })
220 .map_err(|_| invalid_local_rebootstrap_receipt())
221}
222
223fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
224 if value == "v1" {
226 return Ok((0, 0));
227 }
228 let receipt: PersistedLocalDataRebootstrapReceipt =
229 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
230 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
231 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
232 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
233 {
234 return Err(invalid_local_rebootstrap_receipt());
235 }
236 Ok((
237 usize::try_from(receipt.retained_commits)
238 .map_err(|_| invalid_local_rebootstrap_receipt())?,
239 usize::try_from(receipt.reset_subscriptions)
240 .map_err(|_| invalid_local_rebootstrap_receipt())?,
241 ))
242}
243
244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
245enum SubState {
246 Active,
247 Revoked,
248 Failed,
249}
250
251#[cfg(test)]
252mod observation_tests {
253 use super::*;
254 use crate::native_transport::HostTransport;
255 use crate::values::encode_row_json;
256 use crate::BlobUploadGrant;
257 use serde_json::json;
258
259 #[test]
260 fn authority_policy_refuses_encrypted_and_undeclared_columns_before_open() {
261 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
262 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "secret", "type": "bytes", "encrypted": true, "declaredType": "string", "nullable": true}
263 ]}]});
264 for name in [
265 "secret",
266 "missing",
267 "*",
268 "id FROM authority; DELETE FROM authority",
269 "_syncular_version",
270 ] {
271 let limits = ClientLimits {
272 authority_reads: vec![crate::api::AuthorityReadDeclaration {
273 table: "authority".into(),
274 columns: vec!["id".into(), "actor_id".into(), name.into()],
275 scopes: [("actor_id".into(), vec!["a".into()])].into(),
276 }],
277 ..Default::default()
278 };
279 let error = SyncClient::new("authority".into(), &schema, limits)
280 .err()
281 .expect("forbidden policy");
282 assert!(
283 error.starts_with("client.authority_read_forbidden:"),
284 "{error}"
285 );
286 }
287 }
288
289 #[test]
290 fn authority_snapshot_observes_one_file_transaction_during_concurrent_writes() {
291 let path = std::env::temp_dir().join(format!(
292 "syncular-authority-snapshot-{}.db",
293 uuid::Uuid::new_v4()
294 ));
295 let schema = json!({"version": 1, "tables": [{"name": "authority", "primaryKey": "id", "scopes": [{"pattern": "actor:{actor_id}"}], "columns": [
296 {"name": "id", "type": "string", "nullable": false}, {"name": "actor_id", "type": "string", "nullable": false}, {"name": "revision", "type": "integer", "nullable": false}
297 ]}]});
298 let limits = ClientLimits {
299 authority_reads: vec![crate::api::AuthorityReadDeclaration {
300 table: "authority".into(),
301 columns: vec!["id".into(), "actor_id".into(), "revision".into()],
302 scopes: [("actor_id".into(), vec!["a".into()])].into(),
303 }],
304 ..Default::default()
305 };
306 let mut client =
307 SyncClient::open_path("authority".into(), &schema, limits, path.to_str().unwrap())
308 .unwrap();
309 client.begin_security_preflight();
310 let update = |conn: &Connection, revision: i64| {
311 conn.execute_batch("BEGIN IMMEDIATE").unwrap();
312 conn.execute(
313 "INSERT OR REPLACE INTO _syncular_base_authority VALUES('one','a',?1,1)",
314 [revision],
315 )
316 .unwrap();
317 conn.execute(
318 "UPDATE _syncular_meta SET value=?1 WHERE key='localRevision'",
319 [revision.to_string()],
320 )
321 .unwrap();
322 conn.execute("INSERT OR REPLACE INTO _syncular_subscriptions VALUES('s','authority',?1)", [json!({"requested": {"actor_id": ["a"]}, "effectiveScopes": {"actor_id": ["a"]}, "status": "active", "cursor": revision}).to_string()]).unwrap();
323 conn.execute_batch("COMMIT").unwrap();
324 };
325 update(&client.conn, 1);
326 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
327 let writer_barrier = barrier.clone();
328 let writer_path = path.clone();
329 let writer = std::thread::spawn(move || {
330 let conn = Connection::open(writer_path).unwrap();
331 conn.busy_timeout(std::time::Duration::from_secs(5))
332 .unwrap();
333 writer_barrier.wait();
334 for revision in 2..=100 {
335 update(&conn, revision);
336 }
337 });
338 barrier.wait();
339 for _ in 0..100 {
340 let snapshot = client.authority_snapshot().unwrap();
341 let revision = snapshot["revision"]
342 .as_str()
343 .unwrap()
344 .parse::<i64>()
345 .unwrap();
346 assert_eq!(
347 snapshot["tables"][0]["rows"][0]["values"]["revision"],
348 revision
349 );
350 assert_eq!(snapshot["tables"][0]["persisted"][0]["cursor"], revision);
351 assert_eq!(snapshot["complete"], true);
352 }
353 writer.join().unwrap();
354 assert_eq!(client.authority_snapshot().unwrap()["revision"], "100");
355 drop(client);
356 let _ = std::fs::remove_file(path);
357 }
358
359 fn client() -> SyncClient {
360 SyncClient::new(
361 "retry-test".to_owned(),
362 &json!({
363 "version": 1,
364 "tables": [{
365 "name": "tasks",
366 "primaryKey": "id",
367 "columns": [
368 { "name": "id", "type": "string", "nullable": false },
369 { "name": "project_id", "type": "string", "nullable": false }
370 ],
371 "scopes": [{ "pattern": "project:{project_id}" }]
372 }]
373 }),
374 ClientLimits::default(),
375 )
376 .expect("test client")
377 }
378
379 fn round_error(outcome: SyncOutcome) -> TransportError {
380 match outcome {
381 SyncOutcome::Failed {
382 error_code,
383 message,
384 details,
385 ..
386 } => TransportError {
387 code: error_code,
388 message,
389 details,
390 },
391 other => TransportError::new(
392 "client.failed",
393 format!("unexpected round outcome: {other:?}"),
394 ),
395 }
396 }
397
398 fn flush_blob_uploads(
401 client: &mut SyncClient,
402 transport: &mut dyn Transport,
403 ) -> Result<(), TransportError> {
404 let mut prepared = match client.prepare_sync_round(false) {
405 Ok(prepared) => prepared,
406 Err(outcome) => return Err(round_error(*outcome)),
407 };
408 loop {
409 if prepared.uploads.is_empty() {
410 client.cancel_sync_round();
411 return Ok(());
412 }
413 let completed = crate::round::test_support::drive_round(prepared, transport);
414 prepared = match client.apply_sync_round(completed) {
415 crate::AppliedSyncRound::Continue(prepared) => prepared,
416 crate::AppliedSyncRound::Complete { outcome, .. } => {
417 return match outcome {
418 SyncOutcome::Ok(_) => Ok(()),
419 other => Err(round_error(other)),
420 };
421 }
422 };
423 }
424 }
425
426 #[test]
427 fn subscribe_many_registers_all_or_nothing() {
428 let request = |id: &str, project: &str| SubscriptionRequest {
429 id: id.to_owned(),
430 table: "tasks".to_owned(),
431 scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
432 params: None,
433 };
434 let mut client = client();
435 client
436 .subscribe_many(vec![request("a", "p1"), request("b", "p2")])
437 .unwrap();
438 let persisted = |client: &SyncClient| -> i64 {
439 client
440 .conn
441 .query_row("SELECT count(*) FROM _syncular_subscriptions", [], |row| {
442 row.get(0)
443 })
444 .unwrap()
445 };
446 assert_eq!(persisted(&client), 2);
447 let error = client
449 .subscribe_many(vec![request("c", "p3"), request("b", "p9")])
450 .unwrap_err();
451 assert!(error.starts_with("client.subscription_intent_mismatch:"));
452 assert!(client.subscription_state("c").is_none());
453 assert_eq!(persisted(&client), 2);
454 assert!(client.conn.is_autocommit());
455 }
456
457 #[cfg(feature = "e2ee")]
458 #[test]
459 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
460 let schema = json!({
461 "version": 1,
462 "tables": [{
463 "name": "secrets",
464 "primaryKey": "id",
465 "columns": [
466 { "name": "id", "type": "string", "nullable": false },
467 { "name": "encryption_key_id", "type": "string", "nullable": true },
468 { "name": "note", "type": "bytes", "nullable": true,
469 "encrypted": true, "declaredType": "string" }
470 ],
471 "scopes": []
472 }]
473 });
474 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
475 .expect("test client");
476 let mut config = crate::values::EncryptionConfig::default();
477 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
478 config
479 .key_id_columns
480 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
481 client.set_encryption(config);
482 client
485 .write_base_row(
486 "secrets",
487 &vec![Some(ColumnValue::String("ghost".into())), None, None],
488 1,
489 )
490 .unwrap();
491 client.rebuild_overlay().expect("rebuild overlay");
492 let commit_id = client
493 .patch(
494 "secrets",
495 "ghost",
496 Map::from_iter([("note".to_owned(), json!("x"))]),
497 None,
498 )
499 .expect("patch records without an author-time encrypt failure");
500 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
501 assert!(client.drop_unencodable_outbox(true, None).unwrap());
503 assert!(client.pending_commit_ids().is_empty());
504 let rejections = client.rejections();
505 assert_eq!(rejections.len(), 1);
506 assert_eq!(rejections[0].code, "client.encrypt_failed");
507 assert_eq!(rejections[0].client_commit_id, commit_id);
508 assert_eq!(
509 client
510 .commit_outcome(&commit_id)
511 .unwrap()
512 .map(|outcome| outcome.status),
513 Some(CommitOutcomeStatus::Rejected)
514 );
515 }
516
517 #[test]
518 fn patch_scope_read_preserves_sqlite_failure() {
519 let mut client = client();
520 client
521 .conn
522 .execute_batch(
523 "DROP TABLE tasks;
524 CREATE VIEW tasks AS SELECT 't1' AS id, abs(-9223372036854775808) AS project_id;",
525 )
526 .unwrap();
527 let error = client
528 .patch(
529 "tasks",
530 "t1",
531 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
532 None,
533 )
534 .expect_err("the scope read must fail");
535 assert_ne!(error.code, "sync.invalid_request");
536 assert_eq!(error.code, "client.failed");
537 assert!(error.details.as_ref().unwrap()["legacyCause"]
538 .as_str()
539 .unwrap()
540 .contains("integer overflow"));
541 assert!(client.pending_commit_ids().is_empty());
542 assert!(client.conn.is_autocommit());
543 }
544
545 #[test]
546 fn patch_with_a_scope_column_matches_the_stored_local_row() {
547 let mut client = client();
548 client.create_synced_tables().unwrap();
549 client
550 .conn
551 .execute(
552 "INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
553 [],
554 )
555 .unwrap();
556
557 client
560 .patch(
561 "tasks",
562 "t1",
563 Map::from_iter([
564 ("project_id".to_owned(), json!("p1")),
565 ("id".to_owned(), json!("t1")),
566 ]),
567 None,
568 )
569 .expect("an equal scope column is a no-op");
570 let values = client.outbox[0].ops[0]
571 .values
572 .clone()
573 .expect("the patch records values");
574 assert_eq!(values.get("id"), Some(&json!("t1")));
575 assert!(!values.contains_key("project_id"));
576
577 let differing = client.patch(
579 "tasks",
580 "t1",
581 Map::from_iter([("project_id".to_owned(), json!("p2"))]),
582 None,
583 );
584 let differing = differing.expect_err("a differing scope column is rejected");
585 assert_eq!(differing.code, "sync.invalid_request");
586 assert_eq!(differing.message, "the authoring request is invalid");
587 assert!(
588 differing
589 .details
590 .as_ref()
591 .and_then(|details| details["legacyCause"].as_str())
592 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
593 "{differing:?}"
594 );
595
596 let absent = client.patch(
599 "tasks",
600 "ghost",
601 Map::from_iter([("project_id".to_owned(), json!("p1"))]),
602 None,
603 );
604 let absent = absent.expect_err("an absent local row is rejected");
605 assert_eq!(absent.code, "sync.row_missing");
606 assert!(absent.message.contains("requires a local row"));
607 assert_eq!(client.pending_commit_ids().len(), 1);
608 }
609
610 #[test]
611 fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
612 let schema = json!({
613 "version": 1,
614 "tables": [
615 {
616 "name": "tenants",
617 "primaryKey": "tenant_id",
618 "columns": [
619 { "name": "tenant_id", "type": "string", "nullable": false },
620 { "name": "body", "type": "string", "nullable": false }
621 ],
622 "scopes": [{ "pattern": "tenant:{tenant_id}" }]
623 },
624 {
625 "name": "buckets",
626 "primaryKey": "id",
627 "columns": [
628 { "name": "id", "type": "string", "nullable": false },
629 { "name": "bucket", "type": "float", "nullable": false },
630 { "name": "body", "type": "string", "nullable": false }
631 ],
632 "scopes": [{ "pattern": "bucket:{bucket}" }]
633 }
634 ]
635 });
636 let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
637 .expect("test client");
638 client.create_synced_tables().unwrap();
639
640 client
642 .conn
643 .execute(
644 "INSERT INTO tenants (tenant_id, body, _syncular_version) VALUES ('t1', 'seed', 1)",
645 [],
646 )
647 .unwrap();
648 client
649 .patch(
650 "tenants",
651 "t1",
652 Map::from_iter([
653 ("tenant_id".to_owned(), json!("t1")),
654 ("body".to_owned(), json!("created")),
655 ]),
656 None,
657 )
658 .expect("the primary-key scope column is proven equal by construction");
659
660 client
662 .conn
663 .execute(
664 "INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
665 [],
666 )
667 .unwrap();
668 client
669 .patch(
670 "buckets",
671 "b1",
672 Map::from_iter([
673 ("bucket".to_owned(), json!(2)),
674 ("body".to_owned(), json!("coerced")),
675 ]),
676 None,
677 )
678 .expect("the stored REAL value coerces from a supplied integer");
679 let values = client.outbox[1].ops[0]
680 .values
681 .clone()
682 .expect("the patch records values");
683 assert!(!values.contains_key("bucket"));
684
685 let differing = client.patch(
687 "buckets",
688 "b1",
689 Map::from_iter([("bucket".to_owned(), json!(3))]),
690 None,
691 );
692 let differing = differing.expect_err("a differing scope value is rejected");
693 assert_eq!(differing.code, "sync.invalid_request");
694 assert_eq!(differing.message, "the authoring request is invalid");
695 assert!(
696 differing
697 .details
698 .as_ref()
699 .and_then(|details| details["legacyCause"].as_str())
700 .is_some_and(|cause| cause.contains("patch cannot write scope column")),
701 "{differing:?}"
702 );
703 }
704
705 #[cfg(feature = "e2ee")]
706 #[test]
707 fn stored_key_fallback_resolves_an_integer_primary_key() {
708 let schema = json!({
709 "version": 1,
710 "tables": [{
711 "name": "patients",
712 "primaryKey": "id",
713 "columns": [
714 { "name": "id", "type": "integer", "nullable": false },
715 { "name": "encryption_key_id", "type": "string", "nullable": true },
716 { "name": "note", "type": "bytes", "nullable": true,
717 "encrypted": true, "declaredType": "string" }
718 ],
719 "scopes": []
720 }]
721 });
722 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
723 .expect("test client");
724 client.create_synced_tables().unwrap();
725 client
726 .conn
727 .execute(
728 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
729 VALUES (5, 'k1', NULL, 1)",
730 [],
731 )
732 .unwrap();
733 let mut config = crate::values::EncryptionConfig::default();
734 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
735 config
736 .key_id_columns
737 .insert("patients".to_owned(), "encryption_key_id".to_owned());
738 client.set_encryption(config);
739 let table = client.schema.table("patients").unwrap();
740 let fallback = client
741 .stored_key_fallback(table, "5")
742 .expect("the stored integer key resolves the fallback");
743 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
744 }
745
746 #[test]
747 fn patch_types_an_integer_primary_key_from_its_canonical_row_id() {
748 let schema = json!({
749 "version": 1,
750 "tables": [{
751 "name": "counters",
752 "primaryKey": "id",
753 "columns": [
754 { "name": "id", "type": "integer", "nullable": false },
755 { "name": "title", "type": "string", "nullable": false }
756 ],
757 "scopes": []
758 }]
759 });
760 let mut client = SyncClient::new("int-patch".into(), &schema, ClientLimits::default())
761 .expect("test client");
762 client
763 .write_base_row(
764 "counters",
765 &vec![
766 Some(ColumnValue::Integer(42)),
767 Some(ColumnValue::String("before".into())),
768 ],
769 1,
770 )
771 .unwrap();
772 client.rebuild_overlay().expect("rebuild overlay");
773 client
774 .patch(
775 "counters",
776 "42",
777 Map::from_iter([
778 ("id".to_owned(), json!(42)),
779 ("title".to_owned(), json!("after")),
780 ]),
781 None,
782 )
783 .expect("an integer key that renders as the rowId is not a change");
784 let rows = client.query("SELECT id, title FROM counters", &[]).unwrap();
785 assert_eq!(Value::from(rows), json!([{"id": 42, "title": "after"}]));
786 for row_id in ["042", "+42", "42.0"] {
787 let error = client
788 .patch(
789 "counters",
790 row_id,
791 Map::from_iter([("title".to_owned(), json!("x"))]),
792 None,
793 )
794 .expect_err("a non-canonical integer rowId names no row");
795 assert_eq!(error.code, "sync.row_missing", "{row_id}");
796 }
797 }
798
799 #[test]
800 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
801 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
802 {"name":"id","type":"string","nullable":false},
803 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
804 for state in ["absent", "cached", "pinned"] {
805 let path = std::env::temp_dir().join(format!(
806 "syncular-blob-stage-{}.sqlite",
807 uuid::Uuid::new_v4()
808 ));
809 let mut client = SyncClient::open_path(
810 "stage-test".into(),
811 &schema,
812 Default::default(),
813 path.to_str().unwrap(),
814 )
815 .unwrap();
816 let bytes = b"atomic stage";
817 if state != "absent" {
818 client
819 .upload_blob(bytes, Some("text/plain".into()), None)
820 .unwrap();
821 if state == "cached" {
822 client
823 .conn
824 .execute("DELETE FROM _syncular_blob_uploads", [])
825 .unwrap();
826 }
827 }
828 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
829 client
830 .conn
831 .execute_batch(
832 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
833 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
834 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
835 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
836 )
837 .unwrap();
838 let error = client
839 .upload_blob(bytes, Some("application/octet-stream".into()), None)
840 .unwrap_err();
841 assert!(error.contains("injected stage failure"), "{state}: {error}");
842 assert!(client.conn.is_autocommit());
843 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
844 drop(client);
845 let mut reopened = SyncClient::open_path(
846 "stage-test".into(),
847 &schema,
848 Default::default(),
849 path.to_str().unwrap(),
850 )
851 .unwrap();
852 let durable = reopened
853 .query("SELECT * FROM _syncular_blobs", &[])
854 .unwrap();
855 reopened
856 .conn
857 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
858 .unwrap();
859 let retried = reopened.upload_blob(bytes, None, None).unwrap();
860 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
861 assert_eq!(
862 reopened
863 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
864 .unwrap(),
865 vec![Map::from_iter([(
866 "blob_id".into(),
867 Value::from(blob_id_for(bytes)),
868 )])]
869 );
870 drop(reopened);
871 std::fs::remove_file(&path).unwrap();
872 assert_eq!(durable, bodies, "{state}: reopened body state");
873 assert_eq!(immediate, bodies, "{state}: immediate body state");
874 }
875 }
876
877 #[test]
878 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
879 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
880 {"name":"id","type":"string","nullable":false},
881 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
882 for fault in [
883 "missing",
884 "body-type",
885 "length",
886 "hash",
887 "pin-metadata",
888 "read",
889 "pin-delete",
890 ] {
891 let path = std::env::temp_dir().join(format!(
892 "syncular-blob-upload-{}.sqlite",
893 uuid::Uuid::new_v4()
894 ));
895 let mut client = SyncClient::open_path(
896 "upload-test".into(),
897 &schema,
898 Default::default(),
899 path.to_str().unwrap(),
900 )
901 .unwrap();
902 let bytes = b"durable upload";
903 let ref_value = client
904 .upload_blob(bytes, Some("text/plain".into()), None)
905 .unwrap();
906 let commit = client
907 .mutate(vec![Mutation::Upsert {
908 table: "attachments".into(),
909 values: Map::from_iter([
910 ("id".into(), Value::from("pending")),
911 (
912 "file".into(),
913 Value::from(format!(
914 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
915 blob_id_for(bytes),
916 bytes.len()
917 )),
918 ),
919 ]),
920 base_version: None,
921 }])
922 .unwrap();
923 client
924 .conn
925 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
926 .unwrap();
927 client.conn.execute_batch(match fault {
928 "missing" => "DELETE FROM _syncular_blobs",
929 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
930 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
931 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
932 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
933 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
934 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
935 }).unwrap();
936 let state = if fault == "read" {
937 Vec::new()
938 } else {
939 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
940 };
941 client.set_meta(LOG_EPOCH_KEY, "epoch");
942 let mut transport = CountingRealtimeTransport::default();
943 let error = flush_blob_uploads(&mut client, &mut transport).unwrap_err();
944 assert_eq!(
945 error.code,
946 if matches!(fault, "read" | "pin-delete") {
947 "client.failed"
948 } else {
949 "sync.local_corrupt"
950 },
951 "{fault}"
952 );
953 assert_eq!(
954 transport.blob_uploads.len(),
955 usize::from(fault == "pin-delete"),
956 "{fault}: corrupt bytes must never be uploaded"
957 );
958 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
959 if fault != "read" {
960 assert_eq!(
961 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
962 state
963 );
964 } else {
965 client
966 .conn
967 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
968 .unwrap();
969 }
970 drop(client);
971 let mut reopened = SyncClient::open_path(
972 "upload-test".into(),
973 &schema,
974 Default::default(),
975 path.to_str().unwrap(),
976 )
977 .unwrap();
978 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
979 if fault != "read" {
980 assert_eq!(
981 reopened
982 .query("SELECT * FROM _syncular_blobs", &[])
983 .unwrap(),
984 state
985 );
986 }
987 if fault == "pin-delete" {
988 reopened
989 .conn
990 .execute_batch("DROP TRIGGER fail_pin_delete")
991 .unwrap();
992 }
993 if fault == "pin-metadata" {
994 reopened
995 .conn
996 .execute(
997 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
998 [],
999 )
1000 .unwrap();
1001 }
1002 reopened
1003 .conn
1004 .execute_batch(
1005 "DELETE FROM _syncular_blobs;
1006 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
1007 )
1008 .unwrap();
1009 transport.blob_uploads.clear();
1010 flush_blob_uploads(&mut reopened, &mut transport).unwrap();
1011 assert_eq!(
1012 transport.blob_uploads,
1013 [(blob_id_for(bytes), bytes.to_vec())]
1014 );
1015 assert_eq!(
1016 reopened
1017 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
1018 .unwrap(),
1019 Vec::<Map<String, Value>>::new()
1020 );
1021 assert_eq!(reopened.pending_commit_ids(), [commit]);
1022 assert_eq!(
1023 reopened
1024 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
1025 .unwrap()
1026 .unwrap()
1027 .bytes,
1028 bytes
1029 );
1030 drop(reopened);
1031 std::fs::remove_file(path).unwrap();
1032 }
1033 }
1034
1035 fn coalesced_fixture(sections: usize, commits: usize, rows: usize) -> (SyncClient, Message) {
1039 let mut client = client();
1040 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1041 let mut response = Message {
1042 wire_version: WIRE_VERSION,
1043 msg_kind: MsgKind::Response,
1044 frames: vec![Frame::RespHeader {
1045 required_schema_version: None,
1046 latest_schema_version: None,
1047 log_epoch: Some("epoch-1".into()),
1048 reset_required: Some(false),
1049 }],
1050 };
1051 let mut seq = 0;
1052 for section in 0..sections {
1053 let project = format!("p{section}");
1054 let scopes = vec![("project_id".to_owned(), vec![project.clone()])];
1055 client
1056 .subscribe(format!("s{section}"), "tasks".into(), scopes.clone(), None)
1057 .unwrap();
1058 let sub = client.subs.last_mut().unwrap();
1059 sub.cursor = 0;
1060 sub.synced_once = true;
1061 sub.effective = Some(scopes.clone());
1062 let sub = sub.clone();
1063 client.persist_sub(&sub).unwrap();
1064 response.frames.push(Frame::SubStart {
1065 id: format!("s{section}"),
1066 status: SubStatus::Active,
1067 reason_code: "".into(),
1068 effective_scopes: scopes,
1069 bootstrap: false,
1070 });
1071 for _ in 0..commits {
1072 seq += 1;
1073 let changes = (0..rows)
1074 .map(|row| {
1075 let id = format!("{project}-{seq}-{row}");
1076 let values = Map::from_iter([
1077 ("id".into(), Value::from(id.clone())),
1078 ("project_id".into(), Value::from(project.clone())),
1079 ]);
1080 ssp2::model::Change {
1081 table_index: 0,
1082 row_id: id.clone(),
1083 op: Op::Upsert,
1084 row_version: Some(seq),
1085 scopes: vec![("project_id".into(), project.clone())],
1086 row: Some(
1087 encode_row_json(
1088 client.schema.table("tasks").unwrap(),
1089 &id,
1090 &values,
1091 &client.encryption,
1092 )
1093 .unwrap(),
1094 ),
1095 }
1096 })
1097 .collect();
1098 response.frames.push(Frame::Commit {
1099 commit_seq: seq,
1100 actor_id: "writer".into(),
1101 created_at_ms: 1,
1102 tables: vec!["tasks".into()],
1103 changes,
1104 });
1105 }
1106 response.frames.push(Frame::SubEnd {
1107 next_cursor: seq,
1108 bootstrap_state: None,
1109 });
1110 }
1111 client.drain_change_batches();
1112 (client, response)
1113 }
1114
1115 fn count_commits(client: &mut SyncClient) -> std::sync::Arc<std::sync::atomic::AtomicUsize> {
1116 let commits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
1117 let counter = commits.clone();
1118 client
1119 .conn
1120 .commit_hook(Some(move || {
1121 counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1122 false
1123 }))
1124 .expect("commit hook");
1125 commits
1126 }
1127
1128 #[test]
1129 fn a_response_of_many_small_sections_commits_once() {
1130 for realtime in [false, true] {
1131 let (mut client, response) = coalesced_fixture(70, 1, 1);
1132 client.realtime_state = RealtimeState::Connected;
1133 client.realtime_reason_code = None;
1134 let mut transport = CountingRealtimeTransport::default();
1135 let (_, meta) = client.build_request(false).unwrap();
1136 let commits = count_commits(&mut client);
1137 if realtime {
1138 client.on_realtime_binary(&mut transport, &encode_message(&response));
1139 } else {
1140 assert!(matches!(
1141 client.process_response(&mut transport, response, &meta),
1142 SyncOutcome::Ok(_)
1143 ));
1144 }
1145 assert_eq!(
1146 commits.load(std::sync::atomic::Ordering::SeqCst),
1147 1,
1148 "{realtime}"
1149 );
1150 assert!(client.conn.is_autocommit());
1151 assert!(client.subs.iter().all(|sub| sub.cursor > 0));
1152 let revisions = client
1154 .drain_change_batches()
1155 .into_iter()
1156 .filter(|batch| !batch.tables.is_empty())
1157 .map(|batch| batch.revision)
1158 .collect::<Vec<_>>();
1159 assert_eq!(
1160 revisions,
1161 (1..=70).map(|r: u64| r.to_string()).collect::<Vec<_>>()
1162 );
1163 assert_eq!(client.query("SELECT id FROM tasks", &[]).unwrap().len(), 70);
1164 assert_eq!(
1166 transport
1167 .messages
1168 .iter()
1169 .filter(|m| m.contains("\"ack\""))
1170 .count(),
1171 1,
1172 "{realtime}"
1173 );
1174 }
1175 }
1176
1177 #[test]
1178 fn a_response_past_the_row_bound_commits_per_bound() {
1179 let (mut client, response) = coalesced_fixture(1, 3, 600);
1181 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1182 let (_, meta) = client.build_request(false).unwrap();
1183 let commits = count_commits(&mut client);
1184 assert!(matches!(
1185 client.process_response(&mut transport, response, &meta),
1186 SyncOutcome::Ok(_)
1187 ));
1188 assert_eq!(commits.load(std::sync::atomic::Ordering::SeqCst), 3);
1189 assert_eq!(
1190 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1191 1800
1192 );
1193 assert_eq!(client.subs[0].cursor, 3);
1194 }
1195
1196 #[test]
1197 fn a_failed_coalesced_commit_restores_memory_and_publishes_nothing() {
1198 let (mut client, response) = coalesced_fixture(2, 2, 600);
1202 client
1203 .conn
1204 .execute_batch(
1205 "PRAGMA foreign_keys = ON; CREATE TABLE fail_parent(id INTEGER PRIMARY KEY); CREATE TABLE fail_child(id INTEGER REFERENCES fail_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_commit AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'p0-1-0' BEGIN INSERT INTO fail_child VALUES(1); END",
1206 )
1207 .unwrap();
1208 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
1209 let (_, meta) = client.build_request(false).unwrap();
1210 let subs = client.subs.clone();
1211 let revision = client.local_revision();
1212 let outcome = client.process_response(&mut transport, response.clone(), &meta);
1213 assert!(
1214 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == "storage.failed"),
1215 "{outcome:?}"
1216 );
1217 assert!(client.conn.is_autocommit());
1218 assert!(client.coalesced.is_none() && client.coalesced_failure.is_none());
1219 assert_eq!(client.local_revision(), revision);
1220 assert!(client
1221 .drain_change_batches()
1222 .iter()
1223 .all(|batch| batch.tables.is_empty()));
1224 assert!(client
1225 .query("SELECT id FROM tasks", &[])
1226 .unwrap()
1227 .is_empty());
1228 assert_eq!(
1229 client.subs.iter().map(|s| s.cursor).collect::<Vec<_>>(),
1230 subs.iter().map(|s| s.cursor).collect::<Vec<_>>()
1231 );
1232 client
1234 .conn
1235 .execute_batch("DROP TRIGGER fail_commit")
1236 .unwrap();
1237 let (_, meta) = client.build_request(false).unwrap();
1238 assert!(matches!(
1239 client.process_response(&mut transport, response, &meta),
1240 SyncOutcome::Ok(_)
1241 ));
1242 assert_eq!(
1243 client.query("SELECT id FROM tasks", &[]).unwrap().len(),
1244 2400
1245 );
1246 }
1247
1248 #[test]
1249 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
1250 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
1251 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
1252 "scopes":[{"pattern":"project:{project_id}"}]}]});
1253 for realtime in [false, true] {
1254 for fault in [
1255 "none",
1256 "row",
1257 "revision",
1258 "commit",
1259 "cursor",
1260 "error",
1261 "missing-end",
1262 ] {
1263 if realtime && fault == "missing-end" {
1264 continue;
1265 }
1266 let path = std::env::temp_dir()
1267 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
1268 let mut client = SyncClient::open_path(
1269 "frame-test".into(),
1270 &schema,
1271 Default::default(),
1272 path.to_str().unwrap(),
1273 )
1274 .unwrap();
1275 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1276 client
1277 .subscribe(
1278 "tasks".into(),
1279 "tasks".into(),
1280 vec![("project_id".into(), vec!["p1".into()])],
1281 None,
1282 )
1283 .unwrap();
1284 client.subs[0].cursor = 0;
1285 client.subs[0].synced_once = true;
1286 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1287 client.persist_sub(&client.subs[0]).unwrap();
1288 client.realtime_state = RealtimeState::Connected;
1289 client.realtime_reason_code = None;
1290 client.drain_change_batches();
1291 let mut transport = CountingRealtimeTransport::default();
1292 let (_, meta) = client.build_request(false).unwrap();
1293 let mut response = Message {
1294 wire_version: WIRE_VERSION,
1295 msg_kind: MsgKind::Response,
1296 frames: vec![
1297 Frame::RespHeader {
1298 required_schema_version: None,
1299 latest_schema_version: None,
1300 log_epoch: Some("epoch-1".into()),
1301 reset_required: Some(false),
1302 },
1303 Frame::SubStart {
1304 id: "tasks".into(),
1305 status: SubStatus::Active,
1306 reason_code: "".into(),
1307 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1308 bootstrap: false,
1309 },
1310 ],
1311 };
1312 for (index, id) in ["first", "second"].iter().enumerate() {
1313 let values = Map::from_iter([
1314 ("id".into(), Value::from(*id)),
1315 ("project_id".into(), Value::from("p1")),
1316 ]);
1317 response.frames.push(Frame::Commit {
1318 commit_seq: index as i64 + 1,
1319 actor_id: "writer".into(),
1320 created_at_ms: 1,
1321 tables: vec!["tasks".into()],
1322 changes: vec![ssp2::model::Change {
1323 table_index: 0,
1324 row_id: (*id).into(),
1325 op: Op::Upsert,
1326 row_version: Some(index as i64 + 1),
1327 scopes: vec![("project_id".into(), "p1".into())],
1328 row: Some(
1329 encode_row_json(
1330 client.schema.table("tasks").unwrap(),
1331 id,
1332 &values,
1333 &client.encryption,
1334 )
1335 .unwrap(),
1336 ),
1337 }],
1338 });
1339 }
1340 response.frames.push(Frame::SubEnd {
1341 next_cursor: 2,
1342 bootstrap_state: None,
1343 });
1344 let valid = response.clone();
1345 match fault {
1346 "row" => {
1347 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
1348 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
1349 },
1350 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
1351 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
1352 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
1353 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
1354 "missing-end" => {response.frames.truncate(3);},
1355 _ => {}
1356 }
1357 if realtime {
1358 client.on_realtime_binary(&mut transport, &encode_message(&response));
1359 } else {
1360 assert_eq!(
1361 matches!(
1362 client.process_response(&mut transport, response, &meta),
1363 SyncOutcome::Ok(_)
1364 ),
1365 fault == "none",
1366 "{fault}"
1367 );
1368 }
1369 assert!(
1370 client.conn.is_autocommit(),
1371 "no open transaction after {realtime}/{fault}"
1372 );
1373 let count = match fault {
1377 "none" | "cursor" => 2,
1378 "commit" => 0,
1379 _ => 1,
1380 };
1381 let expected_revision = count + u64::from(realtime && fault != "none");
1382 assert_eq!(
1383 client.local_revision(),
1384 expected_revision,
1385 "{realtime}/{fault}"
1386 );
1387 let batches = client.drain_change_batches();
1388 assert_eq!(
1389 batches.len(),
1390 expected_revision as usize,
1391 "{realtime}/{fault}"
1392 );
1393 let row_batches: Vec<_> = batches
1394 .iter()
1395 .filter(|batch| !batch.tables.is_empty())
1396 .collect();
1397 assert_eq!(row_batches.len(), count as usize);
1398 assert_eq!(
1399 row_batches
1400 .iter()
1401 .map(|b| b.revision.clone())
1402 .collect::<Vec<_>>(),
1403 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
1404 );
1405 for table in ["tasks", "_syncular_base_tasks"] {
1406 assert_eq!(
1407 client
1408 .conn
1409 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
1410 .get::<_, u64>(0))
1411 .unwrap(),
1412 count,
1413 "{realtime}/{fault}/{table}"
1414 );
1415 }
1416 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1417 assert_eq!(
1418 transport
1419 .messages
1420 .iter()
1421 .filter(|m| m.contains("\"ack\""))
1422 .count(),
1423 usize::from(fault == "none")
1424 );
1425 if matches!(fault, "revision" | "commit" | "cursor") {
1426 client
1427 .conn
1428 .execute_batch("DROP TRIGGER fail_frame")
1429 .unwrap();
1430 }
1431 drop(client);
1432 let mut reopened = SyncClient::open_path(
1433 "frame-test".into(),
1434 &schema,
1435 Default::default(),
1436 path.to_str().unwrap(),
1437 )
1438 .unwrap();
1439 assert_eq!(reopened.local_revision(), expected_revision);
1440 assert_eq!(
1441 reopened
1442 .query("SELECT id FROM tasks ORDER BY id", &[])
1443 .unwrap()
1444 .len(),
1445 count as usize
1446 );
1447 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
1448 let (_, meta) = reopened.build_request(false).unwrap();
1449 assert!(matches!(
1450 reopened.process_response(&mut transport, valid, &meta),
1451 SyncOutcome::Ok(_)
1452 ));
1453 assert_eq!(
1454 reopened
1455 .query("SELECT id FROM tasks ORDER BY id", &[])
1456 .unwrap()
1457 .len(),
1458 2
1459 );
1460 assert_eq!(reopened.subs[0].cursor, 2);
1461 drop(reopened);
1462 std::fs::remove_file(path).unwrap();
1463 }
1464 }
1465 }
1466
1467 #[test]
1468 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
1469 for status in [SubStatus::Reset, SubStatus::Revoked] {
1470 for fault in ["subscription", "revision"] {
1471 let mut client = client();
1472 client
1473 .subscribe(
1474 "tasks".into(),
1475 "tasks".into(),
1476 vec![("project_id".into(), vec!["p1".into()])],
1477 None,
1478 )
1479 .unwrap();
1480 client.subs[0].cursor = 41;
1481 client.subs[0].synced_once = true;
1482 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1483 client.persist_sub(&client.subs[0]).unwrap();
1484 client
1485 .mutate(vec![Mutation::Upsert {
1486 table: "tasks".into(),
1487 values: Map::from_iter([
1488 ("id".into(), Value::from("pending")),
1489 ("project_id".into(), Value::from("p1")),
1490 ]),
1491 base_version: None,
1492 }])
1493 .unwrap();
1494 let ids = client.pending_commit_ids();
1495 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
1496 let revision = client.local_revision();
1497 client.drain_change_batches();
1498 let (_, meta) = client.build_request(false).unwrap();
1499 client.conn.execute_batch(if fault == "subscription" {
1500 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
1501 } else {
1502 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
1503 }).unwrap();
1504 let mut transport = CountingRealtimeTransport::default();
1505 let mut report = SyncReport::default();
1506 if status == SubStatus::Reset {
1508 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
1509 }
1510 let result = client.process_section(
1511 &mut transport,
1512 "tasks",
1513 status,
1514 "sync.scope_revoked",
1515 vec![],
1516 vec![],
1517 Some((42, None)),
1518 &meta,
1519 &mut report,
1520 &HashSet::new(),
1521 );
1522 assert!(result.is_err(), "{status:?}/{fault}");
1523 assert!(client.conn.is_autocommit());
1524 assert_eq!(client.subs[0].cursor, 41);
1525 assert_eq!(client.subs[0].state, SubState::Active);
1526 assert_eq!(client.pending_commit_ids(), ids);
1527 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
1528 assert!(client.rejections.is_empty());
1529 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
1530 assert_eq!(client.local_revision(), revision);
1531 assert!(client.drain_change_batches().is_empty());
1532 assert!(report.revoked.is_empty() && report.resets.is_empty());
1533 }
1534 }
1535 }
1536
1537 #[test]
1538 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
1539 let mut client = client();
1540 client
1541 .subscribe(
1542 "tasks".into(),
1543 "tasks".into(),
1544 vec![("project_id".into(), vec!["p1".into()])],
1545 None,
1546 )
1547 .unwrap();
1548 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1549 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
1550 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
1551 let row = |id: &str| ssp2::segment::SegmentRow {
1552 server_version: 1,
1553 values: vec![
1554 Some(ColumnValue::String(id.into())),
1555 Some(ColumnValue::String("p1".into())),
1556 ],
1557 };
1558 let segment = RowsSegment {
1559 table: "tasks".into(),
1560 schema_version: 1,
1561 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
1562 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
1563 };
1564 assert!(client.apply_segment(0, &segment, true).is_err());
1565 assert!(client.conn.is_autocommit());
1566 assert_eq!(
1567 client
1568 .query("SELECT id FROM tasks ORDER BY id", &[])
1569 .unwrap(),
1570 vec![Map::from_iter([("id".into(), Value::from("first"))])]
1571 );
1572 assert_eq!(client.local_revision(), 1);
1573 assert_eq!(client.drain_change_batches().len(), 1);
1574 assert_eq!(client.subs[0].cursor, -1);
1575 client
1576 .conn
1577 .execute_batch("DROP TRIGGER fail_block")
1578 .unwrap();
1579 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
1580 assert_eq!(client.local_revision(), 3);
1581 assert_eq!(client.drain_change_batches().len(), 2);
1582 }
1583
1584 struct SegmentBatchTransport {
1588 response: Message,
1589 bodies: BTreeMap<String, Vec<u8>>,
1590 batches: Vec<Vec<String>>,
1591 single_fetches: usize,
1592 }
1593
1594 impl Transport for SegmentBatchTransport {
1595 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1596 Ok(encode_message(&self.response))
1597 }
1598
1599 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1600 self.sync(request)
1601 }
1602
1603 fn download_segment(
1604 &mut self,
1605 _request: &SegmentRequest,
1606 _on_progress: &mut dyn FnMut(u64),
1607 ) -> Result<Vec<u8>, TransportError> {
1608 self.single_fetches += 1;
1609 Err(TransportError::new(
1610 "sync.transport_failed",
1611 "per-descriptor fetch",
1612 ))
1613 }
1614
1615 fn download_segments(
1616 &mut self,
1617 fetches: &[crate::SegmentFetch],
1618 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1619 ) -> Vec<Result<Vec<u8>, TransportError>> {
1620 self.batches.push(
1621 fetches
1622 .iter()
1623 .map(|fetch| match fetch {
1624 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
1625 crate::SegmentFetch::Url(url) => url.clone(),
1626 })
1627 .collect(),
1628 );
1629 fetches
1630 .iter()
1631 .enumerate()
1632 .map(|(index, fetch)| {
1633 let crate::SegmentFetch::Direct(request) = fetch else {
1634 panic!("the descriptors are direct-endpoint requests")
1635 };
1636 let bytes = self.bodies[&request.segment_id].clone();
1637 on_progress(index, bytes.len() as u64);
1638 Ok(bytes)
1639 })
1640 .collect()
1641 }
1642
1643 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1644 Ok(())
1645 }
1646
1647 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1648 Ok(())
1649 }
1650
1651 fn realtime_close(&mut self) -> Result<(), TransportError> {
1652 Ok(())
1653 }
1654 }
1655
1656 #[test]
1657 fn a_response_fetches_all_its_segment_pages_in_one_ordered_batch() {
1658 let mut client = client();
1659 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1660 client
1661 .subscribe(
1662 "tasks".into(),
1663 "tasks".into(),
1664 vec![("project_id".into(), vec!["p1".into()])],
1665 None,
1666 )
1667 .unwrap();
1668 client.subs[0].cursor = 0;
1669 client.subs[0].synced_once = true;
1670 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
1671 client.persist_sub(&client.subs[0]).unwrap();
1672 client.drain_change_batches();
1673 let columns = client.schema.table("tasks").unwrap().wire_columns.clone();
1674
1675 let pages = ["one", "two", "three", "four"].map(|id| {
1679 let bytes = ssp2::segment::encode_rows_segment(&RowsSegment {
1680 table: "tasks".into(),
1681 schema_version: 1,
1682 columns: columns.clone(),
1683 blocks: vec![vec![ssp2::segment::SegmentRow {
1684 server_version: 1,
1685 values: vec![
1686 Some(ColumnValue::String(id.into())),
1687 Some(ColumnValue::String("p1".into())),
1688 ],
1689 }]],
1690 });
1691 (
1692 format!("sha256:{}", bytes_to_hex(&Sha256::digest(&bytes))),
1693 bytes,
1694 )
1695 });
1696
1697 let mut frames = vec![
1698 Frame::RespHeader {
1699 required_schema_version: None,
1700 latest_schema_version: None,
1701 log_epoch: Some("epoch-1".into()),
1702 reset_required: Some(false),
1703 },
1704 Frame::SubStart {
1705 id: "tasks".into(),
1706 status: SubStatus::Active,
1707 reason_code: String::new(),
1708 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
1709 bootstrap: false,
1710 },
1711 ];
1712 for (segment_id, bytes) in &pages {
1713 frames.push(Frame::SegmentRef {
1714 segment_id: segment_id.clone(),
1715 media_type: MediaType::Rows,
1716 table: "tasks".into(),
1717 byte_length: bytes.len() as i64,
1718 row_count: 1,
1719 as_of_commit_seq: 1,
1720 scope_digest: "digest".into(),
1721 row_cursor: None,
1722 next_row_cursor: None,
1723 url: None,
1724 url_expires_at_ms: None,
1725 });
1726 }
1727 frames.push(Frame::SubEnd {
1728 next_cursor: 1,
1729 bootstrap_state: None,
1730 });
1731 let mut transport = SegmentBatchTransport {
1732 response: Message {
1733 wire_version: WIRE_VERSION,
1734 msg_kind: MsgKind::Response,
1735 frames,
1736 },
1737 bodies: pages
1738 .iter()
1739 .map(|(segment_id, bytes)| (segment_id.clone(), bytes.clone()))
1740 .collect(),
1741 batches: Vec::new(),
1742 single_fetches: 0,
1743 };
1744
1745 let outcome = client.sync(&mut transport);
1746 assert!(matches!(outcome, SyncOutcome::Ok(_)), "{outcome:?}");
1747 assert_eq!(
1748 transport.batches,
1749 vec![pages
1750 .iter()
1751 .map(|(segment_id, _)| segment_id.clone())
1752 .collect::<Vec<_>>()],
1753 "one batch per response, in wire order"
1754 );
1755 assert_eq!(
1756 transport.single_fetches, 0,
1757 "the batch seam replaces the per-descriptor calls"
1758 );
1759 assert_eq!(
1760 client
1761 .query("SELECT id FROM tasks ORDER BY id", &[])
1762 .unwrap(),
1763 ["four", "one", "three", "two"]
1764 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
1765 .to_vec()
1766 );
1767 assert_eq!(client.subs[0].cursor, 1);
1768 }
1769
1770 #[test]
1771 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
1772 assert_eq!(
1773 client().diagnostics_storage().blob_cache_bytes_approx,
1774 Some(0)
1775 );
1776 let client = SyncClient::new(
1777 "diagnostic-storage".to_owned(),
1778 &json!({
1779 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
1780 "columns": [{"name": "id", "type": "string", "nullable": false},
1781 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
1782 }),
1783 ClientLimits {
1784 blob_cache_max_bytes: Some(5),
1785 ..Default::default()
1786 },
1787 )
1788 .unwrap();
1789 let empty = client.diagnostics_storage();
1790 assert_eq!(empty.status, "healthy");
1791 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
1792 assert_eq!(empty.retained_outcome_entries, Some(0));
1793 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
1794 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
1795 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
1796 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
1797 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
1798 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
1799 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
1800 let written = client.diagnostics_storage();
1801 assert_eq!(written.status, "pressure");
1802 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
1803 assert_eq!(written.retained_outcome_entries, Some(2));
1804 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
1805 assert_eq!(written.blob_cache_bytes_approx, Some(8));
1806 assert_eq!(
1807 written.pressure_reason_code.as_deref(),
1808 Some("client.blob_cache_over_limit")
1809 );
1810 client
1811 .conn
1812 .execute_batch(
1813 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
1814 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
1815 )
1816 .unwrap();
1817 let staged = client.diagnostics_storage();
1818 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
1819 assert_eq!(staged.retained_outcome_entries, Some(0));
1820 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
1821 assert_eq!(staged.status, "healthy");
1822 client.conn.execute_batch("ROLLBACK").unwrap();
1823 assert_eq!(client.diagnostics_storage(), written);
1824 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
1825 let unreadable = client.diagnostics_storage();
1826 assert_eq!(unreadable.status, "unreadable");
1827 assert!(unreadable.database_bytes_approx.is_none());
1828 assert!(unreadable.pending_outbox_bytes_approx.is_none());
1829 assert!(unreadable.retained_outcome_entries.is_none());
1830 assert!(unreadable.retained_outcome_bytes_approx.is_none());
1831 assert!(unreadable.blob_cache_bytes_approx.is_none());
1832 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
1833 let restored = client.diagnostics_storage();
1834 assert_eq!(restored.status, written.status);
1835 assert_eq!(
1836 restored.retained_outcome_bytes_approx,
1837 written.retained_outcome_bytes_approx
1838 );
1839 client.conn.set_prepared_statement_cache_capacity(0);
1840 assert_eq!(client.diagnostics_storage(), restored);
1841 }
1842
1843 struct ServerErrorTransport {
1845 code: &'static str,
1846 retryable: bool,
1847 }
1848
1849 impl Transport for ServerErrorTransport {
1850 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1851 Ok(encode_message(&Message {
1852 wire_version: WIRE_VERSION,
1853 msg_kind: MsgKind::Response,
1854 frames: vec![
1855 Frame::RespHeader {
1856 required_schema_version: None,
1857 latest_schema_version: None,
1858 log_epoch: Some("epoch-1".to_owned()),
1859 reset_required: Some(false),
1860 },
1861 Frame::Error {
1862 code: self.code.to_owned(),
1863 message: "server error".to_owned(),
1864 category: "internal".to_owned(),
1865 retryable: self.retryable,
1866 recommended_action: "retryLater".to_owned(),
1867 details: None,
1868 },
1869 ],
1870 }))
1871 }
1872
1873 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1874 self.sync(request)
1875 }
1876
1877 fn download_segment(
1878 &mut self,
1879 _request: &SegmentRequest,
1880 _on_progress: &mut dyn FnMut(u64),
1881 ) -> Result<Vec<u8>, TransportError> {
1882 Err(TransportError::new("sync.transport_failed", "offline"))
1883 }
1884
1885 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1886 Ok(())
1887 }
1888
1889 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
1890 Ok(())
1891 }
1892
1893 fn realtime_close(&mut self) -> Result<(), TransportError> {
1894 Ok(())
1895 }
1896 }
1897
1898 #[test]
1899 fn a_retryable_in_band_failure_keeps_the_socket_connected_and_still_asks_for_a_retry() {
1900 let mut client = client();
1901 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1902 let mut transport = ServerErrorTransport {
1903 code: "sync.schema_not_ready",
1904 retryable: true,
1905 };
1906 client.connect_realtime(&mut transport).expect("connect");
1907 assert!(matches!(
1908 client.sync(&mut transport),
1909 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.schema_not_ready"
1910 ));
1911 assert_eq!(
1912 client.realtime_state(),
1913 RealtimeState::Connected,
1914 "the socket delivered the answer, so it is not lost"
1915 );
1916 assert!(
1917 matches!(
1918 client.drain_sync_intents().as_slice(),
1919 [SyncIntent::Background { delay_ms: 250 }]
1920 ),
1921 "the retry deadline is a sync round and no reconnect satisfies it"
1922 );
1923 assert_eq!(
1924 client.progress().snapshot().unwrap().retry_delay_ms,
1925 Some(250)
1926 );
1927 }
1928
1929 #[test]
1930 fn retryable_server_error_schedules_a_background_retry() {
1931 let mut client = client();
1932 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1933 let mut transport = ServerErrorTransport {
1934 code: "sync.internal_error",
1935 retryable: true,
1936 };
1937 assert!(matches!(
1938 client.sync(&mut transport),
1939 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
1940 ));
1941 assert!(matches!(
1942 client.drain_sync_intents().as_slice(),
1943 [SyncIntent::Background { delay_ms: 250 }]
1944 ));
1945 assert_eq!(
1946 client.progress().snapshot().unwrap().retry_delay_ms,
1947 Some(250)
1948 );
1949 let mut transport = ServerErrorTransport {
1950 code: "sync.invalid_request",
1951 retryable: false,
1952 };
1953 assert!(matches!(
1954 client.sync(&mut transport),
1955 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
1956 ));
1957 assert!(client.drain_sync_intents().is_empty());
1958 let progress = client.progress().snapshot().unwrap();
1959 assert_eq!(progress.state, ProgressState::Failed);
1960 assert_eq!(progress.retry_delay_ms, None);
1961 }
1962
1963 #[test]
1964 fn sync_until_idle_reports_budget_exhaustion_or_failure() {
1965 let exhausted = SyncOutcome::BudgetExhausted(SyncReport::default()).to_json();
1968 assert_eq!(exhausted["ok"], json!(true));
1969 assert_eq!(exhausted["budgetExhausted"], json!(true));
1970 assert!(exhausted["report"].is_object());
1971
1972 let mut client = client();
1973 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1974 let mut idle_transport = HostTransport::new_from_config(&json!({})).unwrap();
1976 let zero = client.sync_until_idle(&mut idle_transport, Some(0));
1977 assert!(
1978 matches!(zero, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
1979 "{zero:?}"
1980 );
1981 let mut erroring = ServerErrorTransport {
1983 code: "sync.internal_error",
1984 retryable: false,
1985 };
1986 let failed = client.sync_until_idle(&mut erroring, Some(3));
1987 assert!(
1988 matches!(failed, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"),
1989 "{failed:?}"
1990 );
1991 }
1992
1993 struct CauseTransport(&'static str);
1995
1996 impl Transport for CauseTransport {
1997 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
1998 let mut error = TransportError::new("transport.failed", "no route");
1999 error.details = Some(serde_json::json!({ "causeKind": self.0 }));
2000 Err(error)
2001 }
2002
2003 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2004 self.sync(request)
2005 }
2006
2007 fn download_segment(
2008 &mut self,
2009 _request: &SegmentRequest,
2010 _on_progress: &mut dyn FnMut(u64),
2011 ) -> Result<Vec<u8>, TransportError> {
2012 Err(TransportError::new("sync.transport_failed", "offline"))
2013 }
2014
2015 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2016 Ok(())
2017 }
2018
2019 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2020 Ok(())
2021 }
2022
2023 fn realtime_close(&mut self) -> Result<(), TransportError> {
2024 Ok(())
2025 }
2026 }
2027
2028 #[test]
2029 fn an_unreachable_transport_schedules_no_retry() {
2030 for cause in ["connect", "network"] {
2033 let mut client = client();
2034 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2035 let mut transport = CauseTransport(cause);
2036 assert!(matches!(
2037 client.sync(&mut transport),
2038 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.failed"
2039 ));
2040 assert!(
2041 client.drain_sync_intents().is_empty(),
2042 "{cause} scheduled a retry"
2043 );
2044 }
2045
2046 let mut client = client();
2048 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2049 let mut transport = CauseTransport("timeout");
2050 assert!(matches!(
2051 client.sync(&mut transport),
2052 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.failed"
2053 ));
2054 assert!(matches!(
2055 client.drain_sync_intents().as_slice(),
2056 [SyncIntent::Background { delay_ms: 250 }]
2057 ));
2058 }
2059
2060 #[test]
2061 fn sync_until_idle_reports_its_total_deadline() {
2062 assert_eq!(SyncClient::SYNC_IDLE_TIMEOUT_MS, 60_000);
2063 let mut client = client();
2064 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2065 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2066
2067 let invalid = client.sync_until_idle_with_deadline(&mut transport, None, Some(0));
2069 assert!(
2070 matches!(invalid, SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"),
2071 "{invalid:?}"
2072 );
2073
2074 let now = client.clock_now_ms();
2077 let outcome = client.sync_until_idle_bounded(&mut transport, None, now, Some(now));
2078 let SyncOutcome::Failed {
2079 error_code,
2080 details,
2081 recommended_action,
2082 ..
2083 } = outcome
2084 else {
2085 panic!("expected the idle deadline, got {outcome:?}");
2086 };
2087 assert_eq!(error_code, "sync.idle_timeout");
2088 assert_eq!(recommended_action.as_deref(), Some("retryLater"));
2089 let details = details.expect("deadline details");
2090 assert_eq!(details["timeoutMs"], json!(0));
2091 assert_eq!(details["rounds"], json!(0));
2092 }
2093
2094 struct ScriptedSyncTransport {
2097 responses: std::collections::VecDeque<Message>,
2098 calls: usize,
2099 }
2100
2101 impl Transport for ScriptedSyncTransport {
2102 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
2103 self.calls += 1;
2104 let response = self.responses.pop_front().expect("scripted response");
2105 Ok(encode_message(&response))
2106 }
2107
2108 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2109 self.sync(request)
2110 }
2111
2112 fn download_segment(
2113 &mut self,
2114 _request: &SegmentRequest,
2115 _on_progress: &mut dyn FnMut(u64),
2116 ) -> Result<Vec<u8>, TransportError> {
2117 Err(TransportError::new("sync.transport_failed", "offline"))
2118 }
2119
2120 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2121 Ok(())
2122 }
2123
2124 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2125 Ok(())
2126 }
2127
2128 fn realtime_close(&mut self) -> Result<(), TransportError> {
2129 Ok(())
2130 }
2131 }
2132
2133 #[test]
2134 fn sync_until_idle_success_then_failure_keeps_earlier_work() {
2135 let path = std::env::temp_dir().join(format!(
2136 "syncular-budget-script-{}.sqlite",
2137 uuid::Uuid::new_v4()
2138 ));
2139 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
2140 {"name":"id","type":"string","nullable":false},
2141 {"name":"project_id","type":"string","nullable":false}],
2142 "scopes":[{"pattern":"project:{project_id}"}]}]});
2143 let mut client = SyncClient::open_path(
2144 "budget-script".into(),
2145 &schema,
2146 ClientLimits::default(),
2147 path.to_str().unwrap(),
2148 )
2149 .expect("open client");
2150 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2151 client
2152 .subscribe(
2153 "tasks".into(),
2154 "tasks".into(),
2155 vec![("project_id".into(), vec!["p1".into()])],
2156 None,
2157 )
2158 .unwrap();
2159 client.subs[0].cursor = 0;
2160 client.subs[0].synced_once = true;
2161 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
2162 client.persist_sub(&client.subs[0]).unwrap();
2163 client.drain_change_batches();
2164
2165 let row = encode_row_json(
2166 client.schema.table("tasks").unwrap(),
2167 "kept",
2168 &Map::from_iter([
2169 ("id".into(), json!("kept")),
2170 ("project_id".into(), json!("p1")),
2171 ]),
2172 &client.encryption,
2173 )
2174 .unwrap();
2175 let success = Message {
2176 wire_version: WIRE_VERSION,
2177 msg_kind: MsgKind::Response,
2178 frames: vec![
2179 Frame::RespHeader {
2180 required_schema_version: None,
2181 latest_schema_version: None,
2182 log_epoch: Some("epoch-1".into()),
2183 reset_required: Some(false),
2184 },
2185 Frame::SubStart {
2186 id: "tasks".into(),
2187 status: SubStatus::Active,
2188 reason_code: String::new(),
2189 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
2190 bootstrap: false,
2191 },
2192 Frame::Commit {
2193 commit_seq: 1,
2194 created_at_ms: 1,
2195 actor_id: "writer".into(),
2196 tables: vec!["tasks".into()],
2197 changes: vec![ssp2::model::Change {
2198 table_index: 0,
2199 row_id: "kept".into(),
2200 op: Op::Upsert,
2201 row_version: Some(1),
2202 scopes: vec![("project_id".into(), "p1".into())],
2203 row: Some(row),
2204 }],
2205 },
2206 Frame::SubEnd {
2207 next_cursor: 1,
2208 bootstrap_state: None,
2209 },
2210 ],
2211 };
2212 let failure = Message {
2213 wire_version: WIRE_VERSION,
2214 msg_kind: MsgKind::Response,
2215 frames: vec![
2216 Frame::RespHeader {
2217 required_schema_version: None,
2218 latest_schema_version: None,
2219 log_epoch: Some("epoch-1".into()),
2220 reset_required: Some(false),
2221 },
2222 Frame::Error {
2223 code: "sync.internal_error".into(),
2224 message: "second round failed".into(),
2225 category: "internal".into(),
2226 retryable: false,
2227 recommended_action: "retryLater".into(),
2228 details: None,
2229 },
2230 ],
2231 };
2232 let mut transport = ScriptedSyncTransport {
2233 responses: std::collections::VecDeque::from([success, failure]),
2234 calls: 0,
2235 };
2236 let outcome = client.sync_until_idle(&mut transport, Some(2));
2237 let SyncOutcome::Failed {
2238 error_code,
2239 recommended_action,
2240 ..
2241 } = outcome
2242 else {
2243 panic!("expected the in-band failure");
2244 };
2245 assert_eq!(error_code, "sync.internal_error");
2246 assert_eq!(recommended_action.as_deref(), Some("retryLater"));
2247 assert_eq!(
2248 transport.calls, 2,
2249 "one successful round then the failing round"
2250 );
2251 assert_eq!(
2252 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
2253 json!("kept"),
2254 "the earlier round's durable work survives the later failure"
2255 );
2256 drop(client);
2257 let _ = std::fs::remove_file(&path);
2258 }
2259
2260 #[test]
2261 fn sync_until_idle_stops_on_a_schema_floor() {
2262 let mut client = client();
2263 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2264 let floor = Message {
2265 wire_version: WIRE_VERSION,
2266 msg_kind: MsgKind::Response,
2267 frames: vec![Frame::RespHeader {
2268 required_schema_version: Some(2),
2269 latest_schema_version: Some(2),
2270 log_epoch: Some("epoch-1".into()),
2271 reset_required: Some(false),
2272 }],
2273 };
2274 let mut transport = ScriptedSyncTransport {
2275 responses: std::collections::VecDeque::from([floor.clone(), floor]),
2276 calls: 0,
2277 };
2278 let outcome = client.sync_until_idle(&mut transport, Some(3));
2279 match outcome {
2280 SyncOutcome::Ok(report) => assert_eq!(
2281 report
2282 .schema_floor
2283 .as_ref()
2284 .and_then(|floor| floor.required_schema_version),
2285 Some(2),
2286 "the stop retains the latest schema floor"
2287 ),
2288 other => panic!("expected a schema-floor stop, got {other:?}"),
2289 }
2290 assert_eq!(
2291 transport.calls, 1,
2292 "a schema floor stops the budget loop after one round"
2293 );
2294 }
2295
2296 struct FailureScriptTransport {
2299 rounds: std::collections::VecDeque<Result<Vec<u8>, TransportError>>,
2300 requests: Vec<Message>,
2301 segment_error: Option<TransportError>,
2302 }
2303
2304 impl Transport for FailureScriptTransport {
2305 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2306 self.requests
2307 .push(decode_message(request).expect("the client encodes a legal request"));
2308 self.rounds.pop_front().expect("scripted round")
2309 }
2310
2311 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2312 self.sync(request)
2313 }
2314
2315 fn download_segment(
2316 &mut self,
2317 _request: &SegmentRequest,
2318 _on_progress: &mut dyn FnMut(u64),
2319 ) -> Result<Vec<u8>, TransportError> {
2320 Err(self
2321 .segment_error
2322 .clone()
2323 .unwrap_or_else(|| TransportError::new("transport.timeout", "segment stalled")))
2324 }
2325
2326 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2327 Ok(())
2328 }
2329
2330 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
2331 Ok(())
2332 }
2333
2334 fn realtime_close(&mut self) -> Result<(), TransportError> {
2335 Ok(())
2336 }
2337 }
2338
2339 fn completed_response(frames: Vec<Frame>) -> Vec<u8> {
2341 let mut all = vec![Frame::RespHeader {
2342 required_schema_version: None,
2343 latest_schema_version: None,
2344 log_epoch: Some("epoch-1".into()),
2345 reset_required: Some(false),
2346 }];
2347 all.extend(frames);
2348 encode_message(&Message {
2349 wire_version: WIRE_VERSION,
2350 msg_kind: MsgKind::Response,
2351 frames: all,
2352 })
2353 }
2354
2355 fn carried_round_failures(request: &Message) -> Vec<String> {
2356 request
2357 .frames
2358 .iter()
2359 .filter_map(|frame| match frame {
2360 Frame::RoundFailure { code } => Some(code.clone()),
2361 _ => None,
2362 })
2363 .collect()
2364 }
2365
2366 fn failure_transport(
2367 rounds: impl IntoIterator<Item = Result<Vec<u8>, TransportError>>,
2368 ) -> FailureScriptTransport {
2369 FailureScriptTransport {
2370 rounds: rounds.into_iter().collect(),
2371 requests: Vec::new(),
2372 segment_error: None,
2373 }
2374 }
2375
2376 #[test]
2377 fn transport_diagnostics_bound_evidence_and_clear_it_after_recovery() {
2378 let secret = "https://user:password@example.invalid/private?token=secret";
2379 for (code, details, expected_cause, expected_status, connectivity, retry) in [
2380 (
2381 "transport.failed",
2382 json!({"causeKind":"connect", "secret":secret}),
2383 Some(crate::TransportCauseKind::Connect),
2384 None,
2385 "offline",
2386 None,
2387 ),
2388 (
2389 "transport.failed",
2390 json!({"causeKind":"status", "httpStatus":500, "secret":secret}),
2391 Some(crate::TransportCauseKind::Status),
2392 Some(500),
2393 "online",
2394 Some(250),
2395 ),
2396 (
2397 "sync.auth_required",
2398 json!({"causeKind":"status", "httpStatus":401, "secret":secret}),
2399 Some(crate::TransportCauseKind::Status),
2400 Some(401),
2401 "online",
2402 Some(250),
2403 ),
2404 (
2405 "transport.timeout",
2406 json!({"causeKind":"timeout", "secret":secret}),
2407 Some(crate::TransportCauseKind::Timeout),
2408 None,
2409 "unknown",
2410 Some(250),
2411 ),
2412 (
2413 "transport.failed",
2414 json!({"causeKind":"unknown", "secret":secret}),
2415 Some(crate::TransportCauseKind::Unknown),
2416 None,
2417 "unknown",
2418 Some(250),
2419 ),
2420 (
2421 "transport.failed",
2422 json!({"causeKind":secret, "httpStatus":500}),
2423 None,
2424 None,
2425 "unknown",
2426 Some(250),
2427 ),
2428 (
2429 "transport.failed",
2430 json!({"causeKind":"status", "httpStatus":500.5}),
2431 Some(crate::TransportCauseKind::Status),
2432 None,
2433 "unknown",
2434 Some(250),
2435 ),
2436 (
2437 "transport.failed",
2438 json!({"causeKind":"connect", "httpStatus":500}),
2439 Some(crate::TransportCauseKind::Connect),
2440 None,
2441 "offline",
2442 None,
2443 ),
2444 ] {
2445 let mut client = client();
2446 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2447 let mut error = TransportError::new(code, secret);
2448 error.details = Some(details);
2449 let mut transport = failure_transport([Err(error), Ok(completed_response(vec![]))]);
2450 assert!(matches!(
2451 client.sync(&mut transport),
2452 SyncOutcome::Failed { .. }
2453 ));
2454 let progress = client.progress().snapshot().unwrap();
2455 assert_eq!(progress.cause_kind, expected_cause);
2456 assert_eq!(progress.http_status, expected_status);
2457 assert_eq!(progress.retry_delay_ms, retry);
2458 let snapshot = client
2459 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2460 .unwrap();
2461 assert_eq!(snapshot.host.connectivity, connectivity);
2462 let last_round = snapshot.last_round.as_ref().unwrap();
2463 assert_eq!(last_round.cause_kind, expected_cause);
2464 assert_eq!(last_round.http_status, expected_status);
2465 assert!(!serde_json::to_string(&snapshot).unwrap().contains(secret));
2466 assert!(!serde_json::to_string(&progress).unwrap().contains(secret));
2467
2468 assert!(matches!(client.sync(&mut transport), SyncOutcome::Ok(_)));
2469 let progress = client.progress().snapshot().unwrap();
2470 assert_eq!(progress.cause_kind, None);
2471 assert_eq!(progress.http_status, None);
2472 let snapshot = client
2473 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2474 .unwrap();
2475 assert_eq!(snapshot.host.connectivity, "online");
2476 let last_round = snapshot.last_round.unwrap();
2477 assert_eq!(last_round.cause_kind, None);
2478 assert_eq!(last_round.http_status, None);
2479 }
2480 }
2481
2482 #[test]
2483 fn a_round_without_a_complete_response_reports_its_code_on_every_following_request() {
2484 let mut client = client();
2485 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2486 let mut transport = failure_transport([
2487 Err(TransportError::new("transport.timeout", "round timed out")),
2488 Ok(completed_response(Vec::new())),
2489 Ok(completed_response(Vec::new())),
2490 ]);
2491 for _ in 0..3 {
2492 client.sync(&mut transport);
2493 }
2494 assert!(carried_round_failures(&transport.requests[0]).is_empty());
2495 assert_eq!(
2496 carried_round_failures(&transport.requests[1]),
2497 vec!["transport.timeout"]
2498 );
2499 assert!(
2500 matches!(transport.requests[1].frames[0], Frame::ReqHeader { .. })
2501 && matches!(transport.requests[1].frames[1], Frame::RoundFailure { .. }),
2502 "the report sits immediately after REQ_HEADER"
2503 );
2504 assert!(
2505 carried_round_failures(&transport.requests[2]).is_empty(),
2506 "a complete response ends the report"
2507 );
2508 }
2509
2510 #[test]
2511 fn a_newer_round_failure_replaces_the_kept_code() {
2512 let mut client = client();
2513 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2514 let mut transport = failure_transport([
2515 Err(TransportError::new("transport.timeout", "round timed out")),
2516 Err(TransportError::new("transport.failed", "socket closed")),
2517 Ok(completed_response(Vec::new())),
2518 Ok(completed_response(Vec::new())),
2519 ]);
2520 for _ in 0..4 {
2521 client.sync(&mut transport);
2522 }
2523 assert_eq!(
2524 carried_round_failures(&transport.requests[1]),
2525 vec!["transport.timeout"]
2526 );
2527 assert_eq!(
2528 carried_round_failures(&transport.requests[2]),
2529 vec!["transport.failed"]
2530 );
2531 assert!(carried_round_failures(&transport.requests[3]).is_empty());
2532 }
2533
2534 #[test]
2535 fn an_in_band_error_completes_the_round_and_is_never_reported() {
2536 let mut client = client();
2537 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2538 let mut transport = failure_transport([
2539 Err(TransportError::new("transport.timeout", "round timed out")),
2540 Ok(completed_response(vec![Frame::Error {
2541 code: "sync.internal_error".into(),
2542 message: "server error".into(),
2543 category: "internal".into(),
2544 retryable: false,
2545 recommended_action: "retryLater".into(),
2546 details: None,
2547 }])),
2548 Ok(completed_response(Vec::new())),
2549 ]);
2550 client.sync(&mut transport);
2551 assert!(matches!(
2552 client.sync(&mut transport),
2553 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
2554 ));
2555 client.sync(&mut transport);
2556 assert_eq!(
2557 carried_round_failures(&transport.requests[1]),
2558 vec!["transport.timeout"]
2559 );
2560 assert!(
2561 carried_round_failures(&transport.requests[2]).is_empty(),
2562 "the server already recorded an in-band ERROR"
2563 );
2564 }
2565
2566 #[test]
2567 fn a_silent_segment_fetch_is_retryable_and_reported_on_the_next_request() {
2568 let mut client = client();
2569 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2570 client
2571 .subscribe(
2572 "tasks".into(),
2573 "tasks".into(),
2574 vec![("project_id".into(), vec!["p1".into()])],
2575 None,
2576 )
2577 .unwrap();
2578 client.subs[0].cursor = 0;
2579 client.subs[0].synced_once = true;
2580 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
2581 client.persist_sub(&client.subs[0]).unwrap();
2582 client.drain_change_batches();
2583 client.drain_sync_intents();
2584 let mut transport = failure_transport([
2585 Ok(completed_response(vec![
2586 Frame::SubStart {
2587 id: "tasks".into(),
2588 status: SubStatus::Active,
2589 reason_code: String::new(),
2590 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
2591 bootstrap: false,
2592 },
2593 Frame::SegmentRef {
2594 segment_id: "sha256:stalled".into(),
2595 media_type: MediaType::Rows,
2596 table: "tasks".into(),
2597 byte_length: 1,
2598 row_count: 1,
2599 as_of_commit_seq: 1,
2600 scope_digest: "digest".into(),
2601 row_cursor: None,
2602 next_row_cursor: None,
2603 url: None,
2604 url_expires_at_ms: None,
2605 },
2606 Frame::SubEnd {
2607 next_cursor: 1,
2608 bootstrap_state: None,
2609 },
2610 ])),
2611 Ok(completed_response(Vec::new())),
2612 Ok(completed_response(Vec::new())),
2613 ]);
2614 let mut segment_error = TransportError::new("transport.timeout", "private response body");
2615 segment_error.details = Some(json!({
2616 "causeKind": "status",
2617 "httpStatus": 500,
2618 "url": "https://example.invalid/private"
2619 }));
2620 transport.segment_error = Some(segment_error);
2621
2622 assert!(matches!(
2623 client.sync(&mut transport),
2624 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.timeout"
2625 ));
2626 let progress = client.progress().snapshot().unwrap();
2627 assert_eq!(progress.cause_kind, Some(crate::TransportCauseKind::Status));
2628 assert_eq!(progress.http_status, Some(500));
2629 let diagnostics = client
2630 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
2631 .unwrap();
2632 assert_eq!(diagnostics.host.connectivity, "online");
2633 let round = diagnostics.last_round.as_ref().unwrap();
2634 assert_eq!(round.cause_kind, Some(crate::TransportCauseKind::Status));
2635 assert_eq!(round.http_status, Some(500));
2636 assert!(!serde_json::to_string(&diagnostics)
2637 .unwrap()
2638 .contains("private"));
2639 assert!(!serde_json::to_string(&progress)
2640 .unwrap()
2641 .contains("private"));
2642 assert!(
2643 matches!(
2644 client.drain_sync_intents().as_slice(),
2645 [SyncIntent::Background { delay_ms: 250 }]
2646 ),
2647 "a silent transfer asks the host for a Background retry"
2648 );
2649 client.sync(&mut transport);
2650 client.sync(&mut transport);
2651 assert!(carried_round_failures(&transport.requests[0]).is_empty());
2652 assert_eq!(
2653 carried_round_failures(&transport.requests[1]),
2654 vec!["transport.timeout"],
2655 "the round's own response was complete, so the fetch failure rides the next request"
2656 );
2657 assert!(
2658 carried_round_failures(&transport.requests[2]).is_empty(),
2659 "a complete response ends the report"
2660 );
2661 }
2662
2663 #[test]
2664 fn a_code_the_frame_grammar_rejects_is_not_attached() {
2665 let mut client = client();
2666 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2667 let mut transport = failure_transport([
2668 Err(TransportError::new("offline", "no dotted code")),
2669 Ok(completed_response(Vec::new())),
2670 ]);
2671 client.sync(&mut transport);
2672 client.sync(&mut transport);
2673 assert!(carried_round_failures(&transport.requests[1]).is_empty());
2674 }
2675
2676 #[test]
2677 fn sync_report_merge_schema_floor_is_latest_state() {
2678 let mut aggregate = SyncReport {
2679 schema_floor: Some(SchemaFloor {
2680 required_schema_version: Some(2),
2681 latest_schema_version: Some(2),
2682 }),
2683 ..SyncReport::default()
2684 };
2685 aggregate.merge(&SyncReport::default());
2686 assert!(
2687 aggregate.schema_floor.is_none(),
2688 "a later round without a floor clears it"
2689 );
2690 }
2691
2692 #[test]
2693 fn push_capacity_limits_are_hard_and_keep_intent() {
2694 let mut c = client();
2696 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2697 c.limits.max_push_operations_per_request = Some(1);
2698 let id = c
2699 .mutate(vec![
2700 Mutation::Upsert {
2701 table: "tasks".into(),
2702 values: Map::from_iter([
2703 ("id".into(), json!("a")),
2704 ("project_id".into(), json!("p1")),
2705 ]),
2706 base_version: None,
2707 },
2708 Mutation::Upsert {
2709 table: "tasks".into(),
2710 values: Map::from_iter([
2711 ("id".into(), json!("b")),
2712 ("project_id".into(), json!("p1")),
2713 ]),
2714 base_version: None,
2715 },
2716 ])
2717 .unwrap();
2718 let error = c.build_request(false).unwrap_err();
2719 let SyncOutcome::Failed {
2720 error_code,
2721 details,
2722 ..
2723 } = *error
2724 else {
2725 panic!("expected a capacity failure");
2726 };
2727 assert_eq!(error_code, "client.push_request_too_large");
2728 let details = details.expect("capacity details");
2729 assert_eq!(details["kind"], json!("operations"));
2730 assert_eq!(details["limit"], json!(1));
2731 assert_eq!(details["size"], json!(2));
2732 assert_eq!(details["clientCommitId"], json!(id));
2733 assert_eq!(c.pending_commit_ids(), vec![id]);
2734
2735 let mut c = client();
2737 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2738 c.limits.max_push_commits_per_request = Some(1);
2739 let first = c
2740 .mutate(vec![Mutation::Upsert {
2741 table: "tasks".into(),
2742 values: Map::from_iter([
2743 ("id".into(), json!("one")),
2744 ("project_id".into(), json!("p1")),
2745 ]),
2746 base_version: None,
2747 }])
2748 .unwrap();
2749 let second = c
2750 .mutate(vec![Mutation::Upsert {
2751 table: "tasks".into(),
2752 values: Map::from_iter([
2753 ("id".into(), json!("two")),
2754 ("project_id".into(), json!("p1")),
2755 ]),
2756 base_version: None,
2757 }])
2758 .unwrap();
2759 let (_, meta) = c.build_request(false).unwrap();
2760 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2761 assert_eq!(meta.deferred_commits, 1);
2762 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2763 }
2764
2765 #[test]
2766 fn push_request_byte_cap_defers_the_prefix_boundary_and_errors_the_head() {
2767 let mut c = client();
2770 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2771 let first = c
2772 .mutate(vec![Mutation::Upsert {
2773 table: "tasks".into(),
2774 values: Map::from_iter([
2775 ("id".into(), json!("one")),
2776 ("project_id".into(), json!("p1")),
2777 ]),
2778 base_version: None,
2779 }])
2780 .unwrap();
2781 let second = c
2782 .mutate(vec![Mutation::Upsert {
2783 table: "tasks".into(),
2784 values: Map::from_iter([
2785 ("id".into(), json!("two")),
2786 ("project_id".into(), json!("p1")),
2787 ]),
2788 base_version: None,
2789 }])
2790 .unwrap();
2791 let (message, meta) = c.build_request(false).unwrap();
2792 assert_eq!(meta.pushed_ids.len(), 2);
2793 let full = encode_message(&message).len();
2794 c.limits.max_push_request_bytes = Some(full - 1);
2795 let (_, meta) = c.build_request(false).unwrap();
2796 assert_eq!(meta.pushed_ids, vec![first.clone()]);
2797 assert_eq!(meta.deferred_commits, 1);
2798 assert_eq!(c.pending_commit_ids(), vec![first, second]);
2799
2800 let mut c = client();
2803 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2804 let huge = c
2805 .mutate(vec![Mutation::Upsert {
2806 table: "tasks".into(),
2807 values: Map::from_iter([
2808 ("id".into(), json!("x".repeat(20_000))),
2809 ("project_id".into(), json!("p1")),
2810 ]),
2811 base_version: None,
2812 }])
2813 .unwrap();
2814 let (message, meta) = c.build_request(false).unwrap();
2815 assert_eq!(meta.pushed_ids, vec![huge.clone()]);
2816 let with_huge = encode_message(&message).len();
2817 c.limits.max_push_request_bytes = Some(with_huge - 1);
2818 let error = c.build_request(false).unwrap_err();
2819 let SyncOutcome::Failed {
2820 error_code,
2821 details,
2822 ..
2823 } = *error
2824 else {
2825 panic!("expected a capacity failure");
2826 };
2827 assert_eq!(error_code, "client.push_request_too_large");
2828 let details = details.expect("capacity details");
2829 assert_eq!(details["kind"], json!("bytes"));
2830 assert_eq!(details["limit"], json!((with_huge - 1) as u64));
2831 assert_eq!(details["size"], json!(with_huge as u64));
2832 assert_eq!(details["clientCommitId"], json!(huge));
2833 assert_eq!(c.pending_commit_ids(), vec![huge]);
2834
2835 let mut c = client();
2837 c.set_meta(LOG_EPOCH_KEY, "epoch-1");
2838 c.limits.max_push_request_bytes = Some(1);
2839 let error = c.build_request(false).unwrap_err();
2840 let SyncOutcome::Failed { details, .. } = *error else {
2841 panic!("expected a capacity failure");
2842 };
2843 let details = details.expect("capacity details");
2844 assert_eq!(details["kind"], json!("bytes"));
2845 assert!(details.get("clientCommitId").is_none(), "{details}");
2846 }
2847
2848 #[test]
2849 fn push_limits_validate_at_construction() {
2850 for limits in [
2851 ClientLimits {
2852 max_push_commits_per_request: Some(0),
2853 ..Default::default()
2854 },
2855 ClientLimits {
2856 max_push_operations_per_request: Some(0),
2857 ..Default::default()
2858 },
2859 ClientLimits {
2860 max_push_request_bytes: Some(0),
2861 ..Default::default()
2862 },
2863 ClientLimits {
2864 max_push_request_bytes: Some(u32::MAX as usize + 1),
2865 ..Default::default()
2866 },
2867 ] {
2868 let schema = json!({"version":1,"tables":[]});
2869 let error = SyncClient::with_connection(
2870 "bad-limits".into(),
2871 &schema,
2872 limits,
2873 Connection::open_in_memory().unwrap(),
2874 )
2875 .err();
2876 assert_eq!(
2877 error.as_deref(),
2878 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2879 );
2880 }
2881 }
2882
2883 #[test]
2884 fn open_path_rejects_invalid_push_limits_before_storage() {
2885 for identity in [false, true] {
2886 let path = std::env::temp_dir().join(format!(
2887 "syncular-bad-limits-{}.sqlite",
2888 uuid::Uuid::new_v4()
2889 ));
2890 let schema = json!({"version":1,"tables":[]});
2891 let limits = ClientLimits {
2892 max_push_request_bytes: Some(0),
2893 ..Default::default()
2894 };
2895 let error = if identity {
2896 SyncClient::open_path_with_identity(None, &schema, limits, path.to_str().unwrap())
2897 } else {
2898 SyncClient::open_path("bad-limits".into(), &schema, limits, path.to_str().unwrap())
2899 }
2900 .err();
2901 assert_eq!(
2902 error.as_deref(),
2903 Some("sync.invalid_request: push limit must be an integer in 1..=4294967295")
2904 );
2905 assert!(
2906 !path.exists(),
2907 "an invalid limit is rejected before the database file is created"
2908 );
2909 }
2910 }
2911
2912 #[test]
2913 fn failed_progress_retry_delay_doubles_to_the_cap() {
2914 let mut client = client();
2915 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2916 let mut transport = ServerErrorTransport {
2917 code: "sync.internal_error",
2918 retryable: true,
2919 };
2920 let delays: Vec<Option<u64>> = (0..9)
2921 .map(|_| {
2922 client.sync(&mut transport);
2923 client.progress().snapshot().unwrap().retry_delay_ms
2924 })
2925 .collect();
2926 assert_eq!(
2927 delays,
2928 [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
2929 );
2930 }
2931
2932 #[test]
2933 fn background_retry_deadlines_back_off_and_reset() {
2934 let mut client = client();
2935 client.schedule_background_retry();
2936 assert!(matches!(
2937 client.drain_sync_intents().as_slice(),
2938 [SyncIntent::Background { delay_ms: 250 }]
2939 ));
2940 client.schedule_background_retry();
2941 assert!(matches!(
2942 client.drain_sync_intents().as_slice(),
2943 [SyncIntent::Background { delay_ms: 500 }]
2944 ));
2945 client.reset_background_retry();
2946 client.schedule_background_retry();
2947 assert!(matches!(
2948 client.drain_sync_intents().as_slice(),
2949 [SyncIntent::Background { delay_ms: 250 }]
2950 ));
2951 }
2952
2953 #[test]
2954 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
2955 let mut client = client();
2956 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2957 client
2958 .subscribe(
2959 "epoch-tasks".to_owned(),
2960 "tasks".to_owned(),
2961 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2962 None,
2963 )
2964 .expect("subscribe");
2965 client.drain_sync_intents();
2966
2967 client
2968 .run_log_epoch_reset("epoch-2")
2969 .expect("reset partition log epoch");
2970
2971 assert!(matches!(
2972 client.drain_sync_intents().as_slice(),
2973 [SyncIntent::Interactive]
2974 ));
2975 }
2976
2977 #[derive(Default)]
2978 struct CountingRealtimeTransport {
2979 connects: usize,
2980 closes: usize,
2981 messages: Vec<String>,
2982 blob_uploads: Vec<(String, Vec<u8>)>,
2983 blob_download: Option<Vec<u8>>,
2984 sync_calls: usize,
2985 realtime_sync_calls: usize,
2986 refuse_connect: bool,
2987 }
2988
2989 impl Transport for CountingRealtimeTransport {
2990 fn blob_upload(
2991 &mut self,
2992 blob_id: &str,
2993 bytes: &[u8],
2994 _media_type: Option<&str>,
2995 ) -> Result<(), TransportError> {
2996 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
2997 Ok(())
2998 }
2999
3000 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
3001 self.blob_download
3002 .clone()
3003 .map(BlobDownload::Bytes)
3004 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
3005 }
3006
3007 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
3008 self.sync_calls += 1;
3009 Err(TransportError::new("sync.transport_failed", "offline"))
3010 }
3011
3012 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
3013 self.realtime_sync_calls += 1;
3014 Err(TransportError::new("sync.transport_failed", "offline"))
3015 }
3016
3017 fn download_segment(
3018 &mut self,
3019 _request: &SegmentRequest,
3020 _on_progress: &mut dyn FnMut(u64),
3021 ) -> Result<Vec<u8>, TransportError> {
3022 Err(TransportError::new("sync.transport_failed", "offline"))
3023 }
3024
3025 fn realtime_connect(&mut self) -> Result<(), TransportError> {
3026 self.connects += 1;
3027 if self.refuse_connect {
3028 return Err(TransportError::new("transport.failed", "refused"));
3029 }
3030 Ok(())
3031 }
3032
3033 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
3034 self.messages.push(text.to_owned());
3035 Ok(())
3036 }
3037
3038 fn realtime_close(&mut self) -> Result<(), TransportError> {
3039 self.closes += 1;
3040 Ok(())
3041 }
3042 }
3043
3044 #[test]
3045 fn blob_results_own_downloaded_bytes() {
3046 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
3047 {"name":"id","type":"string","nullable":false},
3048 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
3049 let bytes = b"owned typed result".to_vec();
3050 let blob_id = blob_id_for(&bytes);
3051 let mut transport = CountingRealtimeTransport {
3052 blob_download: Some(bytes.clone()),
3053 ..Default::default()
3054 };
3055 let result = {
3056 let mut client =
3057 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
3058 .unwrap();
3059 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
3060 assert_eq!(result.blob_id, blob_id);
3061 assert_eq!(result.byte_length, bytes.len() as i64);
3062 assert_eq!(result.bytes, bytes);
3063 assert_eq!(result.media_type, None);
3064 assert_eq!(
3065 client
3066 .conn
3067 .query_row(
3068 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
3069 rusqlite::params![blob_id],
3070 |row| row.get::<_, i64>(0),
3071 )
3072 .unwrap(),
3073 0
3074 );
3075 result
3076 };
3077 assert_eq!(result.bytes, bytes);
3078
3079 let mut client =
3080 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
3081 assert_eq!(
3082 client
3083 .fetch_blob_bytes(&mut transport, "not json")
3084 .unwrap_err(),
3085 (
3086 "client.failed".to_owned(),
3087 "blob ref is not JSON".to_owned()
3088 )
3089 );
3090 }
3091
3092 #[test]
3093 fn visible_blob_references_survive_cache_trimming() {
3094 let schema = json!({"version":1,"tables":[
3095 {"name":"attachments","primaryKey":"id","columns":[
3096 {"name":"id","type":"string","nullable":false},
3097 {"name":"body","type":"blob_ref","nullable":true},
3098 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
3099 {"name":"avatars","primaryKey":"id","columns":[
3100 {"name":"id","type":"string","nullable":false},
3101 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
3102 ]});
3103 let bytes = b"referenced above cap".to_vec();
3104 let blob_id = blob_id_for(&bytes);
3105 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
3106 let mut client = SyncClient::new(
3107 "visible-blob-refs".to_owned(),
3108 &schema,
3109 ClientLimits {
3110 blob_cache_max_bytes: Some(1),
3111 ..Default::default()
3112 },
3113 )
3114 .unwrap();
3115 client.conn.execute(
3116 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
3117 rusqlite::params![reference, reference, reference, reference],
3118 ).unwrap();
3119 client
3120 .conn
3121 .execute(
3122 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
3123 rusqlite::params![reference],
3124 )
3125 .unwrap();
3126 client.conn.execute(
3127 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
3128 [],
3129 ).unwrap();
3130 let mut transport = CountingRealtimeTransport {
3131 blob_download: Some(bytes.clone()),
3132 ..Default::default()
3133 };
3134
3135 assert_eq!(
3136 client
3137 .fetch_blob_bytes(&mut transport, &blob_id)
3138 .unwrap()
3139 .bytes,
3140 bytes
3141 );
3142 let rows = client
3143 .conn
3144 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
3145 .unwrap()
3146 .query_map([], |row| row.get::<_, String>(0))
3147 .unwrap()
3148 .collect::<Result<Vec<_>, _>>()
3149 .unwrap();
3150 assert_eq!(rows, vec![blob_id]);
3151 }
3152
3153 #[test]
3154 fn closed_transport_never_invokes_network_and_reopen_is_host_owned() {
3155 let mut transport = CountingRealtimeTransport::default();
3156 let path = std::env::temp_dir().join(format!(
3157 "syncular-transport-gate-{}.db",
3158 uuid::Uuid::new_v4()
3159 ));
3160 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
3161 {"name":"id","type":"string","nullable":false},
3162 {"name":"project_id","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
3163 let mut client = SyncClient::open_path(
3164 "gate".into(),
3165 &schema,
3166 ClientLimits::default(),
3167 path.to_str().unwrap(),
3168 )
3169 .unwrap();
3170 client.connect_realtime(&mut transport).unwrap();
3171 client.set_transport_enabled(&mut transport, false);
3172 assert_eq!(transport.closes, 1);
3173 client.begin_security_preflight();
3174 client.activate_security(Default::default()).unwrap();
3175 client
3176 .subscribe(
3177 "tasks".into(),
3178 "tasks".into(),
3179 vec![("project_id".into(), vec!["p1".into()])],
3180 None,
3181 )
3182 .unwrap();
3183 for id in ["one", "two"] {
3184 client
3185 .mutate(vec![Mutation::Upsert {
3186 table: "tasks".into(),
3187 values: Map::from_iter([
3188 ("id".into(), json!(id)),
3189 ("project_id".into(), json!("p1")),
3190 ]),
3191 base_version: None,
3192 }])
3193 .unwrap();
3194 }
3195 assert_eq!(
3196 client
3197 .query("SELECT id FROM tasks ORDER BY id", &[])
3198 .unwrap()
3199 .len(),
3200 2
3201 );
3202 assert!(
3203 matches!(client.sync(&mut transport), SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.offline")
3204 );
3205 assert!(client
3206 .connect_realtime(&mut transport)
3207 .unwrap_err()
3208 .starts_with("sync.offline:"));
3209 assert!(client
3210 .set_presence(&mut transport, "project:p1", None)
3211 .unwrap_err()
3212 .starts_with("sync.offline:"));
3213 assert_eq!(transport.sync_calls, 0);
3214 assert_eq!(transport.realtime_sync_calls, 0);
3215 assert_eq!(transport.connects, 1);
3216 assert!(transport.messages.is_empty());
3217 assert!(matches!(
3218 client.drain_sync_intents().as_slice(),
3219 [SyncIntent::None]
3220 ));
3221 let pending = client.pending_commit_ids();
3222 drop(client);
3223 let mut reopened = SyncClient::open_path(
3224 "gate".into(),
3225 &schema,
3226 ClientLimits::default(),
3227 path.to_str().unwrap(),
3228 )
3229 .unwrap();
3230 assert!(reopened.transport_enabled());
3231 reopened.set_transport_enabled(&mut transport, false);
3232 assert_eq!(reopened.pending_commit_ids(), pending);
3233 assert_eq!(
3234 reopened.query("SELECT id FROM tasks", &[]).unwrap().len(),
3235 2
3236 );
3237 reopened.set_transport_enabled(&mut transport, true);
3238 assert!(matches!(
3239 reopened.drain_sync_intents().as_slice(),
3240 [SyncIntent::Interactive]
3241 ));
3242 reopened.set_transport_enabled(&mut transport, true);
3243 assert!(reopened.drain_sync_intents().is_empty());
3244 drop(reopened);
3245 std::fs::remove_file(path).unwrap();
3246 }
3247
3248 #[test]
3249 fn realtime_connection_ownership_is_idempotent() {
3250 let mut client = client();
3251 let mut transport = CountingRealtimeTransport {
3252 connects: 0,
3253 closes: 0,
3254 ..Default::default()
3255 };
3256 client
3257 .connect_realtime(&mut transport)
3258 .expect("first connect");
3259 client
3260 .connect_realtime(&mut transport)
3261 .expect("idempotent connect");
3262 assert_eq!(transport.connects, 1);
3263 client.disconnect_realtime(&mut transport);
3264 client.disconnect_realtime(&mut transport);
3265 assert_eq!(transport.closes, 1);
3266 client
3267 .connect_realtime(&mut transport)
3268 .expect("deliberate reconnect");
3269 assert_eq!(transport.connects, 2);
3270 }
3271
3272 #[test]
3273 fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
3274 let mut client = client();
3275 client
3276 .set_realtime_policy(RealtimePolicy::Required)
3277 .expect("set policy");
3278 let mut transport = CountingRealtimeTransport::default();
3279 match client.sync(&mut transport) {
3280 SyncOutcome::RealtimeUnavailable {
3281 state,
3282 reason_code,
3283 retry_delay_ms,
3284 } => {
3285 assert_eq!(state, RealtimeState::Disconnected);
3286 assert_eq!(reason_code, None);
3287 assert_eq!(retry_delay_ms, 250);
3288 }
3289 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3290 }
3291 assert_eq!(transport.sync_calls, 0);
3292 assert!(matches!(
3293 client.drain_sync_intents().as_slice(),
3294 [SyncIntent::Background { delay_ms: 250 }]
3295 ));
3296 let snapshot = client
3297 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3298 .expect("diagnostics");
3299 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
3300 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
3301 assert_eq!(snapshot.host.realtime_reason_code, None);
3302 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3303 assert_eq!(
3304 snapshot
3305 .last_round
3306 .as_ref()
3307 .and_then(|round| round.error_code.clone()),
3308 Some(REALTIME_UNAVAILABLE_CODE.to_owned())
3309 );
3310 }
3311
3312 #[test]
3313 fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
3314 let mut client = client();
3315 client
3316 .set_realtime_policy(RealtimePolicy::Required)
3317 .expect("set policy");
3318 let mut transport = CountingRealtimeTransport {
3319 refuse_connect: true,
3320 ..Default::default()
3321 };
3322 assert!(client.connect_realtime(&mut transport).is_err());
3323 assert_eq!(client.realtime_state(), RealtimeState::Refused);
3324 let snapshot = client
3325 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3326 .expect("diagnostics");
3327 assert_eq!(
3328 snapshot.host.realtime_reason_code.as_deref(),
3329 Some("transport.failed")
3330 );
3331 match client.sync(&mut transport) {
3332 SyncOutcome::RealtimeUnavailable {
3333 state, reason_code, ..
3334 } => {
3335 assert_eq!(state, RealtimeState::Refused);
3336 assert_eq!(reason_code.as_deref(), Some("transport.failed"));
3337 }
3338 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3339 }
3340 assert_eq!(transport.sync_calls, 0);
3341 assert_eq!(transport.realtime_sync_calls, 0);
3342 }
3343
3344 #[test]
3345 fn an_unsolicited_socket_end_is_lost_with_the_transport_code_and_a_retry_intent() {
3346 let mut client = client();
3347 let mut transport = CountingRealtimeTransport::default();
3348 let ended = TransportError::new("transport.failed", "realtime connection closed");
3349 client.realtime_lost(&mut transport, &ended);
3350 assert_eq!(
3351 client.realtime_state(),
3352 RealtimeState::Disconnected,
3353 "a socket that was never connected has nothing to lose"
3354 );
3355 assert!(client.drain_sync_intents().is_empty());
3356
3357 client.connect_realtime(&mut transport).expect("connect");
3358 client.realtime_lost(&mut transport, &ended);
3359 assert_eq!(client.realtime_state(), RealtimeState::Lost);
3360 assert_eq!(transport.closes, 1);
3361 assert!(matches!(
3362 client.drain_sync_intents().as_slice(),
3363 [SyncIntent::Background { delay_ms: 250 }]
3364 ));
3365 let snapshot = client
3366 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3367 .expect("diagnostics");
3368 assert_eq!(snapshot.host.realtime, RealtimeState::Lost);
3369 assert_eq!(
3370 snapshot.host.realtime_reason_code.as_deref(),
3371 Some("transport.failed")
3372 );
3373 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3374
3375 client.realtime_lost(&mut transport, &ended);
3376 assert!(
3377 client.drain_sync_intents().is_empty(),
3378 "a second report for the same loss schedules nothing"
3379 );
3380 client.connect_realtime(&mut transport).expect("reconnect");
3381 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3382 }
3383
3384 #[test]
3385 fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
3386 let mut client = client();
3387 client
3388 .set_realtime_policy(RealtimePolicy::Required)
3389 .expect("set policy");
3390 let mut transport = CountingRealtimeTransport::default();
3391 client.connect_realtime(&mut transport).expect("connect");
3392 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3393 assert!(
3394 client.set_realtime_policy(RealtimePolicy::Off).is_err(),
3395 "a policy change cannot silently re-route a connected socket"
3396 );
3397 assert!(matches!(
3398 client.sync(&mut transport),
3399 SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
3400 ));
3401 assert_eq!(client.realtime_state(), RealtimeState::Lost);
3402 assert_eq!(transport.realtime_sync_calls, 1);
3403 assert_eq!(transport.sync_calls, 0);
3404 let snapshot = client
3405 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3406 .expect("diagnostics");
3407 assert_eq!(
3408 snapshot.host.realtime_reason_code.as_deref(),
3409 Some("sync.transport_failed")
3410 );
3411 assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
3412 match client.sync(&mut transport) {
3413 SyncOutcome::RealtimeUnavailable {
3414 state, reason_code, ..
3415 } => {
3416 assert_eq!(state, RealtimeState::Lost);
3417 assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
3418 }
3419 other => panic!("expected RealtimeUnavailable, got {other:?}"),
3420 }
3421 assert_eq!(transport.sync_calls, 0);
3422 client.disconnect_realtime(&mut transport);
3423 assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
3424 assert_eq!(transport.closes, 1);
3425 client.connect_realtime(&mut transport).expect("reconnect");
3426 assert_eq!(client.realtime_state(), RealtimeState::Connected);
3427 assert_eq!(transport.connects, 2);
3428 }
3429
3430 #[test]
3431 fn off_policy_uses_http_and_refuses_an_explicit_connect() {
3432 let mut client = client();
3433 client
3434 .set_realtime_policy(RealtimePolicy::Off)
3435 .expect("set policy");
3436 let mut transport = CountingRealtimeTransport::default();
3437 assert!(client.connect_realtime(&mut transport).is_err());
3438 assert_eq!(client.realtime_state(), RealtimeState::Disabled);
3439 assert!(matches!(
3440 client.sync(&mut transport),
3441 SyncOutcome::Failed { .. }
3442 ));
3443 assert_eq!(transport.sync_calls, 1);
3444 assert_eq!(transport.realtime_sync_calls, 0);
3445 let snapshot = client
3446 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3447 .expect("diagnostics");
3448 assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
3449 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
3450 }
3451
3452 #[test]
3453 fn optional_policy_keeps_the_http_round() {
3454 let mut client = client();
3455 let mut transport = CountingRealtimeTransport::default();
3456 assert!(matches!(
3457 client.sync(&mut transport),
3458 SyncOutcome::Failed { .. }
3459 ));
3460 assert_eq!(transport.sync_calls, 1);
3461 assert_eq!(transport.realtime_sync_calls, 0);
3462 let snapshot = client
3463 .diagnostics_snapshot(&ClientDiagnosticsRequest::default())
3464 .expect("diagnostics");
3465 assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
3466 assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
3467 }
3468
3469 #[test]
3470 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
3471 let mut client = client();
3472 client
3473 .subscribe(
3474 "repair-tasks".to_owned(),
3475 "tasks".to_owned(),
3476 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3477 None,
3478 )
3479 .expect("subscribe");
3480 {
3481 let sub = client
3482 .subs
3483 .iter_mut()
3484 .find(|sub| sub.id == "repair-tasks")
3485 .expect("subscription");
3486 sub.cursor = 42;
3487 sub.synced_once = true;
3488 let persisted = sub.clone();
3489 client.persist_sub(&persisted).unwrap();
3490 }
3491 for table in ["_syncular_base_tasks", "tasks"] {
3492 client
3493 .conn
3494 .execute(
3495 &format!(
3496 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
3497 ),
3498 rusqlite::params!["server-row", "p1"],
3499 )
3500 .expect("seed server row");
3501 }
3502 let pending = client
3503 .mutate(vec![Mutation::Upsert {
3504 table: "tasks".to_owned(),
3505 values: Map::from_iter([
3506 ("id".to_owned(), Value::from("offline-row")),
3507 ("project_id".to_owned(), Value::from("p1")),
3508 ]),
3509 base_version: None,
3510 }])
3511 .expect("queue offline work");
3512 client.drain_change_batches();
3513 client.drain_sync_intents();
3514
3515 assert_eq!(
3516 client
3517 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3518 rebootstrap_id: "support-case-001".to_owned(),
3519 })
3520 .expect("rebootstrap"),
3521 LocalDataRebootstrapResult {
3522 already_applied: false,
3523 retained_commits: 1,
3524 reset_subscriptions: 1,
3525 }
3526 );
3527 let visible_ids = client
3528 .conn
3529 .prepare("SELECT id FROM tasks ORDER BY id")
3530 .expect("prepare visible ids")
3531 .query_map([], |row| row.get::<_, String>(0))
3532 .expect("query visible ids")
3533 .collect::<Result<Vec<_>, _>>()
3534 .expect("collect visible ids");
3535 assert_eq!(visible_ids, vec!["offline-row"]);
3536 assert_eq!(client.pending_commit_ids(), vec![pending]);
3537 assert_eq!(
3538 client
3539 .subscription_state("repair-tasks")
3540 .expect("subscription")
3541 .cursor,
3542 -1
3543 );
3544 assert!(client.upgrading());
3545 assert!(client.sync_needed());
3546 assert!(matches!(
3547 client.drain_sync_intents().as_slice(),
3548 [SyncIntent::Interactive]
3549 ));
3550 assert_eq!(client.drain_change_batches().len(), 1);
3551
3552 assert_eq!(
3553 client
3554 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3555 rebootstrap_id: "support-case-001".to_owned(),
3556 })
3557 .expect("idempotent retry"),
3558 LocalDataRebootstrapResult {
3559 already_applied: true,
3560 retained_commits: 1,
3561 reset_subscriptions: 1,
3562 }
3563 );
3564 assert!(client.drain_change_batches().is_empty());
3565 }
3566
3567 #[test]
3568 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
3569 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
3570 assert_eq!(
3571 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
3572 (3, 4)
3573 );
3574 assert_eq!(
3575 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
3576 (0, 0)
3577 );
3578 for malformed in [
3579 "",
3580 "{}",
3581 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
3582 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
3583 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
3584 ] {
3585 assert_eq!(
3586 decode_local_rebootstrap_receipt(malformed)
3587 .expect_err("malformed receipt must fail"),
3588 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3589 );
3590 }
3591 }
3592
3593 #[test]
3594 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
3595 let path = std::env::temp_dir().join(format!(
3596 "syncular-rebootstrap-receipt-{}.db",
3597 uuid::Uuid::new_v4()
3598 ));
3599 let schema = json!({
3600 "version": 1,
3601 "tables": [{
3602 "name": "tasks",
3603 "primaryKey": "id",
3604 "columns": [
3605 { "name": "id", "type": "string", "nullable": false },
3606 { "name": "project_id", "type": "string", "nullable": false }
3607 ],
3608 "scopes": [{ "pattern": "project:{project_id}" }]
3609 }]
3610 });
3611 let path_string = path.to_str().expect("UTF-8 temp path");
3612
3613 {
3614 let mut first = SyncClient::open_path(
3615 "repair-restart-client".to_owned(),
3616 &schema,
3617 ClientLimits::default(),
3618 path_string,
3619 )
3620 .expect("first open");
3621 first
3622 .subscribe(
3623 "repair-tasks".to_owned(),
3624 "tasks".to_owned(),
3625 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3626 None,
3627 )
3628 .expect("subscribe");
3629 first
3630 .mutate(vec![Mutation::Upsert {
3631 table: "tasks".to_owned(),
3632 values: Map::from_iter([
3633 ("id".to_owned(), Value::from("offline-row")),
3634 ("project_id".to_owned(), Value::from("p1")),
3635 ]),
3636 base_version: None,
3637 }])
3638 .expect("queue offline work");
3639 assert_eq!(
3640 first
3641 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3642 rebootstrap_id: "restart-receipt".to_owned(),
3643 })
3644 .expect("first rebootstrap"),
3645 LocalDataRebootstrapResult {
3646 already_applied: false,
3647 retained_commits: 1,
3648 reset_subscriptions: 1,
3649 }
3650 );
3651 }
3652
3653 let mut reopened = SyncClient::open_path(
3654 "repair-restart-client".to_owned(),
3655 &schema,
3656 ClientLimits::default(),
3657 path_string,
3658 )
3659 .expect("reopen");
3660 reopened.drain_change_batches();
3661 reopened.drain_sync_intents();
3662 assert_eq!(
3663 reopened
3664 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3665 rebootstrap_id: "restart-receipt".to_owned(),
3666 })
3667 .expect("receipt replay"),
3668 LocalDataRebootstrapResult {
3669 already_applied: true,
3670 retained_commits: 1,
3671 reset_subscriptions: 1,
3672 }
3673 );
3674 assert!(reopened.drain_change_batches().is_empty());
3675 assert!(reopened.drain_sync_intents().is_empty());
3676 drop(reopened);
3677 std::fs::remove_file(path).expect("remove temp database");
3678 }
3679
3680 #[test]
3681 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
3682 let mut malformed = client();
3683 malformed
3684 .subscribe(
3685 "repair-tasks".to_owned(),
3686 "tasks".to_owned(),
3687 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
3688 None,
3689 )
3690 .expect("subscribe");
3691 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
3692 malformed.drain_change_batches();
3693 malformed.drain_sync_intents();
3694 let malformed_error = malformed
3695 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3696 rebootstrap_id: "malformed".to_owned(),
3697 })
3698 .expect_err("malformed receipt must fail closed");
3699 assert_eq!(
3700 malformed_error,
3701 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
3702 );
3703 assert!(!malformed.upgrading());
3704 assert_eq!(
3705 malformed
3706 .subscription_state("repair-tasks")
3707 .expect("unchanged subscription")
3708 .cursor,
3709 -1
3710 );
3711 assert!(malformed.drain_change_batches().is_empty());
3712 assert!(malformed.drain_sync_intents().is_empty());
3713
3714 let mut unreadable = client();
3715 unreadable
3716 .conn
3717 .execute(
3718 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
3719 rusqlite::params!["server-row", "p1"],
3720 )
3721 .expect("seed visible row");
3722 unreadable
3723 .conn
3724 .execute("DROP TABLE _syncular_meta", [])
3725 .expect("break marker storage");
3726 let unreadable_error = unreadable
3727 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3728 rebootstrap_id: "unreadable".to_owned(),
3729 })
3730 .expect_err("unreadable marker storage must fail closed");
3731 assert_eq!(
3732 unreadable_error,
3733 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
3734 );
3735 let visible_rows = unreadable
3736 .conn
3737 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
3738 .expect("visible projection remains");
3739 assert_eq!(visible_rows, 1);
3740 assert!(!unreadable.upgrading());
3741 assert!(unreadable.drain_change_batches().is_empty());
3742 assert!(unreadable.drain_sync_intents().is_empty());
3743 }
3744
3745 #[test]
3746 fn local_rebootstrap_cannot_bypass_schema_floor() {
3747 let mut client = client();
3748 client.set_schema_floor(Some(SchemaFloor {
3749 required_schema_version: Some(2),
3750 latest_schema_version: Some(2),
3751 }));
3752 let error = client
3753 .rebootstrap_local_data(&LocalDataRebootstrapInput {
3754 rebootstrap_id: "blocked-floor".to_owned(),
3755 })
3756 .expect_err("schema floor must block repair");
3757 assert!(error.contains("cannot bypass an active schema-floor stop"));
3758 }
3759
3760 #[test]
3761 fn diego_applied_sparse_moves_stay_visible_until_pull_image_arrives() {
3762 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
3763 "columns":[{"name":"id","type":"string","nullable":false},
3764 {"name":"project_id","type":"string","nullable":false},
3765 {"name":"starts_at_ms","type":"integer","nullable":false},
3766 {"name":"version","type":"integer","nullable":false}],
3767 "scopes":[{"pattern":"project:{project_id}"}]}]});
3768 let mut client =
3769 SyncClient::new("diego-ack-gap".into(), &schema, ClientLimits::default()).unwrap();
3770 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
3771 client
3772 .write_base_row(
3773 "tasks",
3774 &vec![
3775 Some(ColumnValue::String("t1".into())),
3776 Some(ColumnValue::String("p1".into())),
3777 Some(ColumnValue::Integer(100)),
3778 Some(ColumnValue::Integer(1)),
3779 ],
3780 1,
3781 )
3782 .unwrap();
3783 client.rebuild_overlay().expect("rebuild overlay");
3784 let first = client
3785 .patch(
3786 "tasks",
3787 "t1",
3788 Map::from_iter([
3789 ("starts_at_ms".into(), json!(200)),
3790 ("version".into(), json!(2)),
3791 ]),
3792 None,
3793 )
3794 .unwrap();
3795 let (_, first_request) = client.build_request(false).unwrap();
3796 let second = client
3798 .patch(
3799 "tasks",
3800 "t1",
3801 Map::from_iter([
3802 ("starts_at_ms".into(), json!(300)),
3803 ("version".into(), json!(3)),
3804 ]),
3805 None,
3806 )
3807 .unwrap();
3808 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3809 for (id, seq, meta) in [
3810 (first, 2, first_request),
3811 (second, 3, client.build_request(false).unwrap().1),
3812 ] {
3813 let response = Message {
3814 wire_version: WIRE_VERSION,
3815 msg_kind: MsgKind::Response,
3816 frames: vec![
3817 Frame::RespHeader {
3818 required_schema_version: None,
3819 latest_schema_version: None,
3820 log_epoch: Some("epoch-1".into()),
3821 reset_required: Some(false),
3822 },
3823 Frame::PushResult {
3824 client_commit_id: id,
3825 status: PushStatus::Applied,
3826 commit_seq: Some(seq),
3827 results: vec![OpResult::Applied { op_index: 0 }],
3828 },
3829 ],
3830 };
3831 assert!(matches!(
3832 client.process_response(&mut transport, response, &meta),
3833 SyncOutcome::Ok(_)
3834 ));
3835 let row = &client
3836 .query("SELECT starts_at_ms, version FROM tasks WHERE id='t1'", &[])
3837 .unwrap()[0];
3838 assert_eq!(
3839 row["starts_at_ms"],
3840 json!(300),
3841 "an applied ACK without a pull image must not expose the old base"
3842 );
3843 assert_eq!(
3844 row["version"],
3845 json!(3),
3846 "the queued domain version must remain monotonic"
3847 );
3848 }
3849 }
3850
3851 #[test]
3852 fn first_epoch_acquisition_preserves_tables_subscriptions_and_local_intent() {
3853 for stored in [None, Some("epoch-old")] {
3854 let mut client = client();
3855 client
3856 .subscribe(
3857 "tasks".into(),
3858 "tasks".into(),
3859 vec![("project_id".into(), vec!["p1".into()])],
3860 None,
3861 )
3862 .unwrap();
3863 client
3864 .conn
3865 .execute_batch(
3866 "CREATE TRIGGER epoch_table_identity AFTER INSERT ON tasks BEGIN SELECT 1; END",
3867 )
3868 .unwrap();
3869 let id = client
3870 .mutate(vec![Mutation::Upsert {
3871 table: "tasks".into(),
3872 values: Map::from_iter([
3873 ("id".into(), json!("offline")),
3874 ("project_id".into(), json!("p1")),
3875 ]),
3876 base_version: None,
3877 }])
3878 .unwrap();
3879 if let Some(epoch) = stored {
3880 client.set_meta(LOG_EPOCH_KEY, epoch);
3881 }
3882 let before = serde_json::to_value(client.subscription_state("tasks")).unwrap();
3883 let (_, meta) = client.build_request(false).unwrap();
3884 client.drain_change_batches();
3885 client.drain_sync_intents();
3886 let response = Message {
3887 wire_version: WIRE_VERSION,
3888 msg_kind: MsgKind::Response,
3889 frames: vec![Frame::RespHeader {
3890 required_schema_version: None,
3891 latest_schema_version: None,
3892 log_epoch: Some("epoch-new".into()),
3893 reset_required: Some(true),
3894 }],
3895 };
3896 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
3897 let outcome = client.process_response(&mut transport, response, &meta);
3898 let SyncOutcome::Ok(report) = outcome else {
3899 panic!("epoch round failed: {outcome:?}");
3900 };
3901 assert_eq!(
3902 report.resets,
3903 if stored.is_none() {
3904 Vec::<String>::new()
3905 } else {
3906 vec!["tasks".into()]
3907 }
3908 );
3909 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
3910 assert_eq!(client.pending_commit_ids(), vec![id]);
3911 assert_eq!(
3912 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3913 json!("offline")
3914 );
3915 let trigger: i64 = client
3916 .conn
3917 .query_row(
3918 "SELECT count(*) FROM sqlite_master WHERE name='epoch_table_identity'",
3919 [],
3920 |row| row.get(0),
3921 )
3922 .unwrap();
3923 if stored.is_none() {
3924 assert!(!client.upgrading);
3925 assert!(client
3926 .drain_change_batches()
3927 .iter()
3928 .all(|batch| batch.status.as_ref().is_none_or(|status| !status.upgrading)));
3929 assert_eq!(
3930 serde_json::to_value(client.subscription_state("tasks")).unwrap(),
3931 before
3932 );
3933 assert_eq!(trigger, 1, "acquisition must not drop the synced table");
3934 } else {
3935 assert!(client.upgrading);
3936 assert_eq!(trigger, 0, "an epoch change must drop the old table");
3937 }
3938 assert!(matches!(
3939 client.drain_sync_intents().as_slice(),
3940 [SyncIntent::Interactive]
3941 ));
3942 }
3943 }
3944
3945 #[test]
3946 fn epoch_acquisition_storage_failures_preserve_ready_state_and_intent() {
3947 for key in ["logEpoch", "localRevision"] {
3948 let mut client = client();
3949 let id = client
3950 .mutate(vec![Mutation::Upsert {
3951 table: "tasks".into(),
3952 values: Map::from_iter([
3953 ("id".into(), json!("offline")),
3954 ("project_id".into(), json!("p1")),
3955 ]),
3956 base_version: None,
3957 }])
3958 .unwrap();
3959 let revision = client.local_revision();
3960 let needed = client.sync_needed;
3961 client.drain_change_batches();
3962 client.drain_sync_intents();
3963 client.conn.execute_batch(&format!("CREATE TRIGGER fail_acquisition BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected acquisition failure'); END")).unwrap();
3964 assert!(client.run_log_epoch_reset("epoch-new").is_err());
3965 assert!(client.get_meta(LOG_EPOCH_KEY).is_none());
3966 assert!(!client.upgrading);
3967 assert_eq!(client.sync_needed, needed);
3968 assert_eq!(client.local_revision(), revision);
3969 assert_eq!(client.pending_commit_ids(), [id]);
3970 assert!(client.drain_change_batches().is_empty());
3971 assert!(client.drain_sync_intents().is_empty());
3972 }
3973 }
3974
3975 #[test]
3976 fn failed_log_epoch_resets_restore_memory_and_durable_state() {
3977 for key in [
3978 "localSchemaVersion",
3979 "localSchemaDescriptor",
3980 "logEpoch",
3981 "localRevision",
3982 ] {
3983 let mut client = client();
3984 client.set_meta(LOG_EPOCH_KEY, "epoch-old");
3985 let commit = client
3986 .mutate(vec![Mutation::Upsert {
3987 table: "tasks".into(),
3988 values: Map::from_iter([
3989 ("id".into(), json!("offline")),
3990 ("project_id".into(), json!("p1")),
3991 ]),
3992 base_version: None,
3993 }])
3994 .unwrap();
3995 client.active_round = Some(uuid::Uuid::new_v4());
3996 client.overlay_dirty.set(false);
3997 let round = client.active_round;
3998 let needed = client.sync_needed;
3999 let revision = client.local_revision();
4000 let rows = serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap();
4001 client.drain_change_batches();
4002 client.drain_sync_intents();
4003 client.conn.execute_batch(&format!("CREATE TRIGGER fail_reset BEFORE INSERT ON _syncular_meta WHEN NEW.key='{key}' BEGIN SELECT RAISE(FAIL,'injected reset failure'); END")).unwrap();
4004 assert!(client.run_log_epoch_reset("epoch-new").is_err(), "{key}");
4005 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-old"));
4006 assert!(!client.upgrading);
4007 assert_eq!(client.active_round, round);
4008 assert!(!client.overlay_dirty.get());
4009 assert_eq!(client.sync_needed, needed);
4010 assert_eq!(client.local_revision(), revision);
4011 assert_eq!(
4012 client.pending_commit_ids().as_slice(),
4013 std::slice::from_ref(&commit)
4014 );
4015 assert_eq!(
4016 serde_json::to_value(client.read_rows("tasks").unwrap()).unwrap(),
4017 rows
4018 );
4019 assert!(client.drain_change_batches().is_empty());
4020 assert!(client.drain_sync_intents().is_empty());
4021 assert!(client.conn.is_autocommit());
4022 client
4023 .conn
4024 .execute_batch("DROP TRIGGER fail_reset")
4025 .unwrap();
4026 client.run_log_epoch_reset("epoch-new").unwrap();
4027 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("epoch-new"));
4028 assert_eq!(client.pending_commit_ids(), [commit]);
4029 }
4030 }
4031
4032 #[test]
4033 fn acknowledged_restore_and_replay_ignore_unrelated_subscription_count() {
4034 for subscriptions in [0, 2, 6, 12] {
4035 let mut client = SyncClient::new("ack-scope".into(), &json!({ "version": 1, "tables": [
4036 { "name": "tasks", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] },
4037 { "name": "docs", "primaryKey": "id", "columns": [{ "name": "id", "type": "string", "nullable": false }, { "name": "project_id", "type": "string", "nullable": false }], "scopes": [{ "pattern": "project:{project_id}" }] }
4038 ] }), ClientLimits::default()).unwrap();
4039 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4040 client.conn.execute_batch("CREATE TABLE ack_writes(kind TEXT); CREATE TRIGGER count_ack_restore AFTER DELETE ON tasks BEGIN INSERT INTO ack_writes VALUES('restore'); END; CREATE TRIGGER count_ack_replay AFTER INSERT ON tasks BEGIN INSERT INTO ack_writes VALUES('replay'); END;").unwrap();
4041 for index in 0..32 {
4042 client
4043 .mutate(vec![Mutation::Upsert {
4044 table: "tasks".into(),
4045 values: Map::from_iter([
4046 ("id".into(), json!(format!("task-{index}"))),
4047 ("project_id".into(), json!("p1")),
4048 ]),
4049 base_version: None,
4050 }])
4051 .unwrap();
4052 }
4053 let (_, meta) = client.build_request(false).unwrap();
4054 let mut frames = vec![Frame::RespHeader {
4055 required_schema_version: None,
4056 latest_schema_version: None,
4057 log_epoch: Some("epoch-1".into()),
4058 reset_required: Some(false),
4059 }];
4060 frames.extend(meta.pushed_ids.iter().enumerate().map(|(index, id)| {
4061 Frame::PushResult {
4062 client_commit_id: id.clone(),
4063 status: PushStatus::Applied,
4064 commit_seq: Some(index as i64 + 1),
4065 results: vec![OpResult::Applied { op_index: 0 }],
4066 }
4067 }));
4068 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
4069 client.acknowledged_replay_count.set(0);
4070 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
4071 assert!(matches!(
4072 client.process_response(
4073 &mut transport,
4074 Message {
4075 wire_version: WIRE_VERSION,
4076 msg_kind: MsgKind::Response,
4077 frames
4078 },
4079 &meta
4080 ),
4081 SyncOutcome::Ok(_)
4082 ));
4083 assert_eq!(client.acknowledged_replay_count.get(), 32);
4084 assert_eq!(
4085 client
4086 .query(
4087 "SELECT count(*) AS n FROM ack_writes WHERE kind='restore'",
4088 &[]
4089 )
4090 .unwrap()[0]["n"],
4091 json!(32)
4092 );
4093 assert_eq!(
4094 client
4095 .query(
4096 "SELECT count(*) AS n FROM ack_writes WHERE kind='replay'",
4097 &[]
4098 )
4099 .unwrap()[0]["n"],
4100 json!(32)
4101 );
4102 client.conn.execute("DELETE FROM ack_writes", []).unwrap();
4103 client.acknowledged_replay_count.set(0);
4104 for index in 0..subscriptions {
4105 client
4106 .subscribe(
4107 format!("docs-{index}"),
4108 "docs".into(),
4109 vec![("project_id".into(), vec![format!("scope-{index}")])],
4110 None,
4111 )
4112 .unwrap();
4113 let sub = client.subs.len() - 1;
4114 client.subs[sub].effective =
4115 Some(vec![("project_id".into(), vec![format!("scope-{index}")])]);
4116 let segment = RowsSegment {
4117 table: "docs".into(),
4118 schema_version: 1,
4119 columns: client.schema.table("docs").unwrap().wire_columns.clone(),
4120 blocks: if index % 2 == 0 {
4121 vec![vec![ssp2::segment::SegmentRow {
4122 values: vec![
4123 Some(ColumnValue::String(format!("doc-{index}"))),
4124 Some(ColumnValue::String(format!("scope-{index}"))),
4125 ],
4126 server_version: 1,
4127 }]]
4128 } else {
4129 vec![]
4130 },
4131 };
4132 assert!(client.apply_segment(sub, &segment, true).is_ok());
4133 }
4134 assert_eq!(
4135 client
4136 .query("SELECT count(*) AS n FROM ack_writes", &[])
4137 .unwrap()[0]["n"],
4138 json!(0),
4139 "unrelated bootstrap must not restore/replay tasks"
4140 );
4141 assert_eq!(
4142 client.acknowledged_replay_count.get(),
4143 0,
4144 "unrelated bootstrap must not decode task ACKs"
4145 );
4146 assert_eq!(
4147 client
4148 .query("SELECT count(*) AS n FROM tasks", &[])
4149 .unwrap()[0]["n"],
4150 json!(32)
4151 );
4152 assert_eq!(
4153 client
4154 .query("SELECT count(*) AS n FROM _syncular_acked_rows", &[])
4155 .unwrap()[0]["n"],
4156 json!(32)
4157 );
4158 }
4159 }
4160
4161 #[test]
4162 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
4163 let mut client = client();
4164 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
4165 const COMMIT_COUNT: usize = 32;
4166
4167 for index in 0..COMMIT_COUNT {
4168 client
4169 .mutate(vec![Mutation::Upsert {
4170 table: "tasks".to_owned(),
4171 values: Map::from_iter([
4172 ("id".to_owned(), Value::from(format!("task-{index}"))),
4173 ("project_id".to_owned(), Value::from("project-1")),
4174 ]),
4175 base_version: None,
4176 }])
4177 .expect("queue commit");
4178 }
4179
4180 let (_, request_meta) = client.build_request(false).unwrap();
4181 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
4182 let mut frames = vec![Frame::RespHeader {
4183 required_schema_version: None,
4184 latest_schema_version: None,
4185 log_epoch: Some("epoch-1".to_owned()),
4186 reset_required: Some(false),
4187 }];
4188 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
4189 |(index, client_commit_id)| Frame::PushResult {
4190 client_commit_id: client_commit_id.clone(),
4191 status: PushStatus::Applied,
4192 commit_seq: Some(index as i64 + 1),
4193 results: vec![OpResult::Applied { op_index: 0 }],
4194 },
4195 ));
4196 let response = Message {
4197 wire_version: WIRE_VERSION,
4198 msg_kind: MsgKind::Response,
4199 frames,
4200 };
4201 let mut transport =
4202 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
4203
4204 client.overlay_rebuild_count.set(0);
4205 client.outcome_prune_count.set(0);
4206 let outcome = client.process_response(&mut transport, response, &request_meta);
4207 assert!(matches!(outcome, SyncOutcome::Ok(_)));
4208 assert!(client.pending_commit_ids().is_empty());
4209 assert_eq!(
4210 client.overlay_rebuild_count.get(),
4211 1,
4212 "one response must reconcile its acknowledged commits with one overlay rebuild"
4213 );
4214 assert_eq!(
4215 client.outcome_prune_count.get(),
4216 1,
4217 "one response must enforce outcome retention once"
4218 );
4219 }
4220
4221 #[test]
4222 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
4223 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
4224 "columns": [
4225 {"name":"id", "type":"string", "nullable":false},
4226 {"name":"project_id", "type":"string", "nullable":false},
4227 {"name":"title", "type":"string", "nullable":false}
4228 ], "scopes": [{"pattern":"project:{project_id}"}],
4229 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
4230 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
4231 }] });
4232 let mut incremental =
4233 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
4234 let mut reference =
4235 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
4236 let tables = vec!["tasks".to_owned()];
4237 for index in 0..32 {
4238 let id = format!("task-{}", index % 8);
4239 let values = Map::from_iter([
4240 ("id".to_owned(), Value::from(id.clone())),
4241 (
4242 "project_id".to_owned(),
4243 Value::from(format!("p{}", index % 2)),
4244 ),
4245 ("title".to_owned(), Value::from(format!("needle{index}"))),
4246 ]);
4247 let payload = encode_row_json(
4248 incremental.schema.table("tasks").unwrap(),
4249 &id,
4250 &values,
4251 &incremental.encryption,
4252 )
4253 .unwrap();
4254 let change = ssp2::model::Change {
4255 table_index: 0,
4256 row_id: id,
4257 op: if index % 5 == 0 {
4258 Op::Delete
4259 } else {
4260 Op::Upsert
4261 },
4262 row_version: Some(index + 1),
4263 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
4264 row: Some(payload),
4265 };
4266 reference.overlay_dirty.set(true);
4267 for instance in [&mut incremental, &mut reference] {
4268 instance.begin_observation("mirror_test").unwrap();
4269 let mut batch = ChangeAccumulator::default();
4270 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
4271 instance
4272 .apply_commit_changes(&tables, std::slice::from_ref(&change))
4273 .unwrap();
4274 instance
4275 .rebuild_overlay_if_dirty()
4276 .expect("rebuild overlay");
4277 instance.finish_observation("mirror_test", batch).unwrap();
4278 }
4279 for sql in [
4280 "SELECT * FROM tasks ORDER BY id",
4281 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
4282 ] {
4283 assert_eq!(
4284 incremental.query(sql, &[]).unwrap(),
4285 reference.query(sql, &[]).unwrap()
4286 );
4287 }
4288 assert_eq!(incremental.local_revision(), reference.local_revision());
4289 assert_eq!(
4290 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4291 serde_json::to_value(reference.drain_change_batches()).unwrap()
4292 );
4293 }
4294 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
4295 assert_eq!(reference.overlay_rebuild_count.get(), 32);
4296 }
4297
4298 #[test]
4299 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
4300 let cases = [
4301 (
4302 "string",
4303 vec![
4304 json!("1"),
4305 json!("01"),
4306 json!(""),
4307 json!("nul\u{0}id"),
4308 json!("é"),
4309 ],
4310 ),
4311 (
4312 "integer",
4313 vec![
4314 json!(i64::MIN),
4315 json!(-1),
4316 json!(0),
4317 json!(1),
4318 json!(i64::MAX),
4319 ],
4320 ),
4321 ("boolean", vec![json!(false), json!(true)]),
4322 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
4323 ];
4324 for (kind, ids) in cases {
4325 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4326 "columns":[{"name":"id","type":kind,"nullable":false},
4327 {"name":"project_id","type":"string","nullable":false}],
4328 "scopes":[{"pattern":"project:{project_id}"}]}]});
4329 let mut client =
4330 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
4331 for (index, id) in ids.iter().enumerate() {
4332 let table = client.schema.table("tasks").unwrap();
4333 client
4334 .write_base_row(
4335 "tasks",
4336 &vec![
4337 if kind == "integer" {
4338 Some(ColumnValue::Integer(id.as_i64().unwrap()))
4341 } else {
4342 json_to_column_value(&table.columns[0], Some(id)).unwrap()
4343 },
4344 Some(ColumnValue::String(format!("p{index}"))),
4345 ],
4346 1,
4347 )
4348 .unwrap();
4349 }
4350 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4351 for base in [true, false] {
4352 let full = if base {
4353 base_table("tasks")
4354 } else {
4355 visible_table("tasks")
4356 };
4357 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
4358 for row_id in [
4359 "1",
4360 "01",
4361 "1.0",
4362 "1.0000000000000002",
4363 "+1",
4364 "1e0",
4365 " 1",
4366 "-1",
4367 "0",
4368 "9223372036854775807",
4369 "-9223372036854775808",
4370 "true",
4371 "false",
4372 "",
4373 "nul\u{0}id",
4374 "é",
4375 "null",
4376 "{\"a\":1}",
4377 ] {
4378 let old_sql =
4379 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
4380 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
4381 let old = client
4382 .conn
4383 .prepare(&old_sql)
4384 .unwrap()
4385 .query_map([row_id], |row| row.get::<_, String>(0))
4386 .unwrap()
4387 .collect::<Result<Vec<_>, _>>()
4388 .unwrap();
4389 let new = client
4390 .conn
4391 .prepare_cached(&new_sql)
4392 .unwrap()
4393 .query_map([row_id], |row| row.get::<_, String>(0))
4394 .unwrap()
4395 .collect::<Result<Vec<_>, _>>()
4396 .unwrap();
4397 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
4398 let mut batch = ChangeAccumulator::default();
4399 assert_eq!(
4400 client.record_row_scopes(&mut batch, "tasks", row_id, base),
4401 !old.is_empty()
4402 );
4403 let expected = old.first().map(|scope| {
4404 BTreeMap::from([(
4405 "tasks".to_owned(),
4406 Some(BTreeSet::from([format!("project:{scope}")])),
4407 )])
4408 });
4409 assert_eq!(batch.tables, expected.unwrap_or_default());
4410 client
4411 .conn
4412 .execute_batch("SAVEPOINT lookup_delete")
4413 .unwrap();
4414 if base {
4415 client.delete_base_row("tasks", row_id).unwrap();
4416 } else {
4417 client
4418 .apply_outbox_ops(&[OutboxOp {
4419 upsert: false,
4420 insert: false,
4421 table: "tasks".into(),
4422 row_id: row_id.into(),
4423 base_version: None,
4424 values: None,
4425 }])
4426 .expect("replay delete");
4427 }
4428 let remaining: i64 = client
4429 .conn
4430 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
4431 row.get(0)
4432 })
4433 .unwrap();
4434 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
4435 client
4436 .conn
4437 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
4438 .unwrap();
4439 client.overlay_dirty.set(false);
4440 }
4441 for (predicate, expected) in [
4442 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
4443 (predicate, "SEARCH"),
4444 ] {
4445 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
4446 let details = client
4447 .conn
4448 .prepare(&sql)
4449 .unwrap()
4450 .query_map(["1"], |row| row.get::<_, String>(3))
4451 .unwrap()
4452 .collect::<Result<Vec<_>, _>>()
4453 .unwrap();
4454 assert!(
4455 details.iter().any(|detail| detail.contains(expected)),
4456 "{kind}: {details:?}"
4457 );
4458 }
4459 }
4460 }
4461 }
4462
4463 #[test]
4464 fn primary_keys_reject_every_ineligible_column_type() {
4465 for ty in ["float", "bytes", "crdt", "blob_ref"] {
4469 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4470 "columns":[{"name":"id","type":ty,"nullable":false},
4471 {"name":"project_id","type":"string","nullable":false}],
4472 "scopes":[{"pattern":"project:{project_id}"}]}]});
4473 let error =
4474 match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
4475 Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
4476 Err(error) => error,
4477 };
4478 assert!(
4479 error.contains("has an ineligible column type (§2.4)"),
4480 "{ty}: {error}"
4481 );
4482 }
4483 for ty in ["string", "integer", "boolean", "json"] {
4484 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
4485 "columns":[{"name":"id","type":ty,"nullable":false},
4486 {"name":"project_id","type":"string","nullable":false}],
4487 "scopes":[{"pattern":"project:{project_id}"}]}]});
4488 SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
4489 .unwrap_or_else(|error| panic!("{ty}: {error}"));
4490 }
4491 }
4492
4493 #[test]
4494 fn pending_row_reconciliation_matches_full_fifo_replay() {
4495 for unique in [false, true] {
4496 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
4497 "name": name, "primaryKey": "id", "columns": [
4498 {"name":"id", "type":"string", "nullable":false},
4499 {"name":"project_id", "type":"string", "nullable":false},
4500 {"name":"title", "type":"string", "nullable":false}],
4501 "scopes": [{"pattern":"project:{project_id}"}],
4502 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
4503 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
4504 })))});
4505 let mut incremental =
4506 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
4507 let mut reference =
4508 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
4509 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
4510 for instance in [&mut incremental, &mut reference] {
4511 for table in &tables {
4512 for index in 0..12 {
4513 instance
4514 .write_base_row(
4515 table,
4516 &vec![
4517 Some(ColumnValue::String(format!("row{index}"))),
4518 Some(ColumnValue::String("p0".into())),
4519 Some(ColumnValue::String(format!("base{index}"))),
4520 ],
4521 1,
4522 )
4523 .unwrap();
4524 }
4525 }
4526 instance
4527 .rebuild_overlay_if_dirty()
4528 .expect("rebuild overlay");
4529 }
4530 for index in 0..80 {
4531 let mutations = (0..3)
4532 .map(|offset| {
4533 let table = tables[(index + offset) % 2].clone();
4534 let id = format!("row{}", (index + offset) % 16);
4535 if (index + offset) % 7 == 0 {
4536 Mutation::Delete {
4537 table,
4538 row_id: id,
4539 base_version: None,
4540 }
4541 } else {
4542 Mutation::Upsert {
4543 table,
4544 values: Map::from_iter([
4545 ("id".into(), Value::from(id)),
4546 ("project_id".into(), Value::from(format!("p{}", index % 3))),
4547 (
4548 "title".into(),
4549 Value::from(format!("needle{}", (index + offset) % 16)),
4550 ),
4551 ]),
4552 base_version: None,
4553 }
4554 }
4555 })
4556 .collect::<Vec<_>>();
4557 for instance in [&mut incremental, &mut reference] {
4558 instance.mutate(mutations.clone()).unwrap();
4559 }
4560 let changes = (0..3)
4563 .map(|offset| {
4564 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
4565 let id = format!("row{}", (index + offset) % 16);
4566 let values = Map::from_iter([
4567 ("id".into(), Value::from(id.clone())),
4568 (
4569 "project_id".into(),
4570 Value::from(format!("p{}", (index + 1) % 3)),
4571 ),
4572 (
4573 "title".into(),
4574 Value::from(format!("remote{index}_{offset}")),
4575 ),
4576 ]);
4577 ssp2::model::Change {
4578 table_index: table_index as u16,
4579 row_id: id.clone(),
4580 op: if (index + offset) % 5 == 0 {
4581 Op::Delete
4582 } else {
4583 Op::Upsert
4584 },
4585 row_version: Some(index as i64 + 2),
4586 scopes: vec![],
4587 row: Some(
4588 encode_row_json(
4589 incremental.schema.table(&tables[table_index]).unwrap(),
4590 &id,
4591 &values,
4592 &incremental.encryption,
4593 )
4594 .unwrap(),
4595 ),
4596 }
4597 })
4598 .collect::<Vec<_>>();
4599 reference.overlay_dirty.set(true);
4600 incremental
4601 .apply_commit_frame(&tables, &changes, None)
4602 .unwrap();
4603 reference
4604 .apply_commit_frame(&tables, &changes, None)
4605 .unwrap();
4606 for table in &tables {
4607 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
4608 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
4609 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
4610 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
4611 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
4612 }
4613 }
4614 assert_eq!(incremental.local_revision(), reference.local_revision());
4615 assert_eq!(
4616 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
4617 serde_json::to_value(reference.drain_change_batches()).unwrap()
4618 );
4619 }
4620 assert_eq!(
4621 incremental.overlay_rebuild_count.get(),
4622 if unique { 81 } else { 21 }
4623 );
4624 assert_eq!(reference.overlay_rebuild_count.get(), 81);
4625 }
4626 }
4627
4628 #[test]
4629 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
4630 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
4631 "columns":[{"name":"id", "type":"string", "nullable":false},
4632 {"name":"title", "type":"string", "nullable":false}],
4633 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
4634 let mut client =
4635 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
4636 for (id, title) in [("a", "original"), ("b", "occupied")] {
4637 client
4638 .write_base_row(
4639 "tasks",
4640 &vec![
4641 Some(ColumnValue::String(id.into())),
4642 Some(ColumnValue::String(title.into())),
4643 ],
4644 1,
4645 )
4646 .unwrap();
4647 }
4648 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4649 let pending = client
4650 .mutate(vec![Mutation::Upsert {
4651 table: "tasks".into(),
4652 values: Map::from_iter([
4653 ("id".into(), Value::from("a")),
4654 ("title".into(), Value::from("available")),
4655 ]),
4656 base_version: None,
4657 }])
4658 .unwrap();
4659 assert_eq!(
4660 client
4661 .conn
4662 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4663 .get::<_, String>(
4664 0
4665 ))
4666 .unwrap(),
4667 "available"
4668 );
4669 for (title, expected) in [("available", "original"), ("occupied", "available")] {
4670 let change = ssp2::model::Change {
4671 table_index: 0,
4672 row_id: "b".into(),
4673 op: Op::Upsert,
4674 row_version: Some(2),
4675 scopes: vec![],
4676 row: Some(
4677 encode_row_json(
4678 client.schema.table("tasks").unwrap(),
4679 "b",
4680 &Map::from_iter([
4681 ("id".into(), Value::from("b")),
4682 ("title".into(), Value::from(title)),
4683 ]),
4684 &client.encryption,
4685 )
4686 .unwrap(),
4687 ),
4688 };
4689 client
4690 .apply_commit_frame(&["tasks".into()], &[change], None)
4691 .unwrap();
4692 assert_eq!(
4693 client
4694 .conn
4695 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
4696 .get::<_, String>(
4697 0
4698 ))
4699 .unwrap(),
4700 expected
4701 );
4702 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
4703 }
4704 assert_eq!(client.overlay_rebuild_count.get(), 3);
4705 }
4706
4707 fn measure_scoped_overlay(unchanged_rows: i64, rounds: i64) -> Vec<u128> {
4708 let schema = json!({ "version": 1, "tables": [
4709 { "name":"tasks", "primaryKey":"id", "columns":[
4710 {"name":"id","type":"string","nullable":false},
4711 {"name":"project_id","type":"string","nullable":false}],
4712 "scopes":[{"pattern":"project:{project_id}"}] },
4713 { "name":"catalogue", "primaryKey":"id", "columns":[
4714 {"name":"id","type":"string","nullable":false},
4715 {"name":"project_id","type":"string","nullable":false},
4716 {"name":"title","type":"string","nullable":false}],
4717 "scopes":[{"pattern":"project:{project_id}"}],
4718 "ftsIndexes":[{"name":"catalogue_fts","columns":["title"],"tokenize":"unicode61"}] }
4719 ] });
4720 let mut client =
4721 SyncClient::new("overlay-scope".into(), &schema, ClientLimits::default()).unwrap();
4722 if unchanged_rows > 0 {
4723 client.conn.execute_batch(&format!("WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i<{unchanged_rows}) INSERT INTO _syncular_base_catalogue SELECT 'c'||i,'p1','needle '||i,1 FROM n")).unwrap();
4724 }
4725 client.rebuild_overlay().expect("rebuild overlay");
4726 client.conn.execute_batch("CREATE TABLE audit(tbl TEXT); CREATE TRIGGER catalogue_changed AFTER DELETE ON catalogue BEGIN INSERT INTO audit VALUES ('catalogue'); END;").unwrap();
4727 let mut elapsed = Vec::new();
4728 for round in 1..=rounds {
4729 let pending = client
4730 .mutate(vec![Mutation::Upsert {
4731 table: "tasks".into(),
4732 values: Map::from_iter([
4733 ("id".into(), json!("t1")),
4734 ("project_id".into(), json!("p1")),
4735 ]),
4736 base_version: None,
4737 }])
4738 .unwrap();
4739 let started = std::time::Instant::now();
4740 client
4741 .write_base_row(
4742 "tasks",
4743 &vec![
4744 Some(ColumnValue::String("t1".into())),
4745 Some(ColumnValue::String("p1".into())),
4746 ],
4747 round,
4748 )
4749 .unwrap();
4750 let mut report = SyncReport::default();
4751 client
4752 .handle_push_results(
4753 &[Frame::PushResult {
4754 client_commit_id: pending.clone(),
4755 status: PushStatus::Applied,
4756 commit_seq: Some(round),
4757 results: vec![OpResult::Applied { op_index: 0 }],
4758 }],
4759 &HashSet::from([pending.as_str()]),
4760 &HashMap::new(),
4761 &mut report,
4762 Some(&pending),
4763 )
4764 .unwrap();
4765 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
4766 elapsed.push(started.elapsed().as_nanos());
4767 }
4768 assert_eq!(
4769 client
4770 .conn
4771 .query_row("SELECT count(*) FROM audit", [], |r| r.get::<_, i64>(0))
4772 .unwrap(),
4773 0,
4774 "unchanged catalogue must never be copied"
4775 );
4776 assert_eq!(
4777 client
4778 .conn
4779 .query_row(
4780 "SELECT count(*) FROM catalogue_fts WHERE catalogue_fts MATCH 'needle'",
4781 [],
4782 |r| r.get::<_, i64>(0)
4783 )
4784 .unwrap(),
4785 unchanged_rows
4786 );
4787 assert_eq!(
4788 client.query("SELECT * FROM tasks", &[]).unwrap()[0]["_syncular_version"],
4789 json!(rounds)
4790 );
4791 elapsed
4792 }
4793
4794 #[test]
4795 fn changed_overlay_leaves_large_unchanged_table_and_fts_untouched() {
4796 let elapsed = measure_scoped_overlay(100_000, 1);
4797 eprintln!(
4798 "table-scoped overlay with 100k unchanged rows: {} ns",
4799 elapsed[0]
4800 );
4801 }
4802
4803 #[test]
4804 #[ignore = "isolated performance measurement; run with the shared heavy-check lock"]
4805 fn benchmark_scoped_overlay() {
4806 for rows in [0, 100_000, 200_000] {
4807 let mut elapsed = measure_scoped_overlay(rows, 30);
4808 elapsed.sort_unstable();
4809 eprintln!(
4810 "{}",
4811 json!({"unchangedRows": rows, "rounds": elapsed.len(),
4812 "medianNs": elapsed[elapsed.len()/2], "minNs": elapsed[0],
4813 "maxNs": elapsed[elapsed.len()-1], "unchangedDeletes": 0})
4814 );
4815 }
4816 }
4817
4818 #[test]
4819 fn released_context_discards_in_flight_ack_and_server_rows() {
4820 for barrier in [
4821 "preflight",
4822 "purge",
4823 "rebootstrap",
4824 "subscription",
4825 "failed-purge-replay",
4826 "failed-purge-publish",
4827 "failed-rebootstrap",
4828 ] {
4829 let mut client = client();
4830 client.set_meta(LOG_EPOCH_KEY, "epoch");
4831 client
4832 .subscribe(
4833 "tasks".into(),
4834 "tasks".into(),
4835 vec![("project_id".into(), vec!["p1".into()])],
4836 None,
4837 )
4838 .unwrap();
4839 let commit = client
4840 .mutate(vec![Mutation::Upsert {
4841 table: "tasks".into(),
4842 values: Map::from_iter([
4843 ("id".into(), json!("local")),
4844 ("project_id".into(), json!("p1")),
4845 ]),
4846 base_version: None,
4847 }])
4848 .unwrap();
4849 let prepared = client.prepare_sync_round(false).unwrap();
4850 client.set_transport_enabled(&mut CountingRealtimeTransport::default(), false);
4851 let payload = encode_row_json(
4852 &client.schema.tables[0],
4853 "server",
4854 &Map::from_iter([
4855 ("id".into(), json!("server")),
4856 ("project_id".into(), json!("p1")),
4857 ]),
4858 &client.encryption,
4859 )
4860 .unwrap();
4861 let completed = crate::CompletedSyncRound {
4862 prepared,
4863 exchange: crate::round::ExchangeResult::Reply {
4864 transport_failed: false,
4865 downloads: Default::default(),
4866 response: Ok(Message {
4867 wire_version: WIRE_VERSION,
4868 msg_kind: MsgKind::Response,
4869 frames: vec![
4870 Frame::RespHeader {
4871 required_schema_version: None,
4872 latest_schema_version: None,
4873 log_epoch: Some("epoch".into()),
4874 reset_required: Some(false),
4875 },
4876 Frame::PushResult {
4877 client_commit_id: commit.clone(),
4878 status: PushStatus::Applied,
4879 commit_seq: Some(1),
4880 results: vec![OpResult::Applied { op_index: 0 }],
4881 },
4882 Frame::SubStart {
4883 id: "tasks".into(),
4884 status: SubStatus::Active,
4885 reason_code: String::new(),
4886 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
4887 bootstrap: true,
4888 },
4889 Frame::Commit {
4890 commit_seq: 1,
4891 created_at_ms: 0,
4892 actor_id: "actor".into(),
4893 tables: vec!["tasks".into()],
4894 changes: vec![ssp2::model::Change {
4895 table_index: 0,
4896 row_id: "server".into(),
4897 op: Op::Upsert,
4898 row_version: Some(1),
4899 scopes: vec![("project_id".into(), "p1".into())],
4900 row: Some(payload),
4901 }],
4902 },
4903 Frame::SubEnd {
4904 next_cursor: 1,
4905 bootstrap_state: None,
4906 },
4907 ],
4908 }),
4909 },
4910 };
4911 match barrier {
4912 "preflight" => client.begin_security_preflight(),
4913 "purge" => {
4914 client
4915 .purge_local_data(&LocalDataPurgeInput {
4916 purge_id: "pending-round".into(),
4917 targets: vec![LocalDataPurgeTarget {
4918 table: "tasks".into(),
4919 selectors: BTreeMap::from([(
4920 "project_id".into(),
4921 vec!["p1".into()],
4922 )]),
4923 }],
4924 })
4925 .unwrap();
4926 }
4927 "rebootstrap" => {
4928 client
4929 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4930 rebootstrap_id: "pending-round".into(),
4931 })
4932 .unwrap();
4933 }
4934 "failed-purge-replay" | "failed-purge-publish" | "failed-rebootstrap" => {
4935 let trigger = if barrier == "failed-purge-replay" {
4936 "CREATE TRIGGER fail_local BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4937 } else {
4938 "CREATE TRIGGER fail_local BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT, 'local fault'); END"
4939 };
4940 client.conn.execute_batch(trigger).unwrap();
4941 let revision = client.local_revision();
4942 if barrier == "failed-rebootstrap" {
4943 assert!(client
4944 .rebootstrap_local_data(&LocalDataRebootstrapInput {
4945 rebootstrap_id: "failed-round".into(),
4946 })
4947 .is_err());
4948 } else {
4949 assert!(client
4950 .purge_local_data(&LocalDataPurgeInput {
4951 purge_id: "failed-round".into(),
4952 targets: vec![LocalDataPurgeTarget {
4953 table: "tasks".into(),
4954 selectors: BTreeMap::from([(
4955 "project_id".into(),
4956 vec!["p1".into()]
4957 )]),
4958 }],
4959 })
4960 .is_err());
4961 }
4962 assert_eq!(client.local_revision(), revision);
4963 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
4964 client
4965 .conn
4966 .execute_batch("DROP TRIGGER fail_local")
4967 .unwrap();
4968 let applied = client.apply_sync_round(completed);
4969 assert!(
4970 matches!(
4971 applied,
4972 crate::AppliedSyncRound::Complete {
4973 outcome: SyncOutcome::Ok(_),
4974 ..
4975 }
4976 ),
4977 "{barrier}"
4978 );
4979 assert!(client.pending_commit_ids().is_empty());
4980 assert_eq!(
4981 client
4982 .query("SELECT id FROM tasks WHERE id = 'server'", &[])
4983 .unwrap()
4984 .len(),
4985 1
4986 );
4987 continue;
4988 }
4989 "subscription" => {
4990 client.unsubscribe("tasks");
4991 client
4992 .subscribe(
4993 "tasks".into(),
4994 "tasks".into(),
4995 vec![("project_id".into(), vec!["p2".into()])],
4996 None,
4997 )
4998 .unwrap();
4999 }
5000 _ => unreachable!(),
5001 }
5002 let pending_before_reply = client.pending_commit_ids();
5003 let revision = client.local_revision();
5004 let applied = client.apply_sync_round(completed);
5005 assert!(
5006 matches!(applied, crate::AppliedSyncRound::Complete { outcome: SyncOutcome::Failed { ref error_code, .. }, ref controls, .. } if error_code == "client.round_cancelled" && controls.is_empty())
5007 );
5008 assert_eq!(client.local_revision(), revision);
5009 assert!(client
5010 .conn
5011 .query_row(
5012 "SELECT NOT EXISTS(SELECT 1 FROM tasks WHERE id='server')",
5013 [],
5014 |r| r.get::<_, bool>(0)
5015 )
5016 .unwrap());
5017 assert_eq!(client.pending_commit_ids(), pending_before_reply);
5018 }
5019 }
5020
5021 #[test]
5022 fn sync_round_deadline_is_anchored_once_per_round_and_carried_across_continues() {
5023 struct DeadlineRecording {
5024 budget: std::time::Duration,
5025 seen: Vec<Option<std::time::Instant>>,
5026 }
5027 impl Transport for DeadlineRecording {
5028 fn round_deadline(&self) -> Option<std::time::Duration> {
5029 Some(self.budget)
5030 }
5031 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
5032 self.seen.push(deadline);
5033 }
5034 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
5035 Err(TransportError::new("transport.failed", "injected"))
5036 }
5037 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
5038 self.sync(request)
5039 }
5040 fn download_segment(
5041 &mut self,
5042 _: &SegmentRequest,
5043 _: &mut dyn FnMut(u64),
5044 ) -> Result<Vec<u8>, TransportError> {
5045 Err(TransportError::new("transport.failed", "injected"))
5046 }
5047 fn blob_upload_grant(
5048 &mut self,
5049 _: &str,
5050 _: u64,
5051 _: Option<&str>,
5052 ) -> Result<crate::BlobUploadGrant, TransportError> {
5053 Ok(crate::BlobUploadGrant::Present)
5054 }
5055 fn realtime_connect(&mut self) -> Result<(), TransportError> {
5056 Ok(())
5057 }
5058 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
5059 Ok(())
5060 }
5061 fn realtime_close(&mut self) -> Result<(), TransportError> {
5062 Ok(())
5063 }
5064 }
5065
5066 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5067 {"name":"id","type":"string","nullable":false},
5068 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5069 let mut client =
5070 SyncClient::new("deadline-round".into(), &schema, Default::default()).unwrap();
5071 client.set_meta(LOG_EPOCH_KEY, "epoch");
5072 client
5073 .upload_blob(b"pending upload", Some("text/plain".into()), None)
5074 .unwrap();
5075
5076 let mut transport = DeadlineRecording {
5077 budget: std::time::Duration::from_secs(30),
5078 seen: Vec::new(),
5079 };
5080 let prepared = client.prepare_sync_round(false).unwrap();
5081 assert_eq!(prepared.uploads.len(), 1, "one staged upload");
5082 let prepared = match client.apply_sync_round(prepared.exchange(&mut transport)) {
5084 crate::AppliedSyncRound::Continue(prepared) => prepared,
5085 crate::AppliedSyncRound::Complete { .. } => panic!("the upload must continue"),
5086 };
5087 let completed = prepared.exchange(&mut transport);
5089 let anchor = completed
5090 .prepared
5091 .round_deadline_at
5092 .expect("the upload round anchors a deadline");
5093 assert!(matches!(
5094 client.apply_sync_round(completed),
5095 crate::AppliedSyncRound::Complete { .. }
5096 ));
5097 assert_eq!(transport.seen.len(), 4, "one scope per exchange");
5098 assert_eq!(
5099 transport.seen[0],
5100 Some(anchor),
5101 "the upload anchors the round"
5102 );
5103 assert_eq!(
5104 transport.seen[1], None,
5105 "the upload exchange clears its scope"
5106 );
5107 assert_eq!(
5108 transport.seen[2],
5109 Some(anchor),
5110 "a continuation reuses the carried anchor instead of refreshing it"
5111 );
5112 assert_eq!(transport.seen[3], None, "the continuation clears its scope");
5113
5114 transport.budget = std::time::Duration::from_secs(60);
5117 let prepared = client.prepare_sync_round(false).unwrap();
5118 let completed = prepared.exchange(&mut transport);
5119 let fresh = completed
5120 .prepared
5121 .round_deadline_at
5122 .expect("the next round anchors a deadline");
5123 client.apply_sync_round(completed);
5124 assert!(
5125 fresh >= anchor + std::time::Duration::from_secs(30),
5126 "the next round anchors a fresh deadline from the new budget"
5127 );
5128 assert_eq!(transport.seen[4], Some(fresh));
5129 assert_eq!(
5130 transport.seen[5], None,
5131 "the new round clears its scope too"
5132 );
5133 }
5134
5135 #[derive(Default)]
5138 struct RoundDeadlineProbe {
5139 budget: Option<std::time::Duration>,
5140 budget_queries: std::cell::Cell<usize>,
5141 scoped: Vec<Option<std::time::Instant>>,
5142 }
5143
5144 impl Transport for RoundDeadlineProbe {
5145 fn round_deadline(&self) -> Option<std::time::Duration> {
5146 self.budget_queries.set(self.budget_queries.get() + 1);
5147 self.budget
5148 }
5149 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
5150 self.scoped.push(deadline);
5151 }
5152 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
5153 Err(TransportError::new("transport.failed", "injected"))
5154 }
5155 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
5156 self.sync(request)
5157 }
5158 fn download_segment(
5159 &mut self,
5160 _: &SegmentRequest,
5161 _: &mut dyn FnMut(u64),
5162 ) -> Result<Vec<u8>, TransportError> {
5163 Err(TransportError::new("transport.failed", "injected"))
5164 }
5165 fn blob_upload_grant(
5166 &mut self,
5167 _: &str,
5168 _: u64,
5169 _: Option<&str>,
5170 ) -> Result<crate::BlobUploadGrant, TransportError> {
5171 Ok(crate::BlobUploadGrant::Present)
5172 }
5173 fn realtime_connect(&mut self) -> Result<(), TransportError> {
5174 Ok(())
5175 }
5176 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
5177 Ok(())
5178 }
5179 fn realtime_close(&mut self) -> Result<(), TransportError> {
5180 Ok(())
5181 }
5182 }
5183
5184 #[test]
5185 fn an_unusable_upload_body_does_not_anchor_the_round_deadline() {
5186 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5187 {"name":"id","type":"string","nullable":false},
5188 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5189 let mut client =
5190 SyncClient::new("deadline-unusable".into(), &schema, ClientLimits::default()).unwrap();
5191 client.set_meta(LOG_EPOCH_KEY, "epoch");
5192 client
5193 .upload_blob(b"pending upload", Some("text/plain".into()), None)
5194 .unwrap();
5195 client
5196 .conn
5197 .execute_batch("DELETE FROM _syncular_blobs")
5198 .unwrap();
5199 let prepared = client.prepare_sync_round(false).unwrap();
5200 assert!(
5201 prepared.uploads.front().is_some_and(|u| u.bytes.is_err()),
5202 "the staged body must fail locally"
5203 );
5204 let mut transport = RoundDeadlineProbe {
5205 budget: Some(std::time::Duration::from_secs(30)),
5206 ..Default::default()
5207 };
5208 let completed = prepared.exchange(&mut transport);
5209 assert!(matches!(
5210 completed.exchange,
5211 crate::round::ExchangeResult::Upload { result: Err(_), .. }
5212 ));
5213 assert_eq!(
5214 transport.budget_queries.get(),
5215 0,
5216 "a round with no network work never asks for a budget"
5217 );
5218 assert_eq!(transport.scoped, vec![None]);
5219 }
5220
5221 #[test]
5222 fn a_transport_budget_the_clock_cannot_represent_fails_before_network() {
5223 struct HugeBudget;
5224 impl Transport for HugeBudget {
5225 fn round_deadline(&self) -> Option<std::time::Duration> {
5226 Some(std::time::Duration::MAX)
5227 }
5228 fn sync(&mut self, _: &[u8]) -> Result<Vec<u8>, TransportError> {
5229 panic!("no network for an invalid budget")
5230 }
5231 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
5232 self.sync(request)
5233 }
5234 fn download_segment(
5235 &mut self,
5236 _: &SegmentRequest,
5237 _: &mut dyn FnMut(u64),
5238 ) -> Result<Vec<u8>, TransportError> {
5239 panic!("no network for an invalid budget")
5240 }
5241 fn realtime_connect(&mut self) -> Result<(), TransportError> {
5242 Ok(())
5243 }
5244 fn realtime_send(&mut self, _: &str) -> Result<(), TransportError> {
5245 Ok(())
5246 }
5247 fn realtime_close(&mut self) -> Result<(), TransportError> {
5248 Ok(())
5249 }
5250 }
5251
5252 let mut client = client();
5253 client.set_meta(LOG_EPOCH_KEY, "epoch");
5254 let prepared = client.prepare_sync_round(false).unwrap();
5255 let outcome = match client.apply_sync_round(prepared.exchange(&mut HugeBudget)) {
5256 crate::AppliedSyncRound::Complete { outcome, .. } => outcome,
5257 crate::AppliedSyncRound::Continue(_) => {
5258 panic!("an unrepresentable budget must not continue")
5259 }
5260 };
5261 match outcome {
5262 SyncOutcome::Failed {
5263 error_code,
5264 message,
5265 ..
5266 } => {
5267 assert_eq!(error_code, "sync.invalid_request");
5268 assert_eq!(message, "transport round deadline is out of range");
5269 }
5270 _ => panic!("an unrepresentable budget must fail synchronously"),
5271 }
5272 }
5273
5274 #[test]
5275 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
5276 for failure in ["visible", "revision", "commit"] {
5277 let mut client = client();
5278 let id = client
5279 .mutate(vec![Mutation::Upsert {
5280 table: "tasks".into(),
5281 values: Map::from_iter([
5282 ("id".into(), Value::from("t1")),
5283 ("project_id".into(), Value::from("pending")),
5284 ]),
5285 base_version: None,
5286 }])
5287 .unwrap();
5288 client.drain_change_batches();
5289 let revision = client.local_revision();
5290 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
5291 let trigger = match failure {
5292 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
5293 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
5294 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
5295 };
5296 client.conn.execute_batch(&trigger).unwrap();
5297 let change = ssp2::model::Change {
5298 table_index: 0,
5299 row_id: "t1".into(),
5300 op: Op::Upsert,
5301 row_version: Some(2),
5302 scopes: vec![],
5303 row: Some(
5304 encode_row_json(
5305 client.schema.table("tasks").unwrap(),
5306 "t1",
5307 &Map::from_iter([
5308 ("id".into(), Value::from("t1")),
5309 ("project_id".into(), Value::from("remote")),
5310 ]),
5311 &client.encryption,
5312 )
5313 .unwrap(),
5314 ),
5315 };
5316 assert!(client
5317 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), None)
5318 .is_err());
5319 assert!(client.conn.is_autocommit());
5320 assert!(!client.overlay_dirty.get());
5321 assert_eq!(client.local_revision(), revision);
5322 assert!(client.drain_change_batches().is_empty());
5323 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5324 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
5325 assert!(client
5326 .query("SELECT * FROM _syncular_base_tasks", &[])
5327 .unwrap()
5328 .is_empty());
5329 client
5330 .conn
5331 .execute_batch("DROP TRIGGER fail_pending")
5332 .unwrap();
5333 client
5334 .apply_commit_frame(&["tasks".into()], &[change], None)
5335 .unwrap();
5336 assert_eq!(client.local_revision(), revision + 1);
5337 assert_eq!(client.pending_commit_ids(), vec![id]);
5338 let mut expected = before[0].clone();
5341 expected.insert("_syncular_version".to_owned(), Value::from(2));
5342 assert_eq!(
5343 client.query("SELECT * FROM tasks", &[]).unwrap(),
5344 vec![expected]
5345 );
5346 assert_eq!(
5347 client
5348 .conn
5349 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
5350 .get::<_, String>(0))
5351 .unwrap(),
5352 "remote"
5353 );
5354 }
5355 }
5356
5357 #[test]
5358 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
5359 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
5360 "columns": [
5361 {"name":"id", "type":"string", "nullable":false},
5362 {"name":"project_id", "type":"string", "nullable":false},
5363 {"name":"title", "type":"string", "nullable":false}],
5364 "scopes": [{"pattern":"project:{project_id}"}],
5365 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
5366 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
5367 let mut incremental =
5368 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
5369 let mut reference =
5370 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
5371 for instance in [&mut incremental, &mut reference] {
5372 for index in 0..16 {
5373 instance
5374 .write_base_row(
5375 "tasks",
5376 &vec![
5377 Some(ColumnValue::String(format!("task{index}"))),
5378 Some(ColumnValue::String("p0".to_owned())),
5379 Some(ColumnValue::String(format!("base{index}"))),
5380 ],
5381 1,
5382 )
5383 .unwrap();
5384 }
5385 instance
5386 .rebuild_overlay_if_dirty()
5387 .expect("rebuild overlay");
5388 instance.overlay_rebuild_count.set(0);
5389 }
5390 for index in 0..100 {
5391 let mutations = (0..1 + index % 4)
5392 .map(|offset| {
5393 let id = format!("task{}", (index + offset) % 24);
5394 if (index + offset) % 7 == 0 {
5395 Mutation::Delete {
5396 table: "tasks".to_owned(),
5397 row_id: id,
5398 base_version: None,
5399 }
5400 } else {
5401 Mutation::Upsert {
5402 table: "tasks".to_owned(),
5403 values: Map::from_iter([
5404 ("id".to_owned(), Value::from(id)),
5405 (
5406 "project_id".to_owned(),
5407 Value::from(format!("p{}", index % 2)),
5408 ),
5409 (
5410 "title".to_owned(),
5411 Value::from(format!("needle{}", (index + offset) % 24)),
5412 ),
5413 ]),
5414 base_version: None,
5415 }
5416 }
5417 })
5418 .collect::<Vec<_>>();
5419 reference.overlay_dirty.set(true);
5420 incremental.mutate(mutations.clone()).unwrap();
5421 reference.mutate(mutations).unwrap();
5422 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
5423 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
5424 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
5425 }
5426 assert_eq!(incremental.local_revision(), reference.local_revision());
5427 assert_eq!(
5428 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
5429 serde_json::to_value(reference.drain_change_batches()).unwrap()
5430 );
5431 }
5432 assert_eq!(incremental.outbox.len(), 100);
5433 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
5434 assert_eq!(reference.overlay_rebuild_count.get(), 100);
5435 }
5436
5437 #[test]
5438 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
5439 for fail_at in ["outbox", "revision"] {
5440 let mut client = client();
5441 let values = Map::from_iter([
5442 ("id".to_owned(), Value::from("t1")),
5443 ("project_id".to_owned(), Value::from("p1")),
5444 ]);
5445 let mutation = Mutation::Upsert {
5446 table: "tasks".to_owned(),
5447 values,
5448 base_version: None,
5449 };
5450 let trigger = if fail_at == "outbox" {
5451 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
5452 } else {
5453 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
5454 };
5455 client.conn.execute_batch(&trigger).unwrap();
5456 let revision = client.local_revision();
5457 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
5458 assert_eq!(client.local_revision(), revision);
5459 assert!(client.outbox.is_empty());
5460 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
5461 assert_eq!(
5462 client
5463 .conn
5464 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5465 .get::<_, i64>(0))
5466 .unwrap(),
5467 0
5468 );
5469 assert!(client.drain_change_batches().is_empty());
5470 assert!(!client.overlay_dirty.get());
5471 client
5472 .conn
5473 .execute_batch("DROP TRIGGER fail_append")
5474 .unwrap();
5475 client.mutate(vec![mutation]).unwrap();
5476 assert_eq!(client.outbox.len(), 1);
5477 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
5478 }
5479 }
5480
5481 #[test]
5482 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
5483 let mut client = client();
5484 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
5485 let values = Map::from_iter([
5486 ("id".to_owned(), Value::from("t1")),
5487 ("project_id".to_owned(), Value::from("p1")),
5488 ]);
5489 let payload = encode_row_json(
5490 client.schema.table("tasks").unwrap(),
5491 "t1",
5492 &values,
5493 &client.encryption,
5494 )
5495 .unwrap();
5496 let first = ssp2::model::Change {
5497 table_index: 0,
5498 row_id: "t1".to_owned(),
5499 op: Op::Upsert,
5500 row_version: Some(1),
5501 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
5502 row: Some(payload),
5503 };
5504 let mut invalid = first.clone();
5505 invalid.table_index = 1;
5506 let revision = client.local_revision();
5507 client.begin_observation("failed_mirror").unwrap();
5508 assert!(client
5509 .apply_commit_changes(&tables, &[first, invalid])
5510 .is_err());
5511 client.rollback_observation("failed_mirror");
5512 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
5513 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
5514 assert_eq!(
5515 client
5516 .conn
5517 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
5518 .get::<_, i64>(
5519 0
5520 ))
5521 .unwrap(),
5522 0
5523 );
5524 assert_eq!(client.local_revision(), revision);
5525 assert!(client.drain_change_batches().is_empty());
5526 }
5527
5528 #[test]
5529 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
5530 for failure in ["journal", "revision", "commit", "later-frame"] {
5531 let mut client = client();
5532 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5533 for id in ["first", "second"] {
5534 client
5535 .mutate(vec![Mutation::Upsert {
5536 table: "tasks".to_owned(),
5537 values: Map::from_iter([
5538 ("id".to_owned(), Value::from(id)),
5539 ("project_id".to_owned(), Value::from("p1")),
5540 ]),
5541 base_version: None,
5542 }])
5543 .unwrap();
5544 }
5545 let (_, meta) = client.build_request(false).unwrap();
5546 client
5548 .mutate(vec![Mutation::Upsert {
5549 table: "tasks".to_owned(),
5550 values: Map::from_iter([
5551 ("id".to_owned(), Value::from("later")),
5552 ("project_id".to_owned(), Value::from("p1")),
5553 ]),
5554 base_version: None,
5555 }])
5556 .unwrap();
5557 let ids = client.pending_commit_ids();
5558 let rows = client
5559 .query("SELECT * FROM tasks ORDER BY id", &[])
5560 .unwrap();
5561 let revision = client.local_revision();
5562 client.drain_change_batches();
5563 if failure != "later-frame" {
5564 client.conn.execute_batch(match failure {
5565 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
5566 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
5567 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
5568 }).unwrap();
5569 }
5570 let mut response = Message {
5571 wire_version: WIRE_VERSION,
5572 msg_kind: MsgKind::Response,
5573 frames: vec![Frame::RespHeader {
5574 required_schema_version: None,
5575 latest_schema_version: None,
5576 log_epoch: Some("epoch-1".to_owned()),
5577 reset_required: Some(false),
5578 }],
5579 };
5580 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
5581 response.frames.push(Frame::PushResult {
5582 client_commit_id: id.to_owned(),
5583 status: PushStatus::Applied,
5584 commit_seq: Some(1),
5585 results: vec![OpResult::Applied { op_index: 0 }],
5586 });
5587 }
5588 if failure == "later-frame" {
5589 response.frames.push(Frame::Error {
5590 code: "sync.invalid_request".to_owned(),
5591 message: "later error".to_owned(),
5592 category: "protocol".to_owned(),
5593 retryable: false,
5594 recommended_action: "retry".to_owned(),
5595 details: None,
5596 });
5597 }
5598 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5599 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5600 let expected = match failure {
5603 "later-frame" => "sync.invalid_request",
5604 "commit" => "storage.failed",
5605 _ => "client.outcome_persistence_failed",
5606 };
5607 assert!(
5608 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5609 "{failure}: {outcome:?}"
5610 );
5611 if failure == "later-frame" {
5612 assert_eq!(client.pending_commit_ids(), ids[2..]);
5613 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5614 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
5615 let batches = client.drain_change_batches();
5616 assert_eq!(batches.len(), 1);
5617 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
5618 assert_eq!(client.local_revision(), revision + 1);
5619 response.frames.pop();
5620 } else {
5621 assert_eq!(client.pending_commit_ids(), ids);
5622 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5623 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
5624 assert_eq!(client.local_revision(), revision);
5625 assert!(client.drain_change_batches().is_empty());
5626 assert_eq!(
5627 client
5628 .query("SELECT * FROM tasks ORDER BY id", &[])
5629 .unwrap(),
5630 rows
5631 );
5632 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
5633 }
5634 for frame in &mut response.frames {
5635 if let Frame::PushResult { status, .. } = frame {
5636 *status = PushStatus::Cached;
5637 }
5638 }
5639 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
5640 else {
5641 panic!("retry failed");
5642 };
5643 assert_eq!(
5644 report.applied,
5645 if failure == "later-frame" {
5646 vec![]
5647 } else {
5648 ids[..2].to_vec()
5649 }
5650 );
5651 assert_eq!(client.pending_commit_ids(), ids[2..]);
5652 let batches = client.drain_change_batches();
5653 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
5654 if let Some(batch) = batches.first() {
5655 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
5656 }
5657 assert_eq!(client.local_revision(), revision + 1);
5658 }
5659 }
5660
5661 #[test]
5662 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
5663 for failure in ["revision", "journal", "commit"] {
5664 for status in ["applied", "cached", "rejected", "conflict"] {
5665 let mut client = client();
5666 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5667 for id in ["first", "later"] {
5668 client
5669 .mutate(vec![Mutation::Upsert {
5670 table: "tasks".to_owned(),
5671 values: Map::from_iter([
5672 ("id".to_owned(), Value::from(id)),
5673 ("project_id".to_owned(), Value::from("p1")),
5674 ]),
5675 base_version: None,
5676 }])
5677 .unwrap();
5678 }
5679 let (_, meta) = client.build_request(false).unwrap();
5680 let ids = client.pending_commit_ids();
5681 let rows = client
5682 .query("SELECT * FROM tasks ORDER BY id", &[])
5683 .unwrap();
5684 let revision = client.local_revision();
5685 client.drain_change_batches();
5686 let (push_status, results) = match status {
5687 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
5688 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
5689 "conflict" => (
5690 PushStatus::Rejected,
5691 vec![OpResult::Conflict {
5692 op_index: 0,
5693 code: "sync.version_conflict".to_owned(),
5694 message: "conflict".to_owned(),
5695 server_version: 1,
5696 server_row: encode_row_json(
5697 client.schema.table("tasks").unwrap(),
5698 "first",
5699 &Map::from_iter([
5700 ("id".to_owned(), Value::from("first")),
5701 ("project_id".to_owned(), Value::from("p1")),
5702 ]),
5703 &client.encryption,
5704 )
5705 .unwrap(),
5706 conflict_columns: Vec::new(),
5707 }],
5708 ),
5709 _ => (
5710 PushStatus::Rejected,
5711 vec![OpResult::Error {
5712 op_index: 0,
5713 code: "sync.validation_failed".to_owned(),
5714 message: "rejected".to_owned(),
5715 retryable: false,
5716 }],
5717 ),
5718 };
5719 let response = Message {
5720 wire_version: WIRE_VERSION,
5721 msg_kind: MsgKind::Response,
5722 frames: vec![
5723 Frame::RespHeader {
5724 required_schema_version: None,
5725 latest_schema_version: None,
5726 log_epoch: Some("epoch-1".to_owned()),
5727 reset_required: Some(false),
5728 },
5729 Frame::PushResult {
5730 client_commit_id: ids[0].clone(),
5731 status: push_status,
5732 commit_seq: Some(1),
5733 results,
5734 },
5735 ],
5736 };
5737 client.conn.execute_batch(match failure {
5738 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
5739 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
5740 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
5741 }).unwrap();
5742 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
5743 let outcome = client.process_response(&mut transport, response.clone(), &meta);
5744 assert_eq!(
5745 client.pending_commit_ids(),
5746 ids,
5747 "{failure}/{status}: pending identity"
5748 );
5749 assert_eq!(
5750 client.local_revision(),
5751 revision,
5752 "{failure}/{status}: revision"
5753 );
5754 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
5755 assert!(client.conflicts().is_empty());
5756 assert!(client.rejections().is_empty());
5757 assert!(client.drain_change_batches().is_empty());
5758 assert_eq!(
5759 client
5760 .query("SELECT * FROM tasks ORDER BY id", &[])
5761 .unwrap(),
5762 rows
5763 );
5764 let expected = if failure == "commit" {
5767 "storage.failed"
5768 } else {
5769 "client.outcome_persistence_failed"
5770 };
5771 assert!(
5772 matches!(&outcome, SyncOutcome::Failed { error_code, .. } if error_code == expected),
5773 "{failure}/{status}: failed round"
5774 );
5775 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
5776 assert!(matches!(
5777 client.process_response(&mut transport, response, &meta),
5778 SyncOutcome::Ok(_)
5779 ));
5780 assert_eq!(client.pending_commit_ids(), ids[1..]);
5781 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
5782 assert_eq!(client.drain_change_batches().len(), 1);
5783 }
5784 }
5785 }
5786
5787 #[test]
5788 fn mixed_push_results_reconcile_each_boundary_and_prune_once_per_response() {
5789 let mut client = client();
5790 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
5791 for index in 0..4 {
5792 client
5793 .mutate(vec![Mutation::Upsert {
5794 table: "tasks".to_owned(),
5795 values: Map::from_iter([
5796 ("id".to_owned(), Value::from(format!("task-{index}"))),
5797 ("project_id".to_owned(), Value::from("project-1")),
5798 ]),
5799 base_version: None,
5800 }])
5801 .expect("queue commit");
5802 }
5803
5804 let (_, request_meta) = client.build_request(false).unwrap();
5805 let ids = &request_meta.pushed_ids;
5806 assert_eq!(ids.len(), 4);
5807 let response = Message {
5808 wire_version: WIRE_VERSION,
5809 msg_kind: MsgKind::Response,
5810 frames: vec![
5811 Frame::RespHeader {
5812 required_schema_version: None,
5813 latest_schema_version: None,
5814 log_epoch: Some("epoch-1".to_owned()),
5815 reset_required: Some(false),
5816 },
5817 Frame::PushResult {
5818 client_commit_id: ids[0].clone(),
5819 status: PushStatus::Applied,
5820 commit_seq: Some(1),
5821 results: vec![OpResult::Applied { op_index: 0 }],
5822 },
5823 Frame::PushResult {
5824 client_commit_id: ids[1].clone(),
5825 status: PushStatus::Cached,
5826 commit_seq: Some(2),
5827 results: vec![OpResult::Applied { op_index: 0 }],
5828 },
5829 Frame::PushResult {
5830 client_commit_id: ids[2].clone(),
5831 status: PushStatus::Rejected,
5832 commit_seq: None,
5833 results: vec![OpResult::Error {
5834 op_index: 0,
5835 code: "sync.validation_failed".to_owned(),
5836 message: "rejected".to_owned(),
5837 retryable: false,
5838 }],
5839 },
5840 Frame::PushResult {
5841 client_commit_id: ids[3].clone(),
5842 status: PushStatus::Rejected,
5843 commit_seq: None,
5844 results: vec![OpResult::Error {
5845 op_index: 0,
5846 code: "sync.idempotency_cache_miss".to_owned(),
5847 message: "retry".to_owned(),
5848 retryable: true,
5849 }],
5850 },
5851 ],
5852 };
5853 let mut transport =
5854 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
5855
5856 client.overlay_rebuild_count.set(0);
5857 client.outcome_prune_count.set(0);
5858 let outcome = client.process_response(&mut transport, response, &request_meta);
5859 let SyncOutcome::Ok(report) = outcome else {
5860 panic!("mixed push-result response failed");
5861 };
5862
5863 assert_eq!(report.applied, ids[..2]);
5864 assert_eq!(report.rejected, ids[2..3]);
5865 assert_eq!(report.retryable, ids[3..4]);
5866 assert_eq!(client.pending_commit_ids(), ids[3..4]);
5867 assert_eq!(
5868 client
5869 .query("SELECT id FROM tasks ORDER BY id", &[])
5870 .expect("query visible overlay"),
5871 ["task-0", "task-1", "task-3"]
5872 .map(|id| Map::from_iter([("id".to_owned(), Value::from(id))]))
5873 .to_vec()
5874 );
5875 assert_eq!(client.overlay_rebuild_count.get(), 2);
5877 assert_eq!(client.outcome_prune_count.get(), 1);
5878 }
5879
5880 fn replay_schema(fts: bool) -> Value {
5881 json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
5882 {"name":"id","type":"string","nullable":false},
5883 {"name":"project_id","type":"string","nullable":false},
5884 {"name":"title","type":"string","nullable":false},
5885 {"name":"done","type":"boolean","nullable":false}
5886 ],"scopes":[{"pattern":"project:{project_id}"}], "ftsIndexes": if fts { json!([{"name":"tasks_fts","columns":["title"],"tokenize":"unicode61"}]) } else { json!([]) }}]})
5887 }
5888
5889 #[test]
5890 fn aborted_schema_and_epoch_resets_restore_memory_and_durable_intent() {
5891 for schema_reset in [false, true] {
5892 let mut schema = replay_schema(false);
5893 let mut client =
5894 SyncClient::new("reset-failure".into(), &schema, ClientLimits::default()).unwrap();
5895 let id = client
5896 .mutate(vec![Mutation::Upsert {
5897 table: "tasks".into(),
5898 values: Map::from_iter([
5899 ("id".into(), json!("queued")),
5900 ("project_id".into(), json!("p1")),
5901 ("title".into(), json!("old")),
5902 ("done".into(), json!(false)),
5903 ]),
5904 base_version: None,
5905 }])
5906 .unwrap();
5907 client.set_meta(LOG_EPOCH_KEY, "old");
5908 let revision = client.local_revision();
5909 let needed = client.sync_needed;
5910 client.drain_change_batches();
5911 client.drain_sync_intents();
5912 client.conn.execute_batch("CREATE TRIGGER refuse_revision BEFORE INSERT ON _syncular_meta WHEN NEW.key='localRevision' BEGIN SELECT RAISE(ABORT,'reset completion failed'); END;").unwrap();
5913 if schema_reset {
5914 schema["version"] = json!(2);
5915 schema["tables"][0]["columns"]
5916 .as_array_mut()
5917 .unwrap()
5918 .retain(|column| column["name"] != "title");
5919 client.schema = parse_schema_json(&schema).unwrap();
5920 assert!(client.run_schema_reset().is_err());
5921 } else {
5922 assert!(client.run_log_epoch_reset("new").is_err());
5923 }
5924 assert!(!client.upgrading);
5925 assert_eq!(client.sync_needed, needed);
5926 assert_eq!(client.local_revision(), revision);
5927 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
5928 assert!(client.rejections.is_empty());
5929 assert!(client.commit_outcome(&id).unwrap().is_none());
5930 assert_eq!(client.get_meta(LOG_EPOCH_KEY).as_deref(), Some("old"));
5931 assert_eq!(
5932 client
5933 .query("SELECT title FROM tasks WHERE id='queued'", &[])
5934 .unwrap()[0]["title"],
5935 "old"
5936 );
5937 assert_eq!(
5938 client
5939 .conn
5940 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
5941 .get::<_, i64>(0))
5942 .unwrap(),
5943 1
5944 );
5945 assert!(client.drain_change_batches().is_empty());
5946 assert!(client.drain_sync_intents().is_empty());
5947 assert!(client.conn.is_autocommit());
5948 }
5949 }
5950
5951 #[test]
5952 fn cached_blob_cleanup_propagates_delete_failures() {
5953 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
5954 {"name":"id","type":"string","nullable":false},
5955 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
5956 let mut client =
5957 SyncClient::new("blob-cleanup".into(), &schema, ClientLimits::default()).unwrap();
5958 client.conn.execute_batch("CREATE TRIGGER refuse_blob_cleanup BEFORE DELETE ON _syncular_blobs BEGIN SELECT RAISE(ABORT,'blob cleanup failed'); END;").unwrap();
5959 client.conn.execute("INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,created_at_ms) VALUES ('test',X'01',1,0)", []).unwrap();
5960 client
5961 .subscribe("attachments".into(), "attachments".into(), vec![], None)
5962 .unwrap();
5963 client.subs[0].effective = Some(vec![]);
5964 client.persist_sub(&client.subs[0]).unwrap();
5965 let revision = client.local_revision();
5966 let (_, meta) = client.build_request(false).unwrap();
5967 let mut transport = CountingRealtimeTransport::default();
5968 let mut report = SyncReport::default();
5969 assert!(client
5970 .process_section(
5971 &mut transport,
5972 "attachments",
5973 SubStatus::Revoked,
5974 "sync.scope_revoked",
5975 vec![],
5976 vec![],
5977 Some((0, None)),
5978 &meta,
5979 &mut report,
5980 &HashSet::new()
5981 )
5982 .is_err());
5983 assert_eq!(client.subs[0].state, SubState::Active);
5984 assert_eq!(client.local_revision(), revision);
5985 assert!(report.revoked.is_empty());
5986 assert!(client.conn.is_autocommit());
5987 client
5988 .conn
5989 .execute_batch("DROP TRIGGER refuse_blob_cleanup")
5990 .unwrap();
5991 assert!(client
5992 .process_section(
5993 &mut transport,
5994 "attachments",
5995 SubStatus::Revoked,
5996 "sync.scope_revoked",
5997 vec![],
5998 vec![],
5999 Some((0, None)),
6000 &meta,
6001 &mut report,
6002 &HashSet::new()
6003 )
6004 .is_ok());
6005 assert_eq!(client.subs[0].state, SubState::Revoked);
6006 assert_eq!(
6007 client
6008 .conn
6009 .query_row("SELECT count(*) FROM _syncular_blobs", [], |row| row
6010 .get::<_, i64>(0))
6011 .unwrap(),
6012 0
6013 );
6014 }
6015
6016 #[test]
6017 fn overlay_replay_trigger_refuses_reopen_and_preserves_durable_intent() {
6018 let path =
6019 std::env::temp_dir().join(format!("syncular-replay-{}.db", uuid::Uuid::new_v4()));
6020 let schema = replay_schema(false);
6021 let mut client = SyncClient::open_path(
6022 "replay".into(),
6023 &schema,
6024 ClientLimits::default(),
6025 path.to_str().unwrap(),
6026 )
6027 .unwrap();
6028 let ids = ["early", "bad"].map(|id| {
6029 client
6030 .mutate(vec![Mutation::Upsert {
6031 table: "tasks".into(),
6032 values: Map::from_iter([
6033 ("id".into(), json!(id)),
6034 ("project_id".into(), json!("p1")),
6035 ("title".into(), json!(id)),
6036 ("done".into(), json!(false)),
6037 ]),
6038 base_version: None,
6039 }])
6040 .unwrap()
6041 });
6042 client.conn.execute_batch("DELETE FROM tasks; CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.id='bad' BEGIN SELECT RAISE(ABORT,'replay failed'); END;").unwrap();
6043 let revision = client.local_revision();
6044 drop(client);
6045 let error = SyncClient::open_path(
6046 "replay".into(),
6047 &schema,
6048 ClientLimits::default(),
6049 path.to_str().unwrap(),
6050 )
6051 .err()
6052 .expect("reopen must fail");
6053 assert!(error.contains("replay failed"));
6054 let conn = Connection::open(&path).unwrap();
6055 assert_eq!(
6056 conn.query_row("SELECT count(*) FROM tasks", [], |row| row.get::<_, i64>(0))
6057 .unwrap(),
6058 0
6059 );
6060 assert_eq!(
6061 conn.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
6062 .get::<_, i64>(0))
6063 .unwrap(),
6064 2
6065 );
6066 assert_eq!(
6067 meta_get(&conn, LOCAL_REVISION_KEY),
6068 Some(revision.to_string())
6069 );
6070 conn.execute_batch("DROP TRIGGER fail_replay").unwrap();
6071 drop(conn);
6072 let reopened = SyncClient::open_path(
6073 "replay".into(),
6074 &schema,
6075 ClientLimits::default(),
6076 path.to_str().unwrap(),
6077 )
6078 .unwrap();
6079 assert_eq!(reopened.pending_commit_ids(), ids);
6080 assert_eq!(
6081 reopened
6082 .query("SELECT id FROM tasks ORDER BY id", &[])
6083 .unwrap()
6084 .len(),
6085 2
6086 );
6087 drop(reopened);
6088 std::fs::remove_file(path).unwrap();
6089 }
6090
6091 #[test]
6092 fn overlay_replay_failure_rolls_back_push_and_pull_apply_boundaries() {
6093 let schema = replay_schema(false);
6094 let mut client =
6095 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
6096 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
6097 client
6098 .write_base_row(
6099 "tasks",
6100 &vec![
6101 Some(ColumnValue::String("queued".into())),
6102 Some(ColumnValue::String("p1".into())),
6103 Some(ColumnValue::String("server".into())),
6104 Some(ColumnValue::Boolean(false)),
6105 ],
6106 1,
6107 )
6108 .unwrap();
6109 client.rebuild_overlay().unwrap();
6110 let id = client
6111 .patch(
6112 "tasks",
6113 "queued",
6114 Map::from_iter([("title".into(), json!("local"))]),
6115 Some(1),
6116 )
6117 .unwrap();
6118 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
6119 let base = client
6120 .query("SELECT * FROM _syncular_base_tasks", &[])
6121 .unwrap();
6122 let revision = client.local_revision();
6123 client.drain_change_batches();
6124 client.conn.execute_batch("CREATE TRIGGER fail_replay BEFORE INSERT ON tasks WHEN new.title='local' BEGIN SELECT RAISE(ABORT,'replay failed'); END").unwrap();
6125 let remote = vec![
6126 Some(ColumnValue::String("queued".into())),
6127 Some(ColumnValue::String("p1".into())),
6128 Some(ColumnValue::String("remote".into())),
6129 Some(ColumnValue::Boolean(false)),
6130 ];
6131 let bytes = crate::values::encode_row_json(
6132 client.schema.table("tasks").unwrap(),
6133 "queued",
6134 &Map::from_iter(
6135 client
6136 .schema
6137 .table("tasks")
6138 .unwrap()
6139 .columns
6140 .iter()
6141 .zip(&remote)
6142 .map(|(column, value)| (column.name.clone(), column_value_to_json(value))),
6143 ),
6144 &client.encryption,
6145 )
6146 .unwrap();
6147 let change = ssp2::model::Change {
6148 table_index: 0,
6149 op: Op::Upsert,
6150 row_id: "queued".into(),
6151 row_version: Some(2),
6152 scopes: vec![("project_id".into(), "p1".into())],
6153 row: Some(bytes),
6154 };
6155 assert!(client
6156 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change), Some(2))
6157 .is_err());
6158 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
6159 assert_eq!(
6160 client
6161 .query("SELECT * FROM _syncular_base_tasks", &[])
6162 .unwrap(),
6163 base
6164 );
6165 assert_eq!(client.local_revision(), revision);
6166 assert!(client.drain_change_batches().is_empty());
6167 let response = Message {
6168 wire_version: 2,
6169 msg_kind: MsgKind::Response,
6170 frames: vec![
6171 Frame::RespHeader {
6172 required_schema_version: None,
6173 latest_schema_version: None,
6174 log_epoch: Some("epoch-1".into()),
6175 reset_required: Some(false),
6176 },
6177 Frame::PushResult {
6178 client_commit_id: id.clone(),
6179 status: PushStatus::Applied,
6180 commit_seq: Some(2),
6181 results: vec![OpResult::Applied { op_index: 0 }],
6182 },
6183 ],
6184 };
6185 let (_, meta) = client.build_request(false).unwrap();
6186 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
6187 assert!(
6188 matches!(client.process_response(&mut transport,response.clone(),&meta),SyncOutcome::Failed { error_code,.. } if error_code=="client.outcome_persistence_failed")
6189 );
6190 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
6191 assert!(client.commit_outcome(&id).unwrap().is_none());
6192 assert_eq!(
6193 client
6194 .conn
6195 .query_row("SELECT count(*) FROM _syncular_acked_rows", [], |row| row
6196 .get::<_, i64>(
6197 0
6198 ))
6199 .unwrap(),
6200 0
6201 );
6202 assert_eq!(
6203 client
6204 .conn
6205 .query_row("SELECT count(*) FROM _syncular_row_deliveries", [], |row| {
6206 row.get::<_, i64>(0)
6207 })
6208 .unwrap(),
6209 0
6210 );
6211 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
6212 assert_eq!(client.local_revision(), revision);
6213 assert!(client.drain_change_batches().is_empty());
6214 client
6215 .conn
6216 .execute_batch("DROP TRIGGER fail_replay")
6217 .unwrap();
6218 assert!(matches!(
6219 client.process_response(&mut transport, response, &meta),
6220 SyncOutcome::Ok(_)
6221 ));
6222 client
6223 .apply_commit_frame(&["tasks".into()], &[change], Some(2))
6224 .unwrap();
6225 assert!(client.pending_commit_ids().is_empty());
6226 assert_eq!(
6227 client.query("SELECT title FROM tasks", &[]).unwrap()[0]["title"],
6228 json!("remote")
6229 );
6230 }
6231
6232 #[test]
6233 fn overlay_replay_distinguishes_absence_from_sql_and_value_decode_errors() {
6234 let client = SyncClient::new(
6235 "replay".into(),
6236 &replay_schema(false),
6237 ClientLimits::default(),
6238 )
6239 .unwrap();
6240 let table = client.schema.table("tasks").unwrap();
6241 let patch = OutboxOp {
6242 table: "tasks".into(),
6243 row_id: "absent".into(),
6244 upsert: true,
6245 insert: false,
6246 base_version: Some(1),
6247 values: Some(Map::from_iter([
6248 ("id".into(), json!("absent")),
6249 ("title".into(), json!("patch")),
6250 ])),
6251 };
6252 assert!(client.visible_row(table, "absent").unwrap().is_none());
6253 client
6254 .apply_outbox_ops([&patch])
6255 .expect("sparse replay over truly absent row");
6256 client
6257 .conn
6258 .execute_batch("INSERT INTO tasks VALUES ('bad','p1','title','invalid',1)")
6259 .unwrap();
6260 assert!(client
6261 .visible_row(table, "bad")
6262 .unwrap_err()
6263 .starts_with("sync.local_corrupt:"));
6264 let mut invalid = patch.clone();
6265 invalid.values = Some(Map::from_iter([
6266 ("id".into(), json!("bad-value")),
6267 ("project_id".into(), json!("p1")),
6268 ("title".into(), json!("title")),
6269 ("done".into(), json!({"bad":true})),
6270 ]));
6271 assert!(client
6272 .apply_outbox_ops([&invalid])
6273 .unwrap_err()
6274 .starts_with("sync.local_corrupt:"));
6275 client.conn.execute_batch("DROP TABLE tasks").unwrap();
6276 assert!(
6277 client.apply_outbox_ops([&patch]).is_err(),
6278 "SQL failure must not count as absence"
6279 );
6280 }
6281
6282 #[test]
6283 fn overlay_replay_keeps_secondary_unique_deferral_and_lazy_base_precedence() {
6284 let mut schema = replay_schema(false);
6285 schema["tables"][0]["indexes"] =
6286 json!([{"name":"tasks_project_title", "columns":["project_id","title"],"unique":true}]);
6287 let mut client =
6288 SyncClient::new("replay".into(), &schema, ClientLimits::default()).unwrap();
6289 let id = client
6290 .mutate(vec![Mutation::Upsert {
6291 table: "tasks".into(),
6292 values: Map::from_iter([
6293 ("id".into(), json!("queued")),
6294 ("project_id".into(), json!("p1")),
6295 ("title".into(), json!("occupied")),
6296 ("done".into(), json!(false)),
6297 ]),
6298 base_version: None,
6299 }])
6300 .unwrap();
6301 client
6302 .write_base_row(
6303 "tasks",
6304 &vec![
6305 Some(ColumnValue::String("server".into())),
6306 Some(ColumnValue::String("p1".into())),
6307 Some(ColumnValue::String("occupied".into())),
6308 Some(ColumnValue::Boolean(false)),
6309 ],
6310 1,
6311 )
6312 .unwrap();
6313 client.rebuild_overlay_if_dirty().unwrap();
6314 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
6315 assert!(client
6316 .visible_row(client.schema.table("tasks").unwrap(), "queued")
6317 .unwrap()
6318 .is_none());
6319 assert!(client.rejections().is_empty());
6320 client.delete_base_row("tasks", "server").unwrap();
6321 client.rebuild_overlay_if_dirty().unwrap();
6322 assert!(client
6323 .visible_row(client.schema.table("tasks").unwrap(), "queued")
6324 .unwrap()
6325 .is_some());
6326 client
6327 .write_base_row(
6328 "tasks",
6329 &vec![
6330 Some(ColumnValue::String("base".into())),
6331 Some(ColumnValue::String("p1".into())),
6332 Some(ColumnValue::String("base-title".into())),
6333 Some(ColumnValue::Boolean(false)),
6334 ],
6335 1,
6336 )
6337 .unwrap();
6338 client.conn.execute_batch("DROP TABLE tasks").unwrap();
6339 client
6340 .retain_failed_commit(
6341 "retained",
6342 &[OutboxOp {
6343 table: "tasks".into(),
6344 row_id: "base".into(),
6345 upsert: true,
6346 insert: false,
6347 base_version: Some(1),
6348 values: Some(Map::from_iter([
6349 ("id".into(), json!("base")),
6350 ("title".into(), json!("patch")),
6351 ])),
6352 }],
6353 )
6354 .expect("an existing base must not read the unavailable visible table");
6355 }
6356
6357 #[test]
6358 fn overlay_replay_fts_and_savepoint_release_failures_roll_back_all_visible_changes() {
6359 for failure in ["fts", "release"] {
6360 let mut client = SyncClient::new(
6361 "replay".into(),
6362 &replay_schema(true),
6363 ClientLimits::default(),
6364 )
6365 .unwrap();
6366 client
6367 .mutate(vec![Mutation::Upsert {
6368 table: "tasks".into(),
6369 values: Map::from_iter([
6370 ("id".into(), json!("queued")),
6371 ("project_id".into(), json!("p1")),
6372 ("title".into(), json!("needle")),
6373 ("done".into(), json!(false)),
6374 ]),
6375 base_version: None,
6376 }])
6377 .unwrap();
6378 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
6379 let fts_before = client
6380 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
6381 .unwrap();
6382 let triggers_before: Vec<String> = client
6383 .conn
6384 .prepare("SELECT sql FROM sqlite_master WHERE type='trigger' ORDER BY name")
6385 .unwrap()
6386 .query_map([], |row| row.get(0))
6387 .unwrap()
6388 .collect::<Result<_, _>>()
6389 .unwrap();
6390 if failure == "fts" {
6391 client.conn.execute_batch("CREATE TRIGGER fail_fts BEFORE INSERT ON _syncular_fts_tasks_fts BEGIN SELECT RAISE(ABORT,'fts replay failed'); END;").unwrap();
6392 } else {
6393 client.conn.execute_batch("PRAGMA foreign_keys=ON; CREATE TABLE parent(id TEXT PRIMARY KEY); CREATE TABLE deferred(id TEXT REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_release AFTER INSERT ON tasks BEGIN INSERT INTO deferred VALUES ('missing'); END;").unwrap();
6394 }
6395 client.overlay_dirty.set(true);
6396 assert!(client.rebuild_overlay().is_err());
6397 assert!(client.overlay_dirty.get());
6398 assert!(client.conn.is_autocommit());
6399 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
6400 assert_eq!(
6401 client
6402 .query("SELECT _syncular_source_id,title FROM tasks_fts", &[])
6403 .unwrap(),
6404 fts_before
6405 );
6406 let original_triggers: Vec<String> = client.conn.prepare("SELECT sql FROM sqlite_master WHERE type='trigger' AND name NOT LIKE 'fail_%' ORDER BY name").unwrap().query_map([], |row| row.get(0)).unwrap().collect::<Result<_,_>>().unwrap();
6407 assert_eq!(original_triggers, triggers_before);
6408 assert_eq!(client.pending_commit_ids().len(), 1);
6409 client
6410 .conn
6411 .execute_batch(if failure == "fts" {
6412 "DROP TRIGGER fail_fts"
6413 } else {
6414 "DROP TRIGGER fail_release"
6415 })
6416 .unwrap();
6417 client
6418 .rebuild_overlay()
6419 .expect("retry after removing the fault");
6420 }
6421 }
6422
6423 #[test]
6424 fn secondary_unique_collision_preserves_existing_synced_row() {
6425 let client = SyncClient::new(
6426 "unique-upsert-test".to_owned(),
6427 &json!({
6428 "version": 1,
6429 "tables": [{
6430 "name": "tasks",
6431 "primaryKey": "id",
6432 "columns": [
6433 { "name": "id", "type": "string", "nullable": false },
6434 { "name": "project_id", "type": "string", "nullable": false },
6435 { "name": "title", "type": "string", "nullable": false }
6436 ],
6437 "scopes": [{ "pattern": "project:{project_id}" }],
6438 "indexes": [{
6439 "name": "tasks_by_project_title",
6440 "columns": ["project_id", "title"],
6441 "unique": true
6442 }]
6443 }]
6444 }),
6445 ClientLimits::default(),
6446 )
6447 .expect("test client");
6448 let table = client.schema.table("tasks").expect("tasks table");
6449 let sql = client.insert_row_sql(&base_table("tasks"), table);
6450
6451 client
6452 .conn
6453 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
6454 .expect("insert first row");
6455 client
6456 .conn
6457 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
6458 .expect("update same primary key");
6459 client
6460 .conn
6461 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
6462 .expect("insert second row");
6463 assert!(client
6464 .conn
6465 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
6466 .is_err());
6467
6468 let rows = client
6469 .conn
6470 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
6471 .expect("prepare rows")
6472 .query_map([], |row| {
6473 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
6474 })
6475 .expect("query rows")
6476 .collect::<Result<Vec<_>, _>>()
6477 .expect("collect rows");
6478 assert_eq!(
6479 rows,
6480 vec![
6481 ("t1".to_owned(), "updated".to_owned()),
6482 ("t2".to_owned(), "original".to_owned())
6483 ]
6484 );
6485 }
6486
6487 #[test]
6488 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
6489 let path = std::env::temp_dir().join(format!(
6490 "syncular-startup-intent-{}.db",
6491 uuid::Uuid::new_v4()
6492 ));
6493 let schema = json!({
6494 "version": 1,
6495 "tables": [{
6496 "name": "tasks",
6497 "primaryKey": "id",
6498 "columns": [
6499 { "name": "id", "type": "string", "nullable": false },
6500 { "name": "project_id", "type": "string", "nullable": false }
6501 ],
6502 "scopes": [{ "pattern": "project:{project_id}" }]
6503 }]
6504 });
6505
6506 {
6507 let mut first = SyncClient::open_path_with_identity(
6508 None,
6509 &schema,
6510 ClientLimits::default(),
6511 path.to_str().expect("UTF-8 temp path"),
6512 )
6513 .expect("first open");
6514 first
6515 .set_window(
6516 &WindowBase {
6517 table: "tasks".to_owned(),
6518 variable: "project_id".to_owned(),
6519 fixed_scopes: Vec::new(),
6520 params: None,
6521 },
6522 &["persisted".to_owned()],
6523 )
6524 .expect("persist window");
6525 }
6526
6527 let mut reopened = SyncClient::open_path_with_identity(
6528 None,
6529 &schema,
6530 ClientLimits::default(),
6531 path.to_str().expect("UTF-8 temp path"),
6532 )
6533 .expect("reopen");
6534 assert!(reopened.sync_needed());
6535 assert!(matches!(
6536 reopened.drain_sync_intents().as_slice(),
6537 [SyncIntent::Interactive]
6538 ));
6539 drop(reopened);
6540 std::fs::remove_file(path).expect("remove temp database");
6541 }
6542
6543 #[test]
6544 fn reopening_preserves_immutable_subscription_identity_and_progress() {
6545 let path = std::env::temp_dir().join(format!(
6546 "syncular-subscription-identity-{}.db",
6547 uuid::Uuid::new_v4()
6548 ));
6549 let schema = json!({
6550 "version": 1,
6551 "tables": [
6552 {
6553 "name": "tasks",
6554 "primaryKey": "id",
6555 "columns": [
6556 { "name": "id", "type": "string", "nullable": false },
6557 { "name": "project_id", "type": "string", "nullable": false }
6558 ],
6559 "scopes": [{ "pattern": "project:{project_id}" }]
6560 },
6561 {
6562 "name": "docs",
6563 "primaryKey": "id",
6564 "columns": [
6565 { "name": "id", "type": "string", "nullable": false },
6566 { "name": "org_id", "type": "string", "nullable": false },
6567 { "name": "project_id", "type": "string", "nullable": false }
6568 ],
6569 "scopes": [
6570 { "pattern": "org:{org_id}" },
6571 { "pattern": "project:{projectId}", "column": "project_id" }
6572 ]
6573 }
6574 ]
6575 });
6576
6577 {
6578 let mut first = SyncClient::open_path_with_identity(
6579 None,
6580 &schema,
6581 ClientLimits::default(),
6582 path.to_str().expect("UTF-8 temp path"),
6583 )
6584 .expect("first open");
6585 first
6586 .subscribe(
6587 "stable-subscription".to_owned(),
6588 "tasks".to_owned(),
6589 vec![(
6590 "project_id".to_owned(),
6591 vec!["p2".to_owned(), "p1".to_owned()],
6592 )],
6593 Some(r#"{"view":"v1"}"#.to_owned()),
6594 )
6595 .expect("persist subscription");
6596 let persisted = {
6597 let subscription = first
6598 .subs
6599 .iter_mut()
6600 .find(|subscription| subscription.id == "stable-subscription")
6601 .expect("subscription");
6602 subscription.cursor = 41;
6603 subscription.bootstrap_state = Some("resume-token".to_owned());
6604 subscription.effective = Some(vec![(
6605 "project_id".to_owned(),
6606 vec!["p1".to_owned(), "p2".to_owned()],
6607 )]);
6608 subscription.synced_once = true;
6609 subscription.clone()
6610 };
6611 first.persist_sub(&persisted).unwrap();
6612 }
6613
6614 let mut reopened = SyncClient::open_path_with_identity(
6615 None,
6616 &schema,
6617 ClientLimits::default(),
6618 path.to_str().expect("UTF-8 temp path"),
6619 )
6620 .expect("reopen");
6621 let progress = reopened
6622 .subscription_state("stable-subscription")
6623 .expect("persisted state");
6624 assert_eq!(progress.cursor, 41);
6625 assert!(progress.has_resume_token);
6626
6627 reopened
6628 .subscribe(
6629 "stable-subscription".to_owned(),
6630 "tasks".to_owned(),
6631 vec![(
6632 "project_id".to_owned(),
6633 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
6634 )],
6635 Some(r#"{"view":"v1"}"#.to_owned()),
6636 )
6637 .expect("canonical intent is idempotent");
6638 assert_eq!(
6639 reopened
6640 .subscription_state("stable-subscription")
6641 .expect("unchanged state")
6642 .cursor,
6643 progress.cursor
6644 );
6645
6646 for (table, scopes, params) in [
6647 (
6648 "tasks",
6649 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6650 Some(r#"{"view":"v1"}"#.to_owned()),
6651 ),
6652 (
6653 "tasks",
6654 vec![(
6655 "project_id".to_owned(),
6656 vec!["p2".to_owned(), "p1".to_owned()],
6657 )],
6658 Some(r#"{"view":"v2"}"#.to_owned()),
6659 ),
6660 (
6661 "docs",
6662 vec![
6663 ("org_id".to_owned(), vec!["o1".to_owned()]),
6664 ("projectId".to_owned(), vec!["p1".to_owned()]),
6665 ],
6666 Some(r#"{"view":"v1"}"#.to_owned()),
6667 ),
6668 ] {
6669 let error = reopened
6670 .subscribe(
6671 "stable-subscription".to_owned(),
6672 table.to_owned(),
6673 scopes,
6674 params,
6675 )
6676 .expect_err("identity rebind must fail");
6677 assert!(error.starts_with("client.subscription_intent_mismatch:"));
6678 assert_eq!(
6679 reopened
6680 .subscription_state("stable-subscription")
6681 .expect("unchanged state")
6682 .cursor,
6683 progress.cursor
6684 );
6685 }
6686
6687 drop(reopened);
6688 std::fs::remove_file(path).expect("remove temp database");
6689 }
6690
6691 #[test]
6692 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
6693 let path = std::env::temp_dir().join(format!(
6694 "syncular-satisfied-schema-floor-{}.db",
6695 uuid::Uuid::new_v4()
6696 ));
6697 let schema = json!({
6698 "version": 23,
6699 "tables": [{
6700 "name": "tasks",
6701 "primaryKey": "id",
6702 "columns": [
6703 { "name": "id", "type": "string", "nullable": false },
6704 { "name": "project_id", "type": "string", "nullable": false }
6705 ],
6706 "scopes": [{ "pattern": "project:{project_id}" }]
6707 }]
6708 });
6709
6710 {
6711 let mut first = SyncClient::open_path_with_identity(
6712 None,
6713 &schema,
6714 ClientLimits::default(),
6715 path.to_str().expect("UTF-8 temp path"),
6716 )
6717 .expect("first open");
6718 first
6719 .subscribe(
6720 "tasks".to_owned(),
6721 "tasks".to_owned(),
6722 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6723 None,
6724 )
6725 .expect("persist subscription");
6726 first.set_schema_floor(Some(SchemaFloor {
6727 required_schema_version: Some(22),
6728 latest_schema_version: Some(22),
6729 }));
6730 }
6731
6732 let mut reopened = SyncClient::open_path_with_identity(
6733 None,
6734 &schema,
6735 ClientLimits::default(),
6736 path.to_str().expect("UTF-8 temp path"),
6737 )
6738 .expect("reopen");
6739 assert!(reopened.schema_floor().is_none());
6740 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
6741 assert!(reopened.sync_needed());
6742 assert!(matches!(
6743 reopened.drain_sync_intents().as_slice(),
6744 [SyncIntent::Interactive]
6745 ));
6746 drop(reopened);
6747 std::fs::remove_file(path).expect("remove temp database");
6748 }
6749
6750 #[test]
6751 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
6752 let path = std::env::temp_dir().join(format!(
6753 "syncular-unsatisfied-schema-floor-{}.db",
6754 uuid::Uuid::new_v4()
6755 ));
6756 let schema = json!({
6757 "version": 1,
6758 "tables": [{
6759 "name": "tasks",
6760 "primaryKey": "id",
6761 "columns": [
6762 { "name": "id", "type": "string", "nullable": false },
6763 { "name": "project_id", "type": "string", "nullable": false }
6764 ],
6765 "scopes": [{ "pattern": "project:{project_id}" }]
6766 }]
6767 });
6768
6769 {
6770 let mut first = SyncClient::open_path_with_identity(
6771 None,
6772 &schema,
6773 ClientLimits::default(),
6774 path.to_str().expect("UTF-8 temp path"),
6775 )
6776 .expect("first open");
6777 first
6778 .subscribe(
6779 "tasks".to_owned(),
6780 "tasks".to_owned(),
6781 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6782 None,
6783 )
6784 .expect("persist subscription");
6785 first.set_schema_floor(Some(SchemaFloor {
6786 required_schema_version: Some(2),
6787 latest_schema_version: Some(2),
6788 }));
6789 }
6790
6791 let reopened = SyncClient::open_path_with_identity(
6792 None,
6793 &schema,
6794 ClientLimits::default(),
6795 path.to_str().expect("UTF-8 temp path"),
6796 )
6797 .expect("reopen");
6798 assert_eq!(
6799 reopened.schema_floor(),
6800 Some(&SchemaFloor {
6801 required_schema_version: Some(2),
6802 latest_schema_version: Some(2),
6803 })
6804 );
6805 assert!(!reopened.sync_needed());
6806 drop(reopened);
6807 std::fs::remove_file(path).expect("remove temp database");
6808 }
6809
6810 #[test]
6811 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
6812 let path = std::env::temp_dir().join(format!(
6813 "syncular-indexed-column-bump-{}.db",
6814 uuid::Uuid::new_v4()
6815 ));
6816 let old_schema = json!({
6817 "version": 1,
6818 "tables": [
6819 {
6820 "name": "tasks",
6821 "primaryKey": "id",
6822 "columns": [
6823 { "name": "id", "type": "string", "nullable": false },
6824 { "name": "project_id", "type": "string", "nullable": false }
6825 ],
6826 "scopes": [{ "pattern": "project:{project_id}" }]
6827 },
6828 {
6829 "name": "legacy",
6830 "primaryKey": "id",
6831 "columns": [
6832 { "name": "id", "type": "string", "nullable": false },
6833 { "name": "project_id", "type": "string", "nullable": false }
6834 ],
6835 "scopes": [{ "pattern": "project:{project_id}" }]
6836 }
6837 ]
6838 });
6839 {
6840 let mut first = SyncClient::open_path_with_identity(
6841 None,
6842 &old_schema,
6843 ClientLimits::default(),
6844 path.to_str().expect("UTF-8 temp path"),
6845 )
6846 .expect("open old schema");
6847 first
6848 .subscribe(
6849 "legacy-sub".to_owned(),
6850 "legacy".to_owned(),
6851 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
6852 None,
6853 )
6854 .expect("persist legacy subscription");
6855 }
6856
6857 let new_schema = json!({
6858 "version": 2,
6859 "tables": [{
6860 "name": "tasks",
6861 "primaryKey": "id",
6862 "columns": [
6863 { "name": "id", "type": "string", "nullable": false },
6864 { "name": "project_id", "type": "string", "nullable": false },
6865 { "name": "facility_membership_id", "type": "string", "nullable": true }
6866 ],
6867 "scopes": [{ "pattern": "project:{project_id}" }],
6868 "indexes": [{
6869 "name": "tasks_by_membership",
6870 "columns": ["project_id", "facility_membership_id"],
6871 "unique": false
6872 }]
6873 }]
6874 });
6875 let upgraded = SyncClient::open_path_with_identity(
6876 None,
6877 &new_schema,
6878 ClientLimits::default(),
6879 path.to_str().expect("UTF-8 temp path"),
6880 )
6881 .expect("open upgraded schema");
6882 let columns = upgraded
6883 .conn
6884 .prepare("PRAGMA table_info(tasks)")
6885 .expect("prepare columns")
6886 .query_map([], |row| row.get::<_, String>(1))
6887 .expect("query columns")
6888 .collect::<Result<Vec<_>, _>>()
6889 .expect("collect columns");
6890 assert!(columns.contains(&"facility_membership_id".to_owned()));
6891 let index_count: i64 = upgraded
6892 .conn
6893 .query_row(
6894 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
6895 [],
6896 |row| row.get(0),
6897 )
6898 .expect("query index");
6899 assert_eq!(index_count, 1);
6900 assert!(upgraded.subscription_state("legacy-sub").is_none());
6901 drop(upgraded);
6902 std::fs::remove_file(path).expect("remove temp database");
6903 }
6904
6905 #[test]
6906 fn migrates_pre_envelope_outcome_journal_additively() {
6907 let path = std::env::temp_dir().join(format!(
6908 "syncular-outcome-migration-{}.db",
6909 uuid::Uuid::new_v4()
6910 ));
6911 let conn = Connection::open(&path).expect("open legacy database");
6912 conn.execute_batch(
6913 "CREATE TABLE _syncular_commit_outcomes (
6914 seq INTEGER PRIMARY KEY AUTOINCREMENT,
6915 client_commit_id TEXT NOT NULL UNIQUE,
6916 status TEXT NOT NULL,
6917 recorded_at_ms INTEGER NOT NULL,
6918 results_json TEXT NOT NULL,
6919 resolution TEXT NOT NULL DEFAULT 'active',
6920 resolved_at_ms INTEGER,
6921 replacement_client_commit_id TEXT);",
6922 )
6923 .expect("create legacy outcome journal");
6924 drop(conn);
6925 let schema = json!({
6926 "version": 1,
6927 "tables": [{
6928 "name": "tasks",
6929 "primaryKey": "id",
6930 "columns": [
6931 { "name": "id", "type": "string", "nullable": false },
6932 { "name": "project_id", "type": "string", "nullable": false }
6933 ],
6934 "scopes": [{ "pattern": "project:{project_id}" }]
6935 }]
6936 });
6937 let client = SyncClient::open_path(
6938 "migration-native".to_owned(),
6939 &schema,
6940 ClientLimits::default(),
6941 path.to_str().expect("UTF-8 temp path"),
6942 )
6943 .expect("migrate database");
6944 let has_operations = client
6945 .conn
6946 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
6947 .expect("prepare table info")
6948 .query_map([], |row| row.get::<_, String>(1))
6949 .expect("query table info")
6950 .filter_map(Result::ok)
6951 .any(|column| column == "operations_json");
6952 assert!(has_operations);
6953 drop(client);
6954 std::fs::remove_file(path).expect("remove temp database");
6955 }
6956
6957 #[test]
6958 fn retained_rejection_survives_file_reopen_and_security_purge() {
6959 let path =
6960 std::env::temp_dir().join(format!("syncular-retained-{}.db", uuid::Uuid::new_v4()));
6961 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id", "columns": [
6962 { "name": "id", "type": "string", "nullable": false },
6963 { "name": "project_id", "type": "string", "nullable": false },
6964 { "name": "title", "type": "string", "nullable": false }
6965 ], "scopes": [{ "pattern": "project:{project_id}" }] }] });
6966 {
6967 let mut first = SyncClient::open_path(
6968 "retained-native".to_owned(),
6969 &schema,
6970 ClientLimits::default(),
6971 path.to_str().unwrap(),
6972 )
6973 .unwrap();
6974 first
6975 .write_base_row(
6976 "tasks",
6977 &vec![
6978 Some(ColumnValue::String("t1".into())),
6979 Some(ColumnValue::String("p1".into())),
6980 Some(ColumnValue::String("server".into())),
6981 ],
6982 1,
6983 )
6984 .unwrap();
6985 first.rebuild_overlay().expect("rebuild overlay");
6986 let operation = OutboxOp {
6987 table: "tasks".into(),
6988 row_id: "t1".into(),
6989 upsert: true,
6990 insert: false,
6991 base_version: Some(1),
6992 values: Some(Map::from_iter([
6993 ("id".into(), json!("t1")),
6994 ("title".into(), json!("mine")),
6995 ])),
6996 };
6997 first
6998 .retain_failed_commit("retained", std::slice::from_ref(&operation))
6999 .unwrap();
7000 first
7001 .persist_commit_outcome(
7002 "retained",
7003 CommitOutcomeStatus::Rejected,
7004 &[CommitOperationOutcome::Error {
7005 rejection: RejectionRecord {
7006 client_commit_id: "retained".into(),
7007 op_index: 0,
7008 code: "validation.denied".into(),
7009 message: "denied".into(),
7010 retryable: false,
7011 details: None,
7012 operation: None,
7013 },
7014 }],
7015 Some(&[operation]),
7016 )
7017 .unwrap();
7018 first.rebuild_overlay().expect("rebuild overlay");
7019 }
7020 {
7021 let mut reopened = SyncClient::open_path(
7022 "retained-native".to_owned(),
7023 &schema,
7024 ClientLimits::default(),
7025 path.to_str().unwrap(),
7026 )
7027 .unwrap();
7028 let row = reopened
7029 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
7030 .expect("read visible row")
7031 .unwrap();
7032 assert_eq!(row.0[2], Some(ColumnValue::String("mine".into())));
7033 assert_eq!(
7034 reopened
7035 .commit_outcome("retained")
7036 .unwrap()
7037 .unwrap()
7038 .retained_rows
7039 .unwrap()[0]
7040 .server_row
7041 .as_ref()
7042 .unwrap()["title"],
7043 json!("server")
7044 );
7045 reopened
7046 .conn
7047 .execute("DELETE FROM _syncular_base_tasks WHERE id = 't1'", [])
7048 .unwrap();
7049 reopened.rebuild_overlay().expect("rebuild overlay");
7050 assert!(reopened
7051 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
7052 .expect("read visible row")
7053 .is_none());
7054 let retained = reopened.commit_outcome("retained").unwrap().unwrap();
7055 assert!(retained.retained_rows.is_some());
7056 assert_eq!(
7057 retained.operations.unwrap()[0].values.as_ref().unwrap()["title"],
7058 json!("mine")
7059 );
7060 reopened
7061 .purge_local_data(&LocalDataPurgeInput {
7062 purge_id: "protected-purge".into(),
7063 targets: vec![LocalDataPurgeTarget {
7064 table: "tasks".into(),
7065 selectors: BTreeMap::from_iter([("project_id".into(), vec!["p1".into()])]),
7066 }],
7067 })
7068 .unwrap();
7069 assert!(reopened
7070 .visible_row(reopened.schema.table("tasks").unwrap(), "t1")
7071 .expect("read visible row")
7072 .is_none());
7073 assert!(reopened
7074 .commit_outcome("retained")
7075 .unwrap()
7076 .unwrap()
7077 .retained_rows
7078 .is_none());
7079 }
7080 std::fs::remove_file(path).unwrap();
7081 }
7082
7083 #[test]
7084 fn durable_conflict_outcome_and_resolution_survive_reopen() {
7085 let path = std::env::temp_dir().join(format!(
7086 "syncular-durable-outcome-{}.db",
7087 uuid::Uuid::new_v4()
7088 ));
7089 let schema = json!({
7090 "version": 1,
7091 "tables": [{
7092 "name": "tasks",
7093 "primaryKey": "id",
7094 "columns": [
7095 { "name": "id", "type": "string", "nullable": false },
7096 { "name": "project_id", "type": "string", "nullable": false }
7097 ],
7098 "scopes": [{ "pattern": "project:{project_id}" }]
7099 }]
7100 });
7101 let conflict = ConflictRecord {
7102 client_commit_id: "losing-commit".to_owned(),
7103 op_index: 0,
7104 table: "tasks".to_owned(),
7105 row_id: "t1".to_owned(),
7106 code: "sync.version_conflict".to_owned(),
7107 message: "stale base version".to_owned(),
7108 server_version: 2,
7109 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
7110 conflict_columns: vec!["project_id".to_owned()],
7111 operation: Some(CommitOperation {
7112 table: "tasks".to_owned(),
7113 row_id: "t1".to_owned(),
7114 op: "upsert".to_owned(),
7115 base_version: Some(1),
7116 values: Some(Map::from_iter([
7117 ("id".into(), json!("t1")),
7118 ("project_id".into(), json!("p1")),
7119 ])),
7120 insert: false,
7121 }),
7122 };
7123 let failed_operations = vec![
7124 OutboxOp {
7125 upsert: true,
7126 insert: false,
7127 table: "tasks".to_owned(),
7128 row_id: "t1".to_owned(),
7129 base_version: Some(1),
7130 values: Some(Map::from_iter([
7131 ("id".into(), json!("t1")),
7132 ("project_id".into(), json!("p1")),
7133 ])),
7134 },
7135 OutboxOp {
7136 upsert: true,
7137 insert: false,
7138 table: "tasks".to_owned(),
7139 row_id: "status-event-1".to_owned(),
7140 base_version: Some(0),
7141 values: Some(Map::from_iter([
7142 ("id".into(), json!("status-event-1")),
7143 ("project_id".into(), json!("p1")),
7144 ])),
7145 },
7146 ];
7147
7148 {
7149 let mut first = SyncClient::open_path(
7150 "durable-native".to_owned(),
7151 &schema,
7152 ClientLimits::default(),
7153 path.to_str().expect("UTF-8 temp path"),
7154 )
7155 .expect("first open");
7156 first
7157 .begin_observation("test_outcome")
7158 .expect("begin outcome");
7159 first
7160 .persist_commit_outcome(
7161 "losing-commit",
7162 CommitOutcomeStatus::Conflict,
7163 &[CommitOperationOutcome::Conflict {
7164 conflict: conflict.clone(),
7165 }],
7166 Some(&failed_operations),
7167 )
7168 .expect("persist outcome");
7169 first.conflicts.push(conflict);
7170 first
7171 .finish_observation(
7172 "test_outcome",
7173 ChangeAccumulator {
7174 conflicts: true,
7175 outcomes: true,
7176 ..ChangeAccumulator::default()
7177 },
7178 )
7179 .expect("commit outcome");
7180 }
7181
7182 {
7183 let mut reopened = SyncClient::open_path(
7184 "durable-native".to_owned(),
7185 &schema,
7186 ClientLimits::default(),
7187 path.to_str().expect("UTF-8 temp path"),
7188 )
7189 .expect("reopen");
7190 assert_eq!(reopened.conflicts().len(), 1);
7191 assert_eq!(
7192 reopened.conflicts()[0].conflict_columns,
7193 vec!["project_id".to_owned()],
7194 "conflictColumns survives the journal round trip"
7195 );
7196 let outcome = reopened
7197 .commit_outcome("losing-commit")
7198 .expect("read outcome")
7199 .expect("outcome");
7200 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
7201 let operations = outcome.operations.expect("aggregate envelope");
7202 assert_eq!(operations.len(), 2);
7203 assert_eq!(operations[1].row_id, "status-event-1");
7204 let resolved = reopened
7205 .resolve_commit_outcome(ResolveCommitOutcomeInput {
7206 client_commit_id: "losing-commit".to_owned(),
7207 resolution: CommitOutcomeResolution::ResolvedKeepServer,
7208 replacement_client_commit_id: None,
7209 })
7210 .expect("resolve");
7211 assert_eq!(
7212 resolved.resolution,
7213 CommitOutcomeResolution::ResolvedKeepServer
7214 );
7215 assert!(reopened.conflicts().is_empty());
7216 }
7217
7218 let reopened = SyncClient::open_path(
7219 "durable-native".to_owned(),
7220 &schema,
7221 ClientLimits::default(),
7222 path.to_str().expect("UTF-8 temp path"),
7223 )
7224 .expect("second reopen");
7225 assert!(reopened.conflicts().is_empty());
7226 assert_eq!(
7227 reopened
7228 .commit_outcome("losing-commit")
7229 .expect("read outcome")
7230 .expect("outcome")
7231 .resolution,
7232 CommitOutcomeResolution::ResolvedKeepServer
7233 );
7234 drop(reopened);
7235 std::fs::remove_file(path).expect("remove temp database");
7236 }
7237
7238 #[test]
7239 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
7240 let path =
7241 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7242 let schema = json!({
7243 "version": 1,
7244 "tables": [{
7245 "name": "tasks",
7246 "primaryKey": "id",
7247 "columns": [
7248 { "name": "id", "type": "string", "nullable": false },
7249 { "name": "project_id", "type": "string", "nullable": false }
7250 ],
7251 "scopes": [{ "pattern": "project:{project_id}" }]
7252 }]
7253 });
7254 let mut client = SyncClient::open_path_with_identity(
7255 Some("sidecar-client".to_owned()),
7256 &schema,
7257 ClientLimits::default(),
7258 path.to_str().expect("UTF-8 temp path"),
7259 )
7260 .expect("open owner");
7261 let base = WindowBase {
7262 table: "tasks".to_owned(),
7263 variable: "project_id".to_owned(),
7264 fixed_scopes: Vec::new(),
7265 params: None,
7266 };
7267 client
7268 .set_window(&base, &["one".to_owned()])
7269 .expect("set window");
7270 client
7271 .mutate(vec![Mutation::Upsert {
7272 table: "tasks".to_owned(),
7273 values: Map::from_iter([
7274 ("id".to_owned(), Value::from("t1")),
7275 ("project_id".to_owned(), Value::from("one")),
7276 ]),
7277 base_version: None,
7278 }])
7279 .expect("local mutate");
7280
7281 let coverage = [WindowCoverage {
7282 base,
7283 units: vec!["one".to_owned(), "missing".to_owned()],
7284 }];
7285 let owner = client
7286 .query_snapshot(
7287 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
7288 &[],
7289 &coverage,
7290 None,
7291 )
7292 .expect("owner snapshot");
7293 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7294 let sidecar = reader
7295 .query_snapshot(
7296 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
7297 &[],
7298 &coverage,
7299 )
7300 .expect("sidecar snapshot");
7301
7302 assert_eq!(sidecar.revision, owner.revision);
7303 assert_eq!(sidecar.rows, owner.rows);
7304 assert_eq!(
7305 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
7306 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
7307 );
7308 assert_eq!(sidecar.revision, "2");
7309 assert_eq!(sidecar.rows[0]["id"], "t1");
7310 assert_eq!(sidecar.rows[0]["server_version"], -1);
7311 assert!(!sidecar.coverage.complete);
7312 assert_eq!(sidecar.coverage.pending.len(), 1);
7313 assert_eq!(sidecar.coverage.missing.len(), 1);
7314
7315 drop(reader);
7316 drop(client);
7317 std::fs::remove_file(path).expect("remove temp database");
7318 }
7319
7320 #[test]
7321 fn file_snapshot_reader_reports_catchup_delivery_and_multi_statement() {
7322 let path =
7323 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7324 let schema = json!({
7325 "version": 1,
7326 "tables": [{
7327 "name": "tasks",
7328 "primaryKey": "id",
7329 "columns": [
7330 { "name": "id", "type": "string", "nullable": false },
7331 { "name": "project_id", "type": "string", "nullable": false }
7332 ],
7333 "scopes": [{ "pattern": "project:{project_id}" }]
7334 }]
7335 });
7336 let mut client = SyncClient::open_path_with_identity(
7337 Some("sidecar-catchup".to_owned()),
7338 &schema,
7339 ClientLimits::default(),
7340 path.to_str().expect("UTF-8 temp path"),
7341 )
7342 .expect("open owner");
7343 client
7344 .subscribe(
7345 "s1".into(),
7346 "tasks".into(),
7347 vec![("project_id".into(), vec!["one".into()])],
7348 None,
7349 )
7350 .expect("subscribe");
7351 let commit_id = client
7352 .mutate(vec![Mutation::Upsert {
7353 table: "tasks".to_owned(),
7354 values: Map::from_iter([
7355 ("id".to_owned(), Value::from("t1")),
7356 ("project_id".to_owned(), Value::from("one")),
7357 ]),
7358 base_version: None,
7359 }])
7360 .expect("mutate");
7361 let revision = client.local_revision().to_string();
7362
7363 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7364 assert_eq!(reader.local_revision().expect("revision"), revision);
7365
7366 let read = reader
7367 .snapshot_read(&SnapshotReadRequest {
7368 statements: vec![
7369 SnapshotStatement {
7370 sql: "SELECT 1 AS n",
7371 params: &[],
7372 },
7373 SnapshotStatement {
7374 sql: "SELECT id FROM tasks ORDER BY id",
7375 params: &[],
7376 },
7377 ],
7378 coverage: &[],
7379 subscriptions: vec!["s1".to_owned(), "absent".to_owned()],
7380 commit_ids: vec![commit_id.clone(), "never-known".to_owned()],
7381 })
7382 .expect("snapshot read");
7383 assert_eq!(read.revision, revision);
7384 assert_eq!(read.queries.len(), 2);
7385 assert_eq!(read.queries[0][0]["n"], 1);
7386 assert_eq!(read.queries[1][0]["id"], "t1");
7387 match &read.subscriptions[0] {
7388 SubscriptionCatchup::Known {
7389 status,
7390 cursor,
7391 bootstrap_complete,
7392 known_pending_pages,
7393 ..
7394 } => {
7395 assert_eq!(status, "active");
7396 assert_eq!(*cursor, -1);
7397 assert!(!bootstrap_complete);
7398 assert!(known_pending_pages);
7399 }
7400 other => panic!("expected a known subscription: {other:?}"),
7401 }
7402 assert!(matches!(
7403 &read.subscriptions[1],
7404 SubscriptionCatchup::Unknown { id } if id.as_str() == "absent"
7405 ));
7406 assert!(matches!(
7407 &read.deliveries[0],
7408 CommitDelivery::Pending { client_commit_id } if client_commit_id == &commit_id
7409 ));
7410 assert!(matches!(
7411 &read.deliveries[1],
7412 CommitDelivery::Unknown { client_commit_id } if client_commit_id.as_str() == "never-known"
7413 ));
7414
7415 assert!(reader
7417 .query_snapshot("SELECT * FROM does_not_exist", &[], &[])
7418 .is_err());
7419 assert_eq!(
7420 reader.local_revision().expect("revision after rollback"),
7421 revision
7422 );
7423 assert!(reader
7425 .query_snapshot("DELETE FROM tasks", &[], &[])
7426 .is_err());
7427
7428 client
7430 .conn
7431 .execute(
7432 "UPDATE _syncular_meta SET value = '+1' WHERE key = 'localRevision'",
7433 [],
7434 )
7435 .expect("corrupt revision");
7436 let failure = reader.local_revision().expect_err("corrupt revision fails");
7437 assert_eq!(failure.code, Some("sync.local_corrupt"));
7438
7439 drop(reader);
7440 drop(client);
7441 std::fs::remove_file(path).expect("remove temp database");
7442 }
7443
7444 #[test]
7445 fn snapshot_delivery_is_the_persisted_journal_view_without_retained_rows() {
7446 let path =
7447 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
7448 let schema = json!({
7449 "version": 1,
7450 "tables": [{
7451 "name": "tasks",
7452 "primaryKey": "id",
7453 "columns": [
7454 { "name": "id", "type": "string", "nullable": false },
7455 { "name": "project_id", "type": "string", "nullable": false }
7456 ],
7457 "scopes": [{ "pattern": "project:{project_id}" }]
7458 }]
7459 });
7460 let mut client = SyncClient::open_path_with_identity(
7461 Some("sidecar-delivery".to_owned()),
7462 &schema,
7463 ClientLimits::default(),
7464 path.to_str().expect("UTF-8 temp path"),
7465 )
7466 .expect("open owner");
7467 client.begin_observation("test_delivery").expect("observe");
7468 let operations = [OutboxOp {
7469 upsert: true,
7470 insert: false,
7471 table: "tasks".to_owned(),
7472 row_id: "t1".to_owned(),
7473 base_version: Some(1),
7474 values: Some(Map::from_iter([
7475 ("id".into(), json!("t1")),
7476 ("project_id".into(), json!("p1")),
7477 ])),
7478 }];
7479 client
7480 .persist_commit_outcome(
7481 "conflict-commit",
7482 CommitOutcomeStatus::Conflict,
7483 &[CommitOperationOutcome::Conflict {
7484 conflict: ConflictRecord {
7485 client_commit_id: "conflict-commit".to_owned(),
7486 op_index: 0,
7487 table: "tasks".to_owned(),
7488 row_id: "t1".to_owned(),
7489 code: "sync.version_conflict".to_owned(),
7490 message: "stale base version".to_owned(),
7491 server_version: 2,
7492 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
7493 conflict_columns: vec!["project_id".to_owned()],
7494 operation: None,
7495 },
7496 }],
7497 Some(&operations),
7498 )
7499 .expect("persist conflict");
7500 client
7501 .persist_commit_outcome(
7502 "rejected-commit",
7503 CommitOutcomeStatus::Rejected,
7504 &[CommitOperationOutcome::Error {
7505 rejection: RejectionRecord {
7506 client_commit_id: "rejected-commit".to_owned(),
7507 op_index: 0,
7508 code: "sync.unknown_table".to_owned(),
7509 message: "unknown table".to_owned(),
7510 retryable: false,
7511 details: None,
7512 operation: None,
7513 },
7514 }],
7515 Some(&operations),
7516 )
7517 .expect("persist rejection");
7518 client
7519 .finish_observation(
7520 "test_delivery",
7521 ChangeAccumulator {
7522 conflicts: true,
7523 outcomes: true,
7524 ..ChangeAccumulator::default()
7525 },
7526 )
7527 .expect("commit outcomes");
7528
7529 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7530 match reader.commit_delivery("conflict-commit").expect("conflict") {
7531 CommitDelivery::Known { outcome, .. } => {
7532 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
7533 assert!(matches!(
7534 outcome.results.as_slice(),
7535 [CommitOperationOutcome::Conflict { conflict }]
7536 if conflict.code == "sync.version_conflict"
7537 && conflict.server_version == 2
7538 ));
7539 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
7540 assert!(
7541 outcome.retained_rows.is_none(),
7542 "the sidecar reports the persisted view, not owner-derived retained rows"
7543 );
7544 }
7545 other => panic!("expected a known conflict: {other:?}"),
7546 }
7547 match reader
7548 .commit_delivery("rejected-commit")
7549 .expect("rejection")
7550 {
7551 CommitDelivery::Known { outcome, .. } => {
7552 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
7553 assert!(matches!(
7554 outcome.results.as_slice(),
7555 [CommitOperationOutcome::Error { rejection }]
7556 if rejection.code == "sync.unknown_table"
7557 ));
7558 assert_eq!(outcome.operations.as_ref().map(Vec::len), Some(1));
7559 assert!(outcome.retained_rows.is_none());
7560 }
7561 other => panic!("expected a known rejection: {other:?}"),
7562 }
7563 drop(reader);
7564 drop(client);
7565 std::fs::remove_file(path).expect("remove temp database");
7566 }
7567
7568 #[test]
7569 fn snapshot_read_isolates_a_concurrent_writer_commit() {
7570 use std::sync::mpsc::channel;
7571 let path = std::env::temp_dir().join(format!(
7572 "syncular-read-concurrent-{}.db",
7573 uuid::Uuid::new_v4()
7574 ));
7575 let schema = json!({
7576 "version": 1,
7577 "tables": [{
7578 "name": "tasks",
7579 "primaryKey": "id",
7580 "columns": [
7581 { "name": "id", "type": "string", "nullable": false },
7582 { "name": "project_id", "type": "string", "nullable": false }
7583 ],
7584 "scopes": [{ "pattern": "project:{project_id}" }]
7585 }]
7586 });
7587 let mut client = SyncClient::open_path_with_identity(
7588 Some("sidecar-concurrent".to_owned()),
7589 &schema,
7590 ClientLimits::default(),
7591 path.to_str().expect("UTF-8 temp path"),
7592 )
7593 .expect("open owner");
7594 client
7595 .mutate(vec![Mutation::Upsert {
7596 table: "tasks".to_owned(),
7597 values: Map::from_iter([
7598 ("id".to_owned(), Value::from("t1")),
7599 ("project_id".to_owned(), Value::from("one")),
7600 ]),
7601 base_version: None,
7602 }])
7603 .expect("seed row");
7604 let revision_before = client.local_revision().to_string();
7605
7606 let (ready_tx, ready_rx) = channel::<()>();
7609 let (done_tx, done_rx) = channel::<()>();
7610 let writer_path = path.to_string_lossy().into_owned();
7611 let writer_schema = schema.clone();
7612 let writer = std::thread::spawn(move || {
7613 let mut writer = SyncClient::open_path_with_identity(
7614 Some("sidecar-concurrent".to_owned()),
7615 &writer_schema,
7616 ClientLimits::default(),
7617 &writer_path,
7618 )
7619 .expect("open writer");
7620 ready_rx.recv().expect("reader signalled");
7621 writer
7622 .mutate(vec![Mutation::Upsert {
7623 table: "tasks".to_owned(),
7624 values: Map::from_iter([
7625 ("id".to_owned(), Value::from("t2")),
7626 ("project_id".to_owned(), Value::from("one")),
7627 ]),
7628 base_version: None,
7629 }])
7630 .expect("writer commit");
7631 let _ = done_tx.send(());
7632 });
7633 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7634 SNAPSHOT_READ_HOOK.with(|slot| {
7635 *slot.borrow_mut() = Some(Box::new(move || {
7636 ready_tx.send(()).expect("signal writer");
7637 done_rx.recv().expect("writer finished");
7638 }));
7639 });
7640 let read = reader
7641 .snapshot_read(&SnapshotReadRequest {
7642 statements: vec![
7643 SnapshotStatement {
7644 sql: "SELECT count(*) AS n FROM tasks",
7645 params: &[],
7646 },
7647 SnapshotStatement {
7648 sql: "SELECT count(*) AS n FROM tasks",
7649 params: &[],
7650 },
7651 ],
7652 coverage: &[],
7653 subscriptions: Vec::new(),
7654 commit_ids: Vec::new(),
7655 })
7656 .expect("snapshot read");
7657 assert_eq!(
7658 read.queries[0][0]["n"], 1,
7659 "first statement sees the pre-commit row"
7660 );
7661 assert_eq!(
7662 read.queries[1][0]["n"], 1,
7663 "a later statement keeps the one read snapshot"
7664 );
7665 assert_eq!(
7666 read.revision, revision_before,
7667 "the read reports the snapshot revision, not the writer's"
7668 );
7669 writer.join().expect("join writer");
7670
7671 assert_ne!(
7672 reader.local_revision().expect("revision after commit"),
7673 revision_before,
7674 "the next read observes the committed revision"
7675 );
7676 let after = reader
7677 .query_snapshot("SELECT count(*) AS n FROM tasks", &[], &[])
7678 .expect("count after commit");
7679 assert_eq!(
7680 after.rows[0]["n"], 2,
7681 "the committed row is visible afterwards"
7682 );
7683
7684 drop(reader);
7685 drop(client);
7686 std::fs::remove_file(path).expect("remove temp database");
7687 }
7688
7689 #[test]
7690 fn snapshot_read_rejects_every_non_canonical_revision() {
7691 let path = std::env::temp_dir().join(format!(
7692 "syncular-read-revision-{}.db",
7693 uuid::Uuid::new_v4()
7694 ));
7695 let schema = json!({
7696 "version": 1,
7697 "tables": [{
7698 "name": "tasks",
7699 "primaryKey": "id",
7700 "columns": [
7701 { "name": "id", "type": "string", "nullable": false },
7702 { "name": "project_id", "type": "string", "nullable": false }
7703 ],
7704 "scopes": [{ "pattern": "project:{project_id}" }]
7705 }]
7706 });
7707 let client = SyncClient::open_path_with_identity(
7708 Some("sidecar-revision".to_owned()),
7709 &schema,
7710 ClientLimits::default(),
7711 path.to_str().expect("UTF-8 temp path"),
7712 )
7713 .expect("open owner");
7714 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7715 for bad in [
7716 "+1",
7717 "01",
7718 "-1",
7719 " 1",
7720 "1 ",
7721 "18446744073709551616",
7722 "1.0",
7723 "abc",
7724 "",
7725 ] {
7726 client
7727 .conn
7728 .execute(
7729 "UPDATE _syncular_meta SET value = ?1 WHERE key = 'localRevision'",
7730 rusqlite::params![bad],
7731 )
7732 .expect("corrupt revision");
7733 let failure = reader.local_revision().expect_err("corrupt marker fails");
7734 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad:?}");
7735 }
7736 for value in ["X'31'", "CAST(X'FF' AS TEXT)"] {
7737 client
7738 .conn
7739 .execute_batch(&format!(
7740 "UPDATE _syncular_meta SET value={value} WHERE key='localRevision'"
7741 ))
7742 .expect("corrupt stored revision type");
7743 assert_eq!(
7744 reader.local_revision().expect_err("invalid SQL field").code,
7745 Some("sync.local_corrupt")
7746 );
7747 }
7748 client
7750 .conn
7751 .execute("DELETE FROM _syncular_meta WHERE key = 'localRevision'", [])
7752 .expect("delete marker");
7753 assert_eq!(reader.local_revision().expect("legacy zero"), "0");
7754
7755 drop(reader);
7756 drop(client);
7757 std::fs::remove_file(path).expect("remove temp database");
7758 }
7759
7760 #[test]
7761 fn snapshot_read_rejects_corrupt_subscription_state() {
7762 let path = std::env::temp_dir().join(format!(
7763 "syncular-read-substate-{}.db",
7764 uuid::Uuid::new_v4()
7765 ));
7766 let schema = json!({
7767 "version": 1,
7768 "tables": [{
7769 "name": "tasks",
7770 "primaryKey": "id",
7771 "columns": [
7772 { "name": "id", "type": "string", "nullable": false },
7773 { "name": "project_id", "type": "string", "nullable": false }
7774 ],
7775 "scopes": [{ "pattern": "project:{project_id}" }]
7776 }]
7777 });
7778 let mut client = SyncClient::open_path_with_identity(
7779 Some("sidecar-substate".to_owned()),
7780 &schema,
7781 ClientLimits::default(),
7782 path.to_str().expect("UTF-8 temp path"),
7783 )
7784 .expect("open owner");
7785 client
7786 .subscribe(
7787 "s1".into(),
7788 "tasks".into(),
7789 vec![("project_id".into(), vec!["one".into()])],
7790 None,
7791 )
7792 .expect("subscribe");
7793 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7794 for bad in [
7795 "not json".to_owned(),
7796 "{}".to_owned(),
7797 json!({ "status": "bogus", "cursor": 0 }).to_string(),
7798 json!({ "status": "active", "cursor": "x" }).to_string(),
7799 json!({ "status": "active", "cursor": 0, "bootstrapState": 7 }).to_string(),
7800 json!({ "status": "active", "cursor": 0, "reasonCode": 7 }).to_string(),
7801 ] {
7802 client
7803 .conn
7804 .execute(
7805 "UPDATE _syncular_subscriptions SET state_json = ?1 WHERE id = 's1'",
7806 rusqlite::params![bad],
7807 )
7808 .expect("corrupt subscription state");
7809 let failure = reader
7810 .subscription_catchup("s1")
7811 .expect_err("corrupt state fails");
7812 assert_eq!(failure.code, Some("sync.local_corrupt"), "{bad}");
7813 }
7814
7815 for assignment in [
7816 "tbl=X'7461736b73'",
7817 "state_json=X'7b7d'",
7818 "state_json=CAST(X'FF' AS TEXT)",
7819 ] {
7820 client.conn.execute_batch(&format!(
7821 "UPDATE _syncular_subscriptions SET tbl='tasks',state_json='{{\"status\":\"active\",\"cursor\":0}}'; UPDATE _syncular_subscriptions SET {assignment} WHERE id='s1'"
7822 )).expect("corrupt stored subscription type");
7823 assert_eq!(
7824 reader
7825 .subscription_catchup("s1")
7826 .expect_err("invalid SQL field")
7827 .code,
7828 Some("sync.local_corrupt")
7829 );
7830 }
7831
7832 drop(reader);
7833 drop(client);
7834 std::fs::remove_file(path).expect("remove temp database");
7835 }
7836
7837 #[test]
7838 fn snapshot_read_rolls_back_a_later_invalid_statement_and_guards_every_write() {
7839 let path = std::env::temp_dir().join(format!(
7840 "syncular-read-rollback-{}.db",
7841 uuid::Uuid::new_v4()
7842 ));
7843 let schema = json!({
7844 "version": 1,
7845 "tables": [{
7846 "name": "tasks",
7847 "primaryKey": "id",
7848 "columns": [
7849 { "name": "id", "type": "string", "nullable": false },
7850 { "name": "project_id", "type": "string", "nullable": false }
7851 ],
7852 "scopes": [{ "pattern": "project:{project_id}" }]
7853 }]
7854 });
7855 let mut client = SyncClient::open_path_with_identity(
7856 Some("sidecar-rollback".to_owned()),
7857 &schema,
7858 ClientLimits::default(),
7859 path.to_str().expect("UTF-8 temp path"),
7860 )
7861 .expect("open owner");
7862 let commit_id = client
7863 .mutate(vec![Mutation::Upsert {
7864 table: "tasks".to_owned(),
7865 values: Map::from_iter([
7866 ("id".to_owned(), Value::from("t1")),
7867 ("project_id".to_owned(), Value::from("one")),
7868 ]),
7869 base_version: None,
7870 }])
7871 .expect("seed row");
7872 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
7873 let revision = reader.local_revision().expect("revision");
7874
7875 let failure = reader
7878 .snapshot_read(&SnapshotReadRequest {
7879 statements: vec![
7880 SnapshotStatement {
7881 sql: "SELECT id FROM tasks",
7882 params: &[],
7883 },
7884 SnapshotStatement {
7885 sql: "SELECT * FROM does_not_exist",
7886 params: &[],
7887 },
7888 ],
7889 coverage: &[],
7890 subscriptions: Vec::new(),
7891 commit_ids: vec![commit_id.clone()],
7892 })
7893 .expect_err("second statement fails");
7894 assert_eq!(failure.code, None);
7895 assert_eq!(
7896 reader.local_revision().expect("revision after rollback"),
7897 revision,
7898 "a rolled-back read consumes no revision"
7899 );
7900 assert!(
7901 !matches!(
7902 reader
7903 .commit_delivery(&commit_id)
7904 .expect("delivery after rollback"),
7905 CommitDelivery::Unknown { .. }
7906 ),
7907 "the same reader serves the next read after releasing the connection"
7908 );
7909
7910 let guard = reader
7912 .snapshot_read(&SnapshotReadRequest {
7913 statements: vec![SnapshotStatement {
7914 sql: "DELETE FROM tasks",
7915 params: &[],
7916 }],
7917 coverage: &[],
7918 subscriptions: Vec::new(),
7919 commit_ids: Vec::new(),
7920 })
7921 .expect_err("write guard");
7922 assert!(guard.message.contains("read-only"), "{}", guard.message);
7923
7924 drop(reader);
7925 drop(client);
7926 std::fs::remove_file(path).expect("remove temp database");
7927 }
7928
7929 #[test]
7930 fn owned_query_failures_classify_order_bound_and_clear() {
7931 let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7932 rusqlite::ffi::Error::new(266),
7933 Some("private SQLite prose".to_owned()),
7934 ));
7935 assert_eq!(io.code, Some("client.storage_io"));
7936 assert_eq!(io.sqlite_code, Some(266));
7937 assert_eq!(io.message, "local SQLite storage I/O failed");
7938 let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7939 rusqlite::ffi::Error::new(26),
7940 None,
7941 ));
7942 assert_eq!(corrupt.code, Some("client.storage_corrupt"));
7943 assert!(!io.retryable());
7944 assert!(!corrupt.retryable());
7945
7946 for (raw, extended) in [(5, 5), (5 | (1 << 8), 261), (6, 6), (6 | (1 << 8), 262)] {
7949 let busy = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7950 rusqlite::ffi::Error::new(raw),
7951 Some("database is locked".to_owned()),
7952 ));
7953 assert_eq!(busy.code, Some("client.storage_busy"));
7954 assert_eq!(busy.sqlite_code, Some(extended));
7955 assert_eq!(busy.message, "local SQLite storage is busy");
7956 assert!(busy.retryable());
7957 }
7958 let unrelated = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
7959 rusqlite::ffi::Error::new(1),
7960 Some("generic SQLite prose".to_owned()),
7961 ));
7962 assert_eq!(unrelated.code, None);
7963 assert!(!unrelated.retryable());
7964
7965 let mut client = client();
7966 let generic = QueryReadFailure::from("private query prose".to_owned());
7967 client.record_query_read(
7968 QueryOwner {
7969 id: "first",
7970 tables: &["tasks", "tasks"],
7971 },
7972 Some(&generic),
7973 );
7974 client.record_query_read(
7975 QueryOwner {
7976 id: "second",
7977 tables: &["tasks"],
7978 },
7979 Some(&generic),
7980 );
7981 client.record_query_read(
7982 QueryOwner {
7983 id: "first",
7984 tables: &["docs"],
7985 },
7986 Some(&io),
7987 );
7988 let failures = &client
7989 .diagnostics_snapshot(&Default::default())
7990 .expect("diagnostics")
7991 .query_failures;
7992 assert_eq!(
7993 failures
7994 .iter()
7995 .map(|entry| entry.id.as_str())
7996 .collect::<Vec<_>>(),
7997 vec!["second", "first"]
7998 );
7999 assert_eq!(failures[1].tables, vec!["docs"]);
8000 assert_eq!(failures[1].code, "client.storage_io");
8001
8002 for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
8003 client.record_query_read(
8004 QueryOwner {
8005 id: &format!("query-{index}"),
8006 tables: &["tasks"],
8007 },
8008 Some(&generic),
8009 );
8010 }
8011 let failures = &client
8012 .diagnostics_snapshot(&Default::default())
8013 .expect("bounded diagnostics")
8014 .query_failures;
8015 assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
8016 assert_eq!(failures.last().expect("last failure").id, "query-256");
8017
8018 client
8019 .query_snapshot(
8020 "SELECT id FROM tasks",
8021 &[],
8022 &[],
8023 Some(QueryOwner {
8024 id: "query-256",
8025 tables: &["tasks"],
8026 }),
8027 )
8028 .expect("successful owned read");
8029 assert!(client
8030 .diagnostics_snapshot(&Default::default())
8031 .expect("cleared diagnostics")
8032 .query_failures
8033 .iter()
8034 .all(|entry| entry.id != "query-256"));
8035 }
8036
8037 #[test]
8038 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
8039 let mut client = client();
8040 let image = Connection::open_in_memory().unwrap();
8041 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
8042 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
8043 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
8044 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8045 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
8046 let table = client.schema.table("tasks").unwrap().clone();
8047 let scopes = vec![("project_id".into(), vec!["p1".into()])];
8048 assert!(client
8049 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
8050 .is_err());
8051 assert!(client.conn.is_autocommit());
8052 assert_eq!(
8053 client
8054 .query("SELECT count(*) AS n FROM tasks", &[])
8055 .unwrap()[0]["n"],
8056 1024
8057 );
8058 assert_eq!(client.drain_change_batches().len(), 1);
8059 image
8060 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
8061 .unwrap();
8062 assert!(matches!(
8063 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
8064 Ok(2050)
8065 ));
8066 assert_eq!(
8067 client
8068 .query("SELECT count(*) AS n FROM tasks", &[])
8069 .unwrap()[0]["n"],
8070 2050
8071 );
8072 assert_eq!(client.drain_change_batches().len(), 3);
8073 }
8074
8075 #[test]
8076 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
8077 let mut client = client();
8078 let base = WindowBase {
8079 table: "tasks".into(),
8080 variable: "project_id".into(),
8081 fixed_scopes: Vec::new(),
8082 params: None,
8083 };
8084 client
8085 .set_window(&base, &["p1".into(), "p2".into()])
8086 .unwrap();
8087 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
8088 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
8089 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
8090 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
8091 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
8092 client.drain_change_batches();
8093 assert!(client.set_window(&base, &["p2".into()]).is_err());
8094 assert!(client.conn.is_autocommit());
8095 assert_eq!(
8096 client
8097 .query("SELECT count(*) AS n FROM tasks", &[])
8098 .unwrap()[0]["n"],
8099 1027
8100 );
8101 assert_eq!(client.window_state(&base).units, vec!["p2"]);
8102 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
8103 assert_eq!(client.drain_change_batches().len(), 1);
8104 client
8105 .conn
8106 .execute_batch("DROP TRIGGER interrupt_evict")
8107 .unwrap();
8108 client.drain_pending_evictions().unwrap();
8109 assert!(client.load_pending_evictions().unwrap().is_empty());
8110 assert_eq!(
8111 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
8112 "held"
8113 );
8114 assert_eq!(client.drain_change_batches().len(), 2);
8115 }
8116
8117 #[test]
8118 fn eviction_marker_failures_roll_back_window_changes() {
8119 let mut client = client();
8120 let base = WindowBase {
8121 table: "tasks".into(),
8122 variable: "project_id".into(),
8123 fixed_scopes: Vec::new(),
8124 params: None,
8125 };
8126 client.set_window(&base, &["p1".into()]).unwrap();
8127 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
8128 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
8129 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
8130 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
8131 let revision = client.local_revision();
8132 assert!(client.set_window(&base, &[]).is_err());
8133 assert!(client.conn.is_autocommit());
8134 assert_eq!(client.local_revision(), revision);
8135 assert_eq!(
8136 client
8137 .query("SELECT count(*) AS n FROM tasks", &[])
8138 .unwrap()[0]["n"],
8139 2050
8140 );
8141 assert_eq!(client.window_state(&base).units, vec!["p1"]);
8142 client.conn.execute_batch("DROP TRIGGER fail_marker;
8143 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
8144 assert!(client.set_window(&base, &[]).is_err());
8145 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
8146 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
8147 assert!(client.drain_pending_evictions().is_err());
8148 assert!(client.conn.is_autocommit());
8149 assert_eq!(
8150 client
8151 .query("SELECT count(*) AS n FROM tasks", &[])
8152 .unwrap()[0]["n"],
8153 2
8154 );
8155 assert!(client.set_window(&base, &["p1".into()]).is_err());
8156 assert!(client.conn.is_autocommit());
8157 assert!(client.window_state(&base).units.is_empty());
8158 assert!(client.subs.is_empty());
8159 assert_eq!(
8160 client
8161 .query(
8162 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
8163 &[]
8164 )
8165 .unwrap()[0]["n"],
8166 1
8167 );
8168 client
8169 .conn
8170 .execute_batch("DROP TRIGGER fail_marker")
8171 .unwrap();
8172 client.set_window(&base, &["p1".into()]).unwrap();
8173 assert_eq!(client.window_state(&base).units, vec!["p1"]);
8174 }
8175
8176 #[test]
8177 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
8178 let mut client = client();
8179 client
8180 .conn
8181 .execute(
8182 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
8183 [],
8184 )
8185 .unwrap();
8186 assert_eq!(
8187 client.drain_pending_evictions().unwrap_err(),
8188 "sync.local_corrupt: invalid pending eviction scopes"
8189 );
8190 assert!(client.conn.is_autocommit());
8191 assert!(client.enqueue_startup_sync_if_needed().is_err());
8192 }
8193
8194 #[test]
8195 fn image_chunks_reject_null_primary_keys() {
8196 let mut client = client();
8197 let image = Connection::open_in_memory().unwrap();
8198 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8199 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
8200 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
8201 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8202 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
8203 let table = client.schema.table("tasks").unwrap().clone();
8204 assert!(client
8205 .apply_sqlite_image(
8206 &image,
8207 &table,
8208 true,
8209 &[("project_id".into(), vec!["p1".into()])],
8210 1025,
8211 7,
8212 "digest"
8213 )
8214 .is_err());
8215 assert!(client.conn.is_autocommit());
8216 assert_eq!(
8217 client
8218 .query("SELECT count(*) AS n FROM tasks", &[])
8219 .unwrap()[0]["n"],
8220 0
8221 );
8222 }
8223
8224 #[test]
8225 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
8226 for images in [false, true] {
8227 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
8228 "name":name,"primaryKey":"id","columns":[
8229 {"name":"id","type":"string","nullable":false},
8230 {"name":"project_id","type":"string","nullable":false},
8231 {"name":"title","type":"string","nullable":false}],
8232 "scopes":[{"pattern":"project:{project_id}"}],
8233 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
8234 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
8235 })))});
8236 let mut actual =
8237 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
8238 let mut reference =
8239 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
8240 let scopes = vec![("project_id".into(), vec!["p1".into()])];
8241 for client in [&mut actual, &mut reference] {
8242 client
8243 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
8244 .unwrap();
8245 client.subs[0].effective = Some(scopes.clone());
8246 for (table, id, title) in [
8247 ("tasks", "a", "original"),
8248 ("tasks", "b", "occupied"),
8249 ("tasks", "c", "deleted"),
8250 ("notes", "n", "untouched"),
8251 ] {
8252 client
8253 .write_base_row(
8254 table,
8255 &vec![
8256 Some(ColumnValue::String(id.into())),
8257 Some(ColumnValue::String("p1".into())),
8258 Some(ColumnValue::String(title.into())),
8259 ],
8260 1,
8261 )
8262 .unwrap();
8263 }
8264 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
8265 client
8266 .mutate(vec![
8267 Mutation::Upsert {
8268 table: "tasks".into(),
8269 values: Map::from_iter([
8270 ("id".into(), Value::from("a")),
8271 ("project_id".into(), Value::from("p1")),
8272 ("title".into(), Value::from("available")),
8273 ]),
8274 base_version: None,
8275 },
8276 Mutation::Delete {
8277 table: "tasks".into(),
8278 row_id: "c".into(),
8279 base_version: None,
8280 },
8281 Mutation::Upsert {
8282 table: "tasks".into(),
8283 values: Map::from_iter([
8284 ("id".into(), Value::from("local")),
8285 ("project_id".into(), Value::from("p1")),
8286 ("title".into(), Value::from("local value")),
8287 ]),
8288 base_version: None,
8289 },
8290 Mutation::Upsert {
8291 table: "notes".into(),
8292 values: Map::from_iter([
8293 ("id".into(), Value::from("n")),
8294 ("project_id".into(), Value::from("p1")),
8295 ("title".into(), Value::from("pending note")),
8296 ]),
8297 base_version: None,
8298 },
8299 ])
8300 .unwrap();
8301 }
8302 let pending = actual.pending_commit_ids();
8303 actual.overlay_rebuild_count.set(0);
8304 for (id, title) in [
8305 ("b", "available"),
8306 ("b", "occupied"),
8307 ("a", "server original"),
8308 ("d", "local value"),
8309 ("d", "released"),
8310 ("c", "redelivered"),
8311 ] {
8312 let table = actual.schema.table("tasks").unwrap().clone();
8313 let values = vec![
8314 Some(ColumnValue::String(id.into())),
8315 Some(ColumnValue::String("p1".into())),
8316 Some(ColumnValue::String(title.into())),
8317 ];
8318 if images {
8319 let image = Connection::open_in_memory().unwrap();
8320 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8321 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8322 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
8323 image
8324 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
8325 .unwrap();
8326 assert!(matches!(
8327 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
8328 Ok(1)
8329 ));
8330 } else {
8331 let segment = RowsSegment {
8332 table: "tasks".into(),
8333 schema_version: 1,
8334 columns: table.wire_columns.clone(),
8335 blocks: vec![vec![ssp2::segment::SegmentRow {
8336 server_version: 2,
8337 values: values.clone(),
8338 }]],
8339 };
8340 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
8341 }
8342 reference.write_base_row("tasks", &values, 2).unwrap();
8343 reference.rebuild_overlay().expect("rebuild overlay");
8344 for table in ["tasks", "notes"] {
8345 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
8346 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
8347 }
8348 }
8349 assert_eq!(actual.pending_commit_ids(), pending);
8350 }
8351 assert_eq!(
8352 actual.overlay_rebuild_count.get(),
8353 0,
8354 "each chunk must leave unrelated tables alone"
8355 );
8356 let before = actual
8357 .query("SELECT * FROM tasks ORDER BY id", &[])
8358 .unwrap();
8359 let fts_before = actual
8360 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
8361 .unwrap();
8362 let revision = actual.local_revision();
8363 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
8364 let table = actual.schema.table("tasks").unwrap().clone();
8365 let segment = RowsSegment {
8366 table: "tasks".into(),
8367 schema_version: 1,
8368 columns: table.wire_columns.clone(),
8369 blocks: vec![vec![ssp2::segment::SegmentRow {
8370 server_version: 3,
8371 values: vec![
8372 Some(ColumnValue::String("b".into())),
8373 Some(ColumnValue::String("p1".into())),
8374 Some(ColumnValue::String("fresh".into())),
8375 ],
8376 }]],
8377 };
8378 assert!(actual.apply_segment(0, &segment, false).is_err());
8379 assert!(actual.conn.is_autocommit());
8380 assert_eq!(
8381 actual
8382 .query("SELECT * FROM tasks ORDER BY id", &[])
8383 .unwrap(),
8384 before
8385 );
8386 assert_eq!(
8387 actual
8388 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
8389 .unwrap(),
8390 fts_before
8391 );
8392 assert_eq!(actual.local_revision(), revision);
8393 assert_eq!(actual.pending_commit_ids(), pending);
8394 }
8395 }
8396
8397 #[test]
8398 fn import_reconciliation_preserves_typed_pending_primary_keys() {
8399 for (kind, a, b) in [
8400 ("string", json!("a"), json!("b")),
8401 ("integer", json!(1), json!(2)),
8402 ("boolean", json!(true), json!(false)),
8403 ("json", json!("1"), json!("2")),
8404 ] {
8405 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
8406 for images in [false, true] {
8407 let mut client =
8408 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
8409 let table = client.schema.table("tasks").unwrap().clone();
8410 client
8411 .subscribe("tasks".into(), "tasks".into(), vec![], None)
8412 .unwrap();
8413 for (id, title) in [(&a, "original"), (&b, "occupied")] {
8414 client
8415 .write_base_row(
8416 "tasks",
8417 &vec![
8418 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
8419 Some(ColumnValue::String(title.into())),
8420 ],
8421 1,
8422 )
8423 .unwrap();
8424 }
8425 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
8426 client
8427 .mutate(vec![Mutation::Upsert {
8428 table: "tasks".into(),
8429 values: Map::from_iter([
8430 ("id".into(), a.clone()),
8431 ("title".into(), json!("available")),
8432 ]),
8433 base_version: None,
8434 }])
8435 .unwrap();
8436 client.overlay_rebuild_count.set(0);
8437 for (title, expected) in [("available", "original"), ("occupied", "available")] {
8438 let values = vec![
8439 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
8440 Some(ColumnValue::String(title.into())),
8441 ];
8442 if images {
8443 let image = Connection::open_in_memory().unwrap();
8444 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8445 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
8446 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
8447 image
8448 .execute(
8449 "INSERT INTO tasks VALUES (?1,?2,2)",
8450 rusqlite::params![RowParam::Cell(&values[0]), title],
8451 )
8452 .unwrap();
8453 assert!(
8454 matches!(
8455 client.apply_sqlite_image(
8456 &image,
8457 &table,
8458 false,
8459 &[],
8460 1,
8461 7,
8462 "digest"
8463 ),
8464 Ok(1)
8465 ),
8466 "{kind}, images={images}"
8467 );
8468 } else {
8469 let segment = RowsSegment {
8470 table: "tasks".into(),
8471 schema_version: 1,
8472 columns: table.wire_columns.clone(),
8473 blocks: vec![vec![ssp2::segment::SegmentRow {
8474 server_version: 2,
8475 values,
8476 }]],
8477 };
8478 assert!(
8479 matches!(client.apply_segment(0, &segment, false), Ok(1)),
8480 "{kind}"
8481 );
8482 }
8483 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
8484 let actual: String = client
8485 .conn
8486 .query_row(
8487 "SELECT title FROM tasks WHERE id=?1",
8488 [RowParam::Cell(&key)],
8489 |row| row.get(0),
8490 )
8491 .unwrap();
8492 assert_eq!(actual, expected, "{kind}, images={images}");
8493 }
8494 assert_eq!(client.overlay_rebuild_count.get(), 0);
8495 }
8496 }
8497 }
8498
8499 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
8500 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
8501 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
8502 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
8503 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
8504 })))});
8505 let mut client =
8506 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
8507 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
8508 client.rebuild_overlay().expect("rebuild overlay");
8509 client
8510 .mutate(vec![Mutation::Upsert {
8511 table: "tasks".into(),
8512 values: Map::from_iter([
8513 ("id".into(), json!("offline")),
8514 ("project_id".into(), json!("p1")),
8515 ("title".into(), json!("pending edit")),
8516 ]),
8517 base_version: None,
8518 }])
8519 .unwrap();
8520 let image = Connection::open_in_memory().unwrap();
8521 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
8522 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
8523 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
8524 image
8525 .execute(
8526 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
8527 [rows],
8528 )
8529 .unwrap();
8530 client.overlay_rebuild_count.set(0);
8531 let table = client.schema.table("tasks").unwrap().clone();
8532 (client, image, table)
8533 }
8534
8535 #[test]
8536 fn sqlite_full_image_import_retains_the_first_code_and_recovers() {
8537 let (mut client, image, table) = unique_import_fixture(1);
8538 image
8539 .execute("UPDATE tasks SET title = ?1", ["full ".repeat(32768)])
8540 .unwrap();
8541 client
8542 .conn
8543 .execute_batch("PRAGMA max_page_count = 1")
8544 .unwrap();
8545 let effective = [("project_id".into(), vec!["p1".into()])];
8546 assert!(client
8547 .apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest")
8548 .is_err());
8549 let failure = client.storage_failure.borrow().clone().unwrap();
8550 assert_eq!(failure.code, Some("client.storage_full"));
8551 assert_eq!(failure.sqlite_code, Some(13));
8552 assert!(!failure.message.contains("rollback"));
8553 assert!(client.conn.is_autocommit());
8554 client
8555 .conn
8556 .execute_batch("PRAGMA max_page_count = 1073741823")
8557 .unwrap();
8558 assert!(matches!(
8559 client.apply_sqlite_image(&image, &table, true, &effective, 1, 7, "digest"),
8560 Ok(1)
8561 ));
8562 assert!(client.conn.is_autocommit());
8563 }
8564
8565 #[test]
8566 fn authoring_storage_failures_are_structured_and_scoped() {
8567 let path = std::env::temp_dir().join(format!(
8568 "syncular-authoring-failure-{}.db",
8569 uuid::Uuid::new_v4()
8570 ));
8571 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8572 {"name":"id","type":"string","nullable":false},
8573 {"name":"project_id","type":"string","nullable":false},
8574 {"name":"title","type":"string","nullable":false}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8575 let mut client = SyncClient::open_path(
8576 "authoring".into(),
8577 &schema,
8578 ClientLimits::default(),
8579 path.to_str().unwrap(),
8580 )
8581 .expect("open client");
8582 client.create_synced_tables().expect("create synced tables");
8583 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
8585
8586 let upsert = |id: &str| {
8587 vec![Mutation::Upsert {
8588 table: "tasks".into(),
8589 values: Map::from_iter([
8590 ("id".into(), json!(id)),
8591 ("project_id".into(), json!("p1")),
8592 ("title".into(), json!("first")),
8593 ]),
8594 base_version: None,
8595 }]
8596 };
8597
8598 let locker = Connection::open(&path).expect("second connection");
8599 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
8600 let busy = client.mutate(upsert("t1")).expect_err("busy write");
8601 assert_eq!(busy.code, "client.storage_busy");
8602 assert!(busy.retryable);
8603 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
8604 assert!(client.pending_commit_ids().is_empty());
8605 assert_eq!(client.local_revision(), 0);
8606
8607 let invalid = client.mutate(Vec::new()).expect_err("empty commit");
8609 assert_eq!(invalid.code, "sync.invalid_request");
8610 assert_eq!(invalid.message, "the authoring request is invalid");
8611 assert!(
8612 invalid.details.as_ref().unwrap()["legacyCause"]
8613 .as_str()
8614 .unwrap()
8615 .contains("at least one operation"),
8616 "{invalid:?}"
8617 );
8618
8619 locker.execute_batch("ROLLBACK").unwrap();
8620 let id = client.mutate(upsert("t1")).expect("retry after busy");
8621 assert_eq!(client.pending_commit_ids(), vec![id]);
8622 assert_eq!(client.local_revision(), 1);
8623
8624 locker.execute_batch("BEGIN IMMEDIATE").unwrap();
8626 let busy_again = client.mutate(upsert("t2")).expect_err("busy write again");
8627 assert_eq!(busy_again.code, "client.storage_busy");
8628 assert!(busy_again.retryable);
8629 locker.execute_batch("ROLLBACK").unwrap();
8630
8631 let full = ClientError::from(QueryReadFailure::from(rusqlite::Error::SqliteFailure(
8633 rusqlite::ffi::Error::new(13),
8634 Some("database or disk is full".to_owned()),
8635 )));
8636 assert_eq!(full.code, "client.storage_full");
8637 assert!(!full.retryable);
8638 assert_eq!(full.details.as_ref().unwrap()["sqliteCode"], 13);
8639
8640 drop(locker);
8641 drop(client);
8642 let _ = std::fs::remove_file(&path);
8643 }
8644
8645 #[test]
8646 fn caller_value_failures_are_invalid_request_and_leave_no_outbox_entry() {
8647 let path = std::env::temp_dir().join(format!(
8648 "syncular-caller-values-{}.db",
8649 uuid::Uuid::new_v4()
8650 ));
8651 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8652 {"name":"id","type":"string","nullable":false},
8653 {"name":"project_id","type":"string","nullable":false},
8654 {"name":"title","type":"string","nullable":false},
8655 {"name":"priority","type":"integer","nullable":false},
8656 {"name":"payload","type":"bytes","nullable":false},
8657 {"name":"extra","type":"bytes","nullable":true}],"scopes":[{"pattern":"project:{project_id}"}]}]});
8658 let mut client = SyncClient::open_path(
8659 "caller-values".into(),
8660 &schema,
8661 ClientLimits::default(),
8662 path.to_str().unwrap(),
8663 )
8664 .expect("open client");
8665 client.create_synced_tables().expect("create synced tables");
8666
8667 let values = |id: &str| {
8668 Map::from_iter([
8669 ("id".to_owned(), json!(id)),
8670 ("project_id".to_owned(), json!("p1")),
8671 ("title".to_owned(), json!("first")),
8672 ("priority".to_owned(), json!(1)),
8673 ("payload".to_owned(), json!({"$bytes": "0a"})),
8674 ])
8675 };
8676 let upsert = |values: Map<String, Value>| {
8677 vec![Mutation::Upsert {
8678 table: "tasks".into(),
8679 values,
8680 base_version: None,
8681 }]
8682 };
8683 let seed = client.mutate(upsert(values("t1"))).expect("seed row");
8684 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
8685
8686 for (label, override_value, cause) in [
8691 ("unknown column", ("nope", json!(1)), "unknown column"),
8692 (
8693 "internal sync column",
8694 ("_sync_version", json!(1)),
8695 "internal sync column",
8696 ),
8697 ("wrong value type", ("title", json!(7)), "expected a string"),
8698 (
8699 "malformed bytes envelope",
8700 ("payload", json!({"$bytes": "zz"})),
8701 "expected exactly",
8702 ),
8703 (
8704 "odd-length bytes envelope",
8705 ("payload", json!({"$bytes": "abc"})),
8706 "expected exactly",
8707 ),
8708 (
8709 "plus-sign bytes envelope",
8710 ("payload", json!({"$bytes": "+a"})),
8711 "expected exactly",
8712 ),
8713 (
8714 "bytes envelope with a second key",
8715 ("payload", json!({"$bytes": "0a", "extra": 1})),
8716 "expected exactly",
8717 ),
8718 (
8719 "no bytes envelope",
8720 ("payload", json!({"nested": true})),
8721 "expected exactly",
8722 ),
8723 ] {
8724 let mut full = values("t2");
8725 full.insert(override_value.0.to_owned(), override_value.1);
8726 let error = client.mutate(upsert(full)).expect_err(label);
8727 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8728 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8729 assert!(!error.retryable, "{label}");
8730 let legacy = error.details.as_ref().unwrap()["legacyCause"]
8731 .as_str()
8732 .unwrap();
8733 assert!(
8734 legacy.starts_with("sync.invalid_request: ") && legacy.contains(cause),
8735 "{label}: {legacy}"
8736 );
8737 assert_eq!(client.pending_commit_ids(), vec![seed.clone()], "{label}");
8738 assert_eq!(client.local_revision(), 1, "{label}");
8739 assert_eq!(
8740 client.query("SELECT * FROM tasks", &[]).unwrap(),
8741 rows,
8742 "{label}"
8743 );
8744 }
8745
8746 let mut missing = values("t3");
8749 missing.remove("payload");
8750 let missing_error = client.mutate(upsert(missing)).expect_err("missing column");
8751 assert_eq!(missing_error.code, "sync.invalid_request");
8752 assert!(missing_error.details.as_ref().unwrap()["legacyCause"]
8753 .as_str()
8754 .unwrap()
8755 .contains("is not nullable"));
8756
8757 let mut bad_row_id = values("t4");
8760 bad_row_id.insert("id".to_owned(), json!({"nested": true}));
8761 let row_id_error = client.mutate(upsert(bad_row_id)).expect_err("bad rowId");
8762 assert_eq!(row_id_error.code, "sync.invalid_request");
8763
8764 for (label, partial, cause) in [
8766 (
8767 "patch wrong value type",
8768 ("priority", json!("high")),
8769 "expected an integer",
8770 ),
8771 ("patch unknown column", ("nope", json!(1)), "unknown column"),
8772 ] {
8773 let error = client
8774 .patch(
8775 "tasks",
8776 "t1",
8777 Map::from_iter([(partial.0.to_owned(), partial.1)]),
8778 None,
8779 )
8780 .expect_err(label);
8781 assert_eq!(error.code, "sync.invalid_request", "{label}: {error:?}");
8782 assert_eq!(error.message, "the authoring request is invalid", "{label}");
8783 assert!(
8784 error.details.as_ref().unwrap()["legacyCause"]
8785 .as_str()
8786 .unwrap()
8787 .contains(cause),
8788 "{label}"
8789 );
8790 }
8791 assert_eq!(client.pending_commit_ids(), vec![seed.clone()]);
8792 assert_eq!(client.local_revision(), 1);
8793 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
8794
8795 for (label, payload) in [
8798 ("explicit null", json!(null)),
8799 ("empty envelope", json!({"$bytes": ""})),
8800 ("uppercase envelope", json!({"$bytes": "0AFF"})),
8801 ] {
8802 let mut accepted = values("t-ok");
8803 accepted.insert("extra".to_owned(), payload);
8804 let encoded = client.mutate(upsert(accepted)).unwrap_or_else(|error| {
8805 panic!("{label} must be accepted: {error:?}");
8806 });
8807 assert!(!encoded.is_empty(), "{label}");
8808 }
8809
8810 let next = client
8812 .mutate(upsert(values("t5")))
8813 .expect("valid after failures");
8814 assert_ne!(next, seed);
8815
8816 drop(client);
8817 let _ = std::fs::remove_file(&path);
8818 }
8819
8820 #[test]
8821 fn a_legacy_outbox_value_the_codec_refuses_is_dropped_instead_of_wedging() {
8822 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8823 {"name":"id","type":"string","nullable":false},
8824 {"name":"project_id","type":"string","nullable":false},
8825 {"name":"payload","type":"bytes","nullable":true}],
8826 "scopes":[{"pattern":"project:{project_id}"}]}]});
8827 let open = |path: &std::path::Path| {
8828 SyncClient::open_path(
8829 "legacy-values".into(),
8830 &schema,
8831 ClientLimits::default(),
8832 path.to_str().unwrap(),
8833 )
8834 .expect("open client")
8835 };
8836 let row = |id: &str| {
8837 Map::from_iter([
8838 ("id".to_owned(), json!(id)),
8839 ("project_id".to_owned(), json!("p1")),
8840 ("payload".to_owned(), json!({"$bytes": "0a"})),
8841 ])
8842 };
8843 let upsert = |values| {
8844 vec![Mutation::Upsert {
8845 table: "tasks".into(),
8846 values,
8847 base_version: None,
8848 }]
8849 };
8850 for (label, stored_cell, representable) in [
8851 ("plus-sign envelope", json!({"$bytes": "+a"}), false),
8852 ("non-hex envelope", json!({"$bytes": "zz"}), false),
8853 ("odd-length envelope", json!({"$bytes": "abc"}), false),
8854 ("no envelope", json!({"nested": true}), false),
8855 ("wrong declared type", json!("high"), true),
8859 ("stored null for a non-nullable column", json!(null), true),
8860 ] {
8861 let path = std::env::temp_dir().join(format!(
8862 "syncular-legacy-values-{}-{}.db",
8863 label.replace(' ', "-"),
8864 uuid::Uuid::new_v4()
8865 ));
8866 let mut client = open(&path);
8867 client.create_synced_tables().expect("create synced tables");
8868 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
8869 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
8870 drop(client);
8871
8872 let connection = Connection::open(&path).expect("raw connection");
8874 let (column, cell) = if stored_cell.is_null() {
8875 ("project_id", stored_cell.clone())
8876 } else {
8877 ("payload", stored_cell.clone())
8878 };
8879 let mut planted_values = json!({
8880 "id": "t1", "project_id": "p1", "payload": {"$bytes": "0a"}
8881 });
8882 planted_values[column] = cell;
8883 let planted = json!([{
8884 "op": "upsert",
8885 "table": "tasks",
8886 "rowId": "t1",
8887 "baseVersion": null,
8888 "values": planted_values
8889 }]);
8890 connection
8891 .execute(
8892 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
8893 rusqlite::params![planted.to_string(), stale],
8894 )
8895 .expect("plant legacy intent");
8896 drop(connection);
8897
8898 let reopened = open(&path);
8901 assert_eq!(
8902 reopened.pending_commit_ids(),
8903 vec![successor.clone()],
8904 "{label}"
8905 );
8906 let rejection = reopened
8907 .rejections()
8908 .iter()
8909 .find(|rejection| rejection.client_commit_id == stale)
8910 .unwrap_or_else(|| panic!("{label}: rejection"));
8911 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
8912 assert_eq!(
8913 rejection.message, "the persisted commit carries values the current codec refuses",
8914 "{label}"
8915 );
8916 assert_eq!(rejection.operation.is_some(), representable, "{label}");
8917 assert_eq!(
8918 rejection
8919 .details
8920 .as_ref()
8921 .and_then(|details| details.reason.as_deref()),
8922 Some(INVALID_STORED_VALUES_REASON),
8923 "{label}"
8924 );
8925 let journal = reopened
8927 .commit_outcome(&stale)
8928 .expect("read outcome")
8929 .expect("journal entry");
8930 assert!(matches!(journal.status, CommitOutcomeStatus::Rejected));
8931 drop(reopened);
8932 let mut reopened = open(&path);
8933 let reread = reopened
8934 .commit_outcome(&stale)
8935 .expect("read outcome")
8936 .expect("journal entry");
8937 assert!(matches!(reread.status, CommitOutcomeStatus::Rejected));
8938
8939 assert_eq!(
8942 reopened
8943 .query("SELECT id FROM tasks ORDER BY id", &[])
8944 .unwrap(),
8945 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
8946 "{label}"
8947 );
8948 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8949 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
8950 assert!(!reopened
8951 .drop_unencodable_outbox(false, None)
8952 .expect("idempotent prepass"));
8953 let mut transport = CountingRealtimeTransport::default();
8954 let outcome = reopened.sync(&mut transport);
8955 assert!(matches!(outcome, SyncOutcome::Failed { .. }), "{outcome:?}");
8956 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
8957 let next = reopened.mutate(upsert(row("t3"))).expect("after the drop");
8958 assert_eq!(reopened.pending_commit_ids(), vec![successor, next]);
8959 assert_eq!(reopened.pending_payloads().len(), 2, "{label}");
8960 drop(reopened);
8961 let _ = std::fs::remove_file(&path);
8962 }
8963 }
8964
8965 #[test]
8966 fn two_legacy_values_on_the_same_row_recover_in_fifo_order() {
8967 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
8968 {"name":"id","type":"string","nullable":false},
8969 {"name":"project_id","type":"string","nullable":false},
8970 {"name":"payload","type":"bytes","nullable":true}],
8971 "scopes":[{"pattern":"project:{project_id}"}]}]});
8972 for (label, column, planted) in [
8973 ("malformed envelope", "payload", json!({"$bytes": "zz"})),
8974 (
8975 "stored null for a non-nullable column",
8976 "project_id",
8977 json!(null),
8978 ),
8979 ("wrong declared type", "payload", json!(7)),
8980 ] {
8981 let path = std::env::temp_dir().join(format!(
8982 "syncular-shared-row-{}-{}.db",
8983 label.replace(' ', "-"),
8984 uuid::Uuid::new_v4()
8985 ));
8986 let open = |path: &std::path::Path| {
8987 SyncClient::open_path(
8988 "shared-row".into(),
8989 &schema,
8990 ClientLimits::default(),
8991 path.to_str().unwrap(),
8992 )
8993 .expect("open client")
8994 };
8995 let row = |id: &str| {
8996 Map::from_iter([
8997 ("id".to_owned(), json!(id)),
8998 ("project_id".to_owned(), json!("p1")),
8999 ("payload".to_owned(), json!({"$bytes": "0a"})),
9000 ])
9001 };
9002 let upsert = |values| {
9003 vec![Mutation::Upsert {
9004 table: "tasks".into(),
9005 values,
9006 base_version: None,
9007 }]
9008 };
9009 let mut client = open(&path);
9010 client.create_synced_tables().expect("create synced tables");
9011 let older = client.mutate(upsert(row("t1"))).expect("older commit");
9012 let newer = client.mutate(upsert(row("t1"))).expect("newer commit");
9013 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
9014 drop(client);
9015
9016 let connection = Connection::open(&path).expect("raw connection");
9017 for commit_id in [&older, &newer] {
9018 let mut values = row("t1");
9019 values.insert(column.to_owned(), planted.clone());
9020 let planted_ops = json!([{
9021 "op": "upsert",
9022 "table": "tasks",
9023 "rowId": "t1",
9024 "baseVersion": null,
9025 "values": values
9026 }]);
9027 connection
9028 .execute(
9029 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9030 rusqlite::params![planted_ops.to_string(), commit_id],
9031 )
9032 .expect("plant legacy intent");
9033 }
9034 drop(connection);
9035
9036 let mut reopened = open(&path);
9038 assert_eq!(
9039 reopened.pending_commit_ids(),
9040 vec![successor.clone()],
9041 "{label}"
9042 );
9043 assert_eq!(
9044 reopened
9045 .rejections()
9046 .iter()
9047 .map(|rejection| rejection.client_commit_id.clone())
9048 .collect::<Vec<_>>(),
9049 vec![older.clone(), newer.clone()],
9050 "{label}"
9051 );
9052 assert!(
9053 reopened
9054 .rejections()
9055 .iter()
9056 .all(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE
9057 && rejection.details.as_ref().and_then(|d| d.reason.as_deref())
9058 == Some(INVALID_STORED_VALUES_REASON)),
9059 "{label}"
9060 );
9061 for commit_id in [&older, &newer] {
9062 assert!(
9063 matches!(
9064 reopened
9065 .commit_outcome(commit_id)
9066 .expect("outcome")
9067 .map(|o| o.status),
9068 Some(CommitOutcomeStatus::Rejected)
9069 ),
9070 "{label}"
9071 );
9072 }
9073 assert_eq!(
9074 reopened
9075 .query("SELECT id FROM tasks ORDER BY id", &[])
9076 .unwrap(),
9077 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9078 "{label}"
9079 );
9080 let revision = reopened.local_revision();
9081 let batches = reopened.drain_change_batches();
9082 assert!(
9083 batches.iter().any(|batch| batch.rejections_changed
9084 && batch.outcomes_changed
9085 && batch.tables.iter().any(|change| change.table == "tasks")
9086 && batch.revision == revision.to_string()),
9087 "{label}: {batches:?}"
9088 );
9089 drop(reopened);
9090
9091 let mut stable = open(&path);
9094 assert_eq!(stable.local_revision(), revision, "{label}");
9095 assert!(stable.drain_change_batches().is_empty(), "{label}");
9096 assert_eq!(stable.pending_commit_ids(), vec![successor], "{label}");
9097 drop(stable);
9098 let _ = std::fs::remove_file(&path);
9099 }
9100 }
9101
9102 #[test]
9103 fn recovery_omits_a_legacy_operation_the_journal_reader_rejects() {
9104 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9105 {"name":"id","type":"string","nullable":false},
9106 {"name":"project_id","type":"string","nullable":false},
9107 {"name":"payload","type":"bytes","nullable":true}],
9108 "scopes":[{"pattern":"project:{project_id}"}]}]});
9109 for (label, second_op) in [
9110 (
9111 "delete carrying values",
9112 json!({"op": "delete", "table": "tasks", "rowId": "t9", "values": {}}),
9113 ),
9114 (
9115 "baseVersion beyond the safe bound",
9116 json!({"op": "upsert", "table": "tasks", "rowId": "t9",
9117 "baseVersion": MAX_JS_SAFE_INTEGER + 1,
9118 "values": {"id": "t9", "project_id": "p1", "payload": {"$bytes": "0a"}}}),
9119 ),
9120 ] {
9121 let path = std::env::temp_dir().join(format!(
9122 "syncular-journal-shape-{}-{}.db",
9123 label.replace(' ', "-"),
9124 uuid::Uuid::new_v4()
9125 ));
9126 let open = |path: &std::path::Path| {
9127 SyncClient::open_path(
9128 "journal-shape".into(),
9129 &schema,
9130 ClientLimits::default(),
9131 path.to_str().unwrap(),
9132 )
9133 .expect("open client")
9134 };
9135 let values = |id: &str| {
9136 Map::from_iter([
9137 ("id".to_owned(), json!(id)),
9138 ("project_id".to_owned(), json!("p1")),
9139 ("payload".to_owned(), json!({"$bytes": "0a"})),
9140 ])
9141 };
9142 let upsert = |values| {
9143 vec![Mutation::Upsert {
9144 table: "tasks".into(),
9145 values,
9146 base_version: None,
9147 }]
9148 };
9149 let mut client = open(&path);
9150 client.create_synced_tables().expect("create synced tables");
9151 let stale = client.mutate(upsert(values("t1"))).expect("stale commit");
9152 drop(client);
9153
9154 let connection = Connection::open(&path).expect("raw connection");
9155 let planted = json!([
9159 {
9160 "op": "upsert",
9161 "table": "tasks",
9162 "rowId": "t1",
9163 "baseVersion": null,
9164 "values": {"id": "t1", "project_id": null,
9165 "payload": {"$bytes": "0a"}}
9166 },
9167 second_op
9168 ]);
9169 connection
9170 .execute(
9171 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9172 rusqlite::params![planted.to_string(), stale],
9173 )
9174 .expect("plant legacy intent");
9175 drop(connection);
9176
9177 let reopened = open(&path);
9178 let journal = reopened
9179 .commit_outcome(&stale)
9180 .unwrap_or_else(|error| panic!("{label}: journal read: {error}"))
9181 .unwrap_or_else(|| panic!("{label}: journal entry"));
9182 assert!(
9183 matches!(journal.status, CommitOutcomeStatus::Rejected),
9184 "{label}"
9185 );
9186 assert!(journal.operations.is_none(), "{label}");
9187 assert!(
9188 reopened
9189 .commit_outcomes(CommitOutcomeQuery::default())
9190 .is_ok(),
9191 "{label}: outcome list query"
9192 );
9193 assert!(reopened.pending_commit_ids().is_empty(), "{label}");
9194 drop(reopened);
9195 let _ = std::fs::remove_file(&path);
9196 }
9197 }
9198
9199 #[test]
9200 fn legacy_values_recover_at_the_recreate_and_bump_reset_boundaries() {
9201 let schema = |version: i32, extra: bool| {
9202 let mut columns = vec![
9203 json!({"name":"id","type":"string","nullable":false}),
9204 json!({"name":"project_id","type":"string","nullable":false}),
9205 json!({"name":"payload","type":"bytes","nullable":true}),
9206 ];
9207 if extra {
9208 columns.push(json!({"name":"later","type":"string","nullable":true}));
9209 }
9210 json!({"version":version,"tables":[{"name":"tasks","primaryKey":"id","columns":columns,
9211 "scopes":[{"pattern":"project:{project_id}"}]}]})
9212 };
9213 for (label, bump) in [
9214 ("same-version recreation", false),
9215 ("version-bump reset", true),
9216 ("version-only bump reset", true),
9217 ] {
9218 let path = std::env::temp_dir().join(format!(
9219 "syncular-legacy-reset-{}-{}.db",
9220 label.replace(' ', "-"),
9221 uuid::Uuid::new_v4()
9222 ));
9223 let v1 = schema(1, false);
9224 let open = |schema: &Value, path: &std::path::Path| {
9225 SyncClient::open_path(
9226 "legacy-reset".into(),
9227 schema,
9228 ClientLimits::default(),
9229 path.to_str().unwrap(),
9230 )
9231 .expect("open client")
9232 };
9233 let row = |id: &str| {
9234 Map::from_iter([
9235 ("id".to_owned(), json!(id)),
9236 ("project_id".to_owned(), json!("p1")),
9237 ("payload".to_owned(), json!({"$bytes": "0a"})),
9238 ])
9239 };
9240 let upsert = |values| {
9241 vec![Mutation::Upsert {
9242 table: "tasks".into(),
9243 values,
9244 base_version: None,
9245 }]
9246 };
9247 let mut client = open(&v1, &path);
9248 client.create_synced_tables().expect("create synced tables");
9249 let stale = client.mutate(upsert(row("t1"))).expect("stale commit");
9250 let successor = client.mutate(upsert(row("t2"))).expect("successor commit");
9251 drop(client);
9252
9253 let connection = Connection::open(&path).expect("raw connection");
9254 let planted = json!([{
9255 "op": "upsert",
9256 "table": "tasks",
9257 "rowId": "t1",
9258 "baseVersion": null,
9259 "values": {"id": "t1", "project_id": 7, "payload": {"$bytes": "0a"}}
9260 }]);
9261 connection
9262 .execute(
9263 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9264 rusqlite::params![planted.to_string(), stale],
9265 )
9266 .expect("plant legacy intent");
9267 drop(connection);
9268
9269 let target = if bump {
9270 schema(2, label == "version-bump reset")
9271 } else {
9272 v1.clone()
9273 };
9274 let reopened = open(&target, &path);
9275 let rejection = reopened
9276 .rejections()
9277 .iter()
9278 .find(|rejection| rejection.client_commit_id == stale)
9279 .unwrap_or_else(|| panic!("{label}: rejection"));
9280 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
9281 assert_eq!(
9282 rejection
9283 .details
9284 .as_ref()
9285 .and_then(|details| details.reason.as_deref()),
9286 Some(INVALID_STORED_VALUES_REASON),
9287 "{label}"
9288 );
9289 assert!(rejection.operation.is_some(), "{label}");
9290 assert_eq!(
9291 reopened.pending_commit_ids(),
9292 vec![successor.clone()],
9293 "{label}"
9294 );
9295 assert_eq!(reopened.pending_payloads().len(), 1, "{label}");
9299 if label == "version-bump reset" {
9300 assert_eq!(
9301 reopened
9302 .query("SELECT id, later FROM tasks ORDER BY id", &[])
9303 .unwrap(),
9304 vec![Map::from_iter([
9305 ("id".to_owned(), json!("t2")),
9306 ("later".to_owned(), Value::Null),
9307 ])],
9308 "{label}"
9309 );
9310 let mut next = reopened;
9314 let revision = next.local_revision();
9315 let batches = next.drain_change_batches();
9316 assert!(
9317 batches.iter().any(|batch| batch.rejections_changed
9318 && batch.outcomes_changed
9319 && batch.tables.iter().any(|change| change.table == "tasks")
9320 && batch.revision == revision.to_string()),
9321 "{label}: {batches:?}"
9322 );
9323 next.mutate(upsert(Map::from_iter([
9324 ("id".to_owned(), json!("t3")),
9325 ("project_id".to_owned(), json!("p1")),
9326 ("payload".to_owned(), json!({"$bytes": "0b"})),
9327 ("later".to_owned(), json!("v2")),
9328 ])))
9329 .expect("v2 authoring after the reset");
9330 assert_eq!(next.pending_commit_ids().len(), 2, "{label}");
9331 assert_eq!(next.pending_payloads().len(), 2, "{label}");
9332 drop(next);
9333 } else {
9334 assert_eq!(
9335 reopened
9336 .query("SELECT id FROM tasks ORDER BY id", &[])
9337 .unwrap(),
9338 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9339 "{label}"
9340 );
9341 drop(reopened);
9342 }
9343
9344 if !bump {
9347 let mut recreated = open(&v1, &path);
9348 let fresh = recreated
9349 .mutate(upsert(row("t3")))
9350 .expect("fresh commit while open");
9351 recreated.drain_change_batches();
9352 let planted_fresh = json!([{
9353 "op": "upsert",
9354 "table": "tasks",
9355 "rowId": "t3",
9356 "baseVersion": null,
9357 "values": {"id": "t3", "project_id": "p1",
9358 "payload": {"$bytes": "zz"}}
9359 }]);
9360 let connection = Connection::open(&path).expect("raw connection");
9361 let affected = connection
9362 .execute(
9363 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9364 rusqlite::params![planted_fresh.to_string(), fresh],
9365 )
9366 .expect("plant fresh legacy intent");
9367 assert_eq!(
9368 affected, 1,
9369 "{label}: the in-place plant must hit the queued commit"
9370 );
9371 drop(connection);
9372 let revision_before = recreated.local_revision();
9373 recreated
9374 .recreate_with_schema(&v1)
9375 .unwrap_or_else(|error| panic!("{label}: recreation failed: {error}"));
9376 assert_eq!(recreated.pending_commit_ids(), vec![successor], "{label}");
9377 let fresh_rejection = recreated
9378 .rejections()
9379 .iter()
9380 .find(|rejection| rejection.client_commit_id == fresh)
9381 .expect("the in-place recovery rejection");
9382 assert_eq!(fresh_rejection.code, OUTBOX_INCOMPATIBLE_CODE, "{label}");
9383 assert_eq!(
9384 fresh_rejection
9385 .details
9386 .as_ref()
9387 .and_then(|details| details.reason.as_deref()),
9388 Some(INVALID_STORED_VALUES_REASON),
9389 "{label}"
9390 );
9391 assert!(
9392 matches!(
9393 recreated
9394 .commit_outcome(&fresh)
9395 .expect("outcome read")
9396 .map(|outcome| outcome.status),
9397 Some(CommitOutcomeStatus::Rejected)
9398 ),
9399 "{label}"
9400 );
9401 assert!(recreated.local_revision() > revision_before, "{label}");
9402 let revision = recreated.local_revision();
9403 let batches = recreated.drain_change_batches();
9404 assert_eq!(batches.len(), 1, "{label}: one recovery batch");
9405 assert!(
9406 batches.iter().any(|batch| batch.rejections_changed
9407 && batch.outcomes_changed
9408 && batch.tables.iter().any(|change| change.table == "tasks")
9409 && batch.revision == revision.to_string()),
9410 "{label}: {batches:?}"
9411 );
9412 assert_eq!(recreated.pending_payloads().len(), 1, "{label}");
9413 assert_eq!(
9414 recreated
9415 .query("SELECT id FROM tasks ORDER BY id", &[])
9416 .unwrap(),
9417 vec![Map::from_iter([("id".to_owned(), json!("t2"))])],
9418 "{label}: the recovered commit leaves no projection"
9419 );
9420 drop(recreated);
9421 }
9422 let _ = std::fs::remove_file(&path);
9423 }
9424 }
9425
9426 #[test]
9427 fn legacy_values_recover_over_a_synced_authoritative_row() {
9428 let path = std::env::temp_dir().join(format!(
9429 "syncular-authoritative-row-{}.db",
9430 uuid::Uuid::new_v4()
9431 ));
9432 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9433 {"name":"id","type":"string","nullable":false},
9434 {"name":"project_id","type":"string","nullable":false},
9435 {"name":"title","type":"string","nullable":false},
9436 {"name":"payload","type":"bytes","nullable":true}],
9437 "scopes":[{"pattern":"project:{project_id}"}]}]});
9438 let open = |path: &std::path::Path| {
9439 SyncClient::open_path(
9440 "authoritative-row".into(),
9441 &schema,
9442 ClientLimits::default(),
9443 path.to_str().unwrap(),
9444 )
9445 .expect("open client")
9446 };
9447 let row = |title: &str| {
9448 Map::from_iter([
9449 ("id".to_owned(), json!("t1")),
9450 ("project_id".to_owned(), json!("p1")),
9451 ("title".to_owned(), json!(title)),
9452 ("payload".to_owned(), json!({"$bytes": "0a"})),
9453 ])
9454 };
9455 let upsert = |values| {
9456 vec![Mutation::Upsert {
9457 table: "tasks".into(),
9458 values,
9459 base_version: None,
9460 }]
9461 };
9462 let mut client = open(&path);
9463 client.create_synced_tables().expect("create synced tables");
9464 client
9467 .conn
9468 .execute_batch(
9469 "INSERT INTO _syncular_base_tasks VALUES ('t1','p1','authoritative',x'0a',1);
9470 INSERT INTO tasks VALUES ('t1','p1','authoritative',x'0a',1);",
9471 )
9472 .expect("seed authoritative row");
9473 let older = client.mutate(upsert(row("older"))).expect("older");
9474 let newer = client.mutate(upsert(row("newer"))).expect("newer");
9475 let successor = client.mutate(upsert(row("successor"))).expect("successor");
9476 drop(client);
9477
9478 let connection = Connection::open(&path).expect("raw connection");
9479 for commit_id in [&older, &newer] {
9480 let planted = json!([{
9481 "op": "upsert",
9482 "table": "tasks",
9483 "rowId": "t1",
9484 "baseVersion": null,
9485 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
9486 "payload": {"$bytes": "zz"}}
9487 }]);
9488 connection
9489 .execute(
9490 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9491 rusqlite::params![planted.to_string(), commit_id],
9492 )
9493 .expect("plant legacy intent");
9494 }
9495 drop(connection);
9496
9497 let reopened = open(&path);
9498 assert_eq!(reopened.pending_commit_ids(), vec![successor.clone()]);
9499 assert_eq!(
9500 reopened
9501 .rejections()
9502 .iter()
9503 .map(|rejection| rejection.client_commit_id.clone())
9504 .collect::<Vec<_>>(),
9505 vec![older.clone(), newer.clone()]
9506 );
9507 assert_eq!(
9510 reopened.query("SELECT id,title,_syncular_version FROM tasks", &[]),
9511 Ok(vec![Map::from_iter([
9512 ("id".to_owned(), json!("t1")),
9513 ("title".to_owned(), json!("successor")),
9514 ("_syncular_version".to_owned(), json!(1)),
9515 ])])
9516 );
9517
9518 drop(reopened);
9521 let connection = Connection::open(&path).expect("raw connection");
9522 connection
9523 .execute(
9524 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9525 rusqlite::params![
9526 json!([{
9527 "op": "upsert",
9528 "table": "tasks",
9529 "rowId": "t1",
9530 "baseVersion": null,
9531 "values": {"id": "t1", "project_id": "p1", "title": "legacy",
9532 "payload": {"$bytes": "zz"}}
9533 }])
9534 .to_string(),
9535 successor
9536 ],
9537 )
9538 .expect("plant successor intent");
9539 drop(connection);
9540 let late = open(&path);
9541 assert!(late.pending_commit_ids().is_empty());
9542 assert_eq!(
9543 late.query("SELECT id,title,_syncular_version FROM tasks", &[]),
9544 Ok(vec![Map::from_iter([
9545 ("id".to_owned(), json!("t1")),
9546 ("title".to_owned(), json!("authoritative")),
9547 ("_syncular_version".to_owned(), json!(1)),
9548 ])])
9549 );
9550 drop(late);
9551 let _ = std::fs::remove_file(&path);
9552 }
9553
9554 #[test]
9555 fn a_failed_value_recovery_commit_restores_every_side_effect() {
9556 let path = std::env::temp_dir().join(format!(
9557 "syncular-value-recovery-commit-{}.db",
9558 uuid::Uuid::new_v4()
9559 ));
9560 let path_str = path.to_str().expect("path");
9561 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
9562 {"name":"id","type":"string","nullable":false},
9563 {"name":"project_id","type":"string","nullable":false},
9564 {"name":"payload","type":"bytes","nullable":true}],
9565 "scopes":[{"pattern":"project:{project_id}"}]}]});
9566 let conn = Connection::open(path_str).expect("replica connection");
9569 let mut client = SyncClient::with_connection(
9570 "value-recovery-commit".into(),
9571 &schema,
9572 ClientLimits::default(),
9573 conn,
9574 )
9575 .expect("replica");
9576 assert_eq!(
9577 client
9578 .conn
9579 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
9580 .expect("journal mode"),
9581 "delete"
9582 );
9583 client.create_synced_tables().expect("create synced tables");
9584 let stale = client
9585 .mutate(vec![Mutation::Upsert {
9586 table: "tasks".into(),
9587 values: Map::from_iter([
9588 ("id".to_owned(), json!("t1")),
9589 ("project_id".to_owned(), json!("p1")),
9590 ("payload".to_owned(), json!({"$bytes": "0a"})),
9591 ]),
9592 base_version: None,
9593 }])
9594 .expect("queued write");
9595 client.drain_change_batches();
9596 let affected = client
9597 .conn
9598 .execute(
9599 "UPDATE _syncular_outbox SET ops_json=? WHERE commit_id=?",
9600 rusqlite::params![
9601 json!([{
9602 "op": "upsert",
9603 "table": "tasks",
9604 "rowId": "t1",
9605 "baseVersion": null,
9606 "values": {"id": "t1", "project_id": "p1",
9607 "payload": {"$bytes": "zz"}}
9608 }])
9609 .to_string(),
9610 stale
9611 ],
9612 )
9613 .expect("plant legacy intent");
9614 assert_eq!(affected, 1, "the planted row must exist");
9615 let revision = client.local_revision();
9616 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
9617 let prior_outbox = client.pending_commit_ids();
9618 let prior_rejections = client.rejections().len();
9619 let prior_changes = client.change_queue.len();
9620 let prior_intents = client.sync_intent_queue.len();
9621 let prior_failed = client.failed_commits.len();
9622 let reader = Connection::open(path_str).expect("reader");
9625 reader
9626 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9627 .expect("reader snapshot");
9628 client
9629 .conn
9630 .busy_timeout(std::time::Duration::ZERO)
9631 .expect("busy timeout");
9632 let error = client
9633 .recreate_with_schema(&schema)
9634 .expect_err("a blocked COMMIT must refuse the recovery");
9635 assert!(error.starts_with("client.storage_busy:"), "{error}");
9636 assert!(client.conn.is_autocommit(), "the transaction is released");
9637 assert_eq!(client.local_revision(), revision);
9638 assert_eq!(client.pending_commit_ids(), prior_outbox);
9639 assert_eq!(client.rejections().len(), prior_rejections);
9640 assert_eq!(client.change_queue.len(), prior_changes);
9641 assert_eq!(client.sync_intent_queue.len(), prior_intents);
9642 assert_eq!(client.failed_commits.len(), prior_failed);
9643 assert_eq!(
9644 client.query("SELECT * FROM tasks", &[]).expect("rows"),
9645 rows
9646 );
9647 assert!(
9648 client
9649 .commit_outcome(&stale)
9650 .expect("outcome read")
9651 .is_none(),
9652 "the rolled-back recovery kept no outcome"
9653 );
9654 reader.execute_batch("ROLLBACK").expect("release reader");
9656 client
9657 .recreate_with_schema(&schema)
9658 .expect("retry after the blocked commit");
9659 assert!(client.pending_commit_ids().is_empty());
9660 assert_eq!(client.rejections().len(), prior_rejections + 1);
9661 assert!(client.local_revision() > revision);
9662 let batches = client.drain_change_batches();
9663 assert!(
9664 batches.iter().any(|batch| batch.rejections_changed
9665 && batch.outcomes_changed
9666 && batch.tables.iter().any(|change| change.table == "tasks")),
9667 "{batches:?}"
9668 );
9669 drop(client);
9670 let _ = std::fs::remove_file(&path);
9671 }
9672
9673 #[cfg(feature = "e2ee")]
9674 #[test]
9675 fn a_failed_recovery_restores_retained_failed_commits() {
9676 let path = std::env::temp_dir().join(format!(
9677 "syncular-value-recovery-retention-{}.db",
9678 uuid::Uuid::new_v4()
9679 ));
9680 let path_str = path.to_str().expect("path");
9681 let schema = json!({"version":1,"tables":[{"name":"secrets","primaryKey":"id","columns":[
9682 {"name":"id","type":"string","nullable":false},
9683 {"name":"project_id","type":"string","nullable":false},
9684 {"name":"note","type":"bytes","nullable":true,"encrypted":true,"declaredType":"string"}],
9685 "scopes":[{"pattern":"project:{project_id}"}]}]});
9686 let conn = Connection::open(path_str).expect("replica connection");
9687 let mut client = SyncClient::with_connection(
9688 "value-recovery-retention".into(),
9689 &schema,
9690 ClientLimits::default(),
9691 conn,
9692 )
9693 .expect("replica");
9694 client.create_synced_tables().expect("create synced tables");
9695 client.set_retain_failed_commits(true).expect("retention");
9696 client.set_encryption(crate::values::EncryptionConfig::default());
9698 client
9699 .mutate(vec![Mutation::Upsert {
9700 table: "secrets".into(),
9701 values: Map::from_iter([
9702 ("id".to_owned(), json!("s1")),
9703 ("project_id".to_owned(), json!("p1")),
9704 ("note".to_owned(), json!("secret")),
9705 ]),
9706 base_version: None,
9707 }])
9708 .expect("queued write");
9709 client.drain_change_batches();
9710 let revision = client.local_revision();
9711 let prior_outbox = client.pending_commit_ids();
9712 let prior_rejections = client.rejections().len();
9713 let prior_failed = client.failed_commits.len();
9714 let reader = Connection::open(path_str).expect("reader");
9715 reader
9716 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
9717 .expect("reader snapshot");
9718 client
9719 .conn
9720 .busy_timeout(std::time::Duration::ZERO)
9721 .expect("busy timeout");
9722 let error = client
9723 .drop_unencodable_outbox(true, None)
9724 .expect_err("a blocked COMMIT must refuse the recovery");
9725 assert!(error.starts_with("client.storage_busy:"), "{error}");
9726 assert_eq!(client.failed_commits.len(), prior_failed);
9727 assert_eq!(client.pending_commit_ids(), prior_outbox);
9728 assert_eq!(client.rejections().len(), prior_rejections);
9729 assert_eq!(client.local_revision(), revision);
9730 reader.execute_batch("ROLLBACK").expect("release reader");
9732 client
9733 .drop_unencodable_outbox(true, None)
9734 .expect("retry after the blocked commit");
9735 assert_eq!(client.failed_commits.len(), prior_failed + 1);
9736 assert!(client.pending_commit_ids().is_empty());
9737 assert_eq!(client.rejections().len(), prior_rejections + 1);
9738 drop(client);
9739 let _ = std::fs::remove_file(&path);
9740 }
9741
9742 #[test]
9743 fn client_error_maps_legacy_codes_and_static_messages() {
9744 let code_only = ClientError::from("sync.unknown_table".to_owned());
9745 assert_eq!(code_only.code, "sync.unknown_table");
9746 assert_eq!(code_only.message, "the commit targets an unknown table");
9747 assert!(code_only.details.is_none());
9748
9749 let dynamic = ClientError::from(
9750 "sync.invalid_request: table \"tasks\": patch cannot write scope column".to_owned(),
9751 );
9752 assert_eq!(dynamic.code, "sync.invalid_request");
9753 assert_eq!(dynamic.message, "the authoring request is invalid");
9754 assert!(dynamic.details.as_ref().unwrap()["legacyCause"]
9755 .as_str()
9756 .unwrap()
9757 .contains("patch cannot write scope column"));
9758
9759 let unclassified =
9760 ClientError::from("a commit must contain at least one operation".to_owned());
9761 assert_eq!(unclassified.code, "client.failed");
9762 assert_eq!(unclassified.message, "the local authoring operation failed");
9763 assert!(unclassified.details.as_ref().unwrap()["legacyCause"]
9764 .as_str()
9765 .unwrap()
9766 .contains("at least one operation"));
9767
9768 let metadata = ClientError::from(QueryReadFailure {
9770 code: None,
9771 sqlite_code: Some(1),
9772 sqlite_message: Some("generic SQLite prose".to_owned()),
9773 rollback_failure: Some(Box::new(QueryReadFailure::from(
9774 rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(1), None),
9775 ))),
9776 message: "unclassified read".to_owned(),
9777 });
9778 assert_eq!(metadata.code, "client.failed");
9779 assert_eq!(metadata.message, "the local authoring operation failed");
9780 let details = metadata.details.as_ref().unwrap();
9781 assert_eq!(details["sqliteCode"], 1);
9782 assert_eq!(details["sqliteMessage"], "generic SQLite prose");
9783 assert_eq!(details["rollbackFailure"]["sqliteCode"], 1);
9784 assert_eq!(details["legacyCause"], "unclassified read");
9785 }
9786
9787 #[test]
9788 fn schema_marker_read_keeps_storage_classification() {
9789 let busy = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9790 rusqlite::ffi::Error::new(5),
9791 Some("database is locked".to_owned()),
9792 ));
9793 assert!(busy.starts_with("client.storage_busy:"), "{busy}");
9794 let invalid = schema_marker_read_failure(rusqlite::Error::SqliteFailure(
9795 rusqlite::ffi::Error::new(1),
9796 Some("unreadable".to_owned()),
9797 ));
9798 assert_eq!(
9799 invalid,
9800 "sync.local_corrupt: persisted local schema marker is unreadable or invalid"
9801 );
9802
9803 let path =
9806 std::env::temp_dir().join(format!("syncular-marker-busy-{}.db", uuid::Uuid::new_v4()));
9807 let setup = Connection::open(&path).expect("setup connection");
9808 setup
9809 .execute_batch(
9810 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value TEXT);\n INSERT INTO _syncular_meta VALUES ('localSchemaVersion','1');",
9811 )
9812 .unwrap();
9813 let locker = Connection::open(&path).expect("locker connection");
9814 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9815 let reader = Connection::open(&path).expect("reader connection");
9816 reader.busy_timeout(std::time::Duration::ZERO).unwrap();
9817 let locked = read_local_schema_version(&reader, 1).expect_err("locked marker read");
9818 assert!(locked.starts_with("client.storage_busy:"), "{locked}");
9819 locker.execute_batch("ROLLBACK").unwrap();
9820 assert_eq!(read_local_schema_version(&reader, 1).unwrap(), Some(1));
9821 let rows = setup
9822 .query_row("SELECT count(*) FROM _syncular_meta", [], |row| {
9823 row.get::<_, i64>(0)
9824 })
9825 .unwrap();
9826 assert_eq!(rows, 1, "the guard performed no writes");
9827 drop(reader);
9828 drop(locker);
9829 drop(setup);
9830 let _ = std::fs::remove_file(&path);
9831 }
9832
9833 #[cfg(feature = "crdt-yjs")]
9834 #[test]
9835 fn crdt_authoring_pre_read_preserves_storage_classification() {
9836 let path =
9837 std::env::temp_dir().join(format!("syncular-crdt-busy-{}.db", uuid::Uuid::new_v4()));
9838 let schema = json!({"version":1,"tables":[{"name":"notes","primaryKey":"id","columns":[
9839 {"name":"id","type":"string","nullable":false},
9840 {"name":"project_id","type":"string","nullable":false},
9841 {"name":"doc","type":"crdt","nullable":true,"crdtType":"yjs-doc"}],
9842 "scopes":[{"pattern":"project:{project_id}"}]}]});
9843 let mut client = SyncClient::open_path(
9844 "crdt-busy".into(),
9845 &schema,
9846 ClientLimits::default(),
9847 path.to_str().unwrap(),
9848 )
9849 .expect("open client");
9850 client.create_synced_tables().expect("create synced tables");
9851 client.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
9852
9853 let locker = Connection::open(&path).expect("locker connection");
9854 locker.execute_batch("BEGIN EXCLUSIVE").unwrap();
9855 let busy = client
9856 .crdt_apply_update("notes", "n1", "doc", &[])
9857 .expect_err("busy crdt pre-read");
9858 assert_eq!(busy.code, "client.storage_busy");
9859 assert!(busy.retryable);
9860 assert_eq!(busy.details.as_ref().unwrap()["sqliteCode"], 5);
9861 assert!(client.pending_commit_ids().is_empty());
9862 assert_eq!(client.local_revision(), 0);
9863
9864 locker.execute_batch("ROLLBACK").unwrap();
9865 drop(locker);
9866 drop(client);
9867 let _ = std::fs::remove_file(&path);
9868 }
9869
9870 #[test]
9871 fn large_unique_import_copies_no_unrelated_tables() {
9872 let (mut client, image, table) = unique_import_fixture(10240);
9873 let pending = client.pending_commit_ids();
9874 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
9875 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
9876 assert!(matches!(
9877 client.apply_sqlite_image(
9878 &image,
9879 &table,
9880 true,
9881 &[("project_id".into(), vec!["p1".into()])],
9882 10240,
9883 7,
9884 "digest"
9885 ),
9886 Ok(10240)
9887 ));
9888 assert_eq!(client.overlay_rebuild_count.get(), 0);
9889 assert_eq!(client.pending_commit_ids(), pending);
9890 assert_eq!(
9891 client
9892 .query("SELECT count(*) AS n FROM tasks", &[])
9893 .unwrap()[0]["n"],
9894 10241
9895 );
9896 assert_eq!(
9897 client
9898 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9899 .unwrap()[0]["n"],
9900 10241
9901 );
9902 assert_eq!(
9903 client
9904 .query("SELECT count(*) AS n FROM notes_fts", &[])
9905 .unwrap()[0]["n"],
9906 8192
9907 );
9908 }
9909
9910 #[test]
9911 #[ignore = "manual scaling benchmark; see bench/README.md"]
9912 fn benchmark_unique_image_import() {
9913 let mut samples = Vec::new();
9914 for rows in [10240, 51200, 102400] {
9915 for trial in 0..4 {
9916 let (mut client, image, table) = unique_import_fixture(rows);
9917 let start = std::time::Instant::now();
9918 assert!(
9919 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
9920 );
9921 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
9922 assert_eq!(
9923 client
9924 .query("SELECT count(*) AS n FROM tasks", &[])
9925 .unwrap()[0]["n"],
9926 rows + 1
9927 );
9928 assert_eq!(
9929 client
9930 .query("SELECT count(*) AS n FROM tasks_fts", &[])
9931 .unwrap()[0]["n"],
9932 rows + 1
9933 );
9934 assert_eq!(
9935 client
9936 .query("SELECT count(*) AS n FROM notes_fts", &[])
9937 .unwrap()[0]["n"],
9938 8192
9939 );
9940 assert_eq!(client.pending_commit_ids().len(), 1);
9941 if trial > 0 {
9942 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
9943 }
9944 }
9945 }
9946 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
9947 let output = serde_json::to_string_pretty(&artifact).unwrap();
9948 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
9949 use std::io::Write;
9950 let mut file = std::fs::OpenOptions::new()
9951 .write(true)
9952 .create_new(true)
9953 .open(path)
9954 .unwrap();
9955 file.write_all(output.as_bytes()).unwrap();
9956 } else {
9957 println!("{output}");
9958 }
9959 }
9960
9961 #[test]
9962 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
9963 let schema = json!({
9964 "version": 1,
9965 "tables": [{
9966 "name": "catalogue_codes",
9967 "primaryKey": "id",
9968 "columns": [
9969 { "name": "id", "type": "string", "nullable": false },
9970 { "name": "release_id", "type": "string", "nullable": false },
9971 { "name": "code", "type": "string", "nullable": false },
9972 { "name": "title", "type": "string", "nullable": false }
9973 ],
9974 "scopes": [{ "pattern": "release:{release_id}" }],
9975 "ftsIndexes": [{
9976 "name": "catalogue_codes_fts",
9977 "columns": ["code", "title"],
9978 "tokenize": "unicode61 remove_diacritics 2"
9979 }]
9980 }]
9981 });
9982 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
9983 .expect("FTS5 client");
9984 let insert_trigger: String = client
9985 .conn
9986 .query_row(
9987 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
9988 [],
9989 |row| row.get(0),
9990 )
9991 .expect("insert trigger");
9992 let replace_guard: String = client
9993 .conn
9994 .query_row(
9995 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
9996 [],
9997 |row| row.get(0),
9998 )
9999 .expect("replace guard");
10000 assert!(!insert_trigger.contains("DELETE FROM"));
10001 assert!(replace_guard.contains("BEFORE INSERT"));
10002 assert!(replace_guard.contains("WHEN EXISTS"));
10003 let search = |client: &SyncClient, query: &str| {
10004 client
10005 .query(
10006 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
10007 &[Value::from(query)],
10008 )
10009 .expect("FTS query")
10010 };
10011
10012 client
10013 .mutate(vec![Mutation::Upsert {
10014 table: "catalogue_codes".to_owned(),
10015 values: Map::from_iter([
10016 ("id".to_owned(), Value::from("c1")),
10017 ("release_id".to_owned(), Value::from("r1")),
10018 ("code".to_owned(), Value::from("A01")),
10019 ("title".to_owned(), Value::from("Cholera")),
10020 ]),
10021 base_version: None,
10022 }])
10023 .expect("insert code");
10024 assert_eq!(search(&client, "cholera").len(), 1);
10025 let prior = client
10026 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
10027 .unwrap();
10028 client
10029 .conn
10030 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
10031 .unwrap();
10032 client.create_synced_tables().unwrap();
10033 assert_eq!(
10034 client
10035 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
10036 .unwrap(),
10037 prior
10038 );
10039 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
10040 let rendered = serde_json::to_string(&plan).unwrap();
10041 assert!(rendered.contains("COVERING INDEX"));
10042 assert!(rendered.contains("INDEX 0:="));
10043 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
10044 assert_eq!(search(&client, "cholera").len(), 1);
10045
10046 client
10047 .mutate(vec![Mutation::Upsert {
10048 table: "catalogue_codes".to_owned(),
10049 values: Map::from_iter([
10050 ("id".to_owned(), Value::from("c1")),
10051 ("release_id".to_owned(), Value::from("r1")),
10052 ("code".to_owned(), Value::from("A01")),
10053 ("title".to_owned(), Value::from("Enteric infection")),
10054 ]),
10055 base_version: None,
10056 }])
10057 .expect("update code");
10058 assert!(search(&client, "cholera").is_empty());
10059 assert_eq!(search(&client, "enteric").len(), 1);
10060
10061 client
10062 .mutate(vec![Mutation::Delete {
10063 table: "catalogue_codes".to_owned(),
10064 row_id: "c1".to_owned(),
10065 base_version: None,
10066 }])
10067 .expect("delete code");
10068 assert!(search(&client, "enteric").is_empty());
10069 }
10070
10071 #[test]
10072 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
10073 let schema = json!({
10074 "version": 1,
10075 "tables": [{
10076 "name": "patient_notes",
10077 "primaryKey": "id",
10078 "columns": [
10079 { "name": "id", "type": "string", "nullable": false },
10080 { "name": "practice_id", "type": "string", "nullable": false },
10081 { "name": "encryption_key_id", "type": "string", "nullable": false },
10082 { "name": "title", "type": "string", "nullable": false }
10083 ],
10084 "scopes": [{ "pattern": "practice:{practice_id}" }],
10085 "ftsIndexes": [{
10086 "name": "patient_notes_fts",
10087 "columns": ["title"],
10088 "tokenize": "unicode61 remove_diacritics 2"
10089 }]
10090 }]
10091 });
10092 let mut client = SyncClient::new(
10093 "local-purge-test".to_owned(),
10094 &schema,
10095 ClientLimits::default(),
10096 )
10097 .expect("local purge client");
10098 client
10099 .conn
10100 .execute(
10101 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
10102 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
10103 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
10104 [],
10105 )
10106 .expect("seed base rows");
10107 client.overlay_dirty.set(true);
10108 client.rebuild_overlay_if_dirty().expect("rebuild overlay");
10109
10110 let note = |id: &str, key_id: &str, title: &str| {
10111 Map::from_iter([
10112 ("id".to_owned(), Value::from(id)),
10113 ("practice_id".to_owned(), Value::from("practice-1")),
10114 ("encryption_key_id".to_owned(), Value::from(key_id)),
10115 ("title".to_owned(), Value::from(title)),
10116 ])
10117 };
10118 let doomed = client
10119 .mutate(vec![
10120 Mutation::Upsert {
10121 table: "patient_notes".to_owned(),
10122 values: note("target", "key-revoked", "Target changed"),
10123 base_version: None,
10124 },
10125 Mutation::Upsert {
10126 table: "patient_notes".to_owned(),
10127 values: note("unrelated", "key-held", "Unrelated changed"),
10128 base_version: None,
10129 },
10130 ])
10131 .expect("doomed commit");
10132 let kept = client
10133 .mutate(vec![Mutation::Upsert {
10134 table: "patient_notes".to_owned(),
10135 values: note("kept", "key-held", "Kept optimistic"),
10136 base_version: None,
10137 }])
10138 .expect("kept commit");
10139 client.drain_change_batches();
10140
10141 let input = LocalDataPurgeInput {
10142 purge_id: "purge-001".to_owned(),
10143 targets: vec![LocalDataPurgeTarget {
10144 table: "patient_notes".to_owned(),
10145 selectors: BTreeMap::from([(
10146 "encryption_key_id".to_owned(),
10147 vec!["key-revoked".to_owned()],
10148 )]),
10149 }],
10150 };
10151 assert_eq!(
10152 client.purge_local_data(&input).expect("apply purge"),
10153 LocalDataPurgeResult {
10154 already_applied: false,
10155 purged_rows: 1,
10156 dropped_commits: 1,
10157 }
10158 );
10159 let rows = client
10160 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
10161 .expect("visible rows");
10162 assert_eq!(rows.len(), 2);
10163 assert_eq!(rows[0]["id"], "kept");
10164 assert_eq!(rows[1]["id"], "unrelated");
10165 assert_eq!(rows[1]["title"], "Unrelated original");
10166 let fts = client
10167 .query(
10168 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
10169 &[],
10170 )
10171 .expect("fts query");
10172 assert!(fts.is_empty());
10173 assert_eq!(client.outbox.len(), 1);
10174 assert_eq!(client.outbox[0].client_commit_id, kept);
10175 let outcome = client
10176 .commit_outcome(&doomed)
10177 .expect("read doomed outcome")
10178 .expect("doomed outcome");
10179 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
10180 match &outcome.results[0] {
10181 CommitOperationOutcome::Error { rejection } => {
10182 assert_eq!(rejection.code, "client.local_data_purged");
10183 assert_eq!(rejection.client_commit_id, doomed);
10184 }
10185 other => panic!("expected local purge rejection, got {other:?}"),
10186 }
10187 assert_eq!(client.drain_change_batches().len(), 1);
10188 assert_eq!(
10189 client.purge_local_data(&input).expect("retry purge"),
10190 LocalDataPurgeResult {
10191 already_applied: true,
10192 purged_rows: 0,
10193 dropped_commits: 0,
10194 }
10195 );
10196 let conflicting = LocalDataPurgeInput {
10197 purge_id: input.purge_id.clone(),
10198 targets: vec![LocalDataPurgeTarget {
10199 table: "patient_notes".to_owned(),
10200 selectors: BTreeMap::from([(
10201 "encryption_key_id".to_owned(),
10202 vec!["key-held".to_owned()],
10203 )]),
10204 }],
10205 };
10206 assert!(client
10207 .purge_local_data(&conflicting)
10208 .expect_err("id collision must fail")
10209 .contains("already used with a different plan"));
10210 }
10211
10212 fn sync_split(client: &mut SyncClient, transport: &mut dyn Transport) -> SyncOutcome {
10214 let mut next = match client.prepare_sync_round(transport.supports_url_fetch()) {
10215 Ok(prepared) => prepared,
10216 Err(outcome) => return *outcome,
10217 };
10218 loop {
10219 let completed = crate::round::test_support::drive_round(next, transport);
10220 match client.apply_sync_round(completed) {
10221 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
10222 crate::AppliedSyncRound::Complete {
10223 outcome, controls, ..
10224 } => {
10225 if client.transport_enabled() {
10226 for text in controls {
10227 let _ = transport.realtime_send(&text);
10228 }
10229 } else {
10230 client.disconnect_realtime(transport);
10231 }
10232 return outcome;
10233 }
10234 }
10235 }
10236 }
10237
10238 #[derive(Clone)]
10239 enum RoundReplyScript {
10240 Response(Message),
10241 PushAck,
10243 Error(TransportError),
10244 }
10245
10246 struct SplitParityTransport {
10249 script: RoundReplyScript,
10250 segments: BTreeMap<String, Vec<u8>>,
10251 requests: Vec<Message>,
10252 batches: Vec<Vec<String>>,
10253 uploads: Vec<String>,
10254 sync_calls: usize,
10255 realtime_sync_calls: usize,
10256 }
10257
10258 impl SplitParityTransport {
10259 fn new(script: RoundReplyScript) -> Self {
10260 Self {
10261 script,
10262 segments: BTreeMap::new(),
10263 requests: Vec::new(),
10264 batches: Vec::new(),
10265 uploads: Vec::new(),
10266 sync_calls: 0,
10267 realtime_sync_calls: 0,
10268 }
10269 }
10270
10271 fn with_segments(script: RoundReplyScript, segments: BTreeMap<String, Vec<u8>>) -> Self {
10272 Self {
10273 segments,
10274 ..Self::new(script)
10275 }
10276 }
10277
10278 fn respond(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
10279 let message =
10280 decode_message(request).expect("the client encodes a legal round request");
10281 self.requests.push(message.clone());
10282 match &self.script {
10283 RoundReplyScript::Response(response) => Ok(encode_message(response)),
10284 RoundReplyScript::Error(error) => Err(error.clone()),
10285 RoundReplyScript::PushAck => {
10286 let mut frames = vec![Frame::RespHeader {
10287 required_schema_version: None,
10288 latest_schema_version: None,
10289 log_epoch: Some("epoch-1".into()),
10290 reset_required: Some(false),
10291 }];
10292 let mut commit_seq = 0;
10293 for frame in &message.frames {
10294 if let Frame::PushCommit {
10295 client_commit_id,
10296 operations,
10297 } = frame
10298 {
10299 commit_seq += 1;
10300 frames.push(Frame::PushResult {
10301 client_commit_id: client_commit_id.clone(),
10302 status: PushStatus::Applied,
10303 commit_seq: Some(commit_seq),
10304 results: (0..operations.len())
10305 .map(|op_index| OpResult::Applied {
10306 op_index: op_index as i32,
10307 })
10308 .collect(),
10309 });
10310 }
10311 }
10312 frames.push(Frame::SubStart {
10313 id: "tasks".into(),
10314 status: SubStatus::Active,
10315 reason_code: String::new(),
10316 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
10317 bootstrap: false,
10318 });
10319 frames.push(Frame::SubEnd {
10320 next_cursor: 0,
10321 bootstrap_state: None,
10322 });
10323 Ok(encode_message(&Message {
10324 wire_version: WIRE_VERSION,
10325 msg_kind: MsgKind::Response,
10326 frames,
10327 }))
10328 }
10329 }
10330 }
10331 }
10332
10333 impl Transport for SplitParityTransport {
10334 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
10335 self.sync_calls += 1;
10336 self.respond(request)
10337 }
10338
10339 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
10340 self.realtime_sync_calls += 1;
10341 self.respond(request)
10342 }
10343 fn download_segment(
10344 &mut self,
10345 _request: &SegmentRequest,
10346 _on_progress: &mut dyn FnMut(u64),
10347 ) -> Result<Vec<u8>, TransportError> {
10348 Err(TransportError::new(
10349 "sync.transport_failed",
10350 "per-descriptor fetch",
10351 ))
10352 }
10353
10354 fn download_segments(
10355 &mut self,
10356 fetches: &[crate::SegmentFetch],
10357 on_progress: &mut (dyn FnMut(usize, u64) + Send),
10358 ) -> Vec<Result<Vec<u8>, TransportError>> {
10359 self.batches.push(
10360 fetches
10361 .iter()
10362 .map(|fetch| match fetch {
10363 crate::SegmentFetch::Direct(request) => request.segment_id.clone(),
10364 crate::SegmentFetch::Url(url) => url.clone(),
10365 })
10366 .collect(),
10367 );
10368 fetches
10369 .iter()
10370 .enumerate()
10371 .map(|(index, fetch)| {
10372 let crate::SegmentFetch::Direct(request) = fetch else {
10373 panic!("the descriptors are direct-endpoint requests")
10374 };
10375 let bytes =
10376 self.segments
10377 .get(&request.segment_id)
10378 .cloned()
10379 .ok_or_else(|| {
10380 TransportError::new("transport.not_found", "missing segment body")
10381 })?;
10382 on_progress(index, bytes.len() as u64);
10383 Ok(bytes)
10384 })
10385 .collect()
10386 }
10387
10388 fn blob_upload_grant(
10389 &mut self,
10390 _blob_id: &str,
10391 _byte_length: u64,
10392 _media_type: Option<&str>,
10393 ) -> Result<BlobUploadGrant, TransportError> {
10394 Ok(BlobUploadGrant::None)
10395 }
10396
10397 fn blob_upload(
10398 &mut self,
10399 blob_id: &str,
10400 _bytes: &[u8],
10401 _media_type: Option<&str>,
10402 ) -> Result<(), TransportError> {
10403 self.uploads.push(blob_id.to_owned());
10404 Ok(())
10405 }
10406
10407 fn realtime_connect(&mut self) -> Result<(), TransportError> {
10408 Ok(())
10409 }
10410
10411 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
10412 Ok(())
10413 }
10414
10415 fn realtime_close(&mut self) -> Result<(), TransportError> {
10416 Ok(())
10417 }
10418 }
10419
10420 fn parity_clients() -> (SyncClient, SyncClient) {
10421 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
10422 {"name":"id","type":"string","nullable":false},
10423 {"name":"project_id","type":"string","nullable":false}],
10424 "scopes":[{"pattern":"project:{project_id}"}]}]});
10425 let [exchange, split] = ["parity-exchange", "parity-split"].map(|id| {
10426 let mut client =
10427 SyncClient::new(id.to_owned(), &schema, ClientLimits::default()).unwrap();
10428 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
10429 client
10430 .subscribe(
10431 "tasks".into(),
10432 "tasks".into(),
10433 vec![("project_id".into(), vec!["p1".into()])],
10434 None,
10435 )
10436 .unwrap();
10437 client.subs[0].cursor = 0;
10438 client.subs[0].synced_once = true;
10439 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
10440 client.persist_sub(&client.subs[0]).unwrap();
10441 client.drain_change_batches();
10442 client
10443 });
10444 (exchange, split)
10445 }
10446
10447 fn pull_response(client: &SyncClient) -> Message {
10448 let row = encode_row_json(
10449 client.schema.table("tasks").unwrap(),
10450 "kept",
10451 &Map::from_iter([
10452 ("id".into(), json!("kept")),
10453 ("project_id".into(), json!("p1")),
10454 ]),
10455 &client.encryption,
10456 )
10457 .unwrap();
10458 Message {
10459 wire_version: WIRE_VERSION,
10460 msg_kind: MsgKind::Response,
10461 frames: vec![
10462 Frame::RespHeader {
10463 required_schema_version: None,
10464 latest_schema_version: None,
10465 log_epoch: Some("epoch-1".into()),
10466 reset_required: Some(false),
10467 },
10468 Frame::SubStart {
10469 id: "tasks".into(),
10470 status: SubStatus::Active,
10471 reason_code: String::new(),
10472 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
10473 bootstrap: false,
10474 },
10475 Frame::Commit {
10476 commit_seq: 1,
10477 created_at_ms: 1,
10478 actor_id: "writer".into(),
10479 tables: vec!["tasks".into()],
10480 changes: vec![ssp2::model::Change {
10481 table_index: 0,
10482 row_id: "kept".into(),
10483 op: Op::Upsert,
10484 row_version: Some(1),
10485 scopes: vec![("project_id".into(), "p1".into())],
10486 row: Some(row),
10487 }],
10488 },
10489 Frame::SubEnd {
10490 next_cursor: 1,
10491 bootstrap_state: None,
10492 },
10493 ],
10494 }
10495 }
10496
10497 #[test]
10498 fn split_round_matches_exchange_for_a_pull() {
10499 let (mut exchange, mut split) = parity_clients();
10500 let response = pull_response(&exchange);
10501 let mut exchange_transport =
10502 SplitParityTransport::new(RoundReplyScript::Response(response.clone()));
10503 let mut split_transport = SplitParityTransport::new(RoundReplyScript::Response(response));
10504
10505 let exchange_outcome = exchange.sync(&mut exchange_transport);
10506 let split_outcome = sync_split(&mut split, &mut split_transport);
10507 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10508 assert_eq!(
10509 exchange
10510 .query("SELECT id, project_id FROM tasks", &[])
10511 .unwrap(),
10512 split
10513 .query("SELECT id, project_id FROM tasks", &[])
10514 .unwrap()
10515 );
10516 assert_eq!(exchange_transport.requests.len(), 1);
10517 assert_eq!(split_transport.requests.len(), 1);
10518 }
10519
10520 #[test]
10521 fn split_round_matches_exchange_for_a_push_with_an_upload() {
10522 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
10523 {"name":"id","type":"string","nullable":false},
10524 {"name":"project_id","type":"string","nullable":false},
10525 {"name":"file","type":"blob_ref","nullable":true}],
10526 "scopes":[{"pattern":"project:{project_id}"}]}]});
10527 let [mut exchange, mut split] = ["parity-push-exchange", "parity-push-split"].map(|id| {
10528 let mut client =
10529 SyncClient::new(id.to_owned(), &schema, ClientLimits::default()).unwrap();
10530 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
10531 client
10532 .subscribe(
10533 "attachments".into(),
10534 "attachments".into(),
10535 vec![("project_id".into(), vec!["p1".into()])],
10536 None,
10537 )
10538 .unwrap();
10539 client
10540 });
10541 let bytes = b"split upload";
10542 let mut mutations = Vec::new();
10543 for client in [&mut exchange, &mut split] {
10544 client
10545 .upload_blob(bytes, Some("text/plain".into()), None)
10546 .unwrap();
10547 mutations.push(
10548 client
10549 .mutate(vec![Mutation::Upsert {
10550 table: "attachments".into(),
10551 values: Map::from_iter([
10552 ("id".into(), Value::from("a1")),
10553 ("project_id".into(), Value::from("p1")),
10554 (
10555 "file".into(),
10556 Value::from(format!(
10557 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
10558 blob_id_for(bytes),
10559 bytes.len()
10560 )),
10561 ),
10562 ]),
10563 base_version: None,
10564 }])
10565 .unwrap(),
10566 );
10567 }
10568 let mut exchange_transport = SplitParityTransport::new(RoundReplyScript::PushAck);
10569 let mut split_transport = SplitParityTransport::new(RoundReplyScript::PushAck);
10570 let exchange_outcome = exchange.sync(&mut exchange_transport);
10571 let split_outcome = sync_split(&mut split, &mut split_transport);
10572 let SyncOutcome::Ok(exchange_report) = exchange_outcome else {
10573 panic!("the exchange round must succeed: {exchange_outcome:?}")
10574 };
10575 let SyncOutcome::Ok(split_report) = split_outcome else {
10576 panic!("the split round must succeed: {split_outcome:?}")
10577 };
10578 assert_eq!(exchange_report.pushed, 1);
10579 assert_eq!(split_report.pushed, 1);
10580 assert_eq!(
10581 exchange_report.commits_applied,
10582 split_report.commits_applied
10583 );
10584 assert_eq!(
10585 exchange_transport.uploads,
10586 [blob_id_for(bytes)],
10587 "one upload before the metadata request"
10588 );
10589 assert_eq!(split_transport.uploads, exchange_transport.uploads);
10590 assert!(exchange.pending_commit_ids().is_empty());
10591 assert!(split.pending_commit_ids().is_empty());
10592 assert_eq!(
10593 exchange
10594 .query("SELECT id, project_id FROM attachments", &[])
10595 .unwrap(),
10596 split
10597 .query("SELECT id, project_id FROM attachments", &[])
10598 .unwrap()
10599 );
10600 }
10601
10602 #[test]
10603 fn split_round_matches_exchange_for_a_segment_download() {
10604 let (mut exchange, mut split) = parity_clients();
10605 let columns = exchange.schema.table("tasks").unwrap().wire_columns.clone();
10606 let page = ssp2::segment::encode_rows_segment(&RowsSegment {
10607 table: "tasks".into(),
10608 schema_version: 1,
10609 columns,
10610 blocks: vec![vec![ssp2::segment::SegmentRow {
10611 server_version: 1,
10612 values: vec![
10613 Some(ColumnValue::String("page".into())),
10614 Some(ColumnValue::String("p1".into())),
10615 ],
10616 }]],
10617 });
10618 let segment_id = format!("sha256:{}", bytes_to_hex(&Sha256::digest(&page)));
10619 let response = Message {
10620 wire_version: WIRE_VERSION,
10621 msg_kind: MsgKind::Response,
10622 frames: vec![
10623 Frame::RespHeader {
10624 required_schema_version: None,
10625 latest_schema_version: None,
10626 log_epoch: Some("epoch-1".into()),
10627 reset_required: Some(false),
10628 },
10629 Frame::SubStart {
10630 id: "tasks".into(),
10631 status: SubStatus::Active,
10632 reason_code: String::new(),
10633 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
10634 bootstrap: true,
10635 },
10636 Frame::SegmentRef {
10637 segment_id: segment_id.clone(),
10638 media_type: MediaType::Rows,
10639 table: "tasks".into(),
10640 byte_length: page.len() as i64,
10641 row_count: 1,
10642 as_of_commit_seq: 1,
10643 scope_digest: "digest".into(),
10644 row_cursor: None,
10645 next_row_cursor: None,
10646 url: None,
10647 url_expires_at_ms: None,
10648 },
10649 Frame::SubEnd {
10650 next_cursor: 1,
10651 bootstrap_state: None,
10652 },
10653 ],
10654 };
10655 let segments = BTreeMap::from([(segment_id.clone(), page)]);
10656 let mut exchange_transport = SplitParityTransport::with_segments(
10657 RoundReplyScript::Response(response.clone()),
10658 segments.clone(),
10659 );
10660 let mut split_transport =
10661 SplitParityTransport::with_segments(RoundReplyScript::Response(response), segments);
10662 let exchange_outcome = exchange.sync(&mut exchange_transport);
10663 let split_outcome = sync_split(&mut split, &mut split_transport);
10664 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10665 assert_eq!(exchange_transport.batches, [vec![segment_id.clone()]]);
10666 assert_eq!(split_transport.batches, exchange_transport.batches);
10667 assert_eq!(
10668 exchange
10669 .query("SELECT id, project_id FROM tasks", &[])
10670 .unwrap(),
10671 split
10672 .query("SELECT id, project_id FROM tasks", &[])
10673 .unwrap()
10674 );
10675 }
10676
10677 #[test]
10678 fn split_round_matches_exchange_for_a_transport_failure() {
10679 let (mut exchange, mut split) = parity_clients();
10680 let error = TransportError::new("transport.timeout", "injected");
10681 let mut exchange_transport =
10682 SplitParityTransport::new(RoundReplyScript::Error(error.clone()));
10683 let mut split_transport = SplitParityTransport::new(RoundReplyScript::Error(error));
10684 let exchange_outcome = exchange.sync(&mut exchange_transport);
10685 let split_outcome = sync_split(&mut split, &mut split_transport);
10686 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10687 assert!(matches!(
10688 split_outcome,
10689 SyncOutcome::Failed { ref error_code, .. } if error_code == "transport.timeout"
10690 ));
10691 assert_eq!(
10692 exchange.query("SELECT id FROM tasks", &[]).unwrap(),
10693 split.query("SELECT id FROM tasks", &[]).unwrap()
10694 );
10695 }
10696
10697 #[test]
10698 fn split_round_matches_exchange_for_a_failed_realtime_round() {
10699 let (mut exchange, mut split) = parity_clients();
10700 let error = TransportError::new("transport.timeout", "socket round failed");
10701 let mut exchange_transport = SplitParityTransport::new(RoundReplyScript::Error(error));
10702 let mut split_transport = SplitParityTransport::new(RoundReplyScript::Error(
10703 TransportError::new("transport.timeout", "socket round failed"),
10704 ));
10705 exchange.connect_realtime(&mut exchange_transport).unwrap();
10706 split.connect_realtime(&mut split_transport).unwrap();
10707 let exchange_outcome = exchange.sync(&mut exchange_transport);
10708 let split_outcome = sync_split(&mut split, &mut split_transport);
10709 assert_eq!(exchange_outcome.to_json(), split_outcome.to_json());
10710 assert_eq!(exchange_transport.realtime_sync_calls, 1);
10711 assert_eq!(exchange_transport.sync_calls, 0);
10712 assert_eq!(split_transport.realtime_sync_calls, 1);
10713 assert_eq!(split_transport.sync_calls, 0);
10714 assert_eq!(exchange.realtime_state(), RealtimeState::Lost);
10715 assert_eq!(split.realtime_state(), RealtimeState::Lost);
10716 }
10717}
10718
10719impl SubState {
10720 fn name(self) -> &'static str {
10721 match self {
10722 SubState::Active => "active",
10723 SubState::Revoked => "revoked",
10724 SubState::Failed => "failed",
10725 }
10726 }
10727
10728 fn parse(value: &str) -> Self {
10729 match value {
10730 "revoked" => Self::Revoked,
10731 "failed" => Self::Failed,
10732 _ => Self::Active,
10733 }
10734 }
10735}
10736
10737#[derive(Debug, Clone)]
10738struct Subscription {
10739 id: String,
10740 table: String,
10741 requested: Vec<(String, Vec<String>)>,
10742 params: Option<String>,
10743 cursor: i64,
10744 bootstrap_state: Option<String>,
10746 state: SubState,
10747 reason_code: Option<String>,
10748 effective: Option<Vec<(String, Vec<String>)>>,
10751 synced_once: bool,
10752}
10753
10754#[doc(hidden)]
10756pub struct CurrentBasePreparation {
10757 client_commit_id: String,
10758 sequence: i64,
10759 subscriptions: Vec<Subscription>,
10760 transport_epoch: u64,
10761 subscription_epoch: u64,
10762 security_epoch: u64,
10763}
10764
10765#[cfg(test)]
10766mod subscription_rearm_tests {
10767 use super::*;
10768 use serde_json::json;
10769
10770 fn client() -> SyncClient {
10771 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
10772 {"name":"id","type":"string","nullable":false},
10773 {"name":"project_id","type":"string","nullable":false}],
10774 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[],"ftsIndexes":[]}]});
10775 SyncClient::new("rearm".into(), &schema, ClientLimits::default()).expect("client")
10776 }
10777
10778 fn scopes() -> Vec<(String, Vec<String>)> {
10779 vec![("project_id".to_owned(), vec!["p1".to_owned()])]
10780 }
10781
10782 fn store_revoked(client: &mut SyncClient) {
10784 client
10785 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10786 .expect("subscribe");
10787 let sub = &mut client.subs[0];
10788 sub.cursor = 41;
10789 sub.bootstrap_state = Some("resume".to_owned());
10790 sub.effective = Some(scopes());
10791 sub.state = SubState::Revoked;
10792 sub.reason_code = Some("sync.scope_revoked".to_owned());
10793 client.persist_sub(&client.subs[0]).expect("persist");
10794 client.sync_needed = false;
10795 client.sync_intent_queue.clear();
10796 }
10797
10798 #[test]
10799 fn resubscribing_the_same_intent_rearms_a_revoked_registration() {
10800 let mut client = client();
10801 store_revoked(&mut client);
10802
10803 client
10804 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10805 .expect("re-arm");
10806
10807 let view = client.subscription_state("sub").expect("state");
10808 assert_eq!(view.status, "active");
10809 assert_eq!(view.cursor, -1);
10810 assert!(!view.has_resume_token);
10811 assert_eq!(view.effective_scopes, None);
10812 assert_eq!(view.reason_code, None);
10813 assert!(client.sync_needed, "a re-arm asks for a pull");
10814 assert!(matches!(
10815 client.drain_sync_intents().as_slice(),
10816 [SyncIntent::Interactive]
10817 ));
10818 let stored: String = client
10819 .conn
10820 .query_row(
10821 "SELECT state_json FROM _syncular_subscriptions WHERE id = 'sub'",
10822 [],
10823 |row| row.get(0),
10824 )
10825 .expect("persisted row");
10826 let stored: Value = serde_json::from_str(&stored).expect("stored json");
10827 assert_eq!(stored["status"], json!("active"));
10828 assert_eq!(stored["cursor"], json!(-1));
10829 assert_eq!(stored["bootstrapState"], Value::Null);
10830 assert_eq!(stored["effectiveScopes"], Value::Null);
10831 assert_eq!(stored["reasonCode"], Value::Null);
10832 }
10833
10834 #[test]
10835 fn a_failed_registration_is_not_rearmed() {
10836 let mut client = client();
10837 client
10838 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10839 .expect("subscribe");
10840 client.subs[0].state = SubState::Failed;
10841 client.subs[0].reason_code = Some("sync.scope_revoked".to_owned());
10842 client.persist_sub(&client.subs[0]).expect("persist");
10843 client.sync_needed = false;
10844 client.sync_intent_queue.clear();
10845
10846 client
10847 .subscribe("sub".into(), "tasks".into(), scopes(), None)
10848 .expect("subscribe");
10849
10850 let view = client.subscription_state("sub").expect("state");
10851 assert_eq!(view.status, "failed");
10852 assert_eq!(view.reason_code.as_deref(), Some("sync.scope_revoked"));
10853 assert!(!client.sync_needed);
10854 assert!(client.drain_sync_intents().is_empty());
10855 }
10856}
10857
10858#[derive(Debug, Clone)]
10859struct OutboxOp {
10860 upsert: bool,
10861 table: String,
10862 row_id: String,
10863 base_version: Option<i64>,
10864 values: Option<Map<String, Value>>,
10868 insert: bool,
10872}
10873
10874impl OutboxOp {
10875 fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
10880 let values = self.values.as_ref()?;
10881 if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
10882 return Some(Cow::Borrowed(values));
10883 }
10884 full_row_values(table, values.clone()).ok().map(Cow::Owned)
10885 }
10886}
10887
10888impl From<&OutboxOp> for CommitOperation {
10889 fn from(operation: &OutboxOp) -> Self {
10890 Self {
10891 table: operation.table.clone(),
10892 row_id: operation.row_id.clone(),
10893 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
10894 base_version: operation.base_version,
10895 values: operation.values.clone(),
10896 insert: operation.insert,
10897 }
10898 }
10899}
10900
10901#[derive(Debug, Clone)]
10902struct OutboxCommit {
10903 client_commit_id: String,
10904 ops: Vec<OutboxOp>,
10905}
10906
10907fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
10911 commit
10912 .ops
10913 .iter()
10914 .filter(|operation| operation.upsert)
10915 .map(|operation| {
10916 (
10917 operation.table.as_str(),
10918 operation
10919 .values
10920 .as_ref()
10921 .map(|values| values.keys().map(String::as_str).collect())
10922 .unwrap_or_default(),
10923 operation.insert,
10924 )
10925 })
10926 .collect()
10927}
10928
10929type FailedCommit = (OutboxCommit, Map<String, Value>);
10930
10931#[derive(Debug, Clone)]
10932struct CompiledLocalDataPurgeTarget {
10933 table: String,
10934 selectors: Vec<(String, Vec<String>)>,
10935}
10936
10937struct StoredCommitOutcomeRow {
10938 sequence: i64,
10939 client_commit_id: String,
10940 status: String,
10941 recorded_at_ms: i64,
10942 results_json: String,
10943 operations_json: Option<String>,
10944 resolution: String,
10945 resolved_at_ms: Option<i64>,
10946 replacement_client_commit_id: Option<String>,
10947}
10948
10949impl StoredCommitOutcomeRow {
10950 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
10951 Ok(Self {
10952 sequence: row.get(0)?,
10953 client_commit_id: row.get(1)?,
10954 status: row.get(2)?,
10955 recorded_at_ms: row.get(3)?,
10956 results_json: row.get(4)?,
10957 operations_json: row.get(5)?,
10958 resolution: row.get(6)?,
10959 resolved_at_ms: row.get(7)?,
10960 replacement_client_commit_id: row.get(8)?,
10961 })
10962 }
10963}
10964
10965enum SectionError {
10968 FailClosed,
10969 Abort(String, String),
10970 Transport(TransportError),
10971}
10972
10973#[derive(Debug)]
10974pub(crate) struct RequestMeta {
10975 pushed_ids: Vec<String>,
10976 fresh: Vec<(String, bool)>,
10979 pub(crate) accept: u8,
10980 deferred_commits: usize,
10984}
10985
10986#[derive(Default)]
10987struct ChangeAccumulator {
10988 tables: BTreeMap<String, Option<BTreeSet<String>>>,
10990 windows: BTreeMap<(String, String), BTreeSet<String>>,
10991 status: bool,
10992 conflicts: bool,
10993 rejections: bool,
10994 outcomes: bool,
10995}
10996
10997impl ChangeAccumulator {
10998 fn table(&mut self, table: &str) {
10999 self.tables.insert(table.to_owned(), None);
11000 }
11001
11002 fn scope(&mut self, table: &str, key: String) {
11003 match self.tables.get_mut(table) {
11004 Some(None) => {}
11005 Some(Some(keys)) => {
11006 keys.insert(key);
11007 }
11008 None => {
11009 self.tables
11010 .insert(table.to_owned(), Some(BTreeSet::from([key])));
11011 }
11012 }
11013 }
11014
11015 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
11016 self.windows
11017 .entry((base_key.to_owned(), table.to_owned()))
11018 .or_default()
11019 .insert(unit.to_owned());
11020 }
11021
11022 fn touched(&self) -> bool {
11023 !self.tables.is_empty()
11024 || !self.windows.is_empty()
11025 || self.status
11026 || self.conflicts
11027 || self.rejections
11028 || self.outcomes
11029 }
11030}
11031
11032const PUSH_OPS_PER_REQUEST: usize = 500;
11038const MAX_LOCAL_PURGE_TARGETS: usize = 64;
11039const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
11040const MAX_LOCAL_PURGE_VALUES: usize = 128;
11041const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
11042pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
11043
11044struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
11046
11047impl OverlayDirty {
11048 fn clean() -> Self {
11049 Self(RefCell::new(Some(BTreeSet::new())))
11050 }
11051 fn get(&self) -> bool {
11052 self.0
11053 .borrow()
11054 .as_ref()
11055 .is_none_or(|tables| !tables.is_empty())
11056 }
11057 fn set(&self, dirty: bool) {
11058 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
11059 }
11060 fn table(&self, table: &str) {
11061 if let Some(tables) = self.0.borrow_mut().as_mut() {
11062 tables.insert(table.to_owned());
11063 }
11064 }
11065 fn snapshot(&self) -> Option<BTreeSet<String>> {
11066 self.0.borrow().clone()
11067 }
11068 fn restore(&self, tables: Option<BTreeSet<String>>) {
11069 *self.0.borrow_mut() = tables;
11070 }
11071}
11072
11073pub struct SyncClient {
11074 #[cfg(feature = "bench-internals")]
11075 benchmark_phases: Recorder,
11076 conn: Connection,
11077 schema: ClientSchema,
11078 client_id: String,
11079 limits: ClientLimits,
11080 subs: Vec<Subscription>,
11081 outbox: Vec<OutboxCommit>,
11082 failed_commits: Vec<FailedCommit>,
11083 retain_failed_commits: bool,
11084 conflicts: Vec<ConflictRecord>,
11085 rejections: Vec<RejectionRecord>,
11086 schema_floor: Option<SchemaFloor>,
11087 lease_state: Option<LeaseState>,
11089 stopped: bool,
11091 upgrading: bool,
11093 sync_needed: bool,
11095 realtime_policy: RealtimePolicy,
11098 realtime_state: RealtimeState,
11099 realtime_reason_code: Option<String>,
11100 realtime_retry_delay_ms: Option<u64>,
11101 presence: HashMap<String, HashMap<String, PresencePeer>>,
11103 now_ms: Option<i64>,
11106 encryption: crate::values::EncryptionConfig,
11110 security_preflight: bool,
11113 security_epoch: u64,
11114 transport_enabled: bool,
11115 transport_epoch: u64,
11116 subscription_epoch: u64,
11117 previous_version: Option<PreviousVersionContextConfig>,
11121 insert_sql: RefCell<HashMap<String, String>>,
11127 overlay_dirty: OverlayDirty,
11130 storage_failure: RefCell<Option<QueryReadFailure>>,
11132 #[cfg(test)]
11135 overlay_rebuild_count: Cell<usize>,
11136 #[cfg(test)]
11137 acknowledged_replay_count: Cell<usize>,
11138 #[cfg(test)]
11141 outcome_prune_count: Cell<usize>,
11142 progress: ProgressObserver,
11144 change_queue: VecDeque<ClientChangeBatch>,
11145 sync_intent_queue: VecDeque<SyncIntent>,
11146 retry_delay_ms: u64,
11148 round_retry_delay_ms: Option<u64>,
11150 round_failure: Option<String>,
11155 active_round: Option<uuid::Uuid>,
11156 last_round: Option<DiagnosticLastRound>,
11157 last_change: Option<DiagnosticLastChange>,
11158 query_failures: Vec<DiagnosticQueryFailure>,
11160 coalesced: Option<CoalescedApply>,
11162 coalesced_failure: Option<String>,
11165}
11166
11167struct CoalescedApply {
11171 rows: usize,
11172 subs: Vec<Subscription>,
11173 outbox: Vec<OutboxCommit>,
11174 failed_commits: Vec<FailedCommit>,
11175 conflicts: Vec<ConflictRecord>,
11176 rejections: Vec<RejectionRecord>,
11177 schema_floor: Option<SchemaFloor>,
11178 lease_state: Option<LeaseState>,
11179 stopped: bool,
11180 upgrading: bool,
11181 sync_needed: bool,
11182 overlay_dirty: Option<BTreeSet<String>>,
11183 change_queue: usize,
11184 sync_intent_queue: usize,
11185 last_change: Option<DiagnosticLastChange>,
11186}
11187
11188const COALESCED_APPLY_ROWS: usize = 1024;
11189
11190pub(crate) fn quote_ident(name: &str) -> String {
11191 format!("\"{}\"", name.replace('"', "\"\""))
11192}
11193pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
11197 conn.query_row(
11198 "SELECT value FROM _syncular_meta WHERE key = ?1",
11199 rusqlite::params![key],
11200 |row| row.get::<_, String>(0),
11201 )
11202 .ok()
11203}
11204
11205pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
11206 let _ = conn.execute(
11207 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
11208 rusqlite::params![key, value],
11209 );
11210}
11211
11212pub(crate) fn meta_delete(conn: &Connection, key: &str) {
11213 let _ = conn.execute(
11214 "DELETE FROM _syncular_meta WHERE key = ?1",
11215 rusqlite::params![key],
11216 );
11217}
11218
11219fn row_id_predicate(table: &TableSchema) -> String {
11221 let key = quote_ident(&table.primary_key);
11222 let text_match = format!("CAST({key} AS TEXT) = ?1");
11223 match table.columns[table.pk_index].ty {
11224 ColumnType::String | ColumnType::Json => {
11225 format!("{key} = ?1 AND {text_match}")
11226 }
11227 ColumnType::Integer | ColumnType::Boolean => {
11231 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
11232 }
11233 _ => text_match,
11236 }
11237}
11238
11239fn is_local_operation_code_like(value: &str) -> bool {
11240 let bytes = value.as_bytes();
11241 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
11242 && bytes
11243 .iter()
11244 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
11245}
11246
11247fn base_table(name: &str) -> String {
11248 quote_ident(&format!("_syncular_base_{name}"))
11249}
11250
11251type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
11256
11257fn window_base_key(base: &WindowBase) -> String {
11258 format!(
11259 "{}\0{}\0{}",
11260 base.table,
11261 base.variable,
11262 canonical_scope_json(&base.fixed_scopes)
11263 )
11264}
11265
11266fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
11268 let mut scopes = base.fixed_scopes.clone();
11269 scopes.retain(|(k, _)| k != &base.variable);
11270 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
11271 scopes
11272}
11273
11274fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
11278 let canonical = canonical_scope_json(&unit_scopes(base, unit));
11279 let digest = Sha256::digest(canonical.as_bytes());
11280 let hex = bytes_to_hex(&digest);
11281 format!("w:{}:{}", base.table, &hex[..16])
11282}
11283
11284fn visible_table(name: &str) -> String {
11285 quote_ident(name)
11286}
11287
11288const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
11289
11290fn is_synced_table_name(name: &str) -> bool {
11293 if name.starts_with("sqlite_") {
11294 return false;
11295 }
11296 if name.starts_with("_syncular_base_") {
11297 return true; }
11299 !name.starts_with("_syncular_")
11301}
11302
11303fn blob_id_for(bytes: &[u8]) -> String {
11305 let digest = Sha256::digest(bytes);
11306 format!("sha256:{}", bytes_to_hex(&digest))
11307}
11308
11309enum RowParam<'a> {
11313 Cell(&'a Option<ColumnValue>),
11314 Version(i64),
11315}
11316
11317impl rusqlite::ToSql for RowParam<'_> {
11318 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
11319 Ok(match self {
11320 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
11321 RowParam::Cell(cell) => match cell {
11322 None => ToSqlOutput::Owned(SqlValue::Null),
11323 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
11324 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
11325 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
11326 Some(ColumnValue::Boolean(b)) => {
11327 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
11328 }
11329 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
11330 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
11331 }
11332 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
11334 ToSqlOutput::Borrowed(ValueRef::Blob(b))
11335 }
11336 },
11337 })
11338 }
11339}
11340
11341fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
11343 match param.to_sql().map_err(|error| error.to_string())? {
11344 ToSqlOutput::Owned(value) => Ok(value),
11345 ToSqlOutput::Borrowed(value) => {
11346 SqlValue::try_from(value).map_err(|error| error.to_string())
11347 }
11348 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
11349 }
11350}
11351
11352fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
11360 use ssp2::segment::ColumnType;
11361 let mismatch = || {
11362 Err(format!(
11363 "image column {:?} holds a value of the wrong type",
11364 column.name
11365 ))
11366 };
11367 match value {
11368 ValueRef::Null => {
11369 if !column.nullable {
11370 return Err(format!(
11371 "image column {:?} is NULL but not nullable",
11372 column.name
11373 ));
11374 }
11375 Ok(ToSqlOutput::Owned(SqlValue::Null))
11376 }
11377 ValueRef::Integer(i) => match column.ty {
11378 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
11379 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
11380 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
11381 _ => mismatch(),
11382 },
11383 ValueRef::Real(_) => match column.ty {
11384 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
11385 _ => mismatch(),
11386 },
11387 ValueRef::Text(t) => {
11388 std::str::from_utf8(t)
11389 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
11390 match column.ty {
11391 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
11392 Ok(ToSqlOutput::Borrowed(value))
11393 }
11394 _ => mismatch(),
11395 }
11396 }
11397 ValueRef::Blob(_) => match column.ty {
11398 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
11400 _ => mismatch(),
11401 },
11402 }
11403}
11404
11405pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
11406 use rusqlite::types::ValueRef;
11407 match value {
11408 ValueRef::Null => Value::Null,
11409 ValueRef::Integer(i) => match column.ty {
11410 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
11411 ssp2::segment::ColumnType::Float => {
11412 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
11413 }
11414 _ => Value::from(i),
11415 },
11416 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
11417 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
11418 ValueRef::Blob(b) => {
11419 let mut map = Map::new();
11420 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
11421 Value::Object(map)
11422 }
11423 }
11424}
11425
11426fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
11430 Ok(match value {
11431 Value::Null => SqlValue::Null,
11432 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
11433 Value::Number(n) => {
11434 if let Some(i) = n.as_i64() {
11435 SqlValue::Integer(i)
11436 } else if let Some(f) = n.as_f64() {
11437 SqlValue::Real(f)
11438 } else {
11439 return Err(format!("query param number {n} is out of range"));
11440 }
11441 }
11442 Value::String(s) => SqlValue::Text(s.clone()),
11443 Value::Object(_) => {
11444 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
11445 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
11446 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
11447 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
11448 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
11449 })?)
11450 } else {
11451 return Err(
11452 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
11453 .to_owned(),
11454 );
11455 }
11456 }
11457 Value::Array(_) => return Err("query array params are not supported".to_owned()),
11458 })
11459}
11460
11461fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
11466 use rusqlite::types::ValueRef;
11467 match value {
11468 ValueRef::Null => Value::Null,
11469 ValueRef::Integer(i) => {
11470 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
11474 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
11475 Value::from(i)
11476 } else {
11477 let mut map = Map::new();
11478 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
11479 Value::Object(map)
11480 }
11481 }
11482 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
11483 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
11484 ValueRef::Blob(b) => {
11485 let mut map = Map::new();
11486 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
11487 Value::Object(map)
11488 }
11489 }
11490}
11491
11492fn query_connection(
11494 conn: &Connection,
11495 sql: &str,
11496 params: &[QueryValue],
11497) -> Result<Vec<QueryRow>, QueryReadFailure> {
11498 crate::query_guard::assert_read_only_query(sql)?;
11499 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
11500 let bound: Vec<SqlValue> = params
11501 .iter()
11502 .map(json_param_to_sql)
11503 .collect::<Result<_, _>>()?;
11504 let mut stmt = conn.prepare(&lowered_sql)?;
11505 let column_names: Vec<String> = stmt.column_names().map(str::to_owned).collect();
11506 let bound_refs: Vec<&dyn rusqlite::ToSql> =
11507 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
11508 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
11509 let mut out = Vec::new();
11510 while let Some(row) = sql_rows.next()? {
11511 let mut record = Map::new();
11512 for (i, name) in column_names.iter().enumerate() {
11513 let value = row.get_ref(i)?;
11514 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
11515 }
11516 out.push(record);
11517 }
11518 Ok(out)
11519}
11520
11521fn persisted_window_state(
11522 conn: &Connection,
11523 base: &WindowBase,
11524) -> Result<WindowState, QueryReadFailure> {
11525 let mut stmt = conn.prepare(
11526 "SELECT windows.unit, subscriptions.state_json
11527 FROM _syncular_windows AS windows
11528 JOIN _syncular_subscriptions AS subscriptions
11529 ON subscriptions.id = windows.sub_id
11530 WHERE windows.base = ?1
11531 ORDER BY windows.unit ASC",
11532 )?;
11533 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
11534 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11535 })?;
11536 let mut units = Vec::new();
11537 let mut pending = Vec::new();
11538 for row in rows {
11539 let (unit, raw) = row?;
11540 let state: Value = serde_json::from_str(&raw)
11541 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
11542 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
11543 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
11544 || state
11545 .get("bootstrapState")
11546 .is_some_and(|value| !value.is_null());
11547 if is_pending {
11548 pending.push(unit.clone());
11549 }
11550 units.push(unit);
11551 }
11552 Ok(WindowState { units, pending })
11553}
11554
11555#[derive(Debug, Clone, Copy)]
11557pub struct SnapshotStatement<'a> {
11558 pub sql: &'a str,
11559 pub params: &'a [Value],
11560}
11561
11562#[derive(Debug, Clone, Default)]
11566pub struct SnapshotReadRequest<'a> {
11567 pub statements: Vec<SnapshotStatement<'a>>,
11568 pub coverage: &'a [WindowCoverage],
11569 pub subscriptions: Vec<String>,
11570 pub commit_ids: Vec<String>,
11571}
11572
11573#[cfg(test)]
11574thread_local! {
11575 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
11579 std::cell::RefCell::new(None);
11580}
11581
11582#[cfg(test)]
11583fn run_snapshot_read_hook() {
11584 SNAPSHOT_READ_HOOK.with(|slot| {
11585 if let Some(hook) = slot.borrow_mut().take() {
11586 hook();
11587 }
11588 });
11589}
11590
11591fn snapshot_read_connection(
11596 conn: &Connection,
11597 request: &SnapshotReadRequest<'_>,
11598) -> Result<SnapshotRead, QueryReadFailure> {
11599 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
11600 let corrupt = |message: &str| QueryReadFailure {
11601 code: Some("sync.local_corrupt"),
11602 sqlite_code: None,
11603 sqlite_message: None,
11604 rollback_failure: None,
11605 message: message.to_owned(),
11606 };
11607 let stored_failure = |error: rusqlite::Error, message: &str| match error {
11608 rusqlite::Error::Utf8Error(..)
11609 | rusqlite::Error::InvalidColumnType(..)
11610 | rusqlite::Error::FromSqlConversionFailure(..)
11611 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
11612 _ => QueryReadFailure::from(error),
11613 };
11614 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
11615 let revision: Option<String> = conn
11618 .query_row(
11619 "SELECT value FROM _syncular_meta WHERE key = ?1",
11620 rusqlite::params![LOCAL_REVISION_KEY],
11621 |row| row.get(0),
11622 )
11623 .optional()
11624 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
11625 let revision = match revision {
11626 None => "0".to_owned(),
11627 Some(value)
11628 if value == "0"
11629 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
11630 && value.bytes().all(|b| b.is_ascii_digit())
11631 && value.parse::<u64>().is_ok()) =>
11632 {
11633 value
11634 }
11635 Some(_) => return Err(corrupt("persisted local revision is invalid")),
11636 };
11637 let mut queries = Vec::with_capacity(request.statements.len());
11638 for statement in &request.statements {
11639 queries.push(query_connection(conn, statement.sql, statement.params)?);
11640 #[cfg(test)]
11641 run_snapshot_read_hook();
11642 }
11643 let mut pending = Vec::new();
11644 let mut missing = Vec::new();
11645 for requested in request.coverage {
11646 let base_key = window_base_key(&requested.base);
11647 let state = persisted_window_state(conn, &requested.base)?;
11648 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
11649 let reference = WindowUnitRef {
11650 base_key: base_key.clone(),
11651 unit: unit.clone(),
11652 };
11653 if !state.units.iter().any(|held| held == &unit) {
11654 missing.push(reference);
11655 } else if state.pending.iter().any(|held| held == &unit) {
11656 pending.push(reference);
11657 }
11658 }
11659 }
11660 let mut subscriptions = Vec::with_capacity(request.subscriptions.len());
11661 for id in &request.subscriptions {
11662 let row: Option<(String, String)> = conn
11663 .query_row(
11664 "SELECT tbl, state_json FROM _syncular_subscriptions WHERE id = ?1",
11665 rusqlite::params![id],
11666 |row| Ok((row.get(0)?, row.get(1)?)),
11667 )
11668 .optional()
11669 .map_err(|error| {
11670 stored_failure(error, "persisted subscription state is invalid")
11671 })?;
11672 let Some((table, raw)) = row else {
11673 subscriptions.push(SubscriptionCatchup::Unknown { id: id.clone() });
11674 continue;
11675 };
11676 let state: Value = serde_json::from_str(&raw)
11679 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
11680 let status = match state.get("status").and_then(Value::as_str) {
11681 Some(value @ ("active" | "revoked" | "failed")) => value,
11682 _ => return Err(corrupt("persisted subscription state is invalid")),
11683 };
11684 let Some(cursor) = state.get("cursor").and_then(Value::as_i64) else {
11685 return Err(corrupt("persisted subscription state is invalid"));
11686 };
11687 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
11688 return Err(corrupt("persisted subscription state is invalid"));
11689 }
11690 let effective_scopes = state
11691 .get("effectiveScopes")
11692 .filter(|value| !value.is_null());
11693 if let Some(scopes) = effective_scopes {
11694 json_to_scope_map(scopes)
11695 .map_err(|_| corrupt("persisted subscription state is invalid"))?;
11696 }
11697 let has_resume_token = match state.get("bootstrapState") {
11698 None | Some(Value::Null) => false,
11699 Some(Value::String(_)) => true,
11700 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
11701 };
11702 let reason_code = match state.get("reasonCode") {
11705 None | Some(Value::Null) => None,
11706 Some(Value::String(value)) => Some(value.clone()),
11707 Some(_) => return Err(corrupt("persisted subscription state is invalid")),
11708 };
11709 let active = status == "active";
11710 subscriptions.push(SubscriptionCatchup::Known {
11711 id: id.clone(),
11712 table,
11713 status: status.to_owned(),
11714 cursor,
11715 has_resume_token,
11716 bootstrap_complete: active && cursor >= 0 && !has_resume_token,
11717 known_pending_pages: active && (has_resume_token || cursor < 0),
11718 effective_scopes: effective_scopes.cloned(),
11719 reason_code,
11720 });
11721 }
11722 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
11723 for id in &request.commit_ids {
11724 let pending: bool = conn.query_row(
11725 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
11726 rusqlite::params![id],
11727 |row| row.get(0),
11728 )?;
11729 if pending {
11730 deliveries.push(CommitDelivery::Pending {
11731 client_commit_id: id.clone(),
11732 });
11733 continue;
11734 }
11735 let row = conn
11736 .query_row(
11737 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
11738 resolution, resolved_at_ms, replacement_client_commit_id
11739 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
11740 rusqlite::params![id],
11741 StoredCommitOutcomeRow::from_row,
11742 )
11743 .optional()
11744 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
11745 deliveries.push(match row {
11746 Some(row) => CommitDelivery::Known {
11747 client_commit_id: id.clone(),
11748 outcome: SyncClient::outcome_from_row(row)
11749 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
11750 },
11751 None => CommitDelivery::Unknown {
11752 client_commit_id: id.clone(),
11753 },
11754 });
11755 }
11756 Ok(SnapshotRead {
11757 revision,
11758 queries,
11759 coverage: CoverageSnapshot {
11760 complete: pending.is_empty() && missing.is_empty(),
11761 pending,
11762 missing,
11763 },
11764 subscriptions,
11765 deliveries,
11766 })
11767 })();
11768 match result {
11769 Ok(read) => {
11770 conn.execute_batch("RELEASE syncular_snapshot_read")?;
11771 Ok(read)
11772 }
11773 Err(mut error) => {
11774 if let Err(rollback) = conn
11775 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
11776 {
11777 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
11778 }
11779 Err(error)
11780 }
11781 }
11782}
11783
11784fn snapshot_connection(
11785 conn: &Connection,
11786 sql: &str,
11787 params: &[Value],
11788 coverage: &[WindowCoverage],
11789) -> Result<QuerySnapshot, QueryReadFailure> {
11790 let read = snapshot_read_connection(
11791 conn,
11792 &SnapshotReadRequest {
11793 statements: vec![SnapshotStatement { sql, params }],
11794 coverage,
11795 subscriptions: Vec::new(),
11796 commit_ids: Vec::new(),
11797 },
11798 )?;
11799 Ok(QuerySnapshot {
11800 revision: read.revision,
11801 rows: read
11802 .queries
11803 .into_iter()
11804 .next()
11805 .expect("single-query snapshot returns one result"),
11806 coverage: read.coverage,
11807 })
11808}
11809
11810pub struct FileQuerySnapshotReader {
11815 path: String,
11816 conn: Option<Connection>,
11817}
11818
11819impl FileQuerySnapshotReader {
11820 #[must_use]
11821 pub fn new(path: impl Into<String>) -> Self {
11822 Self {
11823 path: path.into(),
11824 conn: None,
11825 }
11826 }
11827
11828 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
11829 if self.conn.is_none() {
11830 let conn = Connection::open_with_flags(
11831 &self.path,
11832 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
11833 )
11834 .map_err(QueryReadFailure::from)?;
11835 conn.busy_timeout(std::time::Duration::from_millis(250))
11836 .map_err(QueryReadFailure::from)?;
11837 self.conn = Some(conn);
11838 }
11839 self.conn
11840 .as_ref()
11841 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
11842 }
11843
11844 pub fn snapshot_read(
11847 &mut self,
11848 request: &SnapshotReadRequest<'_>,
11849 ) -> Result<SnapshotRead, QueryReadFailure> {
11850 snapshot_read_connection(self.connection()?, request)
11851 }
11852
11853 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
11856 Ok(self
11857 .snapshot_read(&SnapshotReadRequest::default())?
11858 .revision)
11859 }
11860
11861 pub fn subscription_catchup(
11863 &mut self,
11864 id: &str,
11865 ) -> Result<SubscriptionCatchup, QueryReadFailure> {
11866 let read = self.snapshot_read(&SnapshotReadRequest {
11867 subscriptions: vec![id.to_owned()],
11868 ..Default::default()
11869 })?;
11870 Ok(read
11871 .subscriptions
11872 .into_iter()
11873 .next()
11874 .expect("one requested catch-up"))
11875 }
11876
11877 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
11880 let read = self.snapshot_read(&SnapshotReadRequest {
11881 commit_ids: vec![id.to_owned()],
11882 ..Default::default()
11883 })?;
11884 Ok(read
11885 .deliveries
11886 .into_iter()
11887 .next()
11888 .expect("one requested delivery"))
11889 }
11890
11891 pub fn query_snapshot(
11894 &mut self,
11895 sql: &str,
11896 params: &[Value],
11897 coverage: &[WindowCoverage],
11898 ) -> Result<QuerySnapshot, QueryReadFailure> {
11899 snapshot_connection(self.connection()?, sql, params, coverage)
11900 }
11901}
11902
11903fn validate_authority_reads(
11904 schema: &ClientSchema,
11905 declarations: &[crate::api::AuthorityReadDeclaration],
11906) -> Result<(), String> {
11907 let mut seen = BTreeSet::new();
11908 for read in declarations {
11909 let table = schema
11910 .table(&read.table)
11911 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
11912 let plain = |name: &str| {
11913 table.columns.iter().enumerate().any(|(i, c)| {
11914 c.name == name
11915 && name != "authority_version"
11916 && !name.starts_with("_sync")
11917 && !table.encrypted_columns.iter().any(|e| e.index == i)
11918 && matches!(
11919 c.ty,
11920 ssp2::segment::ColumnType::String
11921 | ssp2::segment::ColumnType::Integer
11922 | ssp2::segment::ColumnType::Float
11923 | ssp2::segment::ColumnType::Boolean
11924 | ssp2::segment::ColumnType::Json
11925 )
11926 })
11927 };
11928 if !seen.insert(&read.table)
11929 || read.columns.is_empty()
11930 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
11931 || !read.columns.contains(&table.primary_key)
11932 || !read.columns.iter().all(|c| plain(c))
11933 || read.scopes.is_empty()
11934 || !read.scopes.iter().all(|(v, values)| {
11935 table
11936 .scope_column(v)
11937 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
11938 && !values.is_empty()
11939 && values.iter().all(|v| !v.is_empty() && v != "*")
11940 })
11941 {
11942 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
11943 }
11944 }
11945 Ok(())
11946}
11947
11948fn schema_marker_read_failure(error: rusqlite::Error) -> String {
11953 let failure = QueryReadFailure::from(error);
11954 if failure.code.is_some() {
11955 failure.to_string()
11956 } else {
11957 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
11958 }
11959}
11960
11961fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
11965 if limits.outcome_retention_max_entries == Some(0) {
11966 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
11967 }
11968 if [
11969 limits.max_push_commits_per_request,
11970 limits.max_push_operations_per_request,
11971 limits.max_push_request_bytes,
11972 ]
11973 .into_iter()
11974 .flatten()
11975 .any(|value| value == 0 || value > u32::MAX as usize)
11976 {
11977 return Err(
11978 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
11979 );
11980 }
11981 Ok(())
11982}
11983
11984fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
11985 let corrupt =
11986 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
11987 let table_type: Option<String> = conn
11988 .query_row(
11989 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
11990 [],
11991 |row| row.get(0),
11992 )
11993 .optional()
11994 .map_err(schema_marker_read_failure)?;
11995 let Some(table_type) = table_type else {
11996 return Ok(None);
11997 };
11998 if table_type != "table" {
11999 return Err(corrupt());
12000 }
12001 let mut statement = conn
12005 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
12006 .map_err(schema_marker_read_failure)?;
12007 let mut rows = statement
12008 .query([LOCAL_SCHEMA_VERSION_KEY])
12009 .map_err(schema_marker_read_failure)?;
12010 let value: Option<String> = rows
12011 .next()
12012 .map_err(schema_marker_read_failure)?
12013 .map(|row| row.get(0))
12014 .transpose()
12015 .map_err(schema_marker_read_failure)?;
12016 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
12017 return Err(corrupt());
12018 }
12019 let Some(value) = value else {
12020 let has_descriptor = conn
12021 .query_row(
12022 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
12023 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
12024 |_| Ok(()),
12025 )
12026 .optional()
12027 .map_err(schema_marker_read_failure)?
12028 .is_some();
12029 if has_descriptor {
12030 return Err(corrupt());
12031 }
12032 return Ok(None);
12033 };
12034 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
12035 || !value.bytes().all(|b| b.is_ascii_digit())
12036 {
12037 return Err(corrupt());
12038 }
12039 let version = value.parse::<i32>().map_err(|_| corrupt())?;
12040 if version > requested {
12041 return Err(
12042 "client.schema_downgrade: persisted local schema is newer than the requested schema"
12043 .to_owned(),
12044 );
12045 }
12046 Ok(Some(version))
12047}
12048
12049#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
12053fn prepare_wasm_replica(path: &str) -> Result<(), String> {
12054 if path.is_empty() || path == ":memory:" {
12055 return Ok(());
12056 }
12057 crate::wasm_vfs::disable_foreign_lock_check()
12058}
12059
12060#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
12061fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
12062 Ok(())
12063}
12064
12065impl SyncClient {
12066 pub fn new_with_identity(
12067 client_id: Option<String>,
12068 schema_json: &Value,
12069 limits: ClientLimits,
12070 ) -> Result<Self, String> {
12071 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
12072 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
12073 Self::with_connection(resolved, schema_json, limits, conn)
12074 }
12075
12076 pub fn new(
12077 client_id: String,
12078 schema_json: &Value,
12079 limits: ClientLimits,
12080 ) -> Result<Self, String> {
12081 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
12082 Self::with_connection(client_id, schema_json, limits, conn)
12083 }
12084
12085 pub fn open_path(
12091 client_id: String,
12092 schema_json: &Value,
12093 limits: ClientLimits,
12094 path: &str,
12095 ) -> Result<Self, String> {
12096 validate_client_limits(&limits)?;
12097 parse_schema_json(schema_json)?;
12098 prepare_wasm_replica(path)?;
12099 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
12100 Self::with_connection(client_id, schema_json, limits, conn)
12101 }
12102
12103 pub fn open_path_with_identity(
12107 client_id: Option<String>,
12108 schema_json: &Value,
12109 limits: ClientLimits,
12110 path: &str,
12111 ) -> Result<Self, String> {
12112 Self::open_path_with_identity_journal(
12113 client_id,
12114 schema_json,
12115 limits,
12116 path,
12117 !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
12118 )
12119 }
12120
12121 fn open_path_with_identity_journal(
12122 client_id: Option<String>,
12123 schema_json: &Value,
12124 limits: ClientLimits,
12125 path: &str,
12126 require_wal: bool,
12127 ) -> Result<Self, String> {
12128 validate_client_limits(&limits)?;
12129 parse_schema_json(schema_json)?;
12132 prepare_wasm_replica(path)?;
12133 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
12134 conn.busy_timeout(std::time::Duration::from_millis(250))
12142 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
12143 conn.pragma_update(None, "synchronous", "FULL")
12148 .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
12149 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
12150 if require_wal {
12154 let journal_mode: String = client
12155 .conn
12156 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
12157 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
12158 if journal_mode != "wal" {
12159 return Err(
12160 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
12161 );
12162 }
12163 }
12164 Ok(client)
12165 }
12166
12167 pub fn reset_replica(path: &str) -> Result<(), String> {
12181 if path.is_empty() || path == ":memory:" {
12182 return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
12183 }
12184 let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
12188 if vfs.is_null() {
12189 return Err("client.storage_io: no SQLite VFS is registered".to_owned());
12190 }
12191 let delete = unsafe { (*vfs).xDelete }
12192 .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
12193 for suffix in ["-wal", "-shm", "-journal", ""] {
12194 let file_name = format!("{path}{suffix}");
12195 let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
12196 format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
12197 })?;
12198 let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
12201 if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
12204 {
12205 return Err(format!(
12206 "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
12207 ));
12208 }
12209 }
12210 Ok(())
12211 }
12212
12213 #[must_use]
12214 pub fn client_id(&self) -> &str {
12215 &self.client_id
12216 }
12217
12218 pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> String {
12225 self.disconnect_realtime(transport);
12226 let client_id = uuid::Uuid::new_v4().to_string();
12227 self.client_id.clone_from(&client_id);
12228 self.set_meta(CLIENT_ID_KEY, &client_id);
12229 self.round_failure = None;
12230 client_id
12231 }
12232
12233 pub fn with_connection(
12240 client_id: String,
12241 schema_json: &Value,
12242 limits: ClientLimits,
12243 conn: Connection,
12244 ) -> Result<Self, String> {
12245 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
12246 }
12247
12248 fn with_connection_identity(
12251 client_id: Option<String>,
12252 schema_json: &Value,
12253 limits: ClientLimits,
12254 conn: Connection,
12255 ) -> Result<Self, String> {
12256 validate_client_limits(&limits)?;
12257 let schema = parse_schema_json(schema_json)?;
12258 validate_authority_reads(&schema, &limits.authority_reads)?;
12259 let previous_version = match limits.previous_version_context {
12262 Some(config) => {
12263 config.validate()?;
12264 Some(config)
12265 }
12266 None => None,
12267 };
12268 let mut client = SyncClient {
12269 #[cfg(feature = "bench-internals")]
12270 benchmark_phases: Recorder::default(),
12271 conn,
12272 schema,
12273 client_id: client_id
12274 .clone()
12275 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
12276 limits,
12277 subs: Vec::new(),
12278 outbox: Vec::new(),
12279 failed_commits: Vec::new(),
12280 retain_failed_commits: false,
12281 conflicts: Vec::new(),
12282 rejections: Vec::new(),
12283 schema_floor: None,
12284 lease_state: None,
12285 stopped: false,
12286 upgrading: false,
12287 sync_needed: false,
12288 realtime_policy: RealtimePolicy::Optional,
12289 realtime_state: RealtimeState::Disconnected,
12290 realtime_reason_code: None,
12291 realtime_retry_delay_ms: None,
12292 presence: HashMap::new(),
12293 now_ms: None,
12294 encryption: crate::values::EncryptionConfig::default(),
12295 security_preflight: false,
12296 security_epoch: 0,
12297 transport_enabled: true,
12298 transport_epoch: 0,
12299 subscription_epoch: 0,
12300 previous_version,
12301 insert_sql: RefCell::new(HashMap::new()),
12302 overlay_dirty: OverlayDirty::clean(),
12303 storage_failure: RefCell::new(None),
12304 #[cfg(test)]
12305 overlay_rebuild_count: Cell::new(0),
12306 #[cfg(test)]
12307 acknowledged_replay_count: Cell::new(0),
12308 #[cfg(test)]
12309 outcome_prune_count: Cell::new(0),
12310 progress: ProgressObserver::default(),
12311 change_queue: VecDeque::new(),
12312 sync_intent_queue: VecDeque::new(),
12313 retry_delay_ms: 250,
12314 round_retry_delay_ms: None,
12315 round_failure: None,
12316 active_round: None,
12317 last_round: None,
12318 last_change: None,
12319 query_failures: Vec::new(),
12320 coalesced: None,
12321 coalesced_failure: None,
12322 };
12323 client
12327 .conn
12328 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
12329 client
12335 .conn
12336 .execute_batch("BEGIN IMMEDIATE")
12337 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
12338 let startup = (|| -> Result<(), String> {
12339 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
12342 client.create_bookkeeping_tables()?;
12345 match client.get_meta(CLIENT_ID_KEY) {
12346 Some(existing) => match client_id {
12347 Some(requested) if existing != requested => {
12348 return Err(format!(
12349 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
12350 ))
12351 }
12352 _ => client.client_id = existing,
12353 },
12354 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
12355 }
12356 client.restore_persisted_state()?;
12357 client.failed_commits = client.load_failed_commits()?;
12358 match marker {
12359 None => {
12360 client.create_synced_tables()?;
12361 client.persist_schema_version()?;
12362 client.save_subscription_scope_schema();
12363 }
12364 Some(version) if version == client.schema.version => {
12365 client.create_synced_tables()?;
12366 set_local_schema_descriptor(&client.conn, &client.schema)
12370 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
12371 client.save_subscription_scope_schema();
12372 }
12373 Some(_) => client.run_schema_reset()?,
12374 }
12375 client.reconcile_previous_version_at_boot()?;
12381 client.clear_satisfied_persisted_schema_floor();
12382 client.prune_unknown_subscriptions(false)?;
12383 client.drop_unencodable_outbox(false, None)?;
12392 if marker == Some(client.schema.version)
12393 && (!client.outbox.is_empty()
12394 || !client.failed_commits.is_empty()
12395 || client.has_acknowledged_rows())
12396 {
12397 client.overlay_dirty.set(true);
12401 client.rebuild_overlay()?;
12402 }
12403 client.enqueue_startup_sync_if_needed()?;
12410 Ok(())
12411 })();
12412 match startup {
12413 Ok(()) => match client.conn.execute_batch("COMMIT") {
12414 Ok(()) => Ok(client),
12415 Err(error) => {
12416 let failure = Self::sqlite_failure(&client.storage_failure, error);
12417 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
12418 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
12420 }
12421 Err(failure)
12422 }
12423 },
12424 Err(error) => {
12425 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
12426 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
12428 }
12429 Err(error)
12430 }
12431 }
12432 }
12433
12434 pub fn set_now_ms(&mut self, now_ms: i64) {
12437 self.now_ms = Some(now_ms);
12438 }
12439
12440 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
12444 self.encryption = encryption;
12445 }
12446
12447 #[must_use]
12448 pub fn security_lifecycle(&self) -> &'static str {
12449 if self.security_preflight {
12450 "preflight"
12451 } else {
12452 "active"
12453 }
12454 }
12455
12456 #[must_use]
12457 pub fn security_preflight(&self) -> bool {
12458 self.security_preflight
12459 }
12460
12461 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
12463 if self.transport_enabled != enabled {
12464 self.transport_enabled = enabled;
12465 self.transport_epoch = self.transport_epoch.wrapping_add(1);
12466 self.sync_intent_queue.clear();
12467 self.sync_intent_queue
12468 .push_back(if enabled && !self.security_preflight {
12469 SyncIntent::Interactive
12470 } else {
12471 SyncIntent::None
12472 });
12473 }
12474 if !enabled && self.active_round.is_none() {
12476 self.disconnect_realtime(transport);
12477 }
12478 }
12479
12480 pub fn transport_enabled(&self) -> bool {
12481 self.transport_enabled
12482 }
12483
12484 pub fn begin_security_preflight(&mut self) {
12491 self.seal_security_on_teardown();
12492 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
12495 }
12496
12497 pub fn seal_security_on_teardown(&mut self) {
12507 self.cancel_sync_round();
12508 self.security_preflight = true;
12509 self.security_epoch = self.security_epoch.wrapping_add(1);
12510 self.encryption = crate::values::EncryptionConfig::default();
12511 self.sync_intent_queue.clear();
12512 }
12513
12514 pub fn activate_security(
12516 &mut self,
12517 encryption: crate::values::EncryptionConfig,
12518 ) -> Result<(), String> {
12519 if !self.security_preflight {
12520 return Err(
12521 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
12522 );
12523 }
12524 self.enqueue_startup_sync_if_needed()?;
12525 self.encryption = encryption;
12526 self.security_preflight = false;
12527 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
12528 Ok(())
12529 }
12530
12531 fn clock_now_ms(&self) -> i64 {
12532 self.now_ms.unwrap_or_else(crate::wall_clock_ms)
12533 }
12534
12535 fn create_bookkeeping_tables(&self) -> Result<(), String> {
12536 self.conn
12538 .execute_batch(
12539 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
12540 seq INTEGER PRIMARY KEY AUTOINCREMENT,
12541 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
12542 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
12543 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
12544 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
12545 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
12546 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
12547 PRIMARY KEY(commit_id, idx));
12548 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
12549 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
12550 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
12551 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
12552 seq INTEGER PRIMARY KEY AUTOINCREMENT,
12553 client_commit_id TEXT NOT NULL UNIQUE,
12554 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
12555 recorded_at_ms INTEGER NOT NULL,
12556 results_json TEXT NOT NULL,
12557 operations_json TEXT,
12558 resolution TEXT NOT NULL DEFAULT 'active'
12559 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
12560 resolved_at_ms INTEGER,
12561 replacement_client_commit_id TEXT);
12562 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
12563 ON _syncular_commit_outcomes(resolution, seq);
12564 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
12565 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
12566 CREATE TABLE IF NOT EXISTS _syncular_meta (
12567 key TEXT PRIMARY KEY, value TEXT NOT NULL);
12568 CREATE TABLE IF NOT EXISTS _syncular_windows (
12569 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
12570 PRIMARY KEY (base, unit));
12571 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
12572 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
12573 effective_scopes TEXT NOT NULL);
12574 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
12575 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
12576 PRIMARY KEY(commit_id, blob_id));
12577 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
12578 ON _syncular_blob_commit_refs(blob_id);",
12579 )
12580 .map_err(|e| e.to_string())?;
12581 let _ = self
12584 .conn
12585 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
12586 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
12587 self.set_meta(LOCAL_REVISION_KEY, "0");
12588 }
12589 if self.schema_has_blobs() {
12592 let exists = self
12593 .conn
12594 .query_row(
12595 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
12596 [],
12597 |row| row.get::<_, bool>(0),
12598 )
12599 .map_err(|e| e.to_string())?;
12600 if exists {
12601 let mut statement = self
12602 .conn
12603 .prepare("PRAGMA table_info(_syncular_blobs)")
12604 .map_err(|e| e.to_string())?;
12605 let mut columns = statement
12606 .query_map([], |row| row.get::<_, String>(1))
12607 .map_err(|e| e.to_string())?
12608 .collect::<Result<Vec<_>, _>>()
12609 .map_err(|e| e.to_string())?;
12610 columns.sort();
12611 let mut expected = vec![
12612 "blob_id",
12613 "byte_length",
12614 "bytes",
12615 "created_at_ms",
12616 "media_type",
12617 ];
12618 expected.sort_unstable();
12619 if columns != expected {
12620 return Err(
12621 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
12622 );
12623 }
12624 } else {
12625 self.conn
12626 .execute_batch(
12627 "CREATE TABLE _syncular_blobs(
12628 blob_id TEXT PRIMARY KEY,
12629 bytes BLOB NOT NULL,
12630 byte_length INTEGER NOT NULL,
12631 media_type TEXT,
12632 created_at_ms INTEGER NOT NULL);",
12633 )
12634 .map_err(|e| e.to_string())?;
12635 }
12636 self.conn
12637 .execute_batch(
12638 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
12639 blob_id TEXT PRIMARY KEY,
12640 media_type TEXT,
12641 created_at_ms INTEGER NOT NULL);",
12642 )
12643 .map_err(|e| e.to_string())?;
12644 }
12645 Ok(())
12646 }
12647
12648 fn schema_has_blobs(&self) -> bool {
12650 self.schema
12651 .tables
12652 .iter()
12653 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
12654 }
12655
12656 fn get_meta(&self, key: &str) -> Option<String> {
12659 meta_get(&self.conn, key)
12660 }
12661
12662 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
12663 self.conn
12664 .query_row(
12665 "SELECT value FROM _syncular_meta WHERE key = ?1",
12666 rusqlite::params![key],
12667 |row| row.get::<_, String>(0),
12668 )
12669 .optional()
12670 .map_err(|_| {
12671 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
12672 })
12673 }
12674
12675 fn persist_schema_version(&self) -> Result<(), String> {
12678 self.conn
12679 .execute(
12680 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
12681 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
12682 )
12683 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12684 set_local_schema_descriptor(&self.conn, &self.schema)
12685 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12686 }
12687
12688 pub(crate) fn set_meta(&self, key: &str, value: &str) {
12689 meta_set(&self.conn, key, value);
12690 }
12691
12692 fn delete_meta(&self, key: &str) {
12693 meta_delete(&self.conn, key);
12694 }
12695
12696 fn load_outbox(&self) -> Result<Vec<OutboxCommit>, String> {
12698 let mut stmt = self
12699 .conn
12700 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
12701 .map_err(|error| error.to_string())?;
12702 let rows = stmt
12703 .query_map([], |row| {
12704 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12705 })
12706 .map_err(|error| error.to_string())?;
12707 let mut commits = Vec::new();
12708 for row in rows {
12709 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
12710 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
12711 format!("invalid persisted outbox {client_commit_id:?}: {error}")
12712 })?;
12713 let mut ops = Vec::with_capacity(entries.len());
12714 for entry in entries {
12715 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
12716 ops.push(OutboxOp {
12717 upsert: op == "upsert",
12718 table: entry
12719 .get("table")
12720 .and_then(Value::as_str)
12721 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
12722 .to_owned(),
12723 row_id: entry
12724 .get("rowId")
12725 .and_then(Value::as_str)
12726 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
12727 .to_owned(),
12728 base_version: entry.get("baseVersion").and_then(Value::as_i64),
12729 values: entry.get("values").and_then(Value::as_object).cloned(),
12732 insert: entry.get("insert") == Some(&Value::Bool(true)),
12733 });
12734 }
12735 commits.push(OutboxCommit {
12736 client_commit_id,
12737 ops,
12738 });
12739 }
12740 Ok(commits)
12741 }
12742
12743 fn restore_persisted_state(&mut self) -> Result<(), String> {
12744 self.subs = {
12745 let mut stmt = self
12746 .conn
12747 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
12748 .map_err(|error| error.to_string())?;
12749 let rows = stmt
12750 .query_map([], |row| {
12751 Ok((
12752 row.get::<_, String>(0)?,
12753 row.get::<_, String>(1)?,
12754 row.get::<_, String>(2)?,
12755 ))
12756 })
12757 .map_err(|error| error.to_string())?;
12758 let mut subscriptions = Vec::new();
12759 for row in rows {
12760 let (id, table, raw) = row.map_err(|error| error.to_string())?;
12761 let state: Value = serde_json::from_str(&raw)
12762 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
12763 let requested = json_to_scope_map(
12764 state.get("requested").unwrap_or(&Value::Object(Map::new())),
12765 )?;
12766 let effective = state
12767 .get("effectiveScopes")
12768 .filter(|value| !value.is_null())
12769 .map(json_to_scope_map)
12770 .transpose()?;
12771 subscriptions.push(Subscription {
12772 id,
12773 table,
12774 requested,
12775 params: state
12776 .get("params")
12777 .and_then(Value::as_str)
12778 .map(str::to_owned),
12779 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
12780 bootstrap_state: state
12781 .get("bootstrapState")
12782 .and_then(Value::as_str)
12783 .map(str::to_owned),
12784 state: SubState::parse(
12785 state
12786 .get("status")
12787 .and_then(Value::as_str)
12788 .unwrap_or("active"),
12789 ),
12790 reason_code: state
12791 .get("reasonCode")
12792 .and_then(Value::as_str)
12793 .map(str::to_owned),
12794 effective,
12795 synced_once: state
12796 .get("syncedOnce")
12797 .and_then(Value::as_bool)
12798 .unwrap_or(false),
12799 });
12800 }
12801 subscriptions
12802 };
12803
12804 self.outbox = self.load_outbox()?;
12805
12806 self.prune_commit_outcomes()?;
12807 let active = self.commit_outcomes(CommitOutcomeQuery {
12808 active_only: true,
12809 ..CommitOutcomeQuery::default()
12810 })?;
12811 self.conflicts = active
12812 .iter()
12813 .flat_map(|outcome| outcome.results.iter())
12814 .filter_map(|result| match result {
12815 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
12816 _ => None,
12817 })
12818 .collect();
12819 self.rejections = active
12820 .iter()
12821 .flat_map(|outcome| outcome.results.iter())
12822 .filter_map(|result| match result {
12823 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
12824 _ => None,
12825 })
12826 .collect();
12827
12828 self.lease_state = self
12829 .get_meta(LEASE_STATE_KEY)
12830 .map(|raw| serde_json::from_str(&raw))
12831 .transpose()
12832 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
12833 self.schema_floor = self
12834 .get_meta(SCHEMA_FLOOR_KEY)
12835 .map(|raw| serde_json::from_str(&raw))
12836 .transpose()
12837 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
12838 self.stopped = self.schema_floor.is_some();
12839 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
12842 self.security_preflight = true;
12843 }
12844 Ok(())
12845 }
12846
12847 fn save_subscription_scope_schema(&self) {
12848 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
12849 .schema
12850 .tables
12851 .iter()
12852 .map(|table| {
12853 (
12854 table.name.clone(),
12855 table
12856 .scope_variables
12857 .iter()
12858 .map(|scope| {
12859 (
12860 scope.variable.clone(),
12861 [scope.prefix.clone(), scope.column.clone()],
12862 )
12863 })
12864 .collect(),
12865 )
12866 })
12867 .collect();
12868 self.set_meta(
12869 "subscriptionScopeSchema",
12870 &serde_json::to_string(&declarations).expect("scope schema serialization"),
12871 );
12872 }
12873
12874 fn prune_unknown_subscriptions(&mut self, bump: bool) -> Result<(), String> {
12876 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
12877 self.get_meta("subscriptionScopeSchema")
12878 .map(|raw| serde_json::from_str(&raw))
12879 .transpose()
12880 .map_err(|error| format!("invalid persisted scope schema: {error}"))?
12881 } else {
12882 None
12883 };
12884 let compatible = |table_name: &str, requested: &[(String, Vec<String>)]| {
12885 let Some(table) = self.schema.table(table_name) else {
12886 return false;
12887 };
12888 if requested
12889 .iter()
12890 .any(|(variable, _)| table.scope_column(variable).is_none())
12891 {
12892 return false;
12893 }
12894 if !bump {
12895 return true;
12896 }
12897 let Some(old) = previous.as_ref().and_then(|tables| tables.get(table_name)) else {
12898 return false;
12899 };
12900 let variables: BTreeSet<_> = if requested.is_empty() {
12901 old.keys()
12902 .cloned()
12903 .chain(
12904 table
12905 .scope_variables
12906 .iter()
12907 .map(|scope| scope.variable.clone()),
12908 )
12909 .collect()
12910 } else {
12911 requested
12912 .iter()
12913 .map(|(variable, _)| variable.clone())
12914 .collect()
12915 };
12916 variables.iter().all(|variable| {
12917 let current = table
12918 .scope_variables
12919 .iter()
12920 .find(|scope| &scope.variable == variable);
12921 match (old.get(variable), current) {
12922 (Some(old), Some(current)) => {
12923 old == &[current.prefix.clone(), current.column.clone()]
12924 }
12925 _ => false,
12926 }
12927 })
12928 };
12929 let mut stale_ids: BTreeSet<_> = self
12930 .subs
12931 .iter()
12932 .filter(|sub| !compatible(&sub.table, &sub.requested))
12933 .map(|sub| sub.id.clone())
12934 .collect();
12935 for (id, table, effective) in self.load_pending_evictions()? {
12936 if !compatible(&table, &effective) {
12937 stale_ids.insert(id);
12938 }
12939 }
12940 for id in &stale_ids {
12941 for sql in [
12942 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
12943 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
12944 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
12945 ] {
12946 self.conn
12947 .execute(sql, rusqlite::params![id])
12948 .map_err(|error| error.to_string())?;
12949 }
12950 }
12951 self.subs.retain(|sub| !stale_ids.contains(&sub.id));
12952 Ok(())
12953 }
12954
12955 #[must_use]
12956 pub fn local_revision(&self) -> u64 {
12957 self.get_meta(LOCAL_REVISION_KEY)
12958 .and_then(|value| value.parse().ok())
12959 .unwrap_or(0)
12960 }
12961
12962 pub fn snapshot_read(
12968 &mut self,
12969 request: &SnapshotReadRequest<'_>,
12970 owner: Option<QueryOwner<'_>>,
12971 ) -> Result<SnapshotRead, QueryReadFailure> {
12972 let result = snapshot_read_connection(&self.conn, request);
12973 if let Some(owner) = owner {
12974 self.record_query_read(owner, result.as_ref().err());
12975 }
12976 result
12977 }
12978
12979 #[must_use]
12980 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
12981 SyncStatusSnapshot {
12982 current_schema_version: self.schema.version,
12983 outbox: self.outbox.len(),
12984 upgrading: self.upgrading,
12985 lease_state: self.lease_state.clone(),
12986 schema_floor: self.schema_floor.clone(),
12987 sync_needed: self.sync_needed,
12988 previous_version_context: self.previous_version_status(),
12989 }
12990 }
12991
12992 fn previous_version_status(&self) -> PreviousVersionStatus {
12995 let Some(replica_path) = self.previous_version_replica_path() else {
12996 return PreviousVersionStatus {
12997 present: false,
12998 created_at_ms: None,
12999 };
13000 };
13001 match crate::previous_version::read_previous_version_record(&replica_path) {
13002 Ok(Some(record)) => PreviousVersionStatus {
13003 present: true,
13004 created_at_ms: Some(record.created_at_ms),
13005 },
13006 _ => PreviousVersionStatus {
13007 present: std::path::Path::new(
13008 &crate::previous_version::previous_version_container_path(&replica_path),
13009 )
13010 .exists(),
13011 created_at_ms: None,
13012 },
13013 }
13014 }
13015
13016 pub fn previous_version_snapshot(
13022 &self,
13023 spec: &PreviousVersionReadSpec,
13024 ) -> Result<PreviousVersionSnapshot, String> {
13025 let now_ms = self.clock_now_ms();
13026 crate::previous_version::previous_version_snapshot(
13027 &self.conn,
13028 self.previous_version_replica_path().as_deref(),
13029 self.previous_version.as_ref(),
13030 PreviousVersionLifecycle {
13031 lease_inactive: self.previous_version_lease_inactive(now_ms),
13032 scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
13033 coverage_complete: self.previous_version_coverage_complete(),
13034 },
13035 now_ms,
13036 self.schema.version,
13037 spec,
13038 )
13039 }
13040
13041 #[must_use]
13043 pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
13044 crate::previous_version::stored_previous_version_audit(&self.conn)
13045 }
13046
13047 pub fn previous_version_discard(
13051 &self,
13052 ) -> crate::previous_version::PreviousVersionDiscardOutcome {
13053 self.drop_previous_version()
13054 }
13055
13056 fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
13060 let absent = crate::previous_version::PreviousVersionDiscardOutcome {
13061 present: false,
13062 discarded: false,
13063 };
13064 let Some(replica_path) = self.previous_version_replica_path() else {
13065 return absent;
13066 };
13067 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
13068 .unwrap_or(absent)
13069 }
13070
13071 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
13073 let Some(lease) = &self.lease_state else {
13074 return false;
13075 };
13076 if lease.error_code.is_some() {
13077 return true;
13078 }
13079 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
13080 }
13081
13082 fn previous_version_coverage_complete(&self) -> bool {
13086 let active = self
13087 .subs
13088 .iter()
13089 .filter(|sub| sub.state == SubState::Active)
13090 .collect::<Vec<_>>();
13091 if active.is_empty() {
13092 return false;
13093 }
13094 active
13095 .iter()
13096 .all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
13097 }
13098
13099 fn previous_version_lifetime_check(&self) {
13103 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
13104 return;
13105 };
13106 let now_ms = self.clock_now_ms();
13107 if self.previous_version_lease_inactive(now_ms) {
13108 let _ = self.drop_previous_version();
13109 return;
13110 }
13111 let Some(replica_path) = self.previous_version_replica_path() else {
13112 return;
13113 };
13114 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
13115 else {
13116 return;
13117 };
13118 if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
13119 let _ = self.drop_previous_version();
13120 return;
13121 }
13122 if now_ms - record.created_at_ms > config.max_age_ms {
13123 let _ = self.drop_previous_version();
13124 return;
13125 }
13126 if self.previous_version_coverage_complete() {
13127 let _ = self.drop_previous_version();
13128 }
13129 }
13130
13131 pub fn diagnostics_snapshot(
13132 &self,
13133 request: &ClientDiagnosticsRequest,
13134 ) -> Result<ClientDiagnosticsSnapshot, String> {
13135 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
13136 return Err(format!(
13137 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
13138 ));
13139 }
13140 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
13141 for sub in &self.subs {
13142 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
13143 let complete =
13144 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
13145 let state = match sub.state {
13146 SubState::Revoked => "revoked",
13147 SubState::Failed => "failed",
13148 SubState::Active if reset => "reset",
13149 SubState::Active if complete => "complete",
13150 SubState::Active => "bootstrapping",
13151 };
13152 subscriptions.insert(
13153 sub.id.clone(),
13154 DiagnosticSubscription {
13155 id: sub.id.clone(),
13156 table: sub.table.clone(),
13157 state: state.to_owned(),
13158 complete,
13159 cursor: Some(sub.cursor),
13160 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
13161 },
13162 );
13163 }
13164 for expected in &request.expected_subscriptions {
13165 if expected.id.is_empty() || expected.table.is_empty() {
13166 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
13167 }
13168 if subscriptions
13169 .get(&expected.id)
13170 .is_some_and(|registered| registered.table != expected.table)
13171 {
13172 subscriptions.insert(
13173 expected.id.clone(),
13174 DiagnosticSubscription {
13175 id: expected.id.clone(),
13176 table: expected.table.clone(),
13177 state: "failed".to_owned(),
13178 complete: false,
13179 cursor: None,
13180 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
13181 },
13182 );
13183 } else {
13184 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
13185 DiagnosticSubscription {
13186 id: expected.id.clone(),
13187 table: expected.table.clone(),
13188 state: "unregistered".to_owned(),
13189 complete: false,
13190 cursor: None,
13191 reason_code: None,
13192 }
13193 });
13194 }
13195 }
13196 let mut ordered_subscriptions = Vec::new();
13197 let mut included = BTreeSet::new();
13198 for expected in &request.expected_subscriptions {
13199 if included.insert(expected.id.clone()) {
13200 if let Some(subscription) = subscriptions.get(&expected.id) {
13201 ordered_subscriptions.push(subscription.clone());
13202 }
13203 }
13204 }
13205 for (id, subscription) in &subscriptions {
13206 if included.insert(id.clone()) {
13207 ordered_subscriptions.push(subscription.clone());
13208 }
13209 }
13210 let subscriptions_truncated =
13211 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
13212 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
13213 let captured_at_ms = self.clock_now_ms();
13214 let lease = if let Some(error_code) = self
13215 .lease_state
13216 .as_ref()
13217 .and_then(|state| state.error_code.clone())
13218 {
13219 ClientDiagnosticsLease {
13220 state: "stopped".to_owned(),
13221 expires_at_ms: self
13222 .lease_state
13223 .as_ref()
13224 .and_then(|state| state.expires_at_ms),
13225 error_code: Some(Self::diagnostic_code(&error_code)),
13226 }
13227 } else if let Some(expires_at_ms) = self
13228 .lease_state
13229 .as_ref()
13230 .and_then(|state| state.expires_at_ms)
13231 {
13232 ClientDiagnosticsLease {
13233 state: if expires_at_ms <= captured_at_ms {
13234 "expired".to_owned()
13235 } else {
13236 "active".to_owned()
13237 },
13238 expires_at_ms: Some(expires_at_ms),
13239 error_code: None,
13240 }
13241 } else {
13242 ClientDiagnosticsLease {
13243 state: "none".to_owned(),
13244 expires_at_ms: None,
13245 error_code: None,
13246 }
13247 };
13248 let connectivity = match self.last_round.as_ref() {
13249 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
13250 Some(round) if round.status == "succeeded" => "online",
13251 Some(round)
13252 if round.status == "failed"
13253 && matches!(
13254 round.cause_kind,
13255 Some(
13256 crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
13257 )
13258 ) =>
13259 {
13260 "offline"
13261 }
13262 Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
13263 _ => "unknown",
13264 };
13265 Ok(ClientDiagnosticsSnapshot {
13266 version: CLIENT_DIAGNOSTICS_VERSION,
13267 captured_at_ms,
13268 host: ClientDiagnosticsHost {
13269 kind: "direct".to_owned(),
13270 role: "single".to_owned(),
13271 connectivity: connectivity.to_owned(),
13272 realtime: self.realtime_state,
13273 realtime_policy: self.realtime_policy,
13274 realtime_reason_code: self.realtime_reason_code.clone(),
13275 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
13276 },
13277 security_lifecycle: self.security_lifecycle().to_owned(),
13278 schema: ClientDiagnosticsSchema {
13279 current_version: self.schema.version,
13280 upgrading: self.upgrading,
13281 required_version: self
13282 .schema_floor
13283 .as_ref()
13284 .and_then(|floor| floor.required_schema_version),
13285 latest_version: self
13286 .schema_floor
13287 .as_ref()
13288 .and_then(|floor| floor.latest_schema_version),
13289 },
13290 replica: ClientDiagnosticsReplica {
13291 local_revision: self.local_revision().to_string(),
13292 sync_needed: self.sync_needed,
13293 pending_outbox: self.outbox.len(),
13294 },
13295 lease,
13296 subscriptions: ordered_subscriptions,
13297 subscriptions_truncated,
13298 last_round: self.last_round.clone(),
13299 last_change: self.last_change.clone(),
13300 storage: self.diagnostics_storage(),
13301 query_failures: self.query_failures.clone(),
13302 })
13303 }
13304
13305 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
13306 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
13307 let page_count: i64 = self
13308 .conn
13309 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
13310 let page_size: i64 = self
13311 .conn
13312 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
13313 let outbox_bytes: i64 = self
13315 .conn
13316 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
13317 .query_row([], |row| row.get(0))?;
13318 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
13319 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
13320 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
13321 let blob_bytes = if self.schema_has_blobs() {
13322 self.conn
13323 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
13324 .query_row([], |row| row.get(0))?
13325 } else {
13326 0
13327 };
13328 let pressure = self
13329 .limits
13330 .blob_cache_max_bytes
13331 .is_some_and(|limit| blob_bytes > limit);
13332 Ok(ClientDiagnosticsStorage {
13333 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
13334 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
13335 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
13336 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
13337 retained_outcome_entries: Some(outcome_entries.max(0)),
13338 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
13339 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
13340 })
13341 };
13342 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
13343 status: "unreadable".to_owned(),
13344 database_bytes_approx: None,
13345 pending_outbox_bytes_approx: None,
13346 retained_outcome_bytes_approx: None,
13347 retained_outcome_entries: None,
13348 blob_cache_bytes_approx: None,
13349 pressure_reason_code: None,
13350 })
13351 }
13352
13353 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
13354 self.change_queue.drain(..).collect()
13355 }
13356
13357 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
13358 let intents = self.sync_intent_queue.drain(..).collect();
13359 if self.transport_enabled {
13360 intents
13361 } else {
13362 vec![SyncIntent::None]
13363 }
13364 }
13365
13366 fn schedule_background_retry(&mut self) -> u64 {
13367 let delay_ms = self.retry_delay_ms;
13368 self.sync_intent_queue
13369 .push_back(SyncIntent::Background { delay_ms });
13370 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
13371 self.round_retry_delay_ms = Some(delay_ms);
13372 delay_ms
13373 }
13374
13375 fn reset_background_retry(&mut self) {
13376 self.retry_delay_ms = 250;
13377 }
13378
13379 fn retryable_transport_code(code: &str) -> bool {
13380 code == "transport.failed"
13381 || code == "transport.timeout"
13382 || code == "transport.unavailable"
13383 || code == "sync.transport_failed"
13384 }
13385
13386 fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
13391 matches!(
13392 details
13393 .as_ref()
13394 .and_then(|details| details.get("causeKind"))
13395 .and_then(serde_json::Value::as_str),
13396 Some("connect" | "network")
13397 )
13398 }
13399
13400 fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
13405 if ssp2::model::valid_round_failure_code(&error.code) {
13406 self.round_failure = Some(error.code.clone());
13407 }
13408 SectionError::Transport(error)
13409 }
13410
13411 fn retryable_failure_code(code: &str) -> bool {
13416 Self::retryable_transport_code(code)
13417 || matches!(
13418 code,
13419 "sync.auth_required"
13420 | "sync.auth_lease_required"
13421 | "sync.auth_lease_revoked"
13422 | "sync.internal_error"
13423 | "sync.idempotency_cache_miss"
13424 | "sync.schema_not_ready"
13425 | "sync.segment_expired"
13426 | "sync.rate_limited"
13427 | "sync.websocket_connection_limit"
13428 )
13429 }
13430
13431 fn diagnostic_code(code: &str) -> String {
13432 let valid = !code.is_empty()
13433 && code.len() <= 96
13434 && code.contains('.')
13435 && code
13436 .bytes()
13437 .next()
13438 .is_some_and(|byte| byte.is_ascii_lowercase())
13439 && code.bytes().all(|byte| {
13440 byte.is_ascii_lowercase()
13441 || byte.is_ascii_digit()
13442 || matches!(byte, b'.' | b'_' | b'-')
13443 });
13444 if valid {
13445 code.to_owned()
13446 } else {
13447 "client.unknown_failure".to_owned()
13448 }
13449 }
13450
13451 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
13452 if self.sync_needed != value {
13453 if self.begin_observation("syncular_status").is_ok() {
13454 self.sync_needed = value;
13455 let batch = ChangeAccumulator {
13456 status: true,
13457 ..ChangeAccumulator::default()
13458 };
13459 if self.finish_observation("syncular_status", batch).is_err() {
13460 self.rollback_observation("syncular_status");
13461 }
13462 } else {
13463 self.sync_needed = value;
13464 }
13465 }
13466 if value && interactive {
13467 self.sync_intent_queue.push_back(SyncIntent::Interactive);
13468 }
13469 }
13470
13471 fn sqlite_failure(
13472 storage_failure: &RefCell<Option<QueryReadFailure>>,
13473 error: rusqlite::Error,
13474 ) -> String {
13475 let message = error.to_string();
13476 let failure = QueryReadFailure::from(error);
13477 let code = failure.code;
13478 if code.is_some() && storage_failure.borrow().is_none() {
13479 *storage_failure.borrow_mut() = Some(failure);
13480 }
13481 match code {
13482 Some(code) => format!("{code}: {message}"),
13483 None => message,
13484 }
13485 }
13486
13487 fn begin_observation(&self, name: &str) -> Result<(), String> {
13488 if let Some(failure) = &self.coalesced_failure {
13489 return Err(failure.clone());
13490 }
13491 if self.coalesced.is_some() && self.conn.is_autocommit() {
13494 return Err("the coalesced apply transaction was rolled back".to_owned());
13495 }
13496 self.conn
13497 .execute_batch(&format!("SAVEPOINT {name}"))
13498 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13499 }
13500
13501 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
13504 if self.coalesced.is_some() || !self.conn.is_autocommit() {
13505 return Ok(());
13506 }
13507 self.conn
13508 .execute_batch("BEGIN")
13509 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13510 self.coalesced = Some(CoalescedApply {
13511 rows: 0,
13512 subs: self.subs.clone(),
13513 outbox: self.outbox.clone(),
13514 failed_commits: self.failed_commits.clone(),
13515 conflicts: self.conflicts.clone(),
13516 rejections: self.rejections.clone(),
13517 schema_floor: self.schema_floor.clone(),
13518 lease_state: self.lease_state.clone(),
13519 stopped: self.stopped,
13520 upgrading: self.upgrading,
13521 sync_needed: self.sync_needed,
13522 overlay_dirty: self.overlay_dirty.snapshot(),
13523 change_queue: self.change_queue.len(),
13524 sync_intent_queue: self.sync_intent_queue.len(),
13525 last_change: self.last_change.clone(),
13526 });
13527 Ok(())
13528 }
13529
13530 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
13533 let Some(open) = &self.coalesced else {
13534 return Ok(());
13535 };
13536 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
13537 self.commit_coalesced_apply()?;
13538 self.begin_coalesced_apply()?;
13539 }
13540 if let Some(open) = &mut self.coalesced {
13541 open.rows += rows;
13542 }
13543 Ok(())
13544 }
13545
13546 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
13551 let Some(open) = self.coalesced.take() else {
13552 return Ok(());
13553 };
13554 let committed = if self.conn.is_autocommit() {
13555 Err("the coalesced apply transaction was rolled back".to_owned())
13556 } else {
13557 self.conn
13558 .execute_batch("COMMIT")
13559 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13560 };
13561 let Err(error) = committed else {
13562 return Ok(());
13563 };
13564 if !self.conn.is_autocommit() {
13565 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
13566 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
13567 }
13568 }
13569 self.subs = open.subs;
13570 self.outbox = open.outbox;
13571 self.failed_commits = open.failed_commits;
13572 self.conflicts = open.conflicts;
13573 self.rejections = open.rejections;
13574 self.schema_floor = open.schema_floor;
13575 self.lease_state = open.lease_state;
13576 self.stopped = open.stopped;
13577 self.upgrading = open.upgrading;
13578 self.sync_needed = open.sync_needed;
13579 self.overlay_dirty.restore(open.overlay_dirty);
13580 self.change_queue.truncate(open.change_queue);
13581 self.sync_intent_queue.truncate(open.sync_intent_queue);
13582 self.last_change = open.last_change;
13583 self.coalesced_failure = Some(error.clone());
13584 Err(error)
13585 }
13586
13587 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
13589 let committed = self.commit_coalesced_apply();
13590 match self.coalesced_failure.take() {
13591 Some(failure) => Err(failure),
13592 None => committed,
13593 }
13594 }
13595
13596 fn rollback_observation(&self, name: &str) {
13597 if let Err(error) = self
13598 .conn
13599 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
13600 {
13601 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
13602 if failure.rollback_failure.is_none() {
13603 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
13604 }
13605 }
13606 }
13607 }
13608
13609 fn finish_observation(
13610 &mut self,
13611 name: &str,
13612 mut batch: ChangeAccumulator,
13613 ) -> Result<(), String> {
13614 #[cfg(feature = "bench-internals")]
13615 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
13616 if self.failed_commits.iter().any(|(commit, _)| {
13617 commit
13618 .ops
13619 .iter()
13620 .any(|operation| batch.tables.contains_key(&operation.table))
13621 }) {
13622 batch.outcomes = true;
13623 }
13624 if !batch.touched() {
13625 self.conn
13626 .execute_batch(&format!("RELEASE {name}"))
13627 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13628 return Ok(());
13629 }
13630 let revision = self
13631 .local_revision()
13632 .checked_add(1)
13633 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
13634 self.conn
13635 .execute(
13636 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
13637 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
13638 )
13639 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13640 let status = batch.status.then(|| self.status_snapshot());
13641 let event = ClientChangeBatch {
13642 revision: revision.to_string(),
13643 tables: batch
13644 .tables
13645 .into_iter()
13646 .map(|(table, scope_keys)| TableChange {
13647 table,
13648 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
13649 })
13650 .collect(),
13651 windows: batch
13652 .windows
13653 .into_iter()
13654 .map(|((base_key, table), units)| WindowChange {
13655 base_key,
13656 table,
13657 units: units.into_iter().collect(),
13658 })
13659 .collect(),
13660 status,
13661 conflicts_changed: batch.conflicts,
13662 rejections_changed: batch.rejections,
13663 outcomes_changed: batch.outcomes,
13664 applied_commits: Vec::new(),
13665 };
13666 self.conn
13667 .execute_batch(&format!("RELEASE {name}"))
13668 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13669 let mut diagnostic_tables = event
13670 .tables
13671 .iter()
13672 .map(|entry| entry.table.clone())
13673 .collect::<BTreeSet<_>>()
13674 .into_iter()
13675 .collect::<Vec<_>>();
13676 let mut diagnostic_windows = event
13677 .windows
13678 .iter()
13679 .map(|entry| entry.table.clone())
13680 .collect::<BTreeSet<_>>()
13681 .into_iter()
13682 .collect::<Vec<_>>();
13683 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
13684 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
13685 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
13686 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
13687 self.last_change = Some(DiagnosticLastChange {
13688 revision: event.revision.clone(),
13689 recorded_at_ms: self.clock_now_ms(),
13690 tables: diagnostic_tables,
13691 windows: diagnostic_windows,
13692 domains_truncated,
13693 status_changed: event.status.is_some(),
13694 conflicts_changed: event.conflicts_changed,
13695 rejections_changed: event.rejections_changed,
13696 outcomes_changed: event.outcomes_changed,
13697 });
13698 self.change_queue.push_back(event);
13699 Ok(())
13700 }
13701
13702 fn record_scope_map(
13703 &self,
13704 batch: &mut ChangeAccumulator,
13705 table_name: &str,
13706 scopes: &[(String, Vec<String>)],
13707 ) {
13708 let Some(table) = self.schema.table(table_name) else {
13709 return;
13710 };
13711 for (variable, values) in scopes {
13712 let Some(scope) = table
13713 .scope_variables
13714 .iter()
13715 .find(|scope| &scope.variable == variable)
13716 else {
13717 continue;
13718 };
13719 for value in values {
13720 batch.scope(table_name, format!("{}:{value}", scope.prefix));
13721 }
13722 }
13723 }
13724
13725 fn record_row_scopes(
13727 &self,
13728 batch: &mut ChangeAccumulator,
13729 table_name: &str,
13730 row_id: &str,
13731 base: bool,
13732 ) -> bool {
13733 let Some(table) = self.schema.table(table_name) else {
13734 return false;
13735 };
13736 if table.scope_variables.is_empty() {
13737 return false;
13738 }
13739 let columns = table
13740 .scope_variables
13741 .iter()
13742 .map(|scope| quote_ident(&scope.column))
13743 .collect::<Vec<_>>()
13744 .join(", ");
13745 let full_table = if base {
13746 base_table(table_name)
13747 } else {
13748 visible_table(table_name)
13749 };
13750 let sql = format!(
13751 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
13752 row_id_predicate(table)
13753 );
13754 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
13755 return false;
13756 };
13757 let values = stmt.query_row(rusqlite::params![row_id], |row| {
13758 let mut values = Vec::with_capacity(table.scope_variables.len());
13759 for index in 0..table.scope_variables.len() {
13760 values.push(row.get::<_, Option<String>>(index)?);
13761 }
13762 Ok(values)
13763 });
13764 let Ok(values) = values else {
13765 return false;
13766 };
13767 let mut recorded = false;
13768 for (scope, value) in table.scope_variables.iter().zip(values) {
13769 if let Some(value) = value {
13770 batch.scope(table_name, format!("{}:{value}", scope.prefix));
13771 recorded = true;
13772 }
13773 }
13774 recorded
13775 }
13776
13777 fn record_commit_changes(
13778 &self,
13779 batch: &mut ChangeAccumulator,
13780 tables: &[String],
13781 changes: &[ssp2::model::Change],
13782 ) {
13783 #[cfg(feature = "bench-internals")]
13784 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
13785 for change in changes {
13786 let Some(table_name) = tables.get(change.table_index as usize) else {
13787 continue;
13788 };
13789 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
13790 if let Some(table) = self.schema.table(table_name) {
13791 for (variable, value) in &change.scopes {
13792 if let Some(scope) = table
13793 .scope_variables
13794 .iter()
13795 .find(|scope| &scope.variable == variable)
13796 {
13797 batch.scope(table_name, format!("{}:{value}", scope.prefix));
13798 precise = true;
13799 }
13800 }
13801 }
13802 if !precise {
13803 batch.table(table_name);
13804 }
13805 }
13806 }
13807
13808 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
13809 if effective.is_empty() {
13810 return false;
13811 }
13812 let Some(table) = self.schema.table(table_name) else {
13813 return false;
13814 };
13815 let mut clauses = Vec::new();
13816 let mut params = Vec::new();
13817 for (variable, values) in effective {
13818 let Some(column) = table.scope_column(variable) else {
13819 return false;
13820 };
13821 if values.is_empty() {
13822 return false;
13823 }
13824 let placeholders = values
13825 .iter()
13826 .map(|value| {
13827 params.push(SqlValue::Text(value.clone()));
13828 "?"
13829 })
13830 .collect::<Vec<_>>()
13831 .join(", ");
13832 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
13833 }
13834 let sql = format!(
13835 "SELECT 1 FROM {} WHERE {} LIMIT 1",
13836 base_table(table_name),
13837 clauses.join(" AND ")
13838 );
13839 self.conn
13840 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
13841 .is_ok()
13842 }
13843
13844 pub fn upgrading(&self) -> bool {
13846 self.upgrading
13847 }
13848
13849 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
13857 let new_schema = parse_schema_json(schema_json)?;
13858 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
13859 self.conn
13860 .execute_batch("BEGIN IMMEDIATE")
13861 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13862 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
13863 let prior_subs = self.subs.clone();
13869 let prior_outbox = self.outbox.clone();
13870 let prior_rejections = self.rejections.clone();
13871 let prior_upgrading = self.upgrading;
13872 let prior_stopped = self.stopped;
13873 let prior_schema_floor = self.schema_floor.clone();
13874 let prior_overlay_dirty = self.overlay_dirty.snapshot();
13875 let prior_sync_needed = self.sync_needed;
13876 let prior_active_round = self.active_round;
13877 let prior_changes = self.change_queue.len();
13878 let prior_intents = self.sync_intent_queue.len();
13879 let prior_last_change = self.last_change.clone();
13880 let guarded = (|| -> Result<(), String> {
13881 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
13886 self.create_bookkeeping_tables()?;
13887 match marker {
13888 Some(version) if version < self.schema.version => self.run_schema_reset()?,
13889 None => {
13890 self.create_synced_tables()?;
13891 self.persist_schema_version()?;
13892 }
13893 _ => {}
13894 }
13895 self.outbox = self.load_outbox()?;
13900 self.drop_unencodable_outbox(false, None)?;
13905 if marker == Some(self.schema.version)
13906 && (!self.outbox.is_empty()
13907 || !self.failed_commits.is_empty()
13908 || self.has_acknowledged_rows())
13909 {
13910 self.overlay_dirty.set(true);
13911 self.rebuild_overlay()?;
13912 }
13913 self.prune_unknown_subscriptions(false)?;
13914 self.save_subscription_scope_schema();
13915 self.enqueue_startup_sync_if_needed()?;
13919 Ok(())
13920 })();
13921 let outcome = match guarded {
13922 Ok(()) => self
13923 .conn
13924 .execute_batch("COMMIT")
13925 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
13926 Err(error) => Err(error),
13927 };
13928 if let Err(error) = outcome {
13929 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
13930 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
13932 }
13933 self.schema = prior_schema;
13934 self.subs = prior_subs;
13935 self.outbox = prior_outbox;
13936 self.rejections = prior_rejections;
13937 self.upgrading = prior_upgrading;
13938 self.stopped = prior_stopped;
13939 self.schema_floor = prior_schema_floor;
13940 self.overlay_dirty.restore(prior_overlay_dirty);
13941 self.sync_needed = prior_sync_needed;
13942 self.active_round = prior_active_round;
13943 self.change_queue.truncate(prior_changes);
13944 self.sync_intent_queue.truncate(prior_intents);
13945 self.last_change = prior_last_change;
13946 self.insert_sql.borrow_mut().clear();
13948 return Err(error);
13949 }
13950 Ok(())
13951 }
13952
13953 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
13954 let startup_work = !self.stopped
13955 && (!self.outbox.is_empty()
13956 || self.has_acknowledged_rows()
13957 || !self.load_pending_evictions()?.is_empty()
13958 || self.subs.iter().any(|sub| sub.state == SubState::Active));
13959 if startup_work {
13960 self.sync_needed = true;
13961 self.sync_intent_queue.push_back(SyncIntent::Interactive);
13962 }
13963 Ok(())
13964 }
13965
13966 fn clear_satisfied_persisted_schema_floor(&mut self) {
13973 let satisfied = self
13974 .schema_floor
13975 .as_ref()
13976 .and_then(|floor| floor.required_schema_version)
13977 .is_some_and(|required| self.schema.version >= required);
13978 if !satisfied {
13979 return;
13980 }
13981 self.schema_floor = None;
13982 self.stopped = false;
13983 self.delete_meta(SCHEMA_FLOOR_KEY);
13984 }
13985
13986 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
13991 let Some(replica_path) = self.previous_version_replica_path() else {
13992 return Ok(());
13993 };
13994 let max_age_ms = self
13995 .previous_version
13996 .filter(|config| config.enabled)
13997 .map(|config| config.max_age_ms);
13998 reconcile_previous_version_at_boot(
13999 &self.conn,
14000 &replica_path,
14001 self.schema.version,
14002 max_age_ms,
14003 self.clock_now_ms(),
14004 )
14005 }
14006
14007 fn previous_version_replica_path(&self) -> Option<String> {
14011 let path = self.conn.path()?;
14012 if path.is_empty() || path.starts_with("file:") {
14013 return None;
14014 }
14015 Some(path.to_owned())
14016 }
14017
14018 fn run_schema_reset(&mut self) -> Result<(), String> {
14024 self.begin_observation("syncular_schema_reset")?;
14025 let prior_subs = self.subs.clone();
14026 let prior_outbox = self.outbox.clone();
14027 let prior_rejections = self.rejections.clone();
14028 let prior_upgrading = self.upgrading;
14029 let prior_stopped = self.stopped;
14030 let prior_schema_floor = self.schema_floor.clone();
14031 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14032 let prior_sync_needed = self.sync_needed;
14033 let prior_active_round = self.active_round;
14034 let mut batch = ChangeAccumulator::default();
14035 let result = self
14036 .run_schema_reset_observed(&mut batch, true)
14037 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
14038 if let Err(error) = result {
14039 self.rollback_observation("syncular_schema_reset");
14040 self.subs = prior_subs;
14041 self.outbox = prior_outbox;
14042 self.rejections = prior_rejections;
14043 self.upgrading = prior_upgrading;
14044 self.stopped = prior_stopped;
14045 self.schema_floor = prior_schema_floor;
14046 self.overlay_dirty.restore(prior_overlay_dirty);
14047 self.sync_needed = prior_sync_needed;
14048 self.active_round = prior_active_round;
14049 return Err(error);
14050 }
14051 Ok(())
14052 }
14053
14054 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
14055 if self.get_meta(LOG_EPOCH_KEY).is_none() {
14056 self.begin_observation("syncular_epoch_acquisition")?;
14057 let previous_sync_needed = self.sync_needed;
14058 if let Err(error) = self.conn.execute(
14059 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
14060 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
14061 ) {
14062 self.rollback_observation("syncular_epoch_acquisition");
14063 return Err(Self::sqlite_failure(&self.storage_failure, error));
14064 }
14065 self.sync_needed = true;
14066 let batch = ChangeAccumulator {
14067 status: true,
14068 ..ChangeAccumulator::default()
14069 };
14070 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
14071 self.rollback_observation("syncular_epoch_acquisition");
14072 self.sync_needed = previous_sync_needed;
14073 return Err(error);
14074 }
14075 self.sync_intent_queue.push_back(SyncIntent::Interactive);
14076 return Ok(Vec::new());
14077 }
14078 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
14079 self.begin_observation("syncular_log_epoch_reset")?;
14080 let prior_subs = self.subs.clone();
14081 let prior_outbox = self.outbox.clone();
14082 let prior_rejections = self.rejections.clone();
14083 let prior_upgrading = self.upgrading;
14084 let prior_stopped = self.stopped;
14085 let prior_schema_floor = self.schema_floor.clone();
14086 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14087 let prior_sync_needed = self.sync_needed;
14088 let prior_active_round = self.active_round;
14089 let mut batch = ChangeAccumulator::default();
14090 let result = self
14091 .run_schema_reset_observed(&mut batch, false)
14092 .and_then(|()| {
14093 self.conn
14094 .execute(
14095 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
14096 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
14097 )
14098 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14099 self.sync_needed = true;
14100 batch.status = true;
14101 self.finish_observation("syncular_log_epoch_reset", batch)
14102 });
14103 if let Err(error) = result {
14104 self.rollback_observation("syncular_log_epoch_reset");
14105 self.subs = prior_subs;
14106 self.outbox = prior_outbox;
14107 self.rejections = prior_rejections;
14108 self.upgrading = prior_upgrading;
14109 self.stopped = prior_stopped;
14110 self.schema_floor = prior_schema_floor;
14111 self.overlay_dirty.restore(prior_overlay_dirty);
14112 self.sync_needed = prior_sync_needed;
14113 self.active_round = prior_active_round;
14114 return Err(error);
14115 }
14116 self.sync_intent_queue.push_back(SyncIntent::Interactive);
14117 Ok(resets)
14118 }
14119
14120 fn run_schema_reset_observed(
14121 &mut self,
14122 batch: &mut ChangeAccumulator,
14123 capture: bool,
14124 ) -> Result<(), String> {
14125 read_local_schema_version(&self.conn, self.schema.version)?;
14131 self.conn
14132 .execute_batch(
14133 "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
14134 )
14135 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14136 self.cancel_sync_round();
14137 self.upgrading = true;
14138 batch.status = true;
14139 if self.previous_version.is_some_and(|config| config.enabled) {
14147 let previous_version = self
14148 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
14149 .and_then(|value| value.parse::<i32>().ok())
14150 .unwrap_or(self.schema.version);
14151 let pending = self
14152 .outbox
14153 .iter()
14154 .map(|commit| PendingCommitAudit {
14155 commit_id: &commit.client_commit_id,
14156 operations: commit_audit_operations(commit),
14157 })
14158 .collect::<Vec<_>>();
14159 crate::previous_version::write_previous_version_audit(
14160 &self.conn,
14161 &crate::previous_version::build_previous_version_audit(
14162 &self.schema,
14163 &pending,
14164 previous_version,
14165 self.schema.version,
14166 self.clock_now_ms(),
14167 ),
14168 );
14169 }
14170 if let Some(replica_path) = self.previous_version_replica_path() {
14175 sweep_previous_version_container(&self.conn, &replica_path)?;
14176 if capture {
14177 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
14178 let previous_version = self
14179 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
14180 .and_then(|value| value.parse::<i32>().ok())
14181 .unwrap_or(self.schema.version);
14182 capture_previous_version_from_replica(
14183 &self.conn,
14184 &replica_path,
14185 previous_version,
14186 self.schema.version,
14187 &config,
14188 self.clock_now_ms(),
14189 )?;
14190 }
14191 }
14192 }
14193 for table in &self.schema.tables {
14194 batch.table(&table.name);
14195 }
14196 for (base_key, unit, table) in self.load_registered_window_units() {
14197 batch.window(&base_key, &table, &unit);
14198 }
14199 if capture {
14200 self.prune_unknown_subscriptions(true)?;
14201 }
14202 self.insert_sql.borrow_mut().clear();
14204 self.overlay_dirty.set(true);
14205 let virtual_tables: Vec<String> = {
14213 let mut stmt = self
14214 .conn
14215 .prepare(
14216 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
14217 )
14218 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14219 let rows = stmt
14220 .query_map([], |row| row.get::<_, String>(0))
14221 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14222 rows.filter_map(Result::ok)
14223 .filter(|name| is_synced_table_name(name))
14224 .collect()
14225 };
14226 for name in virtual_tables {
14227 self.conn
14228 .execute_batch(&format!(
14229 "DROP TABLE IF EXISTS {}",
14230 quote_ident(&format!("_syncular_fts_{name}"))
14231 ))
14232 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14233 self.conn
14234 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
14235 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14236 }
14237 let existing: Vec<String> = {
14238 let mut stmt = self
14239 .conn
14240 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
14241 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14242 let rows = stmt
14243 .query_map([], |row| row.get::<_, String>(0))
14244 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14245 rows.filter_map(Result::ok)
14246 .filter(|name| is_synced_table_name(name))
14247 .collect()
14248 };
14249 for name in &existing {
14250 if let Some(table) = name.strip_prefix("_syncular_base_") {
14251 batch.table(table);
14252 } else if !name.starts_with("_syncular_") {
14253 batch.table(name);
14254 }
14255 }
14256 for name in existing {
14257 self.conn
14258 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
14259 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14260 }
14261 self.create_synced_tables()?;
14263 for sub in &mut self.subs {
14265 sub.cursor = -1;
14266 sub.bootstrap_state = None;
14267 sub.effective = None;
14268 sub.state = SubState::Active;
14269 sub.reason_code = None;
14270 sub.synced_once = false;
14271 }
14272 let subs = self.subs.clone();
14273 for sub in &subs {
14274 self.persist_sub(sub)?;
14275 }
14276 self.stopped = false;
14278 self.schema_floor = None;
14279 self.delete_meta(SCHEMA_FLOOR_KEY);
14280 self.persist_schema_version()?;
14282 self.save_subscription_scope_schema();
14283 self.drop_unencodable_outbox(false, Some(batch))?;
14294 self.rebuild_overlay()?;
14296 Ok(())
14297 }
14298
14299 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
14308 let incompatible = self
14309 .outbox
14310 .iter()
14311 .filter(|commit| {
14312 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
14313 })
14314 .cloned()
14315 .collect::<Vec<_>>();
14316 if incompatible.is_empty() {
14317 return Ok(false);
14318 }
14319 let prior_outbox = self.outbox.clone();
14325 let prior_rejections = self.rejections.clone();
14326 let prior_overlay_dirty = self.overlay_dirty.snapshot();
14327 let owns_transaction = self.conn.is_autocommit();
14328 self.begin_observation("syncular_outbox_drop")?;
14329 let mut batch = ChangeAccumulator::default();
14330 let dropped = (|| -> Result<(), String> {
14331 let mut rejections = Vec::new();
14332 for commit in &incompatible {
14333 let results = commit
14334 .ops
14335 .iter()
14336 .enumerate()
14337 .map(|(op_index, operation)| {
14338 let rejection = RejectionRecord {
14339 client_commit_id: commit.client_commit_id.clone(),
14340 op_index: op_index as i32,
14341 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
14342 message: "the persisted commit cannot encode under the current schema"
14343 .to_owned(),
14344 retryable: false,
14345 details: None,
14346 operation: Some(CommitOperation::from(operation)),
14347 };
14348 rejections.push(rejection.clone());
14349 CommitOperationOutcome::Error { rejection }
14350 })
14351 .collect::<Vec<_>>();
14352 self.persist_commit_outcome(
14353 &commit.client_commit_id,
14354 CommitOutcomeStatus::Rejected,
14355 &results,
14356 Some(&commit.ops),
14357 )?;
14358 self.delete_outbox_persisted(&commit.client_commit_id)?;
14359 for operation in &commit.ops {
14362 if self.schema.table(&operation.table).is_some() {
14363 batch.table(&operation.table);
14364 }
14365 }
14366 }
14367 self.prune_commit_outcomes()?;
14368 let incompatible_ids = incompatible
14369 .iter()
14370 .map(|commit| commit.client_commit_id.as_str())
14371 .collect::<BTreeSet<_>>();
14372 self.outbox
14373 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
14374 self.rejections.extend(rejections);
14375 self.overlay_dirty.set(true);
14378 self.rebuild_overlay()?;
14379 batch.status = true;
14380 batch.rejections = true;
14381 batch.outcomes = true;
14382 Ok(())
14383 })();
14384 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
14385 if let Err(error) = result {
14386 if owns_transaction {
14390 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
14391 let mut failure = self.storage_failure.borrow_mut();
14392 let failure =
14393 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
14394 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
14395 }
14396 } else {
14397 self.rollback_observation("syncular_outbox_drop");
14398 }
14399 self.outbox = prior_outbox;
14400 self.rejections = prior_rejections;
14401 self.overlay_dirty.restore(prior_overlay_dirty);
14402 return Err(error);
14403 }
14404 Ok(true)
14405 }
14406
14407 fn create_synced_tables(&self) -> Result<(), String> {
14410 for table in &self.schema.tables {
14411 for (full, index_prefix) in [
14415 (base_table(&table.name), "_syncular_base_"),
14416 (visible_table(&table.name), ""),
14417 ] {
14418 let mut cols: Vec<String> =
14419 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
14420 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
14421 let sql = format!(
14422 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
14423 cols.join(", "),
14424 quote_ident(&table.primary_key)
14425 );
14426 self.conn
14427 .execute(&sql, [])
14428 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14429 for index in &table.indexes {
14433 let unique = if index.unique { "UNIQUE " } else { "" };
14434 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
14435 let cols_sql = index
14436 .columns
14437 .iter()
14438 .map(|c| quote_ident(c))
14439 .collect::<Vec<_>>()
14440 .join(", ");
14441 let index_sql = format!(
14442 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
14443 );
14444 self.conn
14445 .execute(&index_sql, [])
14446 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14447 }
14448 }
14449 }
14450 for table in &self.schema.tables {
14453 for index in &table.fts_indexes {
14454 self.create_fts_projection(table, index)?;
14455 }
14456 }
14457 Ok(())
14458 }
14459
14460 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
14461 let count: i64 = self
14462 .conn
14463 .query_row(
14464 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
14465 rusqlite::params![index.name],
14466 |row| row.get(0),
14467 )
14468 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14469 Ok(count > 0)
14470 }
14471
14472 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
14473 for suffix in ["ai", "ad", "au"] {
14474 self.conn
14475 .execute(
14476 &format!(
14477 "DROP TRIGGER IF EXISTS {}",
14478 quote_ident(&format!("{}_{suffix}", index.name))
14479 ),
14480 [],
14481 )
14482 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14483 }
14484 Ok(())
14485 }
14486
14487 fn create_fts_triggers(
14488 &self,
14489 table: &TableSchema,
14490 index: &FtsIndexSchema,
14491 ) -> Result<(), String> {
14492 let fts = quote_ident(&index.name);
14493 let source = visible_table(&table.name);
14494 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
14495 let pk = quote_ident(&table.primary_key);
14496 let projection_columns = std::iter::once(source_id.clone())
14497 .chain(index.columns.iter().map(|column| quote_ident(column)))
14498 .collect::<Vec<_>>()
14499 .join(", ");
14500 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
14501 .chain(
14502 index
14503 .columns
14504 .iter()
14505 .map(|column| format!("new.{}", quote_ident(column))),
14506 )
14507 .collect::<Vec<_>>()
14508 .join(", ");
14509 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
14510 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
14511 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
14512 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
14513 let bi = quote_ident(&format!("{}_bi", index.name));
14514 let ai = quote_ident(&format!("{}_ai", index.name));
14515 let ad = quote_ident(&format!("{}_ad", index.name));
14516 let au = quote_ident(&format!("{}_au", index.name));
14517 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
14518 let sql = format!(
14519 "DROP TRIGGER IF EXISTS {bi};
14520 DROP TRIGGER IF EXISTS {ai};
14521 DROP TRIGGER IF EXISTS {ad};
14522 DROP TRIGGER IF EXISTS {au};
14523 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
14524 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
14525 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
14526 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
14527 );
14528 self.conn
14529 .execute_batch(&sql)
14530 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14531 }
14532
14533 fn rebuild_fts_projection(
14534 &self,
14535 table: &TableSchema,
14536 index: &FtsIndexSchema,
14537 ) -> Result<(), String> {
14538 let fts = quote_ident(&index.name);
14539 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
14540 let pk = quote_ident(&table.primary_key);
14541 let indexed_columns = index
14542 .columns
14543 .iter()
14544 .map(|column| quote_ident(column))
14545 .collect::<Vec<_>>();
14546 let projection_columns = std::iter::once(source_id.clone())
14547 .chain(indexed_columns.iter().cloned())
14548 .collect::<Vec<_>>()
14549 .join(", ");
14550 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
14551 let sql = format!(
14552 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
14553 columns = indexed_columns.join(", "),
14554 source = visible_table(&table.name),
14555 );
14556 self.conn
14557 .execute_batch(&sql)
14558 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14559 }
14560
14561 fn create_fts_projection(
14562 &self,
14563 table: &TableSchema,
14564 index: &FtsIndexSchema,
14565 ) -> Result<(), String> {
14566 self.begin_observation("syncular_fts_schema")?;
14567 let result = (|| {
14568 let existed = self.fts_projection_exists(index)?;
14569 let tokenizer = index.tokenize.replace('\'', "''");
14570 let columns = index
14571 .columns
14572 .iter()
14573 .map(|column| quote_ident(column))
14574 .collect::<Vec<_>>()
14575 .join(", ");
14576 let sql = format!(
14577 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
14578 fts = quote_ident(&index.name),
14579 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
14580 );
14581 self.conn.execute(&sql, []).map_err(|error| {
14582 format!(
14583 "cannot create local FTS5 projection {:?}: {error}",
14584 index.name
14585 )
14586 })?;
14587 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
14588 let mapped: bool = self
14589 .conn
14590 .query_row(
14591 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
14592 [format!("_syncular_fts_{}", index.name)],
14593 |row| row.get(0),
14594 )
14595 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14596 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14597 if !mapped && existed {
14598 self.conn
14599 .execute_batch(&format!(
14600 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
14601 quote_ident(FTS_SOURCE_ID_COLUMN),
14602 quote_ident(&index.name)
14603 ))
14604 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14605 }
14606 self.create_fts_triggers(table, index)?;
14607 if !existed {
14608 self.rebuild_fts_projection(table, index)?;
14609 }
14610 Ok(())
14611 })();
14612 if let Err(error) = result {
14613 self.rollback_observation("syncular_fts_schema");
14614 return Err(error);
14615 }
14616 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
14617 self.rollback_observation("syncular_fts_schema");
14618 return Err(error.to_string());
14619 }
14620 Ok(())
14621 }
14622
14623 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
14626 let state = serde_json::json!({
14627 "requested": scope_map_to_json(&sub.requested),
14628 "params": sub.params,
14629 "cursor": sub.cursor,
14630 "bootstrapState": sub.bootstrap_state,
14631 "status": sub.state.name(),
14632 "reasonCode": sub.reason_code,
14633 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
14634 "syncedOnce": sub.synced_once,
14635 });
14636 self.conn.execute(
14637 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
14638 rusqlite::params![sub.id, sub.table, state.to_string()],
14639 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14640 }
14641
14642 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
14643 let ops: Vec<Value> = commit
14644 .ops
14645 .iter()
14646 .map(|op| {
14647 let mut entry = serde_json::json!({
14648 "op": if op.upsert { "upsert" } else { "delete" },
14649 "table": op.table,
14650 "rowId": op.row_id,
14651 "baseVersion": op.base_version,
14652 "values": op.values.clone().map(Value::Object),
14653 });
14654 if op.insert {
14655 entry["insert"] = Value::Bool(true);
14656 }
14657 entry
14658 })
14659 .collect();
14660 self.conn
14661 .execute(
14662 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
14663 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
14664 )
14665 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14666 for operation in &commit.ops {
14667 let Some(values) = &operation.values else {
14668 continue;
14669 };
14670 let Some(table) = self.schema.table(&operation.table) else {
14671 continue;
14672 };
14673 for column in table
14674 .columns
14675 .iter()
14676 .filter(|column| column.ty == ColumnType::BlobRef)
14677 {
14678 let Some(Value::String(raw)) = values.get(&column.name) else {
14679 continue;
14680 };
14681 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
14682 continue;
14683 };
14684 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
14685 continue;
14686 };
14687 self.conn
14688 .execute(
14689 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
14690 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
14691 rusqlite::params![commit.client_commit_id, blob_id],
14692 )
14693 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14694 }
14695 }
14696 Ok(())
14697 }
14698
14699 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
14700 if let Some(commit) = self
14701 .outbox
14702 .iter()
14703 .find(|commit| commit.client_commit_id == client_commit_id)
14704 {
14705 for operation in &commit.ops {
14706 self.overlay_dirty.table(&operation.table);
14707 }
14708 }
14709
14710 self.conn
14711 .execute(
14712 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
14713 rusqlite::params![client_commit_id],
14714 )
14715 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14716 self.conn
14717 .execute(
14718 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
14719 rusqlite::params![client_commit_id],
14720 )
14721 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14722 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
14723 .map(|_| ())
14724 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14725 }
14726
14727 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
14728 match status {
14729 CommitOutcomeStatus::Applied => "applied",
14730 CommitOutcomeStatus::Cached => "cached",
14731 CommitOutcomeStatus::Conflict => "conflict",
14732 CommitOutcomeStatus::Rejected => "rejected",
14733 }
14734 }
14735
14736 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
14737 match resolution {
14738 CommitOutcomeResolution::Active => "active",
14739 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
14740 CommitOutcomeResolution::Superseded => "superseded",
14741 CommitOutcomeResolution::Dismissed => "dismissed",
14742 }
14743 }
14744
14745 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
14746 match value {
14747 "applied" => Ok(CommitOutcomeStatus::Applied),
14748 "cached" => Ok(CommitOutcomeStatus::Cached),
14749 "conflict" => Ok(CommitOutcomeStatus::Conflict),
14750 "rejected" => Ok(CommitOutcomeStatus::Rejected),
14751 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
14752 }
14753 }
14754
14755 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
14756 match value {
14757 "active" => Ok(CommitOutcomeResolution::Active),
14758 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
14759 "superseded" => Ok(CommitOutcomeResolution::Superseded),
14760 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
14761 _ => Err(format!(
14762 "invalid persisted commit outcome resolution {value:?}"
14763 )),
14764 }
14765 }
14766
14767 fn persist_commit_outcome(
14768 &self,
14769 client_commit_id: &str,
14770 status: CommitOutcomeStatus,
14771 results: &[CommitOperationOutcome],
14772 operations: Option<&[OutboxOp]>,
14773 ) -> Result<(), String> {
14774 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
14775 let operations_json = operations
14776 .map(|items| {
14777 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
14778 })
14779 .transpose()
14780 .map_err(|error| error.to_string())?;
14781 self.conn
14782 .execute(
14783 "INSERT INTO _syncular_commit_outcomes (
14784 client_commit_id, status, recorded_at_ms, results_json,
14785 operations_json, resolution
14786 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
14787 rusqlite::params![
14788 client_commit_id,
14789 Self::outcome_status_name(status),
14790 self.clock_now_ms(),
14791 results_json,
14792 operations_json
14793 ],
14794 )
14795 .map(|_| ())
14796 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
14797 }
14798
14799 fn stored_cell_is_representable(cell: &Value) -> bool {
14802 cell.is_null()
14803 || cell.is_string()
14804 || cell.is_boolean()
14805 || cell.is_number()
14806 || is_canonical_bytes_envelope(cell)
14807 }
14808
14809 fn stored_row_is_representable(value: &Value) -> bool {
14810 value
14811 .as_object()
14812 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
14813 }
14814
14815 fn stored_operation_is_representable(value: &Value) -> bool {
14816 value.get("table").is_some_and(Value::is_string)
14817 && value.get("rowId").is_some_and(Value::is_string)
14818 && value.get("baseVersion").is_none_or(|v| {
14819 v.as_i64()
14820 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
14821 })
14822 && match value.get("op").and_then(Value::as_str) {
14823 Some("upsert") => {
14824 value
14825 .get("values")
14826 .is_some_and(Self::stored_row_is_representable)
14827 && value
14828 .get("insert")
14829 .is_none_or(|insert| *insert == Value::Bool(true))
14830 }
14831 Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
14832 _ => false,
14833 }
14834 }
14835
14836 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
14840 serde_json::to_value(CommitOperation::from(operation))
14841 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
14842 }
14843
14844 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
14845 let StoredCommitOutcomeRow {
14846 sequence,
14847 client_commit_id,
14848 status,
14849 recorded_at_ms,
14850 results_json,
14851 operations_json,
14852 resolution,
14853 resolved_at_ms,
14854 replacement_client_commit_id,
14855 } = row;
14856 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
14857 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
14858 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
14859 {
14860 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
14861 }
14862 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
14863 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
14864 for result in &stored {
14865 let nested = match result.get("status").and_then(Value::as_str) {
14866 Some("conflict") => result.get("conflict"),
14867 Some("error") => result.get("rejection"),
14868 _ => None,
14869 };
14870 if nested
14871 .and_then(|record| record.get("operation"))
14872 .is_some_and(|op| !Self::stored_operation_is_representable(op))
14873 || (result.get("status").and_then(Value::as_str) == Some("conflict")
14874 && nested
14875 .and_then(|record| record.get("serverRow"))
14876 .is_some_and(|row| !Self::stored_row_is_representable(row)))
14877 {
14878 return Err(invalid());
14879 }
14880 }
14881 let results: Vec<CommitOperationOutcome> =
14882 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
14883 let has_conflict = results
14884 .iter()
14885 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
14886 let has_error = results
14887 .iter()
14888 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
14889 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
14890 let redacted = results.is_empty() && operations_json.is_none();
14891 if match status {
14892 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
14893 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
14894 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
14895 } {
14896 return Err(invalid());
14897 }
14898 let operations: Option<Vec<CommitOperation>> = operations_json
14899 .map(|raw| {
14900 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
14901 if !stored.iter().all(Self::stored_operation_is_representable) {
14902 return Err(invalid());
14903 }
14904 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
14905 })
14906 .transpose()?;
14907 for result in &results {
14908 match result {
14909 CommitOperationOutcome::Conflict { conflict }
14910 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
14911 {
14912 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
14913 }
14914 CommitOperationOutcome::Error { rejection } => {
14915 if let Some(details) = &rejection.details {
14916 let encoded = serde_json::to_string(details).map_err(|_| {
14917 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
14918 })?;
14919 RejectionDetails::parse(&encoded).map_err(|_| {
14920 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
14921 })?;
14922 }
14923 }
14924 _ => {}
14925 }
14926 }
14927 Ok(CommitOutcome {
14928 sequence,
14929 client_commit_id,
14930 status,
14931 recorded_at_ms,
14932 results,
14933 operations,
14934 retained_rows: None,
14935 resolution: Self::parse_outcome_resolution(&resolution)?,
14936 resolved_at_ms,
14937 replacement_client_commit_id,
14938 })
14939 }
14940
14941 fn unique_conflicts(
14943 &self,
14944 table: &TableSchema,
14945 values: &Map<String, Value>,
14946 base: bool,
14947 ) -> Result<Vec<RetainedUniqueConflict>, String> {
14948 let source = if base {
14949 base_table(&table.name)
14950 } else {
14951 visible_table(&table.name)
14952 };
14953 let mut conflicts = Vec::new();
14954 for index in table.indexes.iter().filter(|index| index.unique) {
14955 let mut params = vec![json_param_to_sql(
14956 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
14957 )?];
14958 for name in &index.columns {
14959 let column = table
14960 .columns
14961 .iter()
14962 .find(|column| column.name == *name)
14963 .ok_or("sync.local_corrupt")?;
14964 let cell = json_to_column_value(column, values.get(name))?;
14965 params.push(owned_sql_value(RowParam::Cell(&cell))?);
14966 }
14967 let sql = format!(
14968 "SELECT * FROM {source} WHERE {} != ? AND {}",
14969 quote_ident(&table.primary_key),
14970 index
14971 .columns
14972 .iter()
14973 .map(|column| format!("{} = ?", quote_ident(column)))
14974 .collect::<Vec<_>>()
14975 .join(" AND ")
14976 );
14977 let mut statement = self
14978 .conn
14979 .prepare_cached(&sql)
14980 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14981 let records = statement
14982 .query_map(rusqlite::params_from_iter(params), |record| {
14983 let values = table
14984 .columns
14985 .iter()
14986 .enumerate()
14987 .map(|(index, column)| {
14988 let value = sql_ref_to_json(column, record.get_ref(index)?);
14989 json_to_column_value(column, Some(&value)).map_err(|message| {
14990 rusqlite::Error::ToSqlConversionFailure(message.into())
14991 })
14992 })
14993 .collect::<Result<Row, _>>()?;
14994 Ok((values, record.get::<_, i64>(table.columns.len())?))
14995 })
14996 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
14997 for record in records {
14998 let (row, version) =
14999 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15000 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
15001 conflicts.push(RetainedUniqueConflict {
15002 index: index.name.clone(),
15003 columns: index.columns.clone(),
15004 row_id,
15005 server_row: table
15006 .columns
15007 .iter()
15008 .zip(&row)
15009 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
15010 .collect(),
15011 server_version: version,
15012 });
15013 }
15014 }
15015 Ok(conflicts)
15016 }
15017
15018 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
15019 if let Some((commit, initial)) = self
15020 .failed_commits
15021 .iter()
15022 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
15023 {
15024 let mut rows = Vec::new();
15025 for operation in &commit.ops {
15026 let table = self
15027 .schema
15028 .table(&operation.table)
15029 .ok_or_else(|| "sync.unknown_table".to_owned())?;
15030 let key =
15031 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
15032 .map_err(|_| "sync.local_corrupt".to_owned())?;
15033 let base = self.stored_row(table, &operation.row_id, true)?;
15034 rows.push(RetainedCommitRow {
15035 table: operation.table.clone(),
15036 row_id: operation.row_id.clone(),
15037 unique_conflicts: if operation.upsert {
15038 self.unique_conflicts(
15039 table,
15040 initial
15041 .get(&key)
15042 .and_then(Value::as_object)
15043 .or(operation.values.as_ref())
15044 .ok_or("sync.local_corrupt")?,
15045 true,
15046 )?
15047 } else {
15048 Vec::new()
15049 },
15050 local_row: if operation.upsert {
15051 initial
15052 .get(&key)
15053 .and_then(Value::as_object)
15054 .cloned()
15055 .or_else(|| operation.values.clone())
15056 } else {
15057 None
15058 },
15059 server_row: base.as_ref().map(|(values, _)| {
15060 table
15061 .columns
15062 .iter()
15063 .zip(values)
15064 .map(|(column, value)| {
15065 (column.name.clone(), column_value_to_json(value))
15066 })
15067 .collect()
15068 }),
15069 server_version: base.map(|(_, version)| version),
15070 });
15071 }
15072 outcome.retained_rows = Some(rows);
15073 }
15074 Ok(outcome)
15075 }
15076
15077 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
15078 let row = self
15079 .conn
15080 .query_row(
15081 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
15082 resolution, resolved_at_ms, replacement_client_commit_id
15083 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
15084 rusqlite::params![client_commit_id],
15085 |row| {
15086 Ok(StoredCommitOutcomeRow {
15087 sequence: row.get(0)?,
15088 client_commit_id: row.get(1)?,
15089 status: row.get(2)?,
15090 recorded_at_ms: row.get(3)?,
15091 results_json: row.get(4)?,
15092 operations_json: row.get(5)?,
15093 resolution: row.get(6)?,
15094 resolved_at_ms: row.get(7)?,
15095 replacement_client_commit_id: row.get(8)?,
15096 })
15097 },
15098 )
15099 .optional()
15100 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15101 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
15102 .transpose()
15103 }
15104
15105 #[doc(hidden)]
15107 pub fn prepare_current_base(
15108 &self,
15109 client_commit_id: &str,
15110 sequence: i64,
15111 ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
15112 let outcome = self
15113 .commit_outcome(client_commit_id)
15114 .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
15115 let valid = outcome.as_ref().is_some_and(|outcome| {
15116 outcome.sequence == sequence
15117 && outcome.resolution == CommitOutcomeResolution::Active
15118 && matches!(
15119 outcome.status,
15120 CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
15121 )
15122 && outcome
15123 .operations
15124 .as_ref()
15125 .zip(outcome.retained_rows.as_ref())
15126 .is_some_and(|(operations, rows)| {
15127 !rows.is_empty() && operations.len() == rows.len()
15128 })
15129 });
15130 if !valid {
15131 return Err(CurrentBaseProof::failure(
15132 CurrentBaseProofReason::OutcomeUnavailable,
15133 ));
15134 }
15135 if !self.transport_enabled {
15136 return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
15137 }
15138 Ok(CurrentBasePreparation {
15139 client_commit_id: client_commit_id.to_owned(),
15140 sequence,
15141 subscriptions: self.subs.clone(),
15142 transport_epoch: self.transport_epoch,
15143 subscription_epoch: self.subscription_epoch,
15144 security_epoch: self.security_epoch,
15145 })
15146 }
15147
15148 #[doc(hidden)]
15150 pub fn finish_current_base(
15151 &mut self,
15152 prepared: CurrentBasePreparation,
15153 drive: SyncOutcome,
15154 ) -> CurrentBaseProof {
15155 if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
15156 return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
15157 }
15158 if self.subscription_epoch != prepared.subscription_epoch {
15159 return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
15160 }
15161 if self.security_preflight || self.security_epoch != prepared.security_epoch {
15162 return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
15163 }
15164 let report = match drive {
15165 SyncOutcome::Ok(report) => report,
15166 SyncOutcome::BudgetExhausted(_) => {
15167 return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
15168 }
15169 _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
15170 };
15171 if report.schema_floor.is_some() {
15172 return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
15173 }
15174 if self
15175 .conn
15176 .execute_batch("SAVEPOINT syncular_base_proof")
15177 .is_err()
15178 {
15179 return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
15180 }
15181 let proof = (|| {
15182 let snapshot = snapshot_read_connection(
15183 &self.conn,
15184 &SnapshotReadRequest {
15185 subscriptions: prepared
15186 .subscriptions
15187 .iter()
15188 .map(|sub| sub.id.clone())
15189 .collect(),
15190 commit_ids: vec![prepared.client_commit_id.clone()],
15191 ..Default::default()
15192 },
15193 )
15194 .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
15195 let current = self
15196 .commit_outcome(&prepared.client_commit_id)
15197 .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
15198 let unavailable = if report.revoked.is_empty() {
15199 CurrentBaseProofReason::OutcomeUnavailable
15200 } else {
15201 CurrentBaseProofReason::Revoked
15202 };
15203 let Some(outcome) = current else {
15204 return Err(unavailable);
15205 };
15206 if !matches!(
15207 snapshot.deliveries.first(),
15208 Some(CommitDelivery::Known { .. })
15209 ) || outcome.sequence != prepared.sequence
15210 || outcome.resolution != CommitOutcomeResolution::Active
15211 || !matches!(
15212 outcome.status,
15213 CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
15214 )
15215 {
15216 return Err(unavailable);
15217 }
15218 let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
15219 return Err(unavailable);
15220 };
15221 if rows.is_empty() || operations.len() != rows.len() {
15222 return Err(unavailable);
15223 }
15224 let after: HashMap<&str, &Subscription> =
15225 self.subs.iter().map(|sub| (sub.id.as_str(), sub)).collect();
15226 let complete: HashSet<&str> = snapshot
15227 .subscriptions
15228 .iter()
15229 .filter_map(|sub| match sub {
15230 SubscriptionCatchup::Known {
15231 id,
15232 bootstrap_complete: true,
15233 ..
15234 } => Some(id.as_str()),
15235 _ => None,
15236 })
15237 .collect();
15238 let candidates: Vec<_> = prepared
15239 .subscriptions
15240 .iter()
15241 .filter_map(|before| {
15242 let after = after.get(before.id.as_str())?;
15243 if before.table != after.table
15244 || before.params != after.params
15245 || canonical_scope_json(&before.requested)
15246 != canonical_scope_json(&after.requested)
15247 || !complete.contains(before.id.as_str())
15248 {
15249 return None;
15250 }
15251 after
15252 .effective
15253 .as_ref()
15254 .map(|scopes| (after.table.as_str(), scopes))
15255 })
15256 .collect();
15257 let covered = |table_name: &str, row: &Map<String, Value>| {
15258 let Some(table) = self.schema.table(table_name) else {
15259 return false;
15260 };
15261 candidates.iter().any(|(candidate_table, scopes)| {
15262 *candidate_table == table_name
15263 && !scopes.is_empty()
15264 && scopes.iter().all(|(variable, allowed)| {
15265 table
15266 .scope_column(variable)
15267 .and_then(|column| match row.get(column) {
15268 Some(Value::String(value)) => Some(value.clone()),
15269 Some(Value::Bool(value)) => Some(value.to_string()),
15270 Some(Value::Number(value)) => value.as_f64().map(|number| {
15271 ryu_js::Buffer::new().format_finite(number).to_owned()
15272 }),
15273 _ => None,
15274 })
15275 .is_some_and(|value| !value.is_empty() && allowed.contains(&value))
15276 })
15277 })
15278 };
15279 for row in &rows {
15280 let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
15281 return Err(if report.revoked.is_empty() {
15282 CurrentBaseProofReason::MissingCoverage
15283 } else {
15284 CurrentBaseProofReason::Revoked
15285 });
15286 };
15287 if !covered(&row.table, base)
15288 || row
15289 .unique_conflicts
15290 .iter()
15291 .any(|conflict| !covered(&row.table, &conflict.server_row))
15292 {
15293 return Err(if report.revoked.is_empty() {
15294 CurrentBaseProofReason::MissingCoverage
15295 } else {
15296 CurrentBaseProofReason::Revoked
15297 });
15298 }
15299 }
15300 Ok(rows)
15301 })();
15302 if self
15303 .conn
15304 .execute_batch("RELEASE syncular_base_proof")
15305 .is_err()
15306 {
15307 return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
15308 }
15309 match proof {
15310 Ok(rows) => CurrentBaseProof {
15311 proven: true,
15312 rows: Some(rows),
15313 reason: None,
15314 },
15315 Err(reason) => CurrentBaseProof::failure(reason),
15316 }
15317 }
15318
15319 pub fn prove_current_base(
15320 &mut self,
15321 transport: &mut dyn Transport,
15322 client_commit_id: &str,
15323 sequence: i64,
15324 max_rounds: Option<u32>,
15325 ) -> CurrentBaseProof {
15326 let prepared = match self.prepare_current_base(client_commit_id, sequence) {
15327 Ok(prepared) => prepared,
15328 Err(proof) => return proof,
15329 };
15330 let drive = self.sync_until_idle(transport, max_rounds);
15331 self.finish_current_base(prepared, drive)
15332 }
15333
15334 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
15335 if query.limit == Some(0) {
15336 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
15337 }
15338 let mut sql = String::from(
15339 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
15340 resolution, resolved_at_ms, replacement_client_commit_id
15341 FROM _syncular_commit_outcomes",
15342 );
15343 if query.active_only {
15344 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
15345 }
15346 sql.push_str(" ORDER BY seq DESC");
15347 if let Some(limit) = query.limit {
15348 sql.push_str(&format!(" LIMIT {limit}"));
15349 }
15350 let mut stmt = self
15351 .conn
15352 .prepare(&sql)
15353 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15354 let rows = stmt
15355 .query_map([], |row| {
15356 Ok(StoredCommitOutcomeRow {
15357 sequence: row.get(0)?,
15358 client_commit_id: row.get(1)?,
15359 status: row.get(2)?,
15360 recorded_at_ms: row.get(3)?,
15361 results_json: row.get(4)?,
15362 operations_json: row.get(5)?,
15363 resolution: row.get(6)?,
15364 resolved_at_ms: row.get(7)?,
15365 replacement_client_commit_id: row.get(8)?,
15366 })
15367 })
15368 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15369 let mut outcomes = Vec::new();
15370 for row in rows {
15371 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
15372 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
15373 )?)?);
15374 }
15375 Ok(outcomes)
15376 }
15377
15378 fn prune_commit_outcomes(&self) -> Result<(), String> {
15379 #[cfg(test)]
15380 self.outcome_prune_count
15381 .set(self.outcome_prune_count.get() + 1);
15382 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
15383 let count = self
15384 .conn
15385 .query_row(
15386 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
15387 [],
15388 |row| row.get::<_, i64>(0),
15389 )
15390 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15391 as usize;
15392 let excess = count.saturating_sub(max_entries);
15393 if excess == 0 {
15394 return Ok(());
15395 }
15396 let mut stmt = self
15397 .conn
15398 .prepare(
15399 "SELECT seq FROM _syncular_commit_outcomes
15400 WHERE status IN ('applied', 'cached') OR resolution != 'active'
15401 ORDER BY seq ASC LIMIT ?1",
15402 )
15403 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15404 let rows = stmt
15405 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
15406 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15407 let sequences = rows
15408 .collect::<Result<Vec<_>, _>>()
15409 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15410 drop(stmt);
15411 for sequence in sequences {
15412 self.conn
15413 .execute(
15414 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
15415 rusqlite::params![sequence],
15416 )
15417 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15418 }
15419 Ok(())
15420 }
15421
15422 pub fn subscribe(
15425 &mut self,
15426 id: String,
15427 table: String,
15428 scopes: Vec<(String, Vec<String>)>,
15429 params: Option<String>,
15430 ) -> Result<(), String> {
15431 if self.schema.table(&table).is_none() {
15432 return Err(format!("unknown table {table:?}"));
15433 }
15434 let registered_at = self
15435 .subs
15436 .iter()
15437 .position(|subscription| subscription.id == id);
15438 if let Some(index) = registered_at {
15439 let existing = &self.subs[index];
15440 let same_intent = existing.table == table
15441 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
15442 && existing.params == params;
15443 if !same_intent {
15444 return Err(
15445 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
15446 .to_owned(),
15447 );
15448 }
15449 if existing.state != SubState::Revoked {
15454 return Ok(());
15455 }
15456 }
15457 let sub = Subscription {
15458 id: id.clone(),
15459 table,
15460 requested: scopes,
15461 params,
15462 cursor: -1,
15463 bootstrap_state: None,
15464 state: SubState::Active,
15465 reason_code: None,
15466 effective: None,
15467 synced_once: false,
15468 };
15469 self.persist_sub(&sub)?;
15470 match registered_at {
15471 Some(index) => self.subs[index] = sub,
15472 None => self.subs.push(sub),
15473 }
15474 self.subscription_epoch = self.subscription_epoch.wrapping_add(1);
15475 if registered_at.is_some() {
15479 self.set_sync_needed(true, true);
15480 }
15481 Ok(())
15482 }
15483
15484 pub fn subscribe_many(&mut self, requests: Vec<SubscriptionRequest>) -> Result<(), String> {
15490 let prior_subs = self.subs.clone();
15491 self.begin_observation("syncular_subscribe")?;
15492 let registered = (|| {
15493 for request in requests {
15494 self.subscribe(request.id, request.table, request.scopes, request.params)?;
15495 }
15496 self.conn
15497 .execute_batch("RELEASE syncular_subscribe")
15498 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15499 })();
15500 if let Err(error) = registered {
15501 self.rollback_observation("syncular_subscribe");
15502 self.subs = prior_subs;
15503 return Err(error);
15504 }
15505 Ok(())
15506 }
15507
15508 pub fn unsubscribe(&mut self, id: &str) {
15509 if self.subs.iter().any(|s| s.id == id) {
15510 self.cancel_sync_round();
15511 self.subscription_epoch = self.subscription_epoch.wrapping_add(1);
15512 }
15513 self.subs.retain(|s| s.id != id);
15514 let _ = self.conn.execute(
15515 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
15516 rusqlite::params![id],
15517 );
15518 }
15519
15520 pub fn set_window(
15528 &mut self,
15529 base: &WindowBase,
15530 units: &[String],
15531 ) -> Result<CommandEffects, String> {
15532 let table = self
15533 .schema
15534 .table(&base.table)
15535 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
15536 if table.scope_column(&base.variable).is_none() {
15537 return Err(format!(
15538 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
15539 base.table, base.variable
15540 ));
15541 }
15542 let base_key = window_base_key(base);
15543 let wanted: std::collections::HashSet<&String> = units.iter().collect();
15544 let live = self.load_window_units(&base_key);
15545 self.begin_observation("syncular_window")?;
15546 let mut batch = ChangeAccumulator::default();
15547 let mut changed = false;
15548
15549 let prior_subs = self.subs.clone();
15550 let widened = (|| {
15551 for unit in units {
15553 if live.iter().any(|(u, _)| u == unit) {
15554 continue;
15555 }
15556 let sub_id = derive_sub_id(base, unit);
15557 self.delete_pending_evict(&sub_id)?;
15558 self.insert_window_unit(&base_key, unit, &sub_id)?;
15559 self.subscribe(
15560 sub_id,
15561 base.table.clone(),
15562 unit_scopes(base, unit),
15563 base.params.clone(),
15564 )?;
15565 batch.window(&base_key, &base.table, unit);
15566 changed = true;
15567 }
15568
15569 self.finish_observation("syncular_window", batch)
15570 })();
15571 if let Err(error) = widened {
15572 self.rollback_observation("syncular_window");
15573 self.subs = prior_subs;
15574 return Err(error);
15575 }
15576 for (unit, sub_id) in live {
15577 if wanted.contains(&unit) {
15578 continue;
15579 }
15580 self.rebuild_overlay_if_dirty()?;
15581 let prior_subs = self.subs.clone();
15582 self.begin_observation("syncular_window")?;
15583 let effective = self
15584 .subs
15585 .iter()
15586 .find(|sub| sub.id == sub_id)
15587 .and_then(|sub| sub.effective.clone())
15588 .unwrap_or_else(|| unit_scopes(base, &unit));
15589 let mut batch = ChangeAccumulator::default();
15590 self.record_scope_map(&mut batch, &base.table, &effective);
15591 batch.window(&base_key, &base.table, &unit);
15592 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
15593 let remaining = match result {
15594 Ok(remaining) => remaining,
15595 Err(error) => {
15596 self.rollback_observation("syncular_window");
15597 self.subs = prior_subs;
15598 return Err(error);
15599 }
15600 };
15601 if let Err(error) = self.finish_observation("syncular_window", batch) {
15602 self.rollback_observation("syncular_window");
15603 self.subs = prior_subs;
15604 return Err(error);
15605 }
15606 if remaining {
15607 self.drain_pending_evictions()?;
15608 }
15609 changed = true;
15610 }
15611 Ok(if changed {
15612 CommandEffects::interactive()
15613 } else {
15614 CommandEffects::none()
15615 })
15616 }
15617
15618 pub fn window_state(&self, base: &WindowBase) -> WindowState {
15623 let mut units = Vec::new();
15624 let mut pending = Vec::new();
15625 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
15626 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
15627 Some(sub) => {
15628 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
15629 }
15630 None => true,
15631 };
15632 if is_pending {
15633 pending.push(unit.clone());
15634 }
15635 units.push(unit);
15636 }
15637 WindowState { units, pending }
15638 }
15639
15640 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
15641 let mut stmt = match self
15642 .conn
15643 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
15644 {
15645 Ok(stmt) => stmt,
15646 Err(_) => return Vec::new(),
15647 };
15648 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
15649 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
15650 });
15651 match rows {
15652 Ok(rows) => rows.filter_map(Result::ok).collect(),
15653 Err(_) => Vec::new(),
15654 }
15655 }
15656
15657 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
15658 let mut stmt = match self.conn.prepare(
15659 "SELECT windows.base, windows.unit, subscriptions.tbl
15660 FROM _syncular_windows AS windows
15661 JOIN _syncular_subscriptions AS subscriptions
15662 ON subscriptions.id = windows.sub_id
15663 ORDER BY windows.base, windows.unit",
15664 ) {
15665 Ok(stmt) => stmt,
15666 Err(_) => return Vec::new(),
15667 };
15668 let rows = stmt.query_map([], |row| {
15669 Ok((
15670 row.get::<_, String>(0)?,
15671 row.get::<_, String>(1)?,
15672 row.get::<_, String>(2)?,
15673 ))
15674 });
15675 match rows {
15676 Ok(rows) => rows.filter_map(Result::ok).collect(),
15677 Err(_) => Vec::new(),
15678 }
15679 }
15680
15681 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
15682 self.conn
15683 .query_row(
15684 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
15685 rusqlite::params![sub_id],
15686 |row| Ok((row.get(0)?, row.get(1)?)),
15687 )
15688 .ok()
15689 }
15690
15691 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
15692 self.conn
15693 .execute(
15694 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
15695 rusqlite::params![base_key, unit, sub_id],
15696 )
15697 .map(|_| ())
15698 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15699 }
15700
15701 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
15702 self.conn
15703 .execute(
15704 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
15705 rusqlite::params![base_key, unit],
15706 )
15707 .map(|_| ())
15708 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15709 }
15710
15711 fn evict_unit(
15718 &mut self,
15719 base_key: &str,
15720 base: &WindowBase,
15721 unit: &str,
15722 sub_id: &str,
15723 ) -> Result<bool, String> {
15724 let effective = self
15725 .subs
15726 .iter()
15727 .find(|s| s.id == sub_id)
15728 .and_then(|s| s.effective.clone())
15729 .unwrap_or_else(|| unit_scopes(base, unit));
15730 let pinned = self.pinned_row_ids(&base.table);
15731 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
15732 self.delete_window_unit(base_key, unit)?;
15733 self.conn
15734 .execute(
15735 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
15736 [sub_id],
15737 )
15738 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15739 self.subs.retain(|sub| sub.id != sub_id);
15740 self.subscription_epoch = self.subscription_epoch.wrapping_add(1);
15741 if deferred {
15742 self.save_pending_evict(sub_id, &base.table, &effective)?;
15743 } else {
15744 self.delete_pending_evict(sub_id)?;
15745 }
15746 Ok(remaining)
15747 }
15748
15749 fn evict_scope_rows(
15753 &self,
15754 table_name: &str,
15755 effective: &[(String, Vec<String>)],
15756 pinned: &std::collections::HashSet<String>,
15757 ) -> Result<(bool, bool), String> {
15758 if effective.is_empty() {
15759 return Ok((false, false));
15760 }
15761 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
15762 let mut clauses = Vec::new();
15763 let mut params: Vec<SqlValue> = Vec::new();
15764 for (variable, values) in effective {
15765 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
15766 if values.is_empty() {
15767 return Ok((false, false));
15768 }
15769 let holes = values
15770 .iter()
15771 .map(|v| {
15772 params.push(SqlValue::Text(v.clone()));
15773 "?"
15774 })
15775 .collect::<Vec<_>>()
15776 .join(", ");
15777 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
15778 }
15779 let scope = clauses.join(" AND ");
15780 let scope_params = params.len();
15781 let pk = quote_ident(&table.primary_key);
15782 if !pinned.is_empty() {
15783 let holes = pinned
15784 .iter()
15785 .map(|id| {
15786 params.push(SqlValue::Text(id.clone()));
15787 "?"
15788 })
15789 .collect::<Vec<_>>()
15790 .join(", ");
15791 clauses.push(format!("{pk} NOT IN ({holes})"));
15792 }
15793 let predicate = clauses.join(" AND ");
15794 let base = base_table(table_name);
15795 let visible = visible_table(table_name);
15796 let mut ids: Vec<SqlValue> = Vec::new();
15797 for source in [&base, &visible] {
15798 let mut stmt = self
15799 .conn
15800 .prepare_cached(&format!(
15801 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
15802 ))
15803 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15804 ids = stmt
15805 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
15806 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
15807 .collect::<Result<_, _>>()
15808 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15809 if !ids.is_empty() {
15810 break;
15811 }
15812 }
15813 if !ids.is_empty() {
15814 for id in &ids {
15815 self.conn
15816 .execute(
15817 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
15818 rusqlite::params![table_name, id],
15819 )
15820 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15821 }
15822 let holes = vec!["?"; ids.len()].join(", ");
15823 for source in [&base, &visible] {
15824 self.conn
15825 .execute(
15826 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
15827 rusqlite::params_from_iter(&ids),
15828 )
15829 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15830 }
15831 }
15832 let mut remaining = false;
15833 let mut deferred = false;
15834 for source in [&base, &visible] {
15835 remaining |= self
15836 .conn
15837 .query_row(
15838 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
15839 rusqlite::params_from_iter(¶ms),
15840 |row| row.get::<_, bool>(0),
15841 )
15842 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15843 deferred |= self
15844 .conn
15845 .query_row(
15846 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
15847 rusqlite::params_from_iter(¶ms[..scope_params]),
15848 |row| row.get::<_, bool>(0),
15849 )
15850 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15851 }
15852 Ok((remaining, deferred))
15853 }
15854
15855 fn drain_pending_evictions(&mut self) -> Result<(), String> {
15857 let pending = self.load_pending_evictions()?;
15858 if pending.is_empty() {
15859 return Ok(());
15860 }
15861 self.rebuild_overlay_if_dirty()?;
15862 for (sub_id, table_name, effective) in pending {
15863 if self.schema.table(&table_name).is_none() {
15864 self.delete_pending_evict(&sub_id)?;
15865 continue;
15866 }
15867 loop {
15868 std::thread::yield_now();
15869 self.begin_observation("syncular_evict")?;
15870 let pinned = self.pinned_row_ids(&table_name);
15871 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
15872 let (remaining, deferred) = match result {
15873 Ok(result) => result,
15874 Err(error) => {
15875 self.rollback_observation("syncular_evict");
15876 return Err(error);
15877 }
15878 };
15879 if !deferred {
15880 if let Err(error) = self.delete_pending_evict(&sub_id) {
15881 self.rollback_observation("syncular_evict");
15882 return Err(error);
15883 }
15884 }
15885 let mut batch = ChangeAccumulator::default();
15886 self.record_scope_map(&mut batch, &table_name, &effective);
15887 if let Err(error) = self.finish_observation("syncular_evict", batch) {
15888 self.rollback_observation("syncular_evict");
15889 return Err(error);
15890 }
15891 if !remaining {
15892 break;
15893 }
15894 }
15895 }
15896 Ok(())
15897 }
15898
15899 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
15902 let mut pinned = std::collections::HashSet::new();
15903 for commit in self
15904 .outbox
15905 .iter()
15906 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
15907 {
15908 for op in &commit.ops {
15909 if op.table == table {
15910 pinned.insert(op.row_id.clone());
15911 }
15912 }
15913 }
15914 pinned
15915 }
15916
15917 fn save_pending_evict(
15918 &self,
15919 sub_id: &str,
15920 table: &str,
15921 effective: &[(String, Vec<String>)],
15922 ) -> Result<(), String> {
15923 self.conn.execute(
15924 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
15925 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
15926 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15927 }
15928
15929 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
15930 self.conn
15931 .execute(
15932 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
15933 rusqlite::params![sub_id],
15934 )
15935 .map(|_| ())
15936 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
15937 }
15938
15939 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
15940 let mut stmt = self
15941 .conn
15942 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
15943 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15944 let rows = stmt
15945 .query_map([], |row| {
15946 Ok((
15947 row.get::<_, String>(0)?,
15948 row.get::<_, String>(1)?,
15949 row.get::<_, String>(2)?,
15950 ))
15951 })
15952 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15953 rows.map(|row| {
15954 let (sub_id, table, json) =
15955 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15956 let value: Value = serde_json::from_str(&json)
15957 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
15958 let effective = json_to_scope_map(&value)
15959 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
15960 Ok((sub_id, table, effective))
15961 })
15962 .collect()
15963 }
15964
15965 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
15967 self.retain_failed_commits = enabled;
15968 for (commit, _) in &self.failed_commits {
15969 for operation in &commit.ops {
15970 self.overlay_dirty.table(&operation.table);
15971 }
15972 }
15973 self.failed_commits = self.load_failed_commits()?;
15974 for (commit, _) in &self.failed_commits {
15975 for operation in &commit.ops {
15976 self.overlay_dirty.table(&operation.table);
15977 }
15978 }
15979 Ok(())
15980 }
15981
15982 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
15983 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
15984 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15985 let rows = statement
15986 .query_map([], |row| {
15987 Ok((
15988 row.get::<_, String>(0)?,
15989 row.get::<_, String>(1)?,
15990 row.get::<_, String>(2)?,
15991 ))
15992 })
15993 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15994 rows.map(|row| {
15995 let (id, operations, initial) =
15996 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
15997 let operations: Vec<CommitOperation> =
15998 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
15999 let initial =
16000 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
16001 Ok((
16002 OutboxCommit {
16003 client_commit_id: id,
16004 ops: operations
16005 .into_iter()
16006 .map(|op| OutboxOp {
16007 upsert: op.op == "upsert",
16008 table: op.table,
16009 row_id: op.row_id,
16010 base_version: op.base_version,
16011 values: op.values,
16012 insert: op.insert,
16013 })
16014 .collect(),
16015 },
16016 initial,
16017 ))
16018 })
16019 .collect()
16020 }
16021
16022 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
16023 let mut initial = Map::new();
16024 for operation in operations {
16025 let table = self
16026 .schema
16027 .table(&operation.table)
16028 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16029 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
16030 Some(row) => Some(row),
16031 None => self.visible_row(table, &operation.row_id)?,
16032 };
16033 if let Some((values, _)) = initial_row {
16034 let mut intended: Map<String, Value> = table
16035 .columns
16036 .iter()
16037 .zip(values.iter())
16038 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
16039 .collect();
16040 if let Some(patch) = &operation.values {
16041 intended.extend(patch.clone());
16042 }
16043 initial.insert(
16044 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
16045 .map_err(|_| "sync.local_corrupt".to_owned())?,
16046 Value::Object(intended),
16047 );
16048 }
16049 }
16050 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
16051 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
16052 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
16053 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16054 self.failed_commits.push((
16055 OutboxCommit {
16056 client_commit_id: id.to_owned(),
16057 ops: operations.to_vec(),
16058 },
16059 initial,
16060 ));
16061 Ok(())
16062 }
16063
16064 fn drop_failed_commits(
16065 &mut self,
16066 matches: impl Fn(&str, &Map<String, Value>) -> bool,
16067 ) -> Result<Vec<OutboxCommit>, String> {
16068 let mut doomed = self.drop_acknowledged_rows(&matches)?;
16069 for (commit, initial) in &self.failed_commits {
16070 for operation in &commit.ops {
16071 let key =
16072 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
16073 .map_err(|_| "sync.local_corrupt".to_owned())?;
16074 if initial
16075 .get(&key)
16076 .and_then(Value::as_object)
16077 .is_some_and(|values| matches(&operation.table, values))
16078 {
16079 doomed.push(commit.clone());
16080 break;
16081 }
16082 }
16083 }
16084 for commit in &doomed {
16085 self.conn
16086 .execute(
16087 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
16088 rusqlite::params![commit.client_commit_id],
16089 )
16090 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16091 }
16092 for commit in &doomed {
16093 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16094 }
16095 self.conflicts.retain(|record| {
16096 !doomed
16097 .iter()
16098 .any(|commit| commit.client_commit_id == record.client_commit_id)
16099 });
16100 self.rejections.retain(|record| {
16101 !doomed
16102 .iter()
16103 .any(|commit| commit.client_commit_id == record.client_commit_id)
16104 });
16105 self.failed_commits.retain(|(commit, _)| {
16106 !doomed
16107 .iter()
16108 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
16109 });
16110 for commit in &doomed {
16111 for operation in &commit.ops {
16112 self.overlay_dirty.table(&operation.table);
16113 }
16114 }
16115 Ok(doomed)
16116 }
16117
16118 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
16122 let prior = self.storage_failure.borrow_mut().take();
16123 let outcome = (|| -> Result<String, String> {
16124 if mutations.is_empty() {
16125 return Err(
16126 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
16127 .to_owned(),
16128 );
16129 }
16130 let mut ops = Vec::with_capacity(mutations.len());
16131 for mutation in mutations {
16132 match mutation {
16133 Mutation::Patch {
16134 table,
16135 values,
16136 base_version,
16137 } => {
16138 let schema_table = self
16139 .schema
16140 .table(&table)
16141 .ok_or_else(|| "sync.unknown_table".to_owned())?;
16142 let values = normalize_values_casing(schema_table, values)?;
16143 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
16144 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
16145 }
16146 Mutation::Upsert {
16147 table,
16148 values,
16149 base_version,
16150 } => {
16151 let schema_table = self.schema.table(&table).ok_or_else(|| {
16152 format!("sync.unknown_table: unknown table {table:?}")
16153 })?;
16154 let values = normalize_values_casing(schema_table, values)?;
16158 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
16159 let values = full_row_values(schema_table, values)?;
16162 self.validate_author_encode(schema_table, &row_id, &values)?;
16168 let insert = !self.visible_row_exists(schema_table, &row_id)?;
16172 ops.push(OutboxOp {
16173 upsert: true,
16174 table,
16175 row_id,
16176 base_version,
16177 values: Some(values),
16178 insert,
16179 });
16180 }
16181 Mutation::Delete {
16182 table,
16183 row_id,
16184 base_version,
16185 } => {
16186 if self.schema.table(&table).is_none() {
16187 return Err(format!("sync.unknown_table: unknown table {table:?}"));
16188 }
16189 ops.push(OutboxOp {
16190 upsert: false,
16191 table,
16192 row_id,
16193 base_version,
16194 values: None,
16195 insert: false,
16196 });
16197 }
16198 }
16199 }
16200 self.record_outbox_commit(ops)
16201 })();
16202 self.finish_authoring(prior, outcome)
16203 }
16204
16205 fn finish_authoring(
16211 &self,
16212 prior: Option<QueryReadFailure>,
16213 outcome: Result<String, String>,
16214 ) -> Result<String, ClientError> {
16215 let fresh = self.storage_failure.borrow_mut().take();
16216 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
16217 match outcome {
16218 Ok(id) => Ok(id),
16219 Err(message) => Err(fresh
16220 .map(ClientError::from)
16221 .unwrap_or_else(|| ClientError::from(message))),
16222 }
16223 }
16224
16225 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
16226 let commit = OutboxCommit {
16227 client_commit_id: uuid::Uuid::new_v4().to_string(),
16228 ops,
16229 };
16230 self.begin_observation("syncular_mutation")?;
16231 let mut batch = ChangeAccumulator::default();
16232 for op in &commit.ops {
16233 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
16234 if let Some(values) = &op.values {
16235 if let Some(table) = self.schema.table(&op.table) {
16236 for scope in &table.scope_variables {
16237 if let Some(Value::String(value)) = values.get(&scope.column) {
16238 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
16239 precise = true;
16240 }
16241 }
16242 }
16243 }
16244 if !precise {
16245 batch.table(&op.table);
16246 }
16247 }
16248 if let Err(error) = self.persist_outbox_insert(&commit) {
16249 self.rollback_observation("syncular_mutation");
16250 return Err(error);
16251 }
16252 let was_dirty = self.overlay_dirty.snapshot();
16253 if let Err(error) = self.rebuild_overlay_if_dirty() {
16258 self.rollback_observation("syncular_mutation");
16259 self.overlay_dirty.restore(was_dirty.clone());
16260 return Err(error);
16261 }
16262 if let Some(error) = commit
16263 .ops
16264 .iter()
16265 .find_map(|op| self.apply_outbox_op(op).err())
16266 {
16267 self.rollback_observation("syncular_mutation");
16268 self.overlay_dirty.restore(was_dirty.clone());
16269 return Err(error);
16270 }
16271 let id = commit.client_commit_id.clone();
16272 self.outbox.push(commit);
16273 batch.status = true;
16274 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
16275 self.rollback_observation("syncular_mutation");
16276 self.outbox.pop();
16277 self.overlay_dirty.restore(was_dirty.clone());
16278 return Err(error);
16279 }
16280 Ok(id)
16281 }
16282
16283 pub fn patch(
16289 &mut self,
16290 table: &str,
16291 row_id: &str,
16292 partial: Map<String, Value>,
16293 base_version: Option<i64>,
16294 ) -> Result<String, ClientError> {
16295 let prior = self.storage_failure.borrow_mut().take();
16296 let outcome = self.perform_patch(table, row_id, partial, base_version);
16297 self.finish_authoring(prior, outcome)
16298 }
16299
16300 fn perform_patch(
16303 &mut self,
16304 table: &str,
16305 row_id: &str,
16306 partial: Map<String, Value>,
16307 base_version: Option<i64>,
16308 ) -> Result<String, String> {
16309 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
16310 self.record_outbox_commit(vec![operation])
16311 }
16312
16313 fn prepare_patch(
16314 &self,
16315 table: &str,
16316 row_id: &str,
16317 partial: Map<String, Value>,
16318 base_version: Option<i64>,
16319 ) -> Result<OutboxOp, String> {
16320 let schema_table = self
16321 .schema
16322 .table(table)
16323 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
16324 let partial = normalize_values_casing(schema_table, partial)?;
16325 if let Some(pk) = partial.get(&schema_table.primary_key) {
16326 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
16327 return Err(format!(
16328 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
16329 ));
16330 }
16331 }
16332 if !self.visible_row_exists(schema_table, row_id)? {
16333 return Err("sync.row_missing: a sparse patch requires a local row".to_owned());
16334 }
16335 for scope in &schema_table.scope_variables {
16345 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
16346 continue;
16347 }
16348 let Some(column) = schema_table
16349 .columns
16350 .iter()
16351 .find(|column| column.name == scope.column)
16352 else {
16353 continue;
16354 };
16355 let stored = self
16356 .conn
16357 .query_row(
16358 &format!(
16359 "SELECT {} FROM {} WHERE {}",
16360 quote_ident(&scope.column),
16361 visible_table(&schema_table.name),
16362 row_id_predicate(schema_table)
16363 ),
16364 rusqlite::params![row_id],
16365 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
16366 )
16367 .optional()
16368 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16369 let incoming = json_to_column_value(column, partial.get(&scope.column))
16374 .ok()
16375 .map(|value| column_value_to_json(&value));
16376 let equal = stored.as_ref().is_some_and(|stored| {
16380 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
16381 });
16382 if !equal {
16383 return Err(format!(
16384 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
16385 scope.column
16386 ));
16387 }
16388 }
16389 let mut values = partial;
16390 for scope in &schema_table.scope_variables {
16393 values.remove(&scope.column);
16394 }
16395 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
16396 ColumnType::Integer => match row_id.parse::<i64>() {
16398 Ok(v)
16399 if v.to_string() == row_id
16400 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
16401 .contains(&v) =>
16402 {
16403 Value::from(v)
16404 }
16405 _ => {
16406 return Err(format!(
16407 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
16408 ))
16409 }
16410 },
16411 ColumnType::Boolean => match row_id {
16412 "true" => Value::Bool(true),
16413 "false" => Value::Bool(false),
16414 _ => {
16415 return Err(format!(
16416 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
16417 ))
16418 }
16419 },
16420 _ => Value::from(row_id.to_owned()),
16421 };
16422 values.insert(schema_table.primary_key.clone(), pk_value);
16423 self.validate_author_encode(schema_table, row_id, &values)?;
16429 Ok(OutboxOp {
16430 upsert: true,
16431 table: table.to_owned(),
16432 row_id: row_id.to_owned(),
16433 base_version,
16434 values: Some(values),
16435 insert: false,
16436 })
16437 }
16438
16439 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
16444 let selector = self.encryption.key_id_columns.get(&table.name)?;
16445 let index = table
16446 .columns
16447 .iter()
16448 .position(|column| &column.name == selector)?;
16449 let sql = format!(
16450 "SELECT {} FROM {} WHERE {}",
16451 quote_ident(selector),
16452 visible_table(&table.name),
16453 row_id_predicate(table)
16454 );
16455 let key: Option<String> = self
16456 .conn
16457 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
16458 .ok()?;
16459 let key = key.filter(|key| !key.is_empty())?;
16460 let mut fallback: Row = vec![None; table.columns.len()];
16461 fallback[index] = Some(ColumnValue::String(key));
16462 Some(fallback)
16463 }
16464
16465 fn validate_author_encode(
16470 &self,
16471 table: &TableSchema,
16472 row_id: &str,
16473 values: &Map<String, Value>,
16474 ) -> Result<(), String> {
16475 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
16476 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
16477 result => result.map(|_| ()),
16478 }
16479 }
16480
16481 fn drop_unencodable_outbox(
16495 &mut self,
16496 with_encryption: bool,
16497 batch: Option<&mut ChangeAccumulator>,
16498 ) -> Result<bool, String> {
16499 let failures = self
16500 .outbox
16501 .iter()
16502 .filter_map(|commit| {
16503 commit
16504 .ops
16505 .iter()
16506 .find_map(|op| {
16507 let table = self.schema.table(&op.table).filter(|_| op.upsert)?;
16508 let values = op.presented_values(table)?;
16511 let fallback = self.stored_key_fallback(table, &op.row_id);
16512 match encode_sparse_row_json(
16513 table,
16514 &op.row_id,
16515 &values,
16516 &self.encryption,
16517 fallback.as_ref(),
16518 ) {
16519 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
16520 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
16521 }
16522 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
16527 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
16528 OUTBOX_INCOMPATIBLE_CODE,
16529 )),
16530 _ => None,
16531 }
16532 })
16533 .map(|(error, code)| (commit.clone(), error, code))
16534 })
16535 .collect::<Vec<_>>();
16536 if failures.is_empty() {
16537 return Ok(false);
16538 }
16539 let prior_outbox = self.outbox.clone();
16545 let prior_rejections = self.rejections.clone();
16546 let prior_failed_commits = self.failed_commits.clone();
16550 let prior_changes = self.change_queue.len();
16551 let prior_intents = self.sync_intent_queue.len();
16552 let prior_last_change = self.last_change.clone();
16553 let prior_overlay_dirty = self.overlay_dirty.snapshot();
16554 let owns_observation = batch.is_none();
16555 let owns_transaction = self.conn.is_autocommit();
16556 let mut domains = ChangeAccumulator::default();
16557 if owns_observation {
16558 self.begin_observation("syncular_value_recovery")?;
16559 }
16560 let dropped = (|| -> Result<(), String> {
16561 for (commit, message, code) in failures {
16562 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
16563 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
16564 }
16565 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
16566 let rejection = RejectionRecord {
16567 client_commit_id: commit.client_commit_id.clone(),
16568 op_index: 0,
16569 code: code.to_owned(),
16570 message,
16571 retryable: false,
16572 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
16573 field_paths: None,
16574 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
16575 required_action: None,
16576 references: None,
16577 }),
16578 operation: representable
16579 .then(|| commit.ops.first())
16580 .flatten()
16581 .map(CommitOperation::from),
16582 };
16583 self.persist_commit_outcome(
16584 &commit.client_commit_id,
16585 CommitOutcomeStatus::Rejected,
16586 &[CommitOperationOutcome::Error {
16587 rejection: rejection.clone(),
16588 }],
16589 representable.then_some(commit.ops.as_slice()),
16590 )?;
16591 self.delete_outbox_persisted(&commit.client_commit_id)?;
16592 self.outbox
16593 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
16594 self.rejections.push(rejection);
16595 for operation in &commit.ops {
16596 domains.table(operation.table.as_str());
16597 }
16598 }
16599 domains.status = true;
16600 domains.rejections = true;
16601 domains.outcomes = true;
16602 self.prune_commit_outcomes()?;
16603 self.overlay_dirty.set(true);
16605 self.rebuild_overlay()?;
16606 Ok(())
16607 })();
16608 let result = match batch {
16609 Some(batch) => {
16610 if dropped.is_ok() {
16611 for table in domains.tables.keys() {
16612 batch.table(table.as_str());
16613 }
16614 batch.status |= domains.status;
16615 batch.rejections |= domains.rejections;
16616 batch.outcomes |= domains.outcomes;
16617 }
16618 dropped
16619 }
16620 None => {
16621 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
16622 }
16623 };
16624 if let Err(error) = result {
16625 if owns_observation {
16626 if owns_transaction {
16627 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
16628 let mut failure = self.storage_failure.borrow_mut();
16629 let failure =
16630 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
16631 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
16632 }
16633 } else {
16634 self.rollback_observation("syncular_value_recovery");
16635 }
16636 }
16637 self.outbox = prior_outbox;
16638 self.rejections = prior_rejections;
16639 self.failed_commits = prior_failed_commits;
16640 self.change_queue.truncate(prior_changes);
16641 self.sync_intent_queue.truncate(prior_intents);
16642 self.last_change = prior_last_change;
16643 self.overlay_dirty.restore(prior_overlay_dirty);
16644 return Err(error);
16645 }
16646 Ok(true)
16647 }
16648
16649 pub fn pending_commit_ids(&self) -> Vec<String> {
16650 self.outbox
16651 .iter()
16652 .map(|c| c.client_commit_id.clone())
16653 .collect()
16654 }
16655
16656 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
16660 let mut payloads = Vec::new();
16661 for commit in &self.outbox {
16662 for op in &commit.ops {
16663 let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
16664 continue;
16665 };
16666 let Some(values) = op.presented_values(table) else {
16667 continue;
16668 };
16669 let fallback = self.stored_key_fallback(table, &op.row_id);
16670 if let Ok(payload) = encode_sparse_row_json(
16671 table,
16672 &op.row_id,
16673 &values,
16674 &self.encryption,
16675 fallback.as_ref(),
16676 ) {
16677 payloads.push(payload);
16678 }
16679 }
16680 }
16681 payloads
16682 }
16683
16684 pub fn conflicts(&self) -> &[ConflictRecord] {
16685 &self.conflicts
16686 }
16687
16688 pub fn rejections(&self) -> &[RejectionRecord] {
16689 &self.rejections
16690 }
16691
16692 pub fn resolve_commit_outcome(
16693 &mut self,
16694 input: ResolveCommitOutcomeInput,
16695 ) -> Result<CommitOutcome, String> {
16696 let current = self
16697 .commit_outcome(&input.client_commit_id)?
16698 .ok_or_else(|| {
16699 format!(
16700 "sync.outcome_not_found: no durable outcome exists for {:?}",
16701 input.client_commit_id
16702 )
16703 })?;
16704 if current.resolution != CommitOutcomeResolution::Active {
16705 return Ok(current);
16706 }
16707 if input.resolution == CommitOutcomeResolution::Active {
16708 return Err("sync.invalid_request: resolution must leave active state".to_owned());
16709 }
16710 match input.resolution {
16711 CommitOutcomeResolution::Superseded => {
16712 let replacement = input
16713 .replacement_client_commit_id
16714 .as_deref()
16715 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
16716 .ok_or_else(|| {
16717 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
16718 .to_owned()
16719 })?;
16720 let _ = replacement;
16721 }
16722 _ if input.replacement_client_commit_id.is_some() => {
16723 return Err(
16724 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
16725 .to_owned(),
16726 );
16727 }
16728 _ => {}
16729 }
16730 let allowed = match current.status {
16731 CommitOutcomeStatus::Conflict => matches!(
16732 input.resolution,
16733 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
16734 ),
16735 CommitOutcomeStatus::Rejected => {
16736 matches!(
16737 input.resolution,
16738 CommitOutcomeResolution::Superseded
16739 | CommitOutcomeResolution::ResolvedKeepServer
16740 )
16741 }
16742 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
16743 input.resolution == CommitOutcomeResolution::Dismissed
16744 }
16745 };
16746 if !allowed {
16747 return Err(format!(
16748 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
16749 input.resolution, current.status
16750 ));
16751 }
16752
16753 self.begin_observation("syncular_outcome_resolution")?;
16754 let previous_failed = self.failed_commits.clone();
16755 let previous_conflicts = self.conflicts.clone();
16756 let previous_rejections = self.rejections.clone();
16757 let result = (|| {
16758 self.conn
16759 .execute(
16760 "UPDATE _syncular_commit_outcomes
16761 SET resolution = ?1, resolved_at_ms = ?2,
16762 replacement_client_commit_id = ?3
16763 WHERE client_commit_id = ?4 AND resolution = 'active'",
16764 rusqlite::params![
16765 Self::outcome_resolution_name(input.resolution),
16766 self.clock_now_ms(),
16767 input.replacement_client_commit_id,
16768 input.client_commit_id
16769 ],
16770 )
16771 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16772 let resolved = self
16773 .commit_outcome(¤t.client_commit_id)?
16774 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
16775 self.conn
16776 .execute(
16777 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
16778 rusqlite::params![input.client_commit_id],
16779 )
16780 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16781 for (commit, _) in &self.failed_commits {
16782 if commit.client_commit_id == input.client_commit_id {
16783 for operation in &commit.ops {
16784 self.overlay_dirty.table(&operation.table);
16785 }
16786 }
16787 }
16788 self.failed_commits
16789 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
16790 self.prune_commit_outcomes()?;
16791 Ok(resolved)
16792 })();
16793 let resolved = match result {
16794 Ok(outcome) => outcome,
16795 Err(error) => {
16796 self.rollback_observation("syncular_outcome_resolution");
16797 self.failed_commits = previous_failed;
16798 self.conflicts = previous_conflicts;
16799 self.rejections = previous_rejections;
16800 return Err(error);
16801 }
16802 };
16803 self.conflicts
16804 .retain(|record| record.client_commit_id != current.client_commit_id);
16805 self.rejections
16806 .retain(|record| record.client_commit_id != current.client_commit_id);
16807 let mut batch = ChangeAccumulator {
16808 conflicts: current.status == CommitOutcomeStatus::Conflict,
16809 rejections: current.status == CommitOutcomeStatus::Rejected,
16810 outcomes: true,
16811 ..ChangeAccumulator::default()
16812 };
16813 if let Some(operations) = ¤t.operations {
16814 for operation in operations {
16815 batch.table(&operation.table);
16816 }
16817 }
16818 if let Err(error) = self
16819 .rebuild_overlay_if_dirty()
16820 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
16821 {
16822 self.rollback_observation("syncular_outcome_resolution");
16823 self.failed_commits = previous_failed;
16824 self.conflicts = previous_conflicts;
16825 self.rejections = previous_rejections;
16826 return Err(error);
16827 }
16828 Ok(resolved)
16829 }
16830
16831 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
16832 self.schema_floor.as_ref()
16833 }
16834
16835 pub fn lease_state(&self) -> Option<&LeaseState> {
16837 self.lease_state.as_ref()
16838 }
16839
16840 fn record_lease_error(&mut self, code: &str) {
16843 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
16844 return;
16845 }
16846 let mut next = self.lease_state.clone().unwrap_or_default();
16847 next.error_code = Some(code.to_owned());
16848 self.set_lease_state(Some(next));
16849 }
16850
16851 fn set_lease_state(&mut self, next: Option<LeaseState>) {
16852 if self.lease_state == next {
16853 return;
16854 }
16855 if self.begin_observation("syncular_lease").is_err() {
16856 return;
16857 }
16858 self.lease_state = next;
16859 if let Some(lease) = &self.lease_state {
16860 if let Ok(json) = serde_json::to_string(lease) {
16861 self.set_meta(LEASE_STATE_KEY, &json);
16862 }
16863 } else {
16864 self.delete_meta(LEASE_STATE_KEY);
16865 }
16866 let batch = ChangeAccumulator {
16867 status: true,
16868 ..ChangeAccumulator::default()
16869 };
16870 if self.finish_observation("syncular_lease", batch).is_err() {
16871 self.rollback_observation("syncular_lease");
16872 }
16873 }
16874
16875 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
16876 if self.schema_floor == next {
16877 return;
16878 }
16879 if self.begin_observation("syncular_schema_floor").is_err() {
16880 return;
16881 }
16882 self.schema_floor = next;
16883 self.stopped = self.schema_floor.is_some();
16884 if let Some(floor) = &self.schema_floor {
16885 if let Ok(json) = serde_json::to_string(floor) {
16886 self.set_meta(SCHEMA_FLOOR_KEY, &json);
16887 }
16888 } else {
16889 self.delete_meta(SCHEMA_FLOOR_KEY);
16890 }
16891 let batch = ChangeAccumulator {
16892 status: true,
16893 ..ChangeAccumulator::default()
16894 };
16895 if self
16896 .finish_observation("syncular_schema_floor", batch)
16897 .is_err()
16898 {
16899 self.rollback_observation("syncular_schema_floor");
16900 }
16901 }
16902
16903 fn set_upgrading(&mut self, value: bool) {
16904 if self.upgrading == value {
16905 return;
16906 }
16907 if self.begin_observation("syncular_upgrading").is_err() {
16908 return;
16909 }
16910 self.upgrading = value;
16911 let batch = ChangeAccumulator {
16912 status: true,
16913 ..ChangeAccumulator::default()
16914 };
16915 if self
16916 .finish_observation("syncular_upgrading", batch)
16917 .is_err()
16918 {
16919 self.rollback_observation("syncular_upgrading");
16920 }
16921 }
16922
16923 pub fn sync_needed(&self) -> bool {
16924 self.sync_needed
16925 }
16926
16927 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
16928 let sub = self.subs.iter().find(|s| s.id == id)?;
16929 Some(SubscriptionStateView {
16930 id: sub.id.clone(),
16931 table: sub.table.clone(),
16932 status: sub.state.name().to_owned(),
16933 cursor: sub.cursor,
16934 has_resume_token: sub.bootstrap_state.is_some(),
16935 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
16936 reason_code: sub.reason_code.clone(),
16937 })
16938 }
16939
16940 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
16941 let schema_table = self
16942 .schema
16943 .table(table)
16944 .ok_or_else(|| format!("unknown table {table:?}"))?;
16945 let sql = format!(
16946 "SELECT * FROM {} ORDER BY {} ASC",
16947 visible_table(table),
16948 quote_ident(&schema_table.primary_key)
16949 );
16950 let mut stmt = self
16951 .conn
16952 .prepare(&sql)
16953 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16954 let mut rows = stmt
16955 .query([])
16956 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16957 let mut out = Vec::new();
16958 while let Some(row) = rows
16959 .next()
16960 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
16961 {
16962 let mut values = Map::new();
16963 for (i, column) in schema_table.columns.iter().enumerate() {
16964 let value = row
16965 .get_ref(i)
16966 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16967 values.insert(column.name.clone(), sql_ref_to_json(column, value));
16968 }
16969 let version: i64 = row
16970 .get(schema_table.columns.len())
16971 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
16972 let row_id = match values.get(&schema_table.primary_key) {
16973 Some(Value::String(s)) => s.clone(),
16974 Some(Value::Number(n)) => n.to_string(),
16975 Some(Value::Bool(b)) => b.to_string(),
16976 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
16977 };
16978 out.push(RowState {
16979 row_id,
16980 version,
16981 values,
16982 });
16983 }
16984 Ok(out)
16985 }
16986
16987 #[cfg(feature = "crdt-yjs")]
17003 fn crdt_column_bytes(
17004 &self,
17005 table: &str,
17006 row_id: &str,
17007 column: &str,
17008 ) -> Result<Option<Vec<u8>>, String> {
17009 let schema_table = self
17010 .schema
17011 .table(table)
17012 .ok_or_else(|| format!("unknown table {table:?}"))?;
17013 let col = schema_table
17014 .columns
17015 .iter()
17016 .find(|c| c.name == column)
17017 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
17018 if col.ty != ColumnType::Crdt {
17019 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
17020 }
17021 let sql = format!(
17022 "SELECT {} FROM {} WHERE {}",
17023 quote_ident(column),
17024 visible_table(table),
17025 row_id_predicate(schema_table)
17026 );
17027 let bytes: Option<Vec<u8>> = self
17028 .conn
17029 .prepare_cached(&sql)
17030 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
17031 .query_row(rusqlite::params![row_id], |row| {
17032 row.get::<_, Option<Vec<u8>>>(0)
17033 })
17034 .map_err(|e| match e {
17035 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
17036 other => Self::sqlite_failure(&self.storage_failure, other),
17037 })?;
17038 Ok(bytes)
17039 }
17040
17041 #[cfg(feature = "crdt-yjs")]
17045 pub fn crdt_text(
17046 &self,
17047 table: &str,
17048 row_id: &str,
17049 column: &str,
17050 name: &str,
17051 ) -> Result<String, String> {
17052 let bytes = self
17053 .crdt_column_bytes(table, row_id, column)?
17054 .unwrap_or_default();
17055 crate::crdt::text(&bytes, name)
17056 }
17057
17058 #[cfg(feature = "crdt-yjs")]
17062 pub fn crdt_insert_text(
17063 &mut self,
17064 table: &str,
17065 row_id: &str,
17066 column: &str,
17067 name: &str,
17068 index: u32,
17069 value: &str,
17070 ) -> Result<String, ClientError> {
17071 let prior = self.storage_failure.borrow_mut().take();
17072 let outcome = (|| -> Result<String, String> {
17073 let current = self
17074 .crdt_column_bytes(table, row_id, column)?
17075 .unwrap_or_default();
17076 let update = crate::crdt::insert_text(¤t, name, index, value)?;
17077 self.perform_crdt_push_update(table, row_id, column, &update)
17078 })();
17079 self.finish_authoring(prior, outcome)
17080 }
17081
17082 #[cfg(feature = "crdt-yjs")]
17086 pub fn crdt_delete_text(
17087 &mut self,
17088 table: &str,
17089 row_id: &str,
17090 column: &str,
17091 name: &str,
17092 index: u32,
17093 len: u32,
17094 ) -> Result<String, ClientError> {
17095 let prior = self.storage_failure.borrow_mut().take();
17096 let outcome = (|| -> Result<String, String> {
17097 let current = self
17098 .crdt_column_bytes(table, row_id, column)?
17099 .unwrap_or_default();
17100 let update = crate::crdt::delete_text(¤t, name, index, len)?;
17101 self.perform_crdt_push_update(table, row_id, column, &update)
17102 })();
17103 self.finish_authoring(prior, outcome)
17104 }
17105
17106 #[cfg(feature = "crdt-yjs")]
17111 pub fn crdt_apply_update(
17112 &mut self,
17113 table: &str,
17114 row_id: &str,
17115 column: &str,
17116 update: &[u8],
17117 ) -> Result<String, ClientError> {
17118 let prior = self.storage_failure.borrow_mut().take();
17119 let outcome = (|| -> Result<String, String> {
17120 let current = self
17121 .crdt_column_bytes(table, row_id, column)?
17122 .unwrap_or_default();
17123 let next = crate::crdt::apply_update(¤t, update)?;
17124 self.perform_crdt_push_update(table, row_id, column, &next)
17125 })();
17126 self.finish_authoring(prior, outcome)
17127 }
17128
17129 #[cfg(feature = "crdt-yjs")]
17136 fn perform_crdt_push_update(
17137 &mut self,
17138 table: &str,
17139 row_id: &str,
17140 column: &str,
17141 crdt_bytes: &[u8],
17142 ) -> Result<String, String> {
17143 let mut bytes_obj = Map::new();
17144 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
17145 let mut partial = Map::new();
17146 partial.insert(column.to_owned(), Value::Object(bytes_obj));
17147 self.perform_patch(table, row_id, partial, None)
17148 }
17149
17150 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
17165 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
17166 }
17167
17168 #[cfg(feature = "bench-internals")]
17170 #[doc(hidden)]
17171 pub fn benchmark_connection(&mut self) -> &mut Connection {
17172 &mut self.conn
17173 }
17174
17175 #[cfg(feature = "bench-internals")]
17177 #[doc(hidden)]
17178 pub fn benchmark_phases(
17179 &self,
17180 enabled: Option<bool>,
17181 reset: bool,
17182 ) -> Result<Option<Value>, String> {
17183 self.benchmark_phases.configure(enabled, reset)?;
17184 Ok(self.benchmark_phases.snapshot())
17185 }
17186
17187 pub fn authority_snapshot(&self) -> Result<Value, String> {
17189 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
17190 if self.limits.authority_reads.is_empty() {
17191 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
17192 }
17193 self.conn
17194 .execute_batch("SAVEPOINT authority_snapshot")
17195 .map_err(|e| e.to_string())?;
17196 let result = (|| {
17197 let revision = self
17198 .get_meta(LOCAL_REVISION_KEY)
17199 .ok_or("sync.local_corrupt: authority revision is missing")?;
17200 let parsed_revision = revision
17201 .parse::<u64>()
17202 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
17203 if parsed_revision.to_string() != revision {
17204 return Err("sync.local_corrupt: authority revision is invalid".into());
17205 }
17206 let mut tables = Vec::new();
17207 for read in &self.limits.authority_reads {
17208 let table = self
17209 .schema
17210 .table(&read.table)
17211 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
17212 let scalar = |column: &str, value: &Value| -> Result<String, String> {
17213 let ty = table
17214 .columns
17215 .iter()
17216 .find(|c| c.name == column)
17217 .expect("validated authority column")
17218 .ty;
17219 if ty == ColumnType::Boolean {
17220 if let Some(value) = value.as_bool() {
17221 return Ok(value.to_string());
17222 }
17223 return match value.as_i64() {
17224 Some(0) => Ok("false".into()),
17225 Some(1) => Ok("true".into()),
17226 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
17227 };
17228 }
17229 match value {
17230 Value::String(s) => Ok(s.clone()),
17231 Value::Number(n) => Ok(n.to_string()),
17232 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
17233 .get("$bigint")
17234 .and_then(Value::as_str)
17235 .map(str::to_owned)
17236 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
17237 _ => {
17238 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
17239 }
17240 }
17241 };
17242 let matches = |row: &QueryRow| -> Result<bool, String> {
17243 for (variable, values) in &read.scopes {
17244 let column = table
17245 .scope_column(variable)
17246 .expect("validated authority scope");
17247 let value = row
17248 .get(column)
17249 .ok_or("sync.local_corrupt: authority scope column is missing")?;
17250 if value.is_null() || !values.contains(&scalar(column, value)?) {
17251 return Ok(false);
17252 }
17253 }
17254 Ok(true)
17255 };
17256 let selected = read
17257 .columns
17258 .iter()
17259 .map(|c| quote_ident(c))
17260 .collect::<Vec<_>>()
17261 .join(",");
17262 let mut intent = BTreeSet::new();
17263 let mut relevant = BTreeSet::new();
17264 for row in query_connection(&self.conn,
17265 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
17266 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
17267 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
17268 intent.insert(id.clone());
17269 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
17270 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
17271 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
17272 }
17273 }
17274 let mut rows = BTreeMap::new();
17275 for full in [base_table(&read.table), visible_table(&read.table)] {
17276 for mut values in query_connection(
17277 &self.conn,
17278 &format!(
17279 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
17280 ),
17281 &[],
17282 )
17283 .map_err(|e| e.to_string())?
17284 {
17285 if !matches(&values)? {
17286 continue;
17287 }
17288 let id = scalar(
17289 &table.primary_key,
17290 values
17291 .get(&table.primary_key)
17292 .expect("selected primary key"),
17293 )?;
17294 relevant.insert(id.clone());
17295 if full == base_table(&read.table) {
17296 let version = values
17297 .remove("authority_version")
17298 .expect("selected version");
17299 let version = version
17300 .as_i64()
17301 .ok_or("sync.local_corrupt: authority server version is invalid")?;
17302 if version >= 0 {
17303 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
17304 }
17305 }
17306 }
17307 }
17308 intent.retain(|id| relevant.contains(id));
17309 let rows = rows.into_values().collect::<Vec<_>>();
17310 let mut persisted = Vec::new();
17311 for row in query_connection(
17312 &self.conn,
17313 "SELECT state_json FROM _syncular_subscriptions WHERE tbl=? ORDER BY rowid",
17314 &[Value::String(read.table.clone())],
17315 )
17316 .map_err(|e| e.message)?
17317 {
17318 let state: Value = serde_json::from_str(
17319 row.get("state_json")
17320 .and_then(Value::as_str)
17321 .ok_or("sync.local_corrupt: authority subscription state is missing")?,
17322 )
17323 .map_err(|_| "sync.local_corrupt: authority subscription state is invalid")?;
17324 let effective = state.get("effectiveScopes").and_then(Value::as_object);
17325 let status = state
17326 .get("status")
17327 .and_then(Value::as_str)
17328 .ok_or("sync.local_corrupt: authority subscription status is missing")?;
17329 let cursor = state
17330 .get("cursor")
17331 .and_then(Value::as_i64)
17332 .ok_or("sync.local_corrupt: authority subscription cursor is missing")?;
17333 let complete = effective.is_some_and(|s| s.len() == read.scopes.len())
17334 && status == "active"
17335 && cursor >= 0
17336 && state.get("bootstrapState").is_none_or(Value::is_null)
17337 && state.get("params").is_none_or(Value::is_null);
17338 let exposed = effective.map(|s| {
17339 read.scopes
17340 .keys()
17341 .map(|v| (v.clone(), s.get(v).cloned().unwrap_or(json!([]))))
17342 .collect::<Map<_, _>>()
17343 });
17344 let requested = read
17345 .scopes
17346 .keys()
17347 .map(|v| {
17348 (
17349 v.clone(),
17350 state
17351 .get("requested")
17352 .and_then(|s| s.get(v))
17353 .cloned()
17354 .unwrap_or(json!([])),
17355 )
17356 })
17357 .collect::<Map<_, _>>();
17358 persisted.push(json!({"requestedScopes": requested, "status": status, "cursor": cursor, "effectiveScopes": exposed, "complete": complete}));
17359 }
17360 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
17361 match selectors.split_first() {
17362 None => !candidates.is_empty(),
17363 Some(((variable, values), rest)) => values.iter().all(|value| {
17364 covered(
17365 rest,
17366 &candidates
17367 .iter()
17368 .copied()
17369 .filter(|p| {
17370 p["effectiveScopes"][variable.as_str()]
17371 .as_array()
17372 .is_some_and(|held| {
17373 held.contains(&Value::String(value.clone()))
17374 || held.contains(&Value::String("*".into()))
17375 })
17376 })
17377 .collect::<Vec<_>>(),
17378 )
17379 }),
17380 }
17381 }
17382 let coverage = if covered(
17383 &read.scopes.iter().collect::<Vec<_>>(),
17384 &persisted
17385 .iter()
17386 .filter(|p| p["complete"] == true)
17387 .collect::<Vec<_>>(),
17388 ) {
17389 "complete"
17390 } else if persisted.iter().any(|p| {
17391 p["status"] == "active"
17392 && read.scopes.iter().all(|(v, values)| {
17393 values.iter().any(|value| {
17394 p["requestedScopes"][v].as_array().is_some_and(|held| {
17395 held.contains(&Value::String(value.clone()))
17396 || held.contains(&Value::String("*".into()))
17397 })
17398 })
17399 })
17400 }) {
17401 "pending"
17402 } else {
17403 "missing"
17404 };
17405 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
17406 }
17407 Ok(
17408 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
17409 )
17410 })();
17411 match result {
17412 Ok(snapshot) => {
17413 self.conn
17414 .execute_batch("RELEASE authority_snapshot")
17415 .map_err(|e| e.to_string())?;
17416 Ok(snapshot)
17417 }
17418 Err(error) => {
17419 self.conn
17420 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
17421 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
17422 Err(error)
17423 }
17424 }
17425 }
17426
17427 pub fn query_snapshot(
17431 &mut self,
17432 sql: &str,
17433 params: &[QueryValue],
17434 coverage: &[WindowCoverage],
17435 owner: Option<QueryOwner<'_>>,
17436 ) -> Result<QuerySnapshot, String> {
17437 let result = snapshot_connection(&self.conn, sql, params, coverage);
17438 if let Some(owner) = owner {
17439 self.record_query_read(owner, result.as_ref().err());
17440 }
17441 result.map_err(|failure| failure.to_string())
17442 }
17443
17444 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
17448 let position = self
17449 .query_failures
17450 .iter()
17451 .position(|entry| entry.id == owner.id);
17452 let Some(failure) = failure else {
17453 if let Some(index) = position {
17454 self.query_failures.remove(index);
17455 }
17456 return;
17457 };
17458 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
17459 .into_iter()
17460 .map(str::to_owned)
17461 .collect();
17462 let code = failure.code.unwrap_or("client.query_failed").to_owned();
17463 if position.is_some_and(|index| {
17464 let entry = &self.query_failures[index];
17465 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
17466 }) {
17467 return;
17468 }
17469 if let Some(index) = position {
17470 self.query_failures.remove(index);
17471 }
17472 self.query_failures.push(DiagnosticQueryFailure {
17473 id: owner.id.to_owned(),
17474 tables,
17475 code,
17476 sqlite_code: failure.sqlite_code,
17477 at_ms: self.clock_now_ms(),
17478 });
17479 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
17480 self.query_failures.remove(0);
17481 }
17482 }
17483
17484 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
17487 #[cfg(feature = "bench-internals")]
17488 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
17489 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
17490 let max_commits = self.limits.max_push_commits_per_request;
17491 let max_ops = self
17492 .limits
17493 .max_push_operations_per_request
17494 .unwrap_or(PUSH_OPS_PER_REQUEST);
17495 let byte_cap = self.limits.max_push_request_bytes;
17496 let accept = self.limits.accept.unwrap_or(if url_capable {
17499 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
17500 } else {
17501 DEFAULT_ACCEPT
17502 });
17503 let header = Frame::ReqHeader {
17504 client_id: self.client_id.clone(),
17505 schema_version: self.schema.version,
17506 log_epoch: log_epoch.clone(),
17507 };
17508 let mut lead = vec![header];
17512 lead.extend(
17513 self.round_failure
17514 .iter()
17515 .map(|code| Frame::RoundFailure { code: code.clone() }),
17516 );
17517 let pull = Frame::PullHeader {
17518 limit_commits: self.limits.limit_commits.unwrap_or(0),
17519 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
17520 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
17521 accept,
17522 };
17523 let mut fresh = Vec::new();
17524 let mut sub_frames = Vec::new();
17525 for sub in &self.subs {
17526 if sub.state != SubState::Active {
17527 continue;
17528 }
17529 let mut scopes = sub.requested.clone();
17530 sort_scope_map(&mut scopes);
17531 sub_frames.push(Frame::Subscription {
17532 id: sub.id.clone(),
17533 table: sub.table.clone(),
17534 scopes,
17535 params: sub.params.clone().map(RawJson),
17536 cursor: sub.cursor,
17537 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
17538 });
17539 fresh.push((
17540 sub.id.clone(),
17541 sub.cursor < 0 && sub.bootstrap_state.is_none(),
17542 ));
17543 }
17544 let fixed_bytes = match byte_cap {
17547 Some(_) => {
17548 let mut frames = lead.clone();
17549 frames.push(pull.clone());
17550 frames.extend(sub_frames.iter().cloned());
17551 encode_message(&Message {
17552 wire_version: WIRE_VERSION,
17553 msg_kind: MsgKind::Request,
17554 frames,
17555 })
17556 .len()
17557 }
17558 None => 0,
17559 };
17560 if let Some(cap) = byte_cap {
17561 if fixed_bytes > cap {
17562 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
17563 }
17564 }
17565 let base_bytes = match byte_cap {
17566 Some(_) => {
17567 let mut frames = lead.clone();
17568 frames.push(pull.clone());
17569 encode_message(&Message {
17570 wire_version: WIRE_VERSION,
17571 msg_kind: MsgKind::Request,
17572 frames,
17573 })
17574 .len()
17575 }
17576 None => 0,
17577 };
17578 let total_outbox = if log_epoch.is_some() {
17579 self.outbox.len()
17580 } else {
17581 0
17582 };
17583 let mut push_frames: Vec<Frame> = Vec::new();
17584 let mut pushed_ids = Vec::new();
17585 let mut ops_in_request = 0usize;
17586 let mut bytes_in_request = 0usize;
17587 let mut deferred_commits = 0usize;
17588 #[cfg(feature = "bench-internals")]
17589 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
17590 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
17591 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
17595 deferred_commits = total_outbox - index;
17596 break;
17597 }
17598 if ops_in_request + commit.ops.len() > max_ops {
17599 if push_frames.is_empty() {
17600 return Err(Self::push_capacity_outcome(
17601 "operations",
17602 max_ops,
17603 commit.ops.len(),
17604 Some(commit.client_commit_id.as_str()),
17605 ));
17606 }
17607 deferred_commits = total_outbox - index;
17608 break;
17609 }
17610 let operations = commit
17611 .ops
17612 .iter()
17613 .map(|op| {
17614 let payload = self.schema.table(&op.table).and_then(|table| {
17615 let values = op.presented_values(table)?;
17616 let fallback = self.stored_key_fallback(table, &op.row_id);
17626 encode_sparse_row_json(
17627 table,
17628 &op.row_id,
17629 &values,
17630 &self.encryption,
17631 fallback.as_ref(),
17632 )
17633 .ok()
17634 });
17635 ssp2::model::Operation {
17636 table: op.table.clone(),
17637 row_id: op.row_id.clone(),
17638 op: if op.upsert { Op::Upsert } else { Op::Delete },
17639 base_version: op.base_version,
17640 payload,
17641 }
17642 })
17643 .collect();
17644 let frame = Frame::PushCommit {
17645 client_commit_id: commit.client_commit_id.clone(),
17646 operations,
17647 };
17648 let frame_bytes = match byte_cap {
17649 Some(_) => {
17650 let mut frames = lead.clone();
17651 frames.push(frame.clone());
17652 frames.push(pull.clone());
17653 encode_message(&Message {
17654 wire_version: WIRE_VERSION,
17655 msg_kind: MsgKind::Request,
17656 frames,
17657 })
17658 .len()
17659 - base_bytes
17660 }
17661 None => 0,
17662 };
17663 if let Some(cap) = byte_cap {
17667 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
17668 deferred_commits = total_outbox - index;
17669 break;
17670 }
17671 }
17672 push_frames.push(frame);
17673 pushed_ids.push(commit.client_commit_id.clone());
17674 ops_in_request += commit.ops.len();
17675 bytes_in_request += frame_bytes;
17676 }
17677 #[cfg(feature = "bench-internals")]
17678 drop(outbox_phase);
17679 if let Some(cap) = byte_cap {
17680 if fixed_bytes + bytes_in_request > cap {
17681 let blocked = if push_frames.is_empty() {
17684 None
17685 } else {
17686 self.outbox.first().map(|c| c.client_commit_id.as_str())
17687 };
17688 return Err(Self::push_capacity_outcome(
17689 "bytes",
17690 cap,
17691 fixed_bytes + bytes_in_request,
17692 blocked,
17693 ));
17694 }
17695 }
17696 let mut frames = lead;
17697 frames.extend(push_frames);
17698 frames.push(pull);
17699 frames.extend(sub_frames);
17700 let message = Message {
17701 wire_version: WIRE_VERSION,
17702 msg_kind: MsgKind::Request,
17703 frames,
17704 };
17705 Ok((
17706 message,
17707 RequestMeta {
17708 pushed_ids,
17709 fresh,
17710 accept,
17711 deferred_commits,
17712 },
17713 ))
17714 }
17715
17716 fn push_capacity_outcome(
17718 kind: &str,
17719 limit: usize,
17720 size: usize,
17721 client_commit_id: Option<&str>,
17722 ) -> Box<SyncOutcome> {
17723 let mut details = Map::new();
17724 details.insert("kind".to_owned(), Value::from(kind));
17725 details.insert("limit".to_owned(), Value::from(limit as u64));
17726 details.insert("size".to_owned(), Value::from(size as u64));
17727 if let Some(id) = client_commit_id {
17728 details.insert("clientCommitId".to_owned(), Value::from(id));
17729 }
17730 Box::new(SyncOutcome::Failed {
17731 error_code: "client.push_request_too_large".into(),
17732 message: "the push request exceeds a configured capacity limit".into(),
17733 details: Some(Value::Object(details)),
17734 recommended_action: None,
17735 })
17736 }
17737
17738 pub fn progress(&self) -> ProgressObserver {
17742 self.progress.clone()
17743 }
17744
17745 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
17746 let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
17747 Ok(prepared) => prepared,
17748 Err(outcome) => return *outcome,
17749 };
17750 let mut next = prepared;
17755 loop {
17756 match self.apply_sync_round(next.exchange(transport)) {
17757 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
17758 crate::AppliedSyncRound::Complete {
17759 outcome, controls, ..
17760 } => {
17761 if self.transport_enabled {
17762 for text in controls {
17763 let _ = transport.realtime_send(&text);
17764 }
17765 } else {
17766 self.disconnect_realtime(transport);
17767 }
17768 return outcome;
17769 }
17770 }
17771 }
17772 }
17773
17774 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
17775 if let SyncOutcome::Failed {
17776 error_code,
17777 message,
17778 details,
17779 ..
17780 } = &mut outcome
17781 {
17782 if let Some(failure) = self.storage_failure.borrow().as_ref() {
17783 let code = failure
17784 .code
17785 .expect("retained storage failures are classified");
17786 *error_code = code.to_owned();
17787 *message = match code {
17788 "client.storage_busy" => "local SQLite storage is busy",
17789 "client.storage_full" => "local SQLite storage is full",
17790 "client.storage_io" => "local SQLite storage I/O failed",
17791 "client.storage_corrupt" => "local SQLite storage is corrupt",
17792 _ => unreachable!("unrecognized classified storage failure"),
17793 }
17794 .to_owned();
17795 *details = failure.details();
17796 }
17797 }
17798 let details = match &outcome {
17799 SyncOutcome::Failed { details, .. } => details.as_ref(),
17800 _ => None,
17801 };
17802 let cause_kind = details
17803 .and_then(|details| details.get("causeKind"))
17804 .and_then(Value::as_str)
17805 .and_then(|cause| match cause {
17806 "network" => Some(crate::TransportCauseKind::Network),
17807 "connect" => Some(crate::TransportCauseKind::Connect),
17808 "tls" => Some(crate::TransportCauseKind::Tls),
17809 "io" => Some(crate::TransportCauseKind::Io),
17810 "request" => Some(crate::TransportCauseKind::Request),
17811 "protocol" => Some(crate::TransportCauseKind::Protocol),
17812 "timeout" => Some(crate::TransportCauseKind::Timeout),
17813 "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
17814 "aborted" => Some(crate::TransportCauseKind::Aborted),
17815 "body" => Some(crate::TransportCauseKind::Body),
17816 "status" => Some(crate::TransportCauseKind::Status),
17817 "redirect" => Some(crate::TransportCauseKind::Redirect),
17818 "unknown" => Some(crate::TransportCauseKind::Unknown),
17819 _ => None,
17820 });
17821 let http_status = if matches!(
17822 cause_kind,
17823 Some(
17824 crate::TransportCauseKind::Status
17825 | crate::TransportCauseKind::Body
17826 | crate::TransportCauseKind::Redirect
17827 )
17828 ) {
17829 details
17830 .and_then(|details| details.get("httpStatus"))
17831 .and_then(Value::as_f64)
17832 .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
17833 .map(|status| status as u16)
17834 } else {
17835 None
17836 };
17837 let retry_delay_ms = self.round_retry_delay_ms;
17838 self.progress.update(|p| match &outcome {
17839 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
17840 if report.failed.is_empty() =>
17841 {
17842 p.state = ProgressState::Complete
17843 }
17844 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
17845 p.state = ProgressState::Failed;
17846 p.error_code = Some("sync.scope_revoked".into());
17847 }
17848 SyncOutcome::Failed { error_code, .. } => {
17849 p.state = ProgressState::Failed;
17850 p.error_code = Some(Self::diagnostic_code(error_code));
17851 p.cause_kind = cause_kind;
17852 p.http_status = http_status;
17853 p.retry_delay_ms = retry_delay_ms;
17854 }
17855 SyncOutcome::RealtimeUnavailable { .. } => {
17856 p.state = ProgressState::Failed;
17857 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
17858 p.retry_delay_ms = retry_delay_ms;
17859 }
17860 });
17861 let completed_at_ms = self.clock_now_ms();
17862 self.last_round = Some(match &outcome {
17863 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
17864 status: "succeeded".to_owned(),
17865 started_at_ms,
17866 completed_at_ms,
17867 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
17868 counters: Some(DiagnosticRoundCounters {
17869 pushed: report.pushed,
17870 applied: report.applied.len(),
17871 rejected: report.rejected.len(),
17872 retryable: report.retryable.len(),
17873 conflicts: report.conflicts,
17874 commits_applied: report.commits_applied,
17875 segment_rows_applied: report.segment_rows_applied,
17876 bootstrapping: report.bootstrapping.len(),
17877 resets: report.resets.len(),
17878 revoked: report.revoked.len(),
17879 failed: report.failed.len(),
17880 deferred_commits: report.deferred_commits,
17881 }),
17882 error_code: None,
17883 cause_kind: None,
17884 http_status: None,
17885 },
17886 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
17887 status: "failed".to_owned(),
17888 started_at_ms,
17889 completed_at_ms,
17890 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
17891 counters: None,
17892 error_code: Some(Self::diagnostic_code(error_code)),
17893 cause_kind,
17894 http_status,
17895 },
17896 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
17897 status: "failed".to_owned(),
17898 started_at_ms,
17899 completed_at_ms,
17900 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
17901 counters: None,
17902 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
17903 cause_kind: None,
17904 http_status: None,
17905 },
17906 });
17907 outcome
17908 }
17909
17910 pub fn prepare_sync_round(
17913 &mut self,
17914 url_capable: bool,
17915 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
17916 if !self.transport_enabled {
17917 return Err(Box::new(SyncOutcome::Failed {
17918 error_code: "sync.offline".into(),
17919 message: "transport gate is closed".into(),
17920 details: None,
17921 recommended_action: None,
17922 }));
17923 }
17924 if self.active_round.is_some() {
17925 return Err(Box::new(SyncOutcome::Failed {
17926 error_code: "client.round_in_progress".into(),
17927 message: "a sync round is already in flight".into(),
17928 details: None,
17929 recommended_action: None,
17930 }));
17931 }
17932 self.storage_failure.borrow_mut().take();
17933 self.progress.start();
17934 self.round_retry_delay_ms = None;
17935 let started_at_ms = self.clock_now_ms();
17936 let prepared = (|| {
17937 if self.stopped {
17938 return Err(Box::new(SyncOutcome::Ok(SyncReport {
17939 schema_floor: self.schema_floor.clone(),
17940 ..SyncReport::default()
17941 })));
17942 }
17943 if self.security_preflight {
17944 return Err(Box::new(SyncOutcome::Failed {
17945 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
17946 message: "security preflight must complete before syncing".into(),
17947 details: None,
17948 recommended_action: None,
17949 }));
17950 }
17951 self.set_sync_needed(false, false);
17952 self.drain_pending_evictions()
17953 .and_then(|_| self.drop_incompatible_outbox())
17954 .and_then(|_| self.drop_unencodable_outbox(true, None))
17955 .map_err(|message| {
17956 Box::new(SyncOutcome::Failed {
17957 error_code: "storage.failed".into(),
17958 message,
17959 details: None,
17960 recommended_action: None,
17961 })
17962 })?;
17963 let uploads = if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
17964 self.pending_blob_uploads().map_err(|e| {
17965 Box::new(SyncOutcome::Failed {
17966 error_code: e.code,
17967 message: e.message,
17968 details: e.details,
17969 recommended_action: None,
17970 })
17971 })?
17972 } else {
17973 Vec::new()
17974 };
17975 if self.realtime_state != RealtimeState::Connected
17976 && self.realtime_policy == RealtimePolicy::Required
17977 {
17978 let retry_delay_ms = self.schedule_background_retry();
17979 self.realtime_retry_delay_ms = Some(retry_delay_ms);
17980 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
17981 state: self.realtime_state,
17982 reason_code: self.realtime_reason_code.clone(),
17983 retry_delay_ms,
17984 }));
17985 }
17986 let (message, meta) = self.build_request(url_capable)?;
17987 let id = uuid::Uuid::new_v4();
17988 self.active_round = Some(id);
17989 let phase = if uploads.is_empty() {
17990 crate::round::RoundPhase::Sync
17991 } else {
17992 crate::round::RoundPhase::UploadGrant
17993 };
17994 Ok(crate::PreparedSyncRound {
17995 id,
17996 started_at_ms,
17997 message,
17998 meta,
17999 uploads: uploads.into(),
18000 phase,
18001 encoded: None,
18002 continuation: None,
18003 completed: None,
18004 realtime: self.realtime_state == RealtimeState::Connected,
18005 scopes: self
18006 .subs
18007 .iter()
18008 .map(|s| (s.id.clone(), canonical_scope_json(&s.requested)))
18009 .collect(),
18010 #[cfg(feature = "bench-internals")]
18011 benchmark_phases: self.benchmark_phases.clone(),
18012 progress: self.progress.clone(),
18013 fixed_now: self.now_ms,
18014 round_deadline_at: None,
18015 })
18016 })();
18017 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
18018 }
18019
18020 pub fn cancel_sync_round(&mut self) {
18023 self.active_round = None;
18024 }
18025
18026 pub fn apply_sync_round(
18029 &mut self,
18030 completed: crate::CompletedSyncRound,
18031 ) -> crate::AppliedSyncRound {
18032 let prepared = completed.prepared;
18033 if self.active_round != Some(prepared.id) || self.security_preflight {
18034 return crate::AppliedSyncRound::Complete {
18035 outcome: SyncOutcome::Failed {
18036 error_code: "client.round_cancelled".into(),
18037 message: "sync round belongs to a released client context".into(),
18038 details: None,
18039 recommended_action: None,
18040 },
18041 controls: Vec::new(),
18042 more: false,
18043 bootstrap_advanced: false,
18044 };
18045 }
18046 self.storage_failure.borrow_mut().take();
18047 let (response, mut downloads, transport_failed) = match completed.exchange {
18048 crate::round::ExchangeResult::Upload { id, result } => {
18049 let result = result.and_then(|()| {
18050 self.conn
18051 .execute(
18052 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
18053 rusqlite::params![id],
18054 )
18055 .map(|_| ())
18056 .map_err(|error| {
18057 TransportError::new(
18058 "client.failed",
18059 Self::sqlite_failure(&self.storage_failure, error),
18060 )
18061 })
18062 });
18063 match result {
18064 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
18065 Err(TransportError {
18066 code,
18067 message,
18068 details,
18069 }) => {
18070 self.active_round = None;
18071 if Self::retryable_failure_code(&code)
18072 && !Self::unreachable_transport(&details)
18073 {
18074 self.schedule_background_retry();
18075 }
18076 let recommended_action = recommended_action_of(&details);
18077 let outcome = self.finish_sync_round(
18078 prepared.started_at_ms,
18079 SyncOutcome::Failed {
18080 error_code: code,
18081 message,
18082 details,
18083 recommended_action,
18084 },
18085 );
18086 return crate::AppliedSyncRound::Complete {
18087 outcome,
18088 controls: Vec::new(),
18089 more: false,
18090 bootstrap_advanced: false,
18091 };
18092 }
18093 }
18094 }
18095 crate::round::ExchangeResult::Reply {
18096 response,
18097 downloads,
18098 transport_failed,
18099 } => (response, downloads, transport_failed),
18100 };
18101 self.active_round = None;
18102 let before = self
18103 .subs
18104 .iter()
18105 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
18106 .collect::<Vec<_>>();
18107 let outcome = (|| {
18108 let response = match response {
18109 Ok(response) => {
18110 self.round_failure = None;
18113 response
18114 }
18115 Err(TransportError {
18116 code,
18117 message,
18118 details,
18119 }) => {
18120 if ssp2::model::valid_round_failure_code(&code) {
18125 self.round_failure = Some(code.clone());
18126 }
18127 if transport_failed {
18128 if prepared.realtime {
18129 self.set_realtime_state(RealtimeState::Lost, Some(&code));
18130 }
18131 self.record_lease_error(&code);
18132 }
18133 if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
18134 {
18135 let delay = self.schedule_background_retry();
18136 if prepared.realtime && transport_failed {
18137 self.realtime_retry_delay_ms = Some(delay);
18138 }
18139 }
18140 let recommended_action = recommended_action_of(&details);
18141 return SyncOutcome::Failed {
18142 error_code: code,
18143 message,
18144 details,
18145 recommended_action,
18146 };
18147 }
18148 };
18149 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
18150 if let SyncOutcome::Ok(report) = &mut outcome {
18151 report.deferred_commits = prepared.meta.deferred_commits;
18152 }
18153 match &outcome {
18154 SyncOutcome::Ok(_) => self.reset_background_retry(),
18155 SyncOutcome::Failed {
18156 error_code,
18157 details,
18158 ..
18159 } if Self::retryable_failure_code(error_code)
18160 && !Self::unreachable_transport(details) =>
18161 {
18162 self.schedule_background_retry();
18163 }
18164 _ => {}
18165 }
18166 if prepared.meta.deferred_commits > 0 {
18167 self.set_sync_needed(true, true);
18168 }
18169 self.previous_version_lifetime_check();
18170 outcome
18171 })();
18172 let more = self.transport_enabled
18173 && matches!(&outcome, SyncOutcome::Ok(report)
18174 if report.schema_floor.is_none()
18175 && (!report.bootstrapping.is_empty()
18176 || report.commits_applied > 0
18177 || report.segment_rows_applied > 0
18178 || !report.resets.is_empty()
18179 || report.deferred_commits > 0
18180 || self.sync_needed));
18181 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
18182 && before
18183 != self
18184 .subs
18185 .iter()
18186 .map(|s| (s.id.clone(), s.bootstrap_state.clone()))
18187 .collect::<Vec<_>>();
18188 crate::AppliedSyncRound::Complete {
18189 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
18190 controls: if self.transport_enabled {
18191 downloads.controls
18192 } else {
18193 Vec::new()
18194 },
18195 more,
18196 bootstrap_advanced,
18197 }
18198 }
18199
18200 pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
18202
18203 pub fn sync_until_idle(
18204 &mut self,
18205 transport: &mut dyn Transport,
18206 max_rounds: Option<u32>,
18207 ) -> SyncOutcome {
18208 self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
18209 }
18210
18211 pub fn sync_until_idle_with_deadline(
18216 &mut self,
18217 transport: &mut dyn Transport,
18218 max_rounds: Option<u32>,
18219 timeout_ms: Option<u64>,
18220 ) -> SyncOutcome {
18221 if max_rounds == Some(0) {
18224 return SyncOutcome::Failed {
18225 details: None,
18226 error_code: "sync.invalid_request".into(),
18227 message: "maxRounds must be a positive integer".into(),
18228 recommended_action: None,
18229 };
18230 }
18231 if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
18232 return SyncOutcome::Failed {
18233 details: None,
18234 error_code: "sync.invalid_request".into(),
18235 message: "timeoutMs must be a positive integer".into(),
18236 recommended_action: None,
18237 };
18238 }
18239 let started_at_ms = self.clock_now_ms();
18240 let deadline_ms = timeout_ms.map(|ms| started_at_ms.saturating_add(ms as i64));
18241 self.sync_until_idle_bounded(transport, max_rounds, started_at_ms, deadline_ms)
18242 }
18243
18244 fn sync_until_idle_bounded(
18245 &mut self,
18246 transport: &mut dyn Transport,
18247 max_rounds: Option<u32>,
18248 started_at_ms: i64,
18249 deadline_ms: Option<i64>,
18250 ) -> SyncOutcome {
18251 let rounds = max_rounds.unwrap_or(20);
18252 let mut aggregate = SyncReport::default();
18253 let mut spent = 0;
18254 while spent < rounds {
18255 if let Some(deadline_ms) = deadline_ms {
18256 if self.clock_now_ms() >= deadline_ms {
18257 return SyncOutcome::Failed {
18258 error_code: "sync.idle_timeout".into(),
18259 message: "sync_until_idle exceeded its deadline".into(),
18260 details: Some(serde_json::json!({
18261 "timeoutMs": deadline_ms
18262 .saturating_sub(started_at_ms) as u64,
18263 "rounds": spent,
18264 })),
18265 recommended_action: Some("retryLater".into()),
18266 };
18267 }
18268 }
18269 spent += 1;
18270 let before = self
18271 .subs
18272 .iter()
18273 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
18274 .collect::<Vec<_>>();
18275 match self.sync(transport) {
18276 SyncOutcome::Failed {
18277 error_code,
18278 message,
18279 details,
18280 recommended_action,
18281 } => {
18282 return SyncOutcome::Failed {
18283 details,
18284 error_code,
18285 message,
18286 recommended_action,
18287 };
18288 }
18289 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
18290 SyncOutcome::BudgetExhausted(_) => {
18291 unreachable!("sync() never exhausts a round budget")
18292 }
18293 SyncOutcome::Ok(report) => {
18294 if max_rounds.is_none()
18295 && report.segment_rows_applied > 0
18296 && !report.bootstrapping.is_empty()
18297 && before
18298 != self
18299 .subs
18300 .iter()
18301 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
18302 .collect::<Vec<_>>()
18303 {
18304 spent = 0;
18305 }
18306 let stop = !self.transport_enabled() || report.schema_floor.is_some();
18309 aggregate.merge(&report);
18310 if stop {
18311 return SyncOutcome::Ok(aggregate);
18312 }
18313 let more = !report.bootstrapping.is_empty()
18319 || report.commits_applied > 0
18320 || report.segment_rows_applied > 0
18321 || !report.resets.is_empty()
18322 || report.deferred_commits > 0
18323 || self.sync_needed;
18324 if !more {
18325 return SyncOutcome::Ok(aggregate);
18326 }
18327 }
18328 }
18329 }
18330 SyncOutcome::BudgetExhausted(aggregate)
18333 }
18334
18335 fn process_response(
18336 &mut self,
18337 transport: &mut dyn Transport,
18338 response: Message,
18339 meta: &RequestMeta,
18340 ) -> SyncOutcome {
18341 #[cfg(feature = "bench-internals")]
18342 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
18343 let mut report = SyncReport {
18344 pushed: meta.pushed_ids.len() as u32,
18345 ..SyncReport::default()
18346 };
18347 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
18348 HashMap::new();
18349 let pushed_ids = meta
18350 .pushed_ids
18351 .iter()
18352 .map(String::as_str)
18353 .collect::<HashSet<_>>();
18354 let mut incomplete_subscriptions: HashSet<&str> =
18355 meta.fresh.iter().map(|(id, _)| id.as_str()).collect();
18356 let mut last_final_push_result_id: Option<String> = None;
18357 for frame in &response.frames {
18358 match frame {
18359 Frame::PushResultDetails {
18360 client_commit_id,
18361 entries,
18362 } => {
18363 let details = rejection_details_by_commit
18364 .entry(client_commit_id.clone())
18365 .or_default();
18366 for entry in entries {
18367 let parsed = match RejectionDetails::parse(&entry.details.0) {
18368 Ok(value) => value,
18369 Err(message) => {
18370 return SyncOutcome::Failed {
18371 details: None,
18372 error_code: "sync.invalid_request".to_owned(),
18373 message,
18374 recommended_action: None,
18375 };
18376 }
18377 };
18378 details.insert(entry.op_index, parsed);
18379 }
18380 }
18381 Frame::PushResult {
18382 client_commit_id,
18383 status,
18384 results,
18385 ..
18386 } if pushed_ids.contains(client_commit_id.as_str())
18387 && Self::push_result_is_final(*status, results) =>
18388 {
18389 last_final_push_result_id = Some(client_commit_id.clone());
18390 }
18391 _ => {}
18392 }
18393 }
18394 if response.wire_version < 2 {
18395 return SyncOutcome::Failed {
18396 details: None,
18397 error_code: "client.invalid_host_response".to_owned(),
18398 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
18399 recommended_action: None,
18400 };
18401 }
18402 let coalesced_deletes = coalesced_delete_keys(&response.frames, |id, status, _| {
18403 status == SubStatus::Active && self.subs.iter().any(|sub| sub.id == id)
18404 });
18405 let mut frames = response.frames.into_iter().peekable();
18406 match frames.next() {
18407 Some(Frame::RespHeader {
18408 required_schema_version,
18409 latest_schema_version,
18410 log_epoch,
18411 reset_required,
18412 }) => {
18413 let Some(log_epoch) = log_epoch else {
18414 return SyncOutcome::Failed {
18415 details: None,
18416 error_code: "client.invalid_host_response".to_owned(),
18417 message: "response header omits logEpoch".to_owned(),
18418 recommended_action: None,
18419 };
18420 };
18421 let Some(reset_required) = reset_required else {
18422 return SyncOutcome::Failed {
18423 details: None,
18424 error_code: "client.invalid_host_response".to_owned(),
18425 message: "response header omits resetRequired".to_owned(),
18426 recommended_action: None,
18427 };
18428 };
18429 if let Some(required) = required_schema_version {
18430 let floor = SchemaFloor {
18433 required_schema_version: Some(required),
18434 latest_schema_version,
18435 };
18436 self.set_schema_floor(Some(floor.clone()));
18437 report.schema_floor = Some(floor);
18438 return SyncOutcome::Ok(report);
18439 }
18440 if reset_required {
18441 if frames.next().is_some() {
18442 return SyncOutcome::Failed {
18443 details: None,
18444 error_code: "client.invalid_host_response".to_owned(),
18445 message: "log-epoch reset response contains body frames".to_owned(),
18446 recommended_action: None,
18447 };
18448 }
18449 match self.run_log_epoch_reset(&log_epoch) {
18450 Ok(resets) => {
18451 report.resets = resets;
18452 return SyncOutcome::Ok(report);
18453 }
18454 Err(message) => {
18455 return SyncOutcome::Failed {
18456 details: None,
18457 error_code: "sync.local_corrupt".to_owned(),
18458 message,
18459 recommended_action: None,
18460 };
18461 }
18462 }
18463 }
18464 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
18465 return SyncOutcome::Failed {
18466 details: None,
18467 error_code: "client.invalid_host_response".to_owned(),
18468 message: "server changed logEpoch without requiring a reset".to_owned(),
18469 recommended_action: None,
18470 };
18471 }
18472 }
18473 _ => {
18474 return SyncOutcome::Failed {
18475 details: None,
18476 error_code: "sync.invalid_request".to_owned(),
18477 message: "response does not start with RESP_HEADER".to_owned(),
18478 recommended_action: None,
18479 };
18480 }
18481 }
18482
18483 if let Err(message) = self.begin_coalesced_apply() {
18484 return SyncOutcome::Failed {
18485 details: None,
18486 error_code: "storage.failed".into(),
18487 message,
18488 recommended_action: None,
18489 };
18490 }
18491 let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
18492 while let Some(frame) = frames.next() {
18493 match frame {
18494 frame @ Frame::PushResult { .. } => {
18495 let successful = matches!(
18496 &frame,
18497 Frame::PushResult {
18498 status: PushStatus::Applied | PushStatus::Cached,
18499 ..
18500 }
18501 );
18502 let mut results = vec![frame];
18503 if successful {
18504 while matches!(
18505 frames.peek(),
18506 Some(Frame::PushResult {
18507 status: PushStatus::Applied | PushStatus::Cached,
18508 ..
18509 })
18510 ) {
18511 if let Some(next) = frames.next() {
18512 results.push(next);
18513 }
18514 }
18515 }
18516 if self
18517 .handle_push_results(
18518 &results,
18519 &pushed_ids,
18520 &rejection_details_by_commit,
18521 &mut report,
18522 last_final_push_result_id.as_deref(),
18523 )
18524 .is_err()
18525 {
18526 failure = Some((
18527 "client.outcome_persistence_failed".to_owned(),
18528 "local commit outcome could not be persisted".to_owned(),
18529 None,
18530 None,
18531 ));
18532 break;
18533 }
18534 }
18535 Frame::PushResultDetails { .. } => {}
18536 Frame::SubStart {
18537 id,
18538 status,
18539 reason_code,
18540 effective_scopes,
18541 bootstrap: _,
18542 } => {
18543 let mut body = Vec::new();
18544 let mut sub_end: Option<(i64, Option<String>)> = None;
18545 for inner in frames.by_ref() {
18546 match inner {
18547 Frame::SubEnd {
18548 next_cursor,
18549 bootstrap_state,
18550 } => {
18551 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
18552 break;
18553 }
18554 Frame::Unknown { .. } => {}
18555 other => body.push(other),
18556 }
18557 }
18558 if sub_end.is_some() {
18559 incomplete_subscriptions.remove(id.as_str());
18560 }
18561 let section_failure = match self.process_section(
18562 transport,
18563 &id,
18564 status,
18565 &reason_code,
18566 effective_scopes,
18567 body,
18568 sub_end,
18569 meta,
18570 &mut report,
18571 &coalesced_deletes,
18572 ) {
18573 Err(SectionError::Abort(code, message)) => {
18574 Some((code, message, None, None))
18575 }
18576 Err(SectionError::Transport(error)) => {
18577 Some((error.code, error.message, None, error.details))
18578 }
18579 _ => None,
18580 };
18581 if section_failure.is_some() {
18582 failure = section_failure;
18583 break;
18584 }
18585 }
18586 Frame::Lease {
18587 lease_id,
18588 expires_at_ms,
18589 } => {
18590 self.set_lease_state(Some(LeaseState {
18593 lease_id: Some(lease_id),
18594 expires_at_ms: Some(expires_at_ms),
18595 error_code: None,
18596 }));
18597 }
18598 Frame::Error {
18599 code,
18600 message,
18601 recommended_action,
18602 details,
18603 ..
18604 } => {
18605 let recommended_action =
18608 (!recommended_action.is_empty()).then_some(recommended_action);
18609 let details = details
18610 .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
18611 .filter(Value::is_object);
18612 failure = Some((code, message, recommended_action, details));
18613 break;
18614 }
18615 Frame::Unknown { .. } => {}
18616 _ => {
18617 failure = Some((
18618 "sync.invalid_request".to_owned(),
18619 "unexpected frame in response".to_owned(),
18620 None,
18621 None,
18622 ));
18623 break;
18624 }
18625 }
18626 }
18627
18628 let applied = match failure {
18629 Some(failure) => Err(failure),
18630 None => self
18635 .rebuild_overlay_if_dirty()
18636 .and_then(|()| self.drain_pending_evictions())
18637 .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
18638 };
18639 if let Err(message) = self.finish_coalesced_apply() {
18642 return SyncOutcome::Failed {
18643 details: None,
18644 error_code: "storage.failed".into(),
18645 message,
18646 recommended_action: None,
18647 };
18648 }
18649 if let Err((error_code, message, recommended_action, details)) = applied {
18650 return SyncOutcome::Failed {
18651 details,
18652 error_code,
18653 message,
18654 recommended_action,
18655 };
18656 }
18657 if !incomplete_subscriptions.is_empty() {
18658 self.set_sync_needed(true, true);
18659 }
18660 self.ack_after_pull(transport);
18661 if self.upgrading && report.bootstrapping.is_empty() {
18664 self.set_upgrading(false);
18665 }
18666 SyncOutcome::Ok(report)
18667 }
18668
18669 fn handle_push_results(
18672 &mut self,
18673 frames: &[Frame],
18674 pushed_ids: &HashSet<&str>,
18675 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
18676 report: &mut SyncReport,
18677 last_final_push_result_id: Option<&str>,
18678 ) -> Result<(), String> {
18679 self.begin_observation("syncular_push_results")?;
18680 let previous_failed = self.failed_commits.clone();
18681 let previous_sync_needed = self.sync_needed;
18682 let conflict_count = self.conflicts.len();
18683 let rejection_count = self.rejections.len();
18684 let applied_count = report.applied.len();
18685 let rejected_count = report.rejected.len();
18686 let retryable_count = report.retryable.len();
18687 let was_dirty = self.overlay_dirty.snapshot();
18688 let mut removed = Vec::new();
18689 let mut batch = ChangeAccumulator::default();
18690 let persisted = (|| {
18691 for frame in frames {
18692 let Frame::PushResult {
18693 client_commit_id,
18694 status,
18695 results,
18696 commit_seq,
18697 } = frame
18698 else {
18699 return Err("unexpected acknowledgement frame".to_owned());
18700 };
18701 if !pushed_ids.contains(client_commit_id.as_str()) {
18702 continue;
18703 }
18704 let Some(index) = self
18705 .outbox
18706 .iter()
18707 .position(|commit| commit.client_commit_id == *client_commit_id)
18708 else {
18709 continue;
18710 };
18711 let status = *status;
18712 let rejection_details = rejection_details_by_commit.get(client_commit_id);
18713 let operations = self.outbox[index].ops.clone();
18714 match status {
18715 PushStatus::Applied | PushStatus::Cached => {
18716 let sequence =
18719 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
18720 let affected: HashSet<_> = operations
18721 .iter()
18722 .map(|op| (op.table.as_str(), op.row_id.as_str()))
18723 .collect();
18724 let prior = self.acknowledged_operations(Some(&affected), None)?;
18725 for (op_index, operation) in operations.iter().enumerate() {
18726 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
18727 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18728 if delivered {
18729 continue;
18730 }
18731 let table = self
18732 .schema
18733 .table(&operation.table)
18734 .ok_or_else(|| "sync.unknown_table".to_owned())?;
18735 let mut intended: Map<String, Value> = self
18736 .stored_row(table, &operation.row_id, true)?
18737 .map(|(row, _)| {
18738 table
18739 .columns
18740 .iter()
18741 .zip(&row)
18742 .map(|(column, value)| {
18743 (column.name.clone(), column_value_to_json(value))
18744 })
18745 .collect()
18746 })
18747 .unwrap_or_default();
18748 let mut scope_image = intended.clone();
18749 for op in prior
18750 .iter()
18751 .chain(
18752 self.failed_commits
18753 .iter()
18754 .flat_map(|(commit, _)| &commit.ops),
18755 )
18756 .chain(
18757 self.outbox
18758 .iter()
18759 .take(index)
18760 .flat_map(|commit| &commit.ops),
18761 )
18762 .chain(operations.iter().take(op_index + 1))
18763 {
18764 if op.table != operation.table || op.row_id != operation.row_id {
18765 continue;
18766 }
18767 if let Some(values) = &op.values {
18768 intended.extend(values.clone());
18769 } else {
18770 scope_image = intended.clone();
18771 intended.clear();
18772 }
18773 }
18774 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
18775 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
18776 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
18777 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
18778 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
18779 }
18780 let journal_results = results
18781 .iter()
18782 .map(|result| {
18783 let op_index = match result {
18784 OpResult::Applied { op_index }
18785 | OpResult::Conflict { op_index, .. }
18786 | OpResult::Error { op_index, .. } => *op_index,
18787 };
18788 CommitOperationOutcome::Applied { op_index }
18789 })
18790 .collect::<Vec<_>>();
18791 let outcome_status = if status == PushStatus::Applied {
18792 CommitOutcomeStatus::Applied
18793 } else {
18794 CommitOutcomeStatus::Cached
18795 };
18796 let persisted = self
18797 .persist_commit_outcome(
18798 client_commit_id,
18799 outcome_status,
18800 &journal_results,
18801 None,
18802 )
18803 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
18804 persisted?;
18805 for operation in &operations {
18806 self.overlay_dirty.table(&operation.table);
18807 }
18808 self.sync_needed = true;
18809 batch.status = true;
18810 batch.outcomes = true;
18811 }
18812 PushStatus::Rejected => {
18813 if results.iter().any(|result| {
18814 matches!(
18815 result,
18816 OpResult::Error {
18817 code,
18818 retryable: true,
18819 ..
18820 } if code == "sync.idempotency_cache_miss"
18821 )
18822 }) {
18823 report.retryable.push(client_commit_id.to_owned());
18826 continue;
18827 }
18828
18829 let mut journal_results = Vec::with_capacity(results.len());
18830 let mut conflicts = Vec::new();
18831 let mut rejections = Vec::new();
18832 for result in results {
18833 match result {
18834 OpResult::Applied { op_index } => {
18835 journal_results.push(CommitOperationOutcome::Applied {
18836 op_index: *op_index,
18837 });
18838 }
18839 OpResult::Conflict {
18840 op_index,
18841 code,
18842 message,
18843 server_version,
18844 server_row,
18845 conflict_columns: conflict_columns_bitmap,
18846 } => {
18847 let operation = operations
18848 .get(*op_index as usize)
18849 .map(CommitOperation::from);
18850 let (table, row_id) = operation
18851 .as_ref()
18852 .map(|op| (op.table.clone(), op.row_id.clone()))
18853 .unwrap_or_default();
18854 let server_row_json = self
18855 .schema
18856 .table(&table)
18857 .and_then(|t| {
18858 decode_row_bytes(t, server_row, &self.encryption)
18859 .ok()
18860 .map(|row| (t, row))
18861 })
18862 .map(|(t, row)| {
18863 let mut map = Map::new();
18864 for (i, column) in t.columns.iter().enumerate() {
18865 map.insert(
18866 column.name.clone(),
18867 column_value_to_json(
18868 row.get(i).unwrap_or(&None),
18869 ),
18870 );
18871 }
18872 map
18873 })
18874 .unwrap_or_default();
18875 let conflict_columns = self
18878 .schema
18879 .table(&table)
18880 .map(|t| {
18881 t.columns
18882 .iter()
18883 .enumerate()
18884 .filter(|(index, _)| {
18885 conflict_columns_bitmap
18886 .get(index / 8)
18887 .is_some_and(|byte| {
18888 byte & (1 << (index % 8)) != 0
18889 })
18890 })
18891 .map(|(_, column)| column.name.clone())
18892 .collect::<Vec<_>>()
18893 })
18894 .unwrap_or_default();
18895 let conflict = ConflictRecord {
18896 client_commit_id: client_commit_id.to_owned(),
18897 op_index: *op_index,
18898 table,
18899 row_id,
18900 code: code.clone(),
18901 message: message.clone(),
18902 server_version: *server_version,
18903 server_row: server_row_json,
18904 conflict_columns,
18905 operation,
18906 };
18907 journal_results.push(CommitOperationOutcome::Conflict {
18908 conflict: conflict.clone(),
18909 });
18910 conflicts.push(conflict);
18911 }
18912 OpResult::Error {
18913 op_index,
18914 code,
18915 message,
18916 retryable,
18917 } => {
18918 let rejection = RejectionRecord {
18919 client_commit_id: client_commit_id.to_owned(),
18920 op_index: *op_index,
18921 code: code.clone(),
18922 message: message.clone(),
18923 retryable: *retryable,
18924 details: rejection_details
18925 .and_then(|details| details.get(op_index))
18926 .cloned(),
18927 operation: operations
18928 .get(*op_index as usize)
18929 .map(CommitOperation::from),
18930 };
18931 journal_results.push(CommitOperationOutcome::Error {
18932 rejection: rejection.clone(),
18933 });
18934 rejections.push(rejection);
18935 }
18936 }
18937 }
18938 if self.retain_failed_commits {
18939 self.retain_failed_commit(client_commit_id, &operations)?;
18940 }
18941 let outcome_status = if conflicts.is_empty() {
18942 CommitOutcomeStatus::Rejected
18943 } else {
18944 CommitOutcomeStatus::Conflict
18945 };
18946 let persisted = self
18947 .persist_commit_outcome(
18948 client_commit_id,
18949 outcome_status,
18950 &journal_results,
18951 Some(&operations),
18952 )
18953 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
18954 persisted?;
18955 batch.conflicts = !conflicts.is_empty();
18956 batch.rejections = !rejections.is_empty();
18957 batch.status = true;
18958 batch.outcomes = true;
18959 self.conflicts.extend(conflicts);
18960 self.rejections.extend(rejections);
18961 for operation in &operations {
18962 self.overlay_dirty.table(&operation.table);
18963 }
18964 }
18965 }
18966 removed.push((index, self.outbox.remove(index)));
18967 if batch.status {
18968 for operation in &operations {
18969 if !self.record_row_scopes(
18970 &mut batch,
18971 &operation.table,
18972 &operation.row_id,
18973 false,
18974 ) {
18975 batch.table(&operation.table);
18976 }
18977 }
18978 }
18979 match status {
18980 PushStatus::Applied | PushStatus::Cached => {
18981 report.applied.push(client_commit_id.to_owned())
18982 }
18983 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
18984 }
18985 }
18986 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
18987 self.prune_commit_outcomes()?;
18988 }
18989 self.rebuild_overlay_if_dirty()?;
18990 self.finish_observation("syncular_push_results", batch)
18991 })();
18992 if let Err(error) = persisted {
18993 self.rollback_observation("syncular_push_results");
18994 self.failed_commits = previous_failed;
18995 self.sync_needed = previous_sync_needed;
18996 for (index, commit) in removed.into_iter().rev() {
18997 self.outbox.insert(index, commit);
18998 }
18999 self.conflicts.truncate(conflict_count);
19000 self.rejections.truncate(rejection_count);
19001 self.overlay_dirty.restore(was_dirty.clone());
19002 report.applied.truncate(applied_count);
19003 report.rejected.truncate(rejected_count);
19004 report.retryable.truncate(retryable_count);
19005 return Err(error);
19006 }
19007 if !previous_sync_needed && self.sync_needed {
19008 self.sync_intent_queue.push_back(SyncIntent::Interactive);
19009 }
19010 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
19011 Ok(())
19012 }
19013
19014 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
19015 status != PushStatus::Rejected
19016 || !results.iter().any(|result| {
19017 matches!(
19018 result,
19019 OpResult::Error {
19020 code,
19021 retryable: true,
19022 ..
19023 } if code == "sync.idempotency_cache_miss"
19024 )
19025 })
19026 }
19027
19028 #[allow(clippy::too_many_arguments)]
19031 fn process_section(
19032 &mut self,
19033 transport: &mut dyn Transport,
19034 id: &str,
19035 status: SubStatus,
19036 reason_code: &str,
19037 effective_scopes: Vec<(String, Vec<String>)>,
19038 body: Vec<Frame>,
19039 sub_end: Option<(i64, Option<String>)>,
19040 meta: &RequestMeta,
19041 report: &mut SyncReport,
19042 coalesced_deletes: &HashSet<(i64, String, String)>,
19043 ) -> Result<(), SectionError> {
19044 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
19045 return Ok(()); };
19047 if status != SubStatus::Active && sub_end.is_none() {
19048 return Err(SectionError::Abort(
19049 "sync.invalid_request".into(),
19050 "subscription section without SUB_END".into(),
19051 ));
19052 }
19053 match status {
19054 SubStatus::Revoked => {
19055 self.begin_observation("syncular_revocation")
19056 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
19057 let previous = self.subs[sub_index].clone();
19058 let was_dirty = self.overlay_dirty.snapshot();
19059 let previous_outbox = self.outbox.clone();
19060 let previous_failed = self.failed_commits.clone();
19061 let previous_rejections = self.rejections.clone();
19062 let previous_conflicts = self.conflicts.clone();
19063 let revoked_count = report.revoked.len();
19064 let failed_count = report.failed.len();
19065 let result = (|| {
19066 let mut batch = ChangeAccumulator::default();
19067 let registered = self.window_unit_by_sub_id(id);
19068 let (table, effective) = {
19070 let sub = &self.subs[sub_index];
19071 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
19072 };
19073 let scoped_table = self.schema.table(&table).cloned().ok_or_else(|| {
19074 SectionError::Abort(
19075 "sync.unknown_table".to_owned(),
19076 "revoked scope references an unknown table".to_owned(),
19077 )
19078 })?;
19079 let dropped = self
19080 .drop_failed_commits(|name, values| {
19081 name == table
19082 && !effective.is_empty()
19083 && effective.iter().all(|(variable, allowed)| {
19084 scoped_table
19085 .scope_column(variable)
19086 .and_then(|column| values.get(column))
19087 .and_then(Value::as_str)
19088 .is_some_and(|value| {
19089 allowed.iter().any(|allowed| allowed == value)
19090 })
19091 })
19092 })
19093 .map_err(|message| {
19094 SectionError::Abort("storage.failed".to_owned(), message)
19095 })?;
19096 for commit in dropped {
19097 batch.conflicts = true;
19098 batch.rejections = true;
19099 batch.outcomes = true;
19100 for operation in commit.ops {
19101 batch.table(&operation.table);
19102 }
19103 }
19104 let purged = self.purge_scope_rows(&table, &effective);
19105 match purged {
19106 Ok(()) => {
19107 self.record_scope_map(&mut batch, &table, &effective);
19108 let sub = &mut self.subs[sub_index];
19109 sub.state = SubState::Revoked;
19110 sub.reason_code = Some(if reason_code.is_empty() {
19111 "sync.scope_revoked".to_owned()
19112 } else {
19113 reason_code.to_owned()
19114 });
19115 report.revoked.push(id.to_owned());
19116 let doomed_effective = effective;
19117 let sub_table = table;
19118 self.persist_sub(&self.subs[sub_index].clone())
19119 .map_err(|message| {
19120 SectionError::Abort("storage.failed".into(), message)
19121 })?;
19122 let dropped = self
19123 .drop_doomed_outbox(&sub_table, &doomed_effective)
19124 .map_err(|message| {
19125 SectionError::Abort("storage.failed".to_owned(), message)
19126 })?;
19127 if dropped {
19128 batch.status = true;
19129 batch.rejections = true;
19130 batch.outcomes = true;
19131 }
19132 self.delete_unreferenced_cached_blobs().map_err(|message| {
19135 SectionError::Abort("storage.failed".into(), message)
19136 })?;
19137 }
19138 Err(()) => {
19139 let sub = &mut self.subs[sub_index];
19142 sub.state = SubState::Failed;
19143 sub.reason_code = Some("sync.scope_revoked".to_owned());
19144 report.failed.push(id.to_owned());
19145 self.persist_sub(&self.subs[sub_index].clone())
19146 .map_err(|message| {
19147 SectionError::Abort("storage.failed".into(), message)
19148 })?;
19149 }
19150 }
19151 if let Some((base_key, unit)) = registered {
19152 batch.window(&base_key, &self.subs[sub_index].table, &unit);
19153 }
19154 self.rebuild_overlay_if_dirty()
19155 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19156 self.finish_observation("syncular_revocation", batch)
19157 .map_err(|message| {
19158 SectionError::Abort("storage.failed".to_owned(), message)
19159 })?;
19160 Ok(())
19161 })();
19162 if result.is_err() {
19163 self.rollback_observation("syncular_revocation");
19164 self.subs[sub_index] = previous;
19165 self.overlay_dirty.restore(was_dirty.clone());
19166 self.outbox = previous_outbox;
19167 self.failed_commits = previous_failed;
19168 self.rejections = previous_rejections;
19169 self.conflicts = previous_conflicts;
19170 report.revoked.truncate(revoked_count);
19171 report.failed.truncate(failed_count);
19172 }
19173 result
19174 }
19175 SubStatus::Reset => {
19176 self.begin_observation("syncular_reset")
19177 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
19178 let previous = self.subs[sub_index].clone();
19179 let was_dirty = self.overlay_dirty.snapshot();
19180 let reset_count = report.resets.len();
19181 let result = (|| {
19182 let mut batch = ChangeAccumulator::default();
19183 let registered = self.window_unit_by_sub_id(id);
19184 let sub = &mut self.subs[sub_index];
19187 sub.cursor = -1;
19188 sub.bootstrap_state = None;
19189 sub.reason_code = Some(if reason_code.is_empty() {
19190 "sync.cursor_expired".to_owned()
19191 } else {
19192 reason_code.to_owned()
19193 });
19194 report.resets.push(id.to_owned());
19195 self.persist_sub(&self.subs[sub_index].clone())
19196 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19197 if let Some((base_key, unit)) = registered {
19198 batch.window(&base_key, &self.subs[sub_index].table, &unit);
19199 }
19200 self.finish_observation("syncular_reset", batch)
19201 .map_err(|message| {
19202 SectionError::Abort("storage.failed".to_owned(), message)
19203 })?;
19204 Ok(())
19205 })();
19206 if result.is_err() {
19207 self.rollback_observation("syncular_reset");
19208 self.subs[sub_index] = previous;
19209 self.overlay_dirty.restore(was_dirty.clone());
19210 report.resets.truncate(reset_count);
19211 }
19212 result
19213 }
19214 SubStatus::Active => {
19215 let fresh = meta
19216 .fresh
19217 .iter()
19218 .find(|(fid, _)| fid == id)
19219 .map(|(_, fresh)| *fresh)
19220 .unwrap_or(false);
19221 let previous = self.subs[sub_index].clone();
19222 self.subs[sub_index].effective = Some(effective_scopes);
19223 let outcome = self
19224 .apply_section_body(
19225 transport,
19226 sub_index,
19227 body,
19228 fresh,
19229 meta,
19230 report,
19231 coalesced_deletes,
19232 )
19233 .and_then(|()| {
19234 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
19235 SectionError::Abort(
19236 "sync.invalid_request".into(),
19237 "subscription section without SUB_END".into(),
19238 )
19239 })?;
19240 self.apply_sub_end(sub_index, cursor, bootstrap_state)
19241 .map_err(|message| {
19242 SectionError::Abort("storage.failed".into(), message)
19243 })
19244 });
19245 match outcome {
19246 Ok(()) => {
19247 if self.subs[sub_index].bootstrap_state.is_some() {
19248 report.bootstrapping.push(id.to_owned());
19249 }
19250 Ok(())
19251 }
19252 Err(SectionError::FailClosed) => {
19253 self.subs[sub_index] = previous.clone();
19254 self.begin_observation("syncular_section_failure")
19255 .map_err(|message| {
19256 SectionError::Abort("storage.failed".into(), message)
19257 })?;
19258 let persisted = (|| {
19259 let mut batch = ChangeAccumulator::default();
19260 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
19261 batch.window(&base_key, &self.subs[sub_index].table, &unit);
19262 }
19263 self.subs[sub_index].state = SubState::Failed;
19264 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
19265 self.persist_sub(&self.subs[sub_index])?;
19266 self.finish_observation("syncular_section_failure", batch)
19267 })();
19268 if let Err(message) = persisted {
19269 self.rollback_observation("syncular_section_failure");
19270 self.subs[sub_index] = previous;
19271 return Err(SectionError::Abort("storage.failed".into(), message));
19272 }
19273 report.failed.push(id.to_owned());
19274 Ok(())
19275 }
19276 Err(error) => {
19277 self.subs[sub_index] = previous;
19280 Err(error)
19281 }
19282 }
19283 }
19284 }
19285 }
19286
19287 fn apply_sub_end(
19288 &mut self,
19289 sub_index: usize,
19290 cursor: i64,
19291 bootstrap_state: Option<String>,
19292 ) -> Result<(), String> {
19293 #[cfg(feature = "bench-internals")]
19294 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
19295 let previous = self.subs[sub_index].clone();
19296 self.begin_observation("syncular_sub_end")?;
19297 let persisted = (|| {
19298 let mut batch = ChangeAccumulator::default();
19299 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
19300 && cursor >= 0
19301 && bootstrap_state.is_none();
19302 if completed {
19303 let table = self
19304 .schema
19305 .table(&previous.table)
19306 .ok_or_else(|| "sync.unknown_table".to_owned())?;
19307 let effective = previous.effective.as_deref().unwrap_or(&[]);
19308 let mut statement = self.conn.prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
19309 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19310 let rows = statement
19311 .query_map(rusqlite::params![previous.table, cursor], |row| {
19312 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
19313 })
19314 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19315 let mut covered = HashSet::new();
19316 for row in rows {
19317 let (id, intent) =
19318 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19319 let values: Map<String, Value> = serde_json::from_str(&intent)
19320 .map_err(|_| "sync.local_corrupt".to_owned())?;
19321 if !effective.is_empty()
19322 && effective.iter().all(|(variable, allowed)| {
19323 table
19324 .scope_column(variable)
19325 .and_then(|column| values.get(column))
19326 .is_some_and(|value| {
19327 allowed.contains(
19328 &value
19329 .as_str()
19330 .map_or_else(|| value.to_string(), str::to_owned),
19331 )
19332 })
19333 })
19334 {
19335 covered.insert(id);
19336 }
19337 }
19338 drop(statement);
19339 for id in covered {
19340 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![previous.table, id, cursor])
19341 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
19342 self.overlay_dirty.table(&previous.table);
19343 batch.table(&previous.table);
19344 }
19345 self.rebuild_overlay_if_dirty()?;
19346 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
19347 batch.window(&base_key, &previous.table, &unit);
19348 }
19349 }
19350 let sub = &mut self.subs[sub_index];
19351 sub.cursor = cursor;
19352 sub.bootstrap_state = bootstrap_state;
19353 sub.reason_code = None;
19356 sub.synced_once = true;
19357 self.persist_sub(&self.subs[sub_index])?;
19358 self.finish_observation("syncular_sub_end", batch)
19359 })();
19360 if persisted.is_err() {
19361 self.rollback_observation("syncular_sub_end");
19362 self.subs[sub_index] = previous;
19363 }
19364 persisted
19365 }
19366
19367 #[allow(clippy::too_many_arguments)]
19368 fn apply_section_body(
19369 &mut self,
19370 transport: &mut dyn Transport,
19371 sub_index: usize,
19372 body: Vec<Frame>,
19373 fresh: bool,
19374 meta: &RequestMeta,
19375 report: &mut SyncReport,
19376 coalesced_deletes: &HashSet<(i64, String, String)>,
19377 ) -> Result<(), SectionError> {
19378 let mut saw_segment = false;
19379 for frame in body {
19380 match frame {
19381 Frame::Commit {
19382 tables,
19383 changes,
19384 commit_seq,
19385 ..
19386 } => {
19387 if self
19388 .apply_commit_frame_coalesced(
19389 &tables,
19390 changes,
19391 commit_seq,
19392 coalesced_deletes,
19393 sub_index,
19394 AppliedServerCommitSource::Pull,
19395 )
19396 .map_err(|(c, m)| SectionError::Abort(c, m))?
19397 {
19398 report.commits_applied += 1;
19399 }
19400 }
19401 Frame::SegmentInline { payload } => {
19402 let segment = decode_rows_segment(&payload)
19403 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
19404 self.progress.update(|p| {
19405 p.phase = ProgressPhase::Import;
19406 p.subscription_id = Some(self.subs[sub_index].id.clone());
19407 p.table = Some(self.subs[sub_index].table.clone());
19408 p.segment_id = None;
19409 p.bytes_received = payload.len() as u64;
19410 p.bytes_total = Some(payload.len() as u64);
19411 p.rows_processed = 0;
19412 p.rows_total =
19413 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
19414 });
19415 let first = !saw_segment;
19416 saw_segment = true;
19417 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
19418 report.segment_rows_applied += applied;
19419 }
19420 Frame::SegmentRef {
19421 segment_id,
19422 byte_length,
19423 media_type,
19424 table,
19425 row_count,
19426 as_of_commit_seq,
19427 scope_digest,
19428 row_cursor,
19429 next_row_cursor,
19430 url,
19431 url_expires_at_ms,
19432 ..
19433 } => {
19434 let advertised = match media_type {
19437 MediaType::Rows => {
19438 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
19439 || meta.accept & ACCEPT_INLINE_ROWS != 0
19440 }
19441 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
19442 };
19443 if !advertised {
19444 return Err(SectionError::Abort(
19445 "sync.invalid_request".to_owned(),
19446 format!(
19447 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
19448 media_type.name()
19449 ),
19450 ));
19451 }
19452 self.progress.update(|p| {
19453 p.phase = ProgressPhase::Download;
19454 p.subscription_id = Some(self.subs[sub_index].id.clone());
19455 p.table = Some(table.clone());
19456 p.segment_id = Some(segment_id.clone());
19457 p.bytes_received = 0;
19458 p.bytes_total = u64::try_from(byte_length).ok();
19459 p.rows_processed = 0;
19460 p.rows_total = u64::try_from(row_count).ok();
19461 });
19462 let observer = self.progress.clone();
19463 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
19464 let bytes = if let Some(url) = url {
19465 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
19470 return Err(SectionError::Abort(
19471 "sync.invalid_request".to_owned(),
19472 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
19473 .to_owned(),
19474 ));
19475 }
19476 if url_expires_at_ms.is_some_and(|exp| {
19478 exp <= transport
19479 .segment_fetch_started_at(&url)
19480 .unwrap_or_else(|| self.clock_now_ms())
19481 }) {
19482 return Err(SectionError::Abort(
19483 "sync.segment_expired".to_owned(),
19484 format!(
19485 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
19486 ),
19487 ));
19488 }
19489 transport
19490 .fetch_url(&url, &mut on_progress)
19491 .map_err(|e| self.segment_fetch_failed(e))?
19492 } else {
19493 let requested_scopes_json =
19494 canonical_scope_json(&self.subs[sub_index].requested);
19495 transport
19496 .download_segment(
19497 &SegmentRequest {
19498 segment_id: segment_id.clone(),
19499 table,
19500 requested_scopes_json,
19501 },
19502 &mut on_progress,
19503 )
19504 .map_err(|e| self.segment_fetch_failed(e))?
19505 };
19506 on_progress(bytes.len() as u64);
19508 let digest = Sha256::digest(&bytes);
19509 let expected = segment_id
19510 .strip_prefix("sha256:")
19511 .unwrap_or(segment_id.as_str());
19512 if bytes_to_hex(&digest) != expected {
19513 self.schedule_background_retry();
19515 return Err(SectionError::Abort(
19516 "sync.invalid_request".to_owned(),
19517 "segment bytes do not match the content address (§5.1)".to_owned(),
19518 ));
19519 }
19520 self.progress.update(|p| p.phase = ProgressPhase::Import);
19521 if media_type == MediaType::Sqlite {
19522 if row_cursor.is_some() || next_row_cursor.is_some() {
19525 return Err(SectionError::Abort(
19526 "sync.invalid_request".to_owned(),
19527 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
19528 .to_owned(),
19529 ));
19530 }
19531 let first = !saw_segment;
19532 saw_segment = true;
19533 let applied = self.apply_sqlite_segment(
19534 sub_index,
19535 &bytes,
19536 fresh && first,
19537 row_count,
19538 as_of_commit_seq,
19539 &scope_digest,
19540 )?;
19541 report.segment_rows_applied += applied;
19542 } else {
19543 let segment = decode_rows_segment(&bytes).map_err(|e| {
19544 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
19545 })?;
19546 let first = row_cursor.is_none();
19547 saw_segment = true;
19548 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
19549 report.segment_rows_applied += applied;
19550 }
19551 }
19552 Frame::Unknown { .. } => {}
19553 Frame::Error { code, message, .. } => {
19554 return Err(SectionError::Abort(code, message))
19555 }
19556 _ => {
19557 return Err(SectionError::Abort(
19558 "sync.invalid_request".to_owned(),
19559 "unexpected frame inside a subscription section".to_owned(),
19560 ));
19561 }
19562 }
19563 }
19564 Ok(())
19565 }
19566
19567 fn apply_commit_frame_coalesced(
19571 &mut self,
19572 tables: &[String],
19573 changes: Vec<ssp2::model::Change>,
19574 commit_seq: i64,
19575 coalesced_deletes: &HashSet<(i64, String, String)>,
19576 sub_index: usize,
19577 source: AppliedServerCommitSource,
19578 ) -> Result<bool, (String, String)> {
19579 let changes: Vec<ssp2::model::Change> = changes
19580 .into_iter()
19581 .filter(|change| {
19582 if change.op != Op::Delete {
19583 return true;
19584 }
19585 let Some(table) = tables.get(change.table_index as usize) else {
19586 return true;
19587 };
19588 !coalesced_deletes.contains(&(commit_seq, table.clone(), change.row_id.clone()))
19589 })
19590 .collect();
19591 if changes.is_empty() {
19592 return Ok(false);
19593 }
19594 let affected_tables = changes
19595 .iter()
19596 .filter_map(|change| tables.get(change.table_index as usize))
19597 .cloned()
19598 .collect::<BTreeSet<_>>();
19599 let previous_batches = self.change_queue.len();
19600 self.apply_commit_frame(tables, &changes, Some(commit_seq))?;
19601 if self.change_queue.len() > previous_batches {
19602 let subscription_id = self.subs[sub_index].id.clone();
19603 if let Some(batch) = self.change_queue.back_mut() {
19604 batch
19605 .applied_commits
19606 .extend(
19607 affected_tables
19608 .into_iter()
19609 .map(|table| AppliedServerCommit {
19610 subscription_id: subscription_id.clone(),
19611 table,
19612 commit_seq,
19613 source,
19614 }),
19615 );
19616 }
19617 }
19618 Ok(true)
19619 }
19620
19621 fn apply_commit_frame(
19623 &mut self,
19624 tables: &[String],
19625 changes: &[ssp2::model::Change],
19626 commit_seq: Option<i64>,
19627 ) -> Result<(), (String, String)> {
19628 #[cfg(feature = "bench-internals")]
19629 let _phase = self.benchmark_phases.start(Phase::CommitApply);
19630 let was_dirty = self.overlay_dirty.snapshot();
19631 self.coalesce_unit(changes.len())
19632 .and_then(|()| self.begin_observation("syncular_commit"))
19633 .map_err(|message| ("storage.failed".into(), message))?;
19634 let applied = (|| {
19635 let mut batch = ChangeAccumulator::default();
19636 self.record_commit_changes(&mut batch, tables, changes);
19637 if let Some(sequence) = commit_seq {
19638 for change in changes {
19639 let table = tables.get(change.table_index as usize).ok_or_else(|| {
19640 (
19641 "sync.invalid_request".to_owned(),
19642 "change tableIndex out of range".to_owned(),
19643 )
19644 })?;
19645 if self
19646 .outbox
19647 .iter()
19648 .flat_map(|commit| &commit.ops)
19649 .any(|op| op.table == *table && op.row_id == change.row_id)
19650 {
19651 self.conn.execute("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)", rusqlite::params![table, change.row_id, sequence])
19652 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
19653 }
19654 self.conn.execute("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3", rusqlite::params![table, change.row_id, sequence])
19655 .map_err(|error| ("storage.failed".to_owned(), Self::sqlite_failure(&self.storage_failure, error)))?;
19656 }
19657 }
19658 self.apply_commit_changes(tables, changes)?;
19659 self.rebuild_overlay_if_dirty()
19660 .map_err(|message| ("storage.failed".into(), message))?;
19661 self.finish_observation("syncular_commit", batch)
19662 .map_err(|message| ("storage.failed".into(), message))
19663 })();
19664 if applied.is_err() {
19665 self.rollback_observation("syncular_commit");
19666 self.overlay_dirty.restore(was_dirty.clone());
19667 }
19668 applied
19669 }
19670
19671 fn apply_commit_changes(
19672 &mut self,
19673 tables: &[String],
19674 changes: &[ssp2::model::Change],
19675 ) -> Result<(), (String, String)> {
19676 let mirror_visible = !self.overlay_dirty.get()
19681 && self.failed_commits.is_empty()
19682 && ((self.outbox.is_empty() && !self.has_acknowledged_rows())
19683 || changes.iter().all(|change| {
19684 tables
19685 .get(change.table_index as usize)
19686 .and_then(|name| self.schema.table(name))
19687 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
19688 }));
19689 for change in changes {
19690 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
19691 (
19692 "sync.invalid_request".to_owned(),
19693 "change tableIndex out of range".to_owned(),
19694 )
19695 })?;
19696 let table = self.schema.table(table_name).ok_or_else(|| {
19697 (
19698 "sync.schema_mismatch".to_owned(),
19699 format!("change targets unknown table {table_name:?}"),
19700 )
19701 })?;
19702 match change.op {
19703 Op::Upsert => {
19704 let payload = change.row.as_ref().ok_or_else(|| {
19705 (
19706 "sync.invalid_request".to_owned(),
19707 "upsert change without row payload".to_owned(),
19708 )
19709 })?;
19710 let row = {
19712 #[cfg(feature = "bench-internals")]
19713 let _phase = self.benchmark_phases.start(Phase::RowDecode);
19714 decode_row_bytes(table, payload, &self.encryption)
19715 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
19716 };
19717 let version = change.row_version.unwrap_or(0);
19718 let table_name = table.name.clone();
19719 self.write_base_row(&table_name, &row, version)
19720 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
19721 if mirror_visible {
19722 self.write_row(&visible_table(&table_name), &table_name, &row, version)
19723 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
19724 }
19725 }
19726 Op::Delete => {
19727 self.delete_base_row(table_name, &change.row_id)
19728 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
19729 if mirror_visible {
19730 let sql = format!(
19731 "DELETE FROM {} WHERE {}",
19732 visible_table(table_name),
19733 row_id_predicate(table)
19734 );
19735 self.conn
19736 .prepare_cached(&sql)
19737 .and_then(|mut statement| {
19738 statement.execute(rusqlite::params![change.row_id])
19739 })
19740 .map_err(|error| {
19741 ("sync.invalid_request".to_owned(), error.to_string())
19742 })?;
19743 }
19744 }
19745 }
19746 }
19747 if mirror_visible {
19748 {
19749 let changed_rows: HashSet<(&str, &str)> = changes
19750 .iter()
19751 .map(|change| {
19752 (
19753 tables[change.table_index as usize].as_str(),
19754 change.row_id.as_str(),
19755 )
19756 })
19757 .collect();
19758 let acknowledged = self
19759 .acknowledged_operations(Some(&changed_rows), None)
19760 .map_err(|error| ("storage.failed".to_owned(), error))?;
19761 self.apply_outbox_ops(&acknowledged)
19762 .map_err(|message| ("storage.failed".into(), message))?;
19763 self.apply_outbox_ops(
19764 self.outbox
19765 .iter()
19766 .flat_map(|commit| &commit.ops)
19767 .filter(|op| {
19768 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
19769 }),
19770 )
19771 .map_err(|message| ("storage.failed".into(), message))?;
19772 }
19773 self.overlay_dirty.set(false);
19774 }
19775 Ok(())
19776 }
19777
19778 fn apply_segment(
19783 &mut self,
19784 sub_index: usize,
19785 segment: &RowsSegment,
19786 first_fresh_page: bool,
19787 ) -> Result<u32, SectionError> {
19788 let (sub_table, effective) = {
19789 let sub = &self.subs[sub_index];
19790 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
19791 };
19792 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
19793 SectionError::Abort(
19794 "sync.schema_mismatch".to_owned(),
19795 format!("subscription table {sub_table:?} is not in the client schema"),
19796 )
19797 })?;
19798 let matches = segment.table == table.name
19803 && segment.schema_version == self.schema.version
19804 && segment.columns.len() == table.wire_columns.len()
19805 && segment
19806 .columns
19807 .iter()
19808 .zip(table.wire_columns.iter())
19809 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
19810 if !matches {
19811 return Err(SectionError::Abort(
19812 "sync.schema_mismatch".to_owned(),
19813 "segment column table does not match the generated schema (§5.2)".to_owned(),
19814 ));
19815 }
19816 let mut applied = 0u32;
19817 let empty = Vec::new();
19818 let blocks = if segment.blocks.is_empty() {
19819 std::slice::from_ref(&empty)
19820 } else {
19821 &segment.blocks
19822 };
19823 for (index, block) in blocks.iter().enumerate() {
19824 let clear = first_fresh_page && index == 0;
19825 if !clear && block.is_empty() {
19826 continue;
19827 }
19828 let was_dirty = self.overlay_dirty.snapshot();
19829 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
19830 && self
19831 .outbox
19832 .iter()
19833 .flat_map(|commit| &commit.ops)
19834 .any(|op| op.table == table.name);
19835 let mirror_visible = !reconcile_pending
19836 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
19837 self.coalesce_unit(block.len())
19838 .and_then(|()| self.begin_observation("syncular_segment_block"))
19839 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19840 let outcome = (|| {
19841 self.rebuild_overlay_if_dirty()
19842 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19843 let mut changed_keys = Vec::new();
19844 let mut batch = ChangeAccumulator::default();
19845 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
19846 batch.table(&table.name);
19847 }
19848 if clear {
19849 self.purge_scope_rows(&table.name, &effective)
19850 .map_err(|()| SectionError::FailClosed)?;
19851 }
19852 for (row_index, row) in block.iter().enumerate() {
19853 let decrypted;
19854 let values = if table.has_encrypted_columns() {
19855 let mut values = row.values.clone();
19856 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
19857 .map_err(|message| {
19858 SectionError::Abort("client.decrypt_failed".into(), message)
19859 })?;
19860 decrypted = values;
19861 &decrypted
19862 } else {
19863 &row.values
19864 };
19865 self.write_base_row(&table.name, values, row.server_version)
19866 .map_err(|message| {
19867 SectionError::Abort("sync.invalid_request".into(), message)
19868 })?;
19869 if reconcile_pending {
19870 changed_keys.push(
19871 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
19872 |message| SectionError::Abort("storage.failed".into(), message),
19873 )?,
19874 );
19875 }
19876 if mirror_visible {
19877 self.write_row(
19878 &visible_table(&table.name),
19879 &table.name,
19880 values,
19881 row.server_version,
19882 )
19883 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19884 }
19885 let processed = u64::from(applied) + row_index as u64 + 1;
19886 if processed.is_multiple_of(1024) {
19887 self.progress.update(|p| p.rows_processed = processed);
19888 }
19889 }
19890 if reconcile_pending {
19891 self.reconcile_imported_rows(&table, &mut changed_keys)
19892 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19893 }
19894 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
19895 self.apply_outbox_ops(
19896 self.outbox
19897 .iter()
19898 .flat_map(|commit| &commit.ops)
19899 .filter(|op| op.table == table.name),
19900 )
19901 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19902 self.overlay_dirty.set(false);
19903 } else {
19904 self.rebuild_overlay_if_dirty()
19905 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
19906 }
19907 self.finish_observation("syncular_segment_block", batch)
19908 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
19909 })();
19910 if let Err(error) = outcome {
19911 self.rollback_observation("syncular_segment_block");
19912 self.overlay_dirty.restore(was_dirty.clone());
19913 return Err(error);
19914 }
19915 applied += block.len() as u32;
19916 self.progress
19917 .update(|p| p.rows_processed = u64::from(applied));
19918 std::thread::yield_now();
19919 }
19920 self.progress
19921 .update(|p| p.rows_processed = u64::from(applied));
19922 Ok(applied)
19923 }
19924
19925 fn apply_sqlite_segment(
19933 &mut self,
19934 sub_index: usize,
19935 bytes: &[u8],
19936 first_fresh_page: bool,
19937 row_count: i64,
19938 as_of_commit_seq: i64,
19939 scope_digest: &str,
19940 ) -> Result<u32, SectionError> {
19941 let invalid = |detail: &str| {
19942 SectionError::Abort(
19943 "sync.invalid_request".to_owned(),
19944 format!("sqlite segment rejected: {detail} (§5.3)"),
19945 )
19946 };
19947 let (sub_table, effective) = {
19948 let sub = &self.subs[sub_index];
19949 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
19950 };
19951 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
19952 SectionError::Abort(
19953 "sync.schema_mismatch".to_owned(),
19954 format!("subscription table {sub_table:?} is not in the client schema"),
19955 )
19956 })?;
19957
19958 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
19959 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
19960 let img = match rusqlite::Connection::open_with_flags(
19961 &path,
19962 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
19963 ) {
19964 Ok(conn) => conn,
19965 Err(_) => {
19966 let _ = std::fs::remove_file(&path);
19967 return Err(invalid("bytes do not open as a SQLite database"));
19968 }
19969 };
19970 let outcome = self.apply_sqlite_image(
19971 &img,
19972 &table,
19973 first_fresh_page,
19974 &effective,
19975 row_count,
19976 as_of_commit_seq,
19977 scope_digest,
19978 );
19979 drop(img);
19980 let _ = std::fs::remove_file(&path);
19981 outcome
19982 }
19983
19984 #[allow(clippy::too_many_arguments)]
19985 fn apply_sqlite_image(
19986 &mut self,
19987 img: &rusqlite::Connection,
19988 table: &crate::schema::TableSchema,
19989 first_fresh_page: bool,
19990 effective: &[(String, Vec<String>)],
19991 row_count: i64,
19992 as_of_commit_seq: i64,
19993 scope_digest: &str,
19994 ) -> Result<u32, SectionError> {
19995 let invalid = |detail: String| {
19996 SectionError::Abort(
19997 "sync.invalid_request".to_owned(),
19998 format!("sqlite segment rejected: {detail} (§5.3)"),
19999 )
20000 };
20001
20002 type MetaRow = (i64, String, i64, i64, String, i64, i64);
20005 let meta: MetaRow = img
20006 .query_row(
20007 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
20008 \"scopeDigest\", \"rowCount\",
20009 (SELECT count(*) FROM _syncular_segment)
20010 FROM _syncular_segment",
20011 [],
20012 |row| {
20013 Ok((
20014 row.get(0)?,
20015 row.get(1)?,
20016 row.get(2)?,
20017 row.get(3)?,
20018 row.get(4)?,
20019 row.get(5)?,
20020 row.get(6)?,
20021 ))
20022 },
20023 )
20024 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
20025 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
20026 if meta_count != 1 {
20027 return Err(invalid(format!(
20028 "_syncular_segment must contain exactly one row, found {meta_count}"
20029 )));
20030 }
20031 if format != 1 {
20032 return Err(invalid(format!("format {format}")));
20033 }
20034 if meta_table != table.name {
20035 return Err(invalid(format!("image table {meta_table:?}")));
20036 }
20037 if schema_version != i64::from(self.schema.version) {
20038 return Err(invalid(format!("schemaVersion {schema_version}")));
20039 }
20040 if pin != as_of_commit_seq {
20041 return Err(invalid(format!("asOfCommitSeq {pin}")));
20042 }
20043 if digest != scope_digest {
20044 return Err(invalid("scopeDigest mismatch".to_owned()));
20045 }
20046 if meta_rows != row_count {
20047 return Err(invalid(format!("rowCount {meta_rows}")));
20048 }
20049
20050 let mut names: Vec<String> = Vec::new();
20052 let mut primary_keys: Vec<String> = Vec::new();
20053 {
20054 let mut stmt = img
20055 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
20056 .map_err(|_| invalid("image data table missing".to_owned()))?;
20057 let mut rows = stmt
20058 .query([])
20059 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
20060 while let Some(row) = rows
20061 .next()
20062 .map_err(|_| invalid("image data table unreadable".to_owned()))?
20063 {
20064 if row
20065 .get::<_, i64>(5)
20066 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
20067 > 0
20068 {
20069 primary_keys.push(row.get(1).map_err(|error| {
20070 invalid(Self::sqlite_failure(&self.storage_failure, error))
20071 })?);
20072 }
20073 names.push(
20074 row.get::<_, String>(1)
20075 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
20076 );
20077 }
20078 }
20079 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
20080 expected.push("_syncular_version");
20081 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
20082 return Err(SectionError::Abort(
20083 "sync.schema_mismatch".to_owned(),
20084 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
20085 ));
20086 }
20087
20088 if primary_keys != vec![table.primary_key.clone()] {
20089 return Err(invalid(
20090 "image primary key does not match generated schema".into(),
20091 ));
20092 }
20093 let actual_count: i64 = img
20094 .query_row(
20095 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
20096 [],
20097 |row| row.get(0),
20098 )
20099 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
20100 if actual_count != row_count {
20101 return Err(invalid("image row count does not match descriptor".into()));
20102 }
20103 let insert = self.insert_row_sql(&base_table(&table.name), table);
20104 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
20105 let column_list = names
20106 .iter()
20107 .map(|n| quote_ident(n))
20108 .collect::<Vec<_>>()
20109 .join(", ");
20110 let mut stmt = img
20111 .prepare(&format!(
20112 "SELECT {column_list} FROM {} ORDER BY {}",
20113 quote_ident(&table.name),
20114 quote_ident(&table.primary_key)
20115 ))
20116 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
20117 let mut rows = stmt
20118 .query([])
20119 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
20120 let mut applied = 0u32;
20121 loop {
20122 let clear = applied == 0 && first_fresh_page;
20123 let was_dirty = self.overlay_dirty.snapshot();
20124 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
20125 && self
20126 .outbox
20127 .iter()
20128 .flat_map(|commit| &commit.ops)
20129 .any(|op| op.table == table.name);
20130 let mirror_visible = !reconcile_pending
20131 && (self.failed_commits.is_empty() && !self.has_acknowledged_rows());
20132 self.coalesce_unit(1024)
20134 .and_then(|()| self.begin_observation("syncular_image_chunk"))
20135 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20136 let result = (|| {
20137 self.rebuild_overlay_if_dirty()
20138 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20139 let mut changed_keys = Vec::new();
20140 let mut batch = ChangeAccumulator::default();
20141 if clear {
20142 if self.scoped_rows_exist(&table.name, effective) {
20143 batch.table(&table.name);
20144 }
20145 self.purge_scope_rows(&table.name, effective)
20146 .map_err(|()| SectionError::FailClosed)?;
20147 }
20148 let mut processed = 0u32;
20149 {
20150 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
20151 invalid(Self::sqlite_failure(&self.storage_failure, error))
20152 })?;
20153 let mut mirror = if mirror_visible {
20154 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
20155 invalid(Self::sqlite_failure(&self.storage_failure, error))
20156 })?)
20157 } else {
20158 None
20159 };
20160 for _ in 0..1024 {
20161 let Some(row) = rows.next().map_err(|error| {
20162 invalid(Self::sqlite_failure(&self.storage_failure, error))
20163 })?
20164 else {
20165 break;
20166 };
20167 for (i, column) in table.columns.iter().enumerate() {
20168 let cell = row.get_ref(i).map_err(|error| {
20169 invalid(Self::sqlite_failure(&self.storage_failure, error))
20170 })?;
20171 let param = image_cell_param(column, cell).map_err(invalid)?;
20172 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
20173 invalid(Self::sqlite_failure(&self.storage_failure, error))
20174 })?;
20175 if let Some(mirror) = &mut mirror {
20176 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
20177 invalid(Self::sqlite_failure(&self.storage_failure, error))
20178 })?;
20179 }
20180 }
20181 let version_index = table.columns.len();
20182 let version: i64 = row.get(version_index).map_err(|error| {
20183 invalid(Self::sqlite_failure(&self.storage_failure, error))
20184 })?;
20185 if version < 1 {
20186 return Err(invalid("image row version must be positive".into()));
20187 }
20188 ins.raw_bind_parameter(version_index + 1, version)
20189 .map_err(|error| {
20190 invalid(Self::sqlite_failure(&self.storage_failure, error))
20191 })?;
20192 ins.raw_execute().map_err(|error| {
20193 invalid(Self::sqlite_failure(&self.storage_failure, error))
20194 })?;
20195 if let Some(mirror) = &mut mirror {
20196 mirror
20197 .raw_bind_parameter(version_index + 1, version)
20198 .map_err(|error| {
20199 invalid(Self::sqlite_failure(&self.storage_failure, error))
20200 })?;
20201 mirror.raw_execute().map_err(|error| {
20202 invalid(Self::sqlite_failure(&self.storage_failure, error))
20203 })?;
20204 }
20205 if reconcile_pending {
20206 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
20207 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
20208 )?);
20209 }
20210 processed += 1;
20211 }
20212 }
20213 if processed > 0 {
20214 batch.table(&table.name);
20215 self.overlay_dirty.table(&table.name);
20216 }
20217 if reconcile_pending {
20218 self.reconcile_imported_rows(table, &mut changed_keys)
20219 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20220 }
20221 if self.failed_commits.is_empty() && !self.has_acknowledged_rows() {
20222 self.apply_outbox_ops(
20223 self.outbox
20224 .iter()
20225 .flat_map(|commit| &commit.ops)
20226 .filter(|op| op.table == table.name),
20227 )
20228 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20229 self.overlay_dirty.set(false);
20230 } else {
20231 self.rebuild_overlay_if_dirty()
20232 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20233 }
20234 self.finish_observation("syncular_image_chunk", batch)
20235 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
20236 Ok(processed)
20237 })();
20238 let processed = match result {
20239 Ok(processed) => processed,
20240 Err(error) => {
20241 self.rollback_observation("syncular_image_chunk");
20242 self.overlay_dirty.restore(was_dirty.clone());
20243 return Err(error);
20244 }
20245 };
20246 applied += processed;
20247 self.progress
20248 .update(|p| p.rows_processed = u64::from(applied));
20249 std::thread::yield_now();
20250 if i64::from(applied) == actual_count {
20251 break;
20252 }
20253 }
20254 Ok(applied)
20255 }
20256
20257 fn reconcile_imported_rows(
20261 &self,
20262 table: &TableSchema,
20263 keys: &mut Vec<SqlValue>,
20264 ) -> Result<(), String> {
20265 let visible = visible_table(&table.name);
20266 let base = base_table(&table.name);
20267 let pk = quote_ident(&table.primary_key);
20268 let predicate = row_id_predicate(table);
20269 let mut lookup = self.conn.prepare_cached(&format!(
20270 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
20271 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20272 let mut deleted = HashSet::new();
20273 for op in self
20274 .outbox
20275 .iter()
20276 .flat_map(|commit| &commit.ops)
20277 .filter(|op| op.table == table.name)
20278 {
20279 if op.upsert {
20280 if let Some(values) = &op.values {
20281 let key = json_to_column_value(
20282 &table.columns[table.pk_index],
20283 values.get(&table.primary_key),
20284 )?;
20285 keys.push(owned_sql_value(RowParam::Cell(&key))?);
20286 }
20287 } else if deleted.insert(&op.row_id) {
20288 keys.extend(
20289 lookup
20290 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
20291 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
20292 .collect::<Result<Vec<_>, _>>()
20293 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
20294 );
20295 }
20296 }
20297 for chunk in keys.chunks(400) {
20300 let holes = vec!["?"; chunk.len()].join(",");
20301 self.conn
20302 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
20303 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
20304 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20305 }
20306 let updates = table
20307 .columns
20308 .iter()
20309 .map(|column| column.name.as_str())
20310 .chain(std::iter::once("_syncular_version"))
20311 .filter(|name| *name != table.primary_key)
20312 .map(|name| {
20313 let name = quote_ident(name);
20314 format!("{name}=excluded.{name}")
20315 })
20316 .collect::<Vec<_>>()
20317 .join(",");
20318 for chunk in keys.chunks(400) {
20319 let holes = vec!["?"; chunk.len()].join(",");
20320 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
20321 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
20322 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20323 }
20324 Ok(())
20325 }
20326
20327 fn compile_local_data_purge(
20330 &self,
20331 input: &LocalDataPurgeInput,
20332 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
20333 let invalid = |message: String| format!("sync.invalid_request: {message}");
20334 if input.purge_id.is_empty()
20335 || input.purge_id.len() > 128
20336 || !is_local_operation_code_like(&input.purge_id)
20337 {
20338 return Err(invalid(
20339 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
20340 ));
20341 }
20342 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
20343 return Err(invalid(format!(
20344 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
20345 )));
20346 }
20347
20348 let mut deduplicated: BTreeMap<
20349 String,
20350 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
20351 > = BTreeMap::new();
20352 for target in &input.targets {
20353 let table = self.schema.table(&target.table).ok_or_else(|| {
20354 invalid(format!(
20355 "local purge names unknown table {:?}",
20356 target.table
20357 ))
20358 })?;
20359 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
20360 return Err(invalid(format!(
20361 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
20362 target.table
20363 )));
20364 }
20365 let mut selectors = Vec::with_capacity(target.selectors.len());
20366 let mut canonical_selectors = BTreeMap::new();
20367 for (column_name, raw_values) in &target.selectors {
20368 let Some((column_index, column)) = table
20369 .columns
20370 .iter()
20371 .enumerate()
20372 .find(|(_, column)| column.name == *column_name)
20373 else {
20374 return Err(invalid(format!(
20375 "local purge target {:?} names unknown column {:?}",
20376 target.table, column_name
20377 )));
20378 };
20379 let encrypted = table
20380 .encrypted_columns
20381 .iter()
20382 .any(|candidate| candidate.index == column_index);
20383 if column.ty != ColumnType::String || encrypted {
20384 return Err(invalid(format!(
20385 "local purge selector {:?}.{:?} must be a plaintext string column",
20386 target.table, column_name
20387 )));
20388 }
20389 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
20390 return Err(invalid(format!(
20391 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
20392 target.table, column_name
20393 )));
20394 }
20395 let mut values = raw_values.clone();
20396 values.sort();
20397 values.dedup();
20398 if values.iter().any(|value| {
20399 value.is_empty()
20400 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
20401 || !is_local_operation_code_like(value)
20402 }) {
20403 return Err(invalid(format!(
20404 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
20405 )));
20406 }
20407 selectors.push((column_name.clone(), values.clone()));
20408 canonical_selectors.insert(column_name.clone(), values);
20409 }
20410 let canonical = LocalDataPurgeTarget {
20411 table: target.table.clone(),
20412 selectors: canonical_selectors,
20413 };
20414 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
20415 deduplicated.insert(
20416 key,
20417 (
20418 CompiledLocalDataPurgeTarget {
20419 table: target.table.clone(),
20420 selectors,
20421 },
20422 canonical,
20423 ),
20424 );
20425 }
20426 let targets = deduplicated
20427 .values()
20428 .map(|(compiled, _)| compiled.clone())
20429 .collect::<Vec<_>>();
20430 let canonical = deduplicated
20431 .values()
20432 .map(|(_, target)| target.clone())
20433 .collect::<Vec<_>>();
20434 let canonical_plan =
20435 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
20436 Ok((targets, canonical_plan))
20437 }
20438
20439 fn local_purge_base_row_ids(
20440 &self,
20441 targets: &[CompiledLocalDataPurgeTarget],
20442 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
20443 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
20444 for target in targets {
20445 let table = self
20446 .schema
20447 .table(&target.table)
20448 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
20449 let mut clauses = Vec::with_capacity(target.selectors.len());
20450 let mut params = Vec::new();
20451 for (column, values) in &target.selectors {
20452 clauses.push(format!(
20453 "{} IN ({})",
20454 quote_ident(column),
20455 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
20456 ));
20457 params.extend(values.iter().cloned().map(SqlValue::Text));
20458 }
20459 let sql = format!(
20460 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
20461 quote_ident(&table.primary_key),
20462 base_table(&target.table),
20463 clauses.join(" AND ")
20464 );
20465 let mut statement = self
20466 .conn
20467 .prepare(&sql)
20468 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20469 let rows = statement
20470 .query_map(rusqlite::params_from_iter(params), |row| {
20471 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
20472 render_row_id_json(Some(&value))
20473 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
20474 })
20475 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20476 let ids = by_table.entry(target.table.clone()).or_default();
20477 for row in rows {
20478 ids.insert(
20479 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
20480 );
20481 }
20482 }
20483 Ok(by_table)
20484 }
20485
20486 fn local_purge_values_match(
20487 target: &CompiledLocalDataPurgeTarget,
20488 values: &Map<String, Value>,
20489 ) -> bool {
20490 target.selectors.iter().all(|(column, allowed)| {
20491 values
20492 .get(column)
20493 .and_then(Value::as_str)
20494 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
20495 })
20496 }
20497
20498 pub fn purge_local_data(
20502 &mut self,
20503 input: &LocalDataPurgeInput,
20504 ) -> Result<LocalDataPurgeResult, String> {
20505 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
20506 let meta_key = format!("localPurge:{}", input.purge_id);
20507 let applied_plan = self
20508 .conn
20509 .query_row(
20510 "SELECT value FROM _syncular_meta WHERE key = ?1",
20511 rusqlite::params![meta_key],
20512 |row| row.get::<_, String>(0),
20513 )
20514 .optional()
20515 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20516 if let Some(applied_plan) = applied_plan {
20517 if applied_plan != canonical_plan {
20518 return Err(format!(
20519 "sync.invalid_request: local purge id {:?} was already used with a different plan",
20520 input.purge_id
20521 ));
20522 }
20523 return Ok(LocalDataPurgeResult {
20524 already_applied: true,
20525 purged_rows: 0,
20526 dropped_commits: 0,
20527 });
20528 }
20529
20530 let _ = self.drop_previous_version();
20534
20535 let prior_outbox = self.outbox.clone();
20536 let prior_failed = self.failed_commits.clone();
20537 let prior_rejections = self.rejections.clone();
20538 let prior_conflicts = self.conflicts.clone();
20539 let prior_overlay_dirty = self.overlay_dirty.snapshot();
20540 self.begin_observation("syncular_local_purge")?;
20541 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
20542 let row_ids = self.local_purge_base_row_ids(&targets)?;
20543 let retained_dropped = self.drop_failed_commits(|table, values| {
20544 targets.iter().any(|target| {
20545 target.table == table && Self::local_purge_values_match(target, values)
20546 })
20547 })?;
20548 let doomed = self
20549 .outbox
20550 .iter()
20551 .filter(|commit| {
20552 commit.ops.iter().any(|operation| {
20553 let matching_targets = targets
20554 .iter()
20555 .filter(|target| target.table == operation.table)
20556 .collect::<Vec<_>>();
20557 if matching_targets.is_empty() {
20558 return false;
20559 }
20560 if row_ids
20561 .get(&operation.table)
20562 .is_some_and(|ids| ids.contains(&operation.row_id))
20563 {
20564 return true;
20565 }
20566 operation.values.as_ref().is_some_and(|values| {
20567 matching_targets
20568 .iter()
20569 .any(|target| Self::local_purge_values_match(target, values))
20570 })
20571 })
20572 })
20573 .cloned()
20574 .collect::<Vec<_>>();
20575 let doomed_ids = doomed
20576 .iter()
20577 .map(|commit| commit.client_commit_id.clone())
20578 .collect::<BTreeSet<_>>();
20579 let mut batch = ChangeAccumulator::default();
20580 for commit in &retained_dropped {
20581 batch.conflicts = true;
20582 batch.rejections = true;
20583 batch.outcomes = true;
20584 for operation in &commit.ops {
20585 batch.table(&operation.table);
20586 }
20587 }
20588 let mut rejections = Vec::new();
20589 for commit in &doomed {
20590 for operation in &commit.ops {
20591 batch.table(&operation.table);
20592 }
20593 let results = commit
20594 .ops
20595 .iter()
20596 .enumerate()
20597 .map(|(op_index, operation)| {
20598 let rejection = RejectionRecord {
20599 client_commit_id: commit.client_commit_id.clone(),
20600 op_index: op_index as i32,
20601 code: "client.local_data_purged".to_owned(),
20602 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
20603 retryable: false,
20604 details: None,
20605 operation: Some(CommitOperation::from(operation)),
20606 };
20607 rejections.push(rejection.clone());
20608 CommitOperationOutcome::Error { rejection }
20609 })
20610 .collect::<Vec<_>>();
20611 self.persist_commit_outcome(
20612 &commit.client_commit_id,
20613 CommitOutcomeStatus::Rejected,
20614 &results,
20615 Some(&commit.ops),
20616 )?;
20617 self.delete_outbox_persisted(&commit.client_commit_id)?;
20618 }
20619 if !doomed.is_empty() {
20620 self.outbox
20621 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
20622 self.rejections.extend(rejections);
20623 self.prune_commit_outcomes()?;
20624 batch.status = true;
20625 batch.rejections = true;
20626 batch.outcomes = true;
20627 }
20628
20629 let mut purged_rows = 0usize;
20630 for (table, ids) in &row_ids {
20631 if ids.is_empty() {
20632 continue;
20633 }
20634 batch.table(table);
20635 for row_id in ids {
20636 self.delete_base_row(table, row_id)?;
20637 purged_rows += 1;
20638 }
20639 }
20640 self.rebuild_overlay_if_dirty()?;
20641 self.delete_unreferenced_cached_blobs()?;
20642 self.conn
20643 .execute(
20644 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
20645 rusqlite::params![meta_key, canonical_plan],
20646 )
20647 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20648 Ok((
20649 batch,
20650 LocalDataPurgeResult {
20651 already_applied: false,
20652 purged_rows,
20653 dropped_commits: doomed.len(),
20654 },
20655 ))
20656 })();
20657 let (batch, result) = match applied {
20658 Ok(value) => value,
20659 Err(error) => {
20660 self.rollback_observation("syncular_local_purge");
20661 self.outbox = prior_outbox;
20662 self.failed_commits = prior_failed;
20663 self.rejections = prior_rejections;
20664 self.conflicts = prior_conflicts;
20665 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20666 return Err(error);
20667 }
20668 };
20669 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
20670 self.rollback_observation("syncular_local_purge");
20671 self.outbox = prior_outbox;
20672 self.failed_commits = prior_failed;
20673 self.rejections = prior_rejections;
20674 self.conflicts = prior_conflicts;
20675 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20676 return Err(error);
20677 }
20678 self.cancel_sync_round();
20679 Ok(result)
20680 }
20681
20682 pub fn rebootstrap_local_data(
20688 &mut self,
20689 input: &LocalDataRebootstrapInput,
20690 ) -> Result<LocalDataRebootstrapResult, String> {
20691 if input.rebootstrap_id.is_empty()
20692 || input.rebootstrap_id.len() > 128
20693 || !is_local_operation_code_like(&input.rebootstrap_id)
20694 {
20695 return Err(
20696 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
20697 .to_owned(),
20698 );
20699 }
20700 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
20701 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
20702 let (retained_commits, reset_subscriptions) =
20703 decode_local_rebootstrap_receipt(&persisted)?;
20704 return Ok(LocalDataRebootstrapResult {
20705 already_applied: true,
20706 retained_commits,
20707 reset_subscriptions,
20708 });
20709 }
20710 if self.stopped || self.schema_floor.is_some() {
20711 return Err(
20712 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
20713 .to_owned(),
20714 );
20715 }
20716
20717 let retained_commits = self.outbox.len();
20718 let reset_subscriptions = self.subs.len();
20719 let prior_subs = self.subs.clone();
20720 let prior_upgrading = self.upgrading;
20721 let prior_stopped = self.stopped;
20722 let prior_schema_floor = self.schema_floor.clone();
20723 let prior_overlay_dirty = self.overlay_dirty.snapshot();
20724 let prior_sync_needed = self.sync_needed;
20725 let prior_sync_intents = self.sync_intent_queue.clone();
20726 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
20727
20728 let prior_active_round = self.active_round;
20729 self.begin_observation("syncular_local_rebootstrap")?;
20730 let mut batch = ChangeAccumulator::default();
20731 let applied = (|| -> Result<(), String> {
20732 self.run_schema_reset_observed(&mut batch, false)?;
20733 self.conn
20734 .execute(
20735 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
20736 rusqlite::params![meta_key, receipt],
20737 )
20738 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20739 self.sync_needed = true;
20740 self.sync_intent_queue.push_back(SyncIntent::Interactive);
20741 batch.status = true;
20742 Ok(())
20743 })();
20744 if let Err(error) = applied {
20745 self.rollback_observation("syncular_local_rebootstrap");
20746 self.active_round = prior_active_round;
20747 self.subs = prior_subs;
20748 self.upgrading = prior_upgrading;
20749 self.stopped = prior_stopped;
20750 self.schema_floor = prior_schema_floor;
20751 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20752 self.sync_needed = prior_sync_needed;
20753 self.sync_intent_queue = prior_sync_intents;
20754 return Err(error);
20755 }
20756 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
20757 self.rollback_observation("syncular_local_rebootstrap");
20758 self.active_round = prior_active_round;
20759 self.subs = prior_subs;
20760 self.upgrading = prior_upgrading;
20761 self.stopped = prior_stopped;
20762 self.schema_floor = prior_schema_floor;
20763 self.overlay_dirty.restore(prior_overlay_dirty.clone());
20764 self.sync_needed = prior_sync_needed;
20765 self.sync_intent_queue = prior_sync_intents;
20766 return Err(error);
20767 }
20768
20769 Ok(LocalDataRebootstrapResult {
20770 already_applied: false,
20771 retained_commits,
20772 reset_subscriptions,
20773 })
20774 }
20775
20776 fn purge_scope_rows(
20781 &mut self,
20782 table_name: &str,
20783 effective: &[(String, Vec<String>)],
20784 ) -> Result<(), ()> {
20785 if effective.is_empty() {
20786 return Ok(());
20787 }
20788 let table = self.schema.table(table_name).ok_or(())?.clone();
20789 let mut clauses = Vec::new();
20790 let mut params: Vec<SqlValue> = Vec::new();
20791 for (variable, values) in effective {
20792 let column = table.scope_column(variable).ok_or(())?;
20793 let placeholders: Vec<String> = values
20794 .iter()
20795 .map(|v| {
20796 params.push(SqlValue::Text(v.clone()));
20797 "?".to_owned()
20798 })
20799 .collect();
20800 clauses.push(format!(
20801 "{} IN ({})",
20802 quote_ident(column),
20803 placeholders.join(", ")
20804 ));
20805 }
20806 let sql = format!(
20807 "DELETE FROM {} WHERE {}",
20808 base_table(table_name),
20809 clauses.join(" AND ")
20810 );
20811 self.overlay_dirty.table(table_name);
20812 self.conn
20813 .execute(&sql, rusqlite::params_from_iter(¶ms))
20814 .map_err(|_| ())?;
20815 self.conn
20816 .execute(
20817 &format!(
20818 "DELETE FROM {} WHERE {}",
20819 visible_table(table_name),
20820 clauses.join(" AND ")
20821 ),
20822 rusqlite::params_from_iter(¶ms),
20823 )
20824 .map_err(|_| ())?;
20825 Ok(())
20826 }
20827
20828 fn drop_doomed_outbox(
20831 &mut self,
20832 table_name: &str,
20833 effective: &[(String, Vec<String>)],
20834 ) -> Result<bool, String> {
20835 if effective.is_empty() {
20836 return Ok(false);
20837 }
20838 let Some(table) = self.schema.table(table_name).cloned() else {
20839 return Ok(false);
20840 };
20841 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
20842 for (variable, values) in effective {
20843 match table.scope_column(variable) {
20844 Some(column) => mappings.push((column, values)),
20845 None => return Ok(false), }
20847 }
20848 let doomed: Vec<OutboxCommit> = self
20849 .outbox
20850 .iter()
20851 .filter(|commit| {
20852 commit.ops.iter().any(|op| {
20853 op.upsert
20854 && op.table == table_name
20855 && op.values.as_ref().is_some_and(|values| {
20856 mappings
20857 .iter()
20858 .all(|(column, allowed)| match values.get(*column) {
20859 Some(Value::String(s)) => allowed.contains(s),
20860 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
20861 Some(Value::Bool(b)) => allowed.contains(&b.to_string()),
20862 _ => false,
20863 })
20864 })
20865 })
20866 })
20867 .cloned()
20868 .collect();
20869 if doomed.is_empty() {
20870 return Ok(false);
20871 }
20872 let mut rejections = Vec::new();
20873 for commit in &doomed {
20874 let results = commit
20875 .ops
20876 .iter()
20877 .enumerate()
20878 .map(|(op_index, operation)| {
20879 let rejection = RejectionRecord {
20880 client_commit_id: commit.client_commit_id.clone(),
20881 op_index: op_index as i32,
20882 code: "sync.scope_revoked".to_owned(),
20883 message: "the commit was dropped because its effective scope was revoked"
20884 .to_owned(),
20885 retryable: false,
20886 details: None,
20887 operation: Some(CommitOperation::from(operation)),
20888 };
20889 rejections.push(rejection.clone());
20890 CommitOperationOutcome::Error { rejection }
20891 })
20892 .collect::<Vec<_>>();
20893 self.persist_commit_outcome(
20894 &commit.client_commit_id,
20895 CommitOutcomeStatus::Rejected,
20896 &results,
20897 Some(&commit.ops),
20898 )?;
20899 self.delete_outbox_persisted(&commit.client_commit_id)?;
20900 }
20901 self.prune_commit_outcomes()?;
20902 let doomed_ids = doomed
20903 .iter()
20904 .map(|commit| commit.client_commit_id.as_str())
20905 .collect::<BTreeSet<_>>();
20906 self.outbox
20907 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
20908 self.rejections.extend(rejections);
20909 Ok(true)
20910 }
20911
20912 pub fn upload_blob(
20919 &mut self,
20920 bytes: &[u8],
20921 media_type: Option<String>,
20922 name: Option<String>,
20923 ) -> Result<Value, String> {
20924 let blob_id = blob_id_for(bytes);
20925 let now = self.clock_now_ms();
20926 let tx = self
20927 .conn
20928 .transaction()
20929 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20930 tx.execute(
20931 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
20932 ON CONFLICT(blob_id) DO NOTHING",
20933 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
20934 )
20935 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20936 tx.execute(
20937 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
20938 rusqlite::params![blob_id, media_type, now],
20939 )
20940 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20941 tx.commit()
20942 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
20943 self.enforce_blob_cache_cap();
20944 let mut obj = Map::new();
20945 obj.insert("blobId".to_owned(), Value::from(blob_id));
20946 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
20947 if let Some(mt) = media_type {
20948 obj.insert("mediaType".to_owned(), Value::from(mt));
20949 }
20950 if let Some(n) = name {
20951 obj.insert("name".to_owned(), Value::from(n));
20952 }
20953 Ok(Value::Object(obj))
20954 }
20955
20956 pub fn fetch_blob_bytes(
20960 &mut self,
20961 transport: &mut dyn Transport,
20962 blob_id_or_ref: &str,
20963 ) -> Result<FetchedBlob, (String, String)> {
20964 let simple = |m: String| ("client.failed".to_owned(), m);
20965 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
20966 blob_id_or_ref.to_owned()
20967 } else {
20968 let value: Value = serde_json::from_str(blob_id_or_ref)
20969 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
20970 value
20971 .get("blobId")
20972 .and_then(Value::as_str)
20973 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
20974 .to_owned()
20975 };
20976 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
20977 return Ok(cached);
20978 }
20979 if !self.transport_enabled {
20980 return Err(("sync.offline".into(), "transport gate is closed".into()));
20981 }
20982 #[cfg(feature = "bench-internals")]
20988 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
20989 let bytes = match transport
20990 .blob_download(&blob_id)
20991 .map_err(|e| (e.code, e.message))?
20992 {
20993 BlobDownload::Bytes(bytes) => bytes,
20994 BlobDownload::Url {
20995 url,
20996 url_expires_at_ms,
20997 } => {
20998 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
21000 return Err((
21001 "sync.segment_expired".to_owned(),
21002 format!(
21003 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
21004 ),
21005 ));
21006 }
21007 transport
21008 .fetch_blob_url(&url)
21009 .map_err(|e| (e.code, e.message))?
21010 }
21011 };
21012 #[cfg(feature = "bench-internals")]
21013 drop(download_phase);
21014 #[cfg(feature = "bench-internals")]
21015 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
21016 if blob_id_for(&bytes) != blob_id {
21018 return Err(simple(format!(
21019 "blob content address mismatch for {blob_id}"
21020 )));
21021 }
21022 #[cfg(feature = "bench-internals")]
21023 drop(validate_phase);
21024 #[cfg(feature = "bench-internals")]
21025 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
21026 let now = self.clock_now_ms();
21027 self.conn
21028 .execute(
21029 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
21030 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
21031 )
21032 .map_err(|e| simple(e.to_string()))?;
21033 self.conn
21034 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
21035 .map_err(|e| simple(e.to_string()))?;
21036 self.enforce_blob_cache_cap();
21037 #[cfg(feature = "bench-internals")]
21038 drop(insert_phase);
21039 self.get_cached_blob_bytes(&blob_id)
21040 .map_err(simple)?
21041 .ok_or_else(|| simple("blob cache write failed".to_owned()))
21042 }
21043
21044 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
21045 #[cfg(feature = "bench-internals")]
21046 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
21047 let mut stmt = self
21048 .conn
21049 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
21050 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21051 let mut rows = stmt
21052 .query(rusqlite::params![blob_id])
21053 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21054 if let Some(row) = rows
21055 .next()
21056 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
21057 {
21058 let bytes: Vec<u8> = row
21059 .get(0)
21060 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21061 let byte_length: i64 = row
21062 .get(1)
21063 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21064 let media_type: Option<String> = row
21065 .get(2)
21066 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21067 return Ok(Some(FetchedBlob {
21068 blob_id: blob_id.to_owned(),
21069 byte_length,
21070 bytes,
21071 media_type,
21072 }));
21073 }
21074 Ok(None)
21075 }
21076
21077 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
21079 let pending: Vec<(String, Option<String>)> = {
21080 let mut stmt = self
21081 .conn
21082 .prepare(
21083 "SELECT blob_id, media_type FROM (
21084 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
21085 UNION ALL
21086 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
21087 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
21088 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
21089 ) ORDER BY created_at_ms, blob_id",
21090 )
21091 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
21092 let rows = stmt
21093 .query_map([], |row| {
21094 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
21095 })
21096 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
21097 rows.collect::<Result<_, _>>()
21098 .map_err(|error| match error {
21099 rusqlite::Error::InvalidColumnType(..)
21100 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
21101 "sync.local_corrupt",
21102 "Pending blob upload metadata is invalid",
21103 ),
21104 _ => TransportError::new("client.failed", error.to_string()),
21105 })?
21106 };
21107 let mut uploads = Vec::new();
21108 for (blob_id, media_type) in pending {
21109 let bytes = (|| {
21110 let stored: Option<(SqlValue, SqlValue)> = self
21111 .conn
21112 .query_row(
21113 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
21114 rusqlite::params![blob_id],
21115 |row| Ok((row.get(0)?, row.get(1)?)),
21116 )
21117 .optional()
21118 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
21119 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
21120 return Err(TransportError::new(
21121 "sync.local_corrupt",
21122 "Pending blob upload body is missing or corrupt",
21123 ));
21124 };
21125 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
21126 return Err(TransportError::new(
21127 "sync.local_corrupt",
21128 "Pending blob upload body is missing or corrupt",
21129 ));
21130 }
21131 Ok(bytes)
21132 })();
21133 let failed = bytes.is_err();
21134 uploads.push(crate::round::PendingUpload {
21135 id: blob_id,
21136 bytes,
21137 media_type,
21138 });
21139 if failed {
21140 break;
21141 }
21142 }
21143 Ok(uploads)
21144 }
21145
21146 fn visible_blob_ids_sql(&self) -> String {
21147 let mut selects = Vec::new();
21148 for table in &self.schema.tables {
21149 for column in table
21150 .columns
21151 .iter()
21152 .filter(|column| column.ty == ColumnType::BlobRef)
21153 {
21154 let identifier = quote_ident(&column.name);
21155 selects.push(format!(
21156 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
21157 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
21158 AND json_type({identifier}, '$.blobId') = 'text'",
21159 quote_ident(&table.name),
21160 ));
21161 }
21162 }
21163 if selects.is_empty() {
21164 "SELECT NULL AS blob_id WHERE 0".to_owned()
21165 } else {
21166 selects.join(" UNION ")
21167 }
21168 }
21169
21170 fn enforce_blob_cache_cap(&self) {
21173 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
21174 return;
21175 };
21176 let mut total: i64 = self
21177 .conn
21178 .query_row(
21179 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
21180 [],
21181 |row| row.get(0),
21182 )
21183 .unwrap_or(0);
21184 if total <= max_bytes {
21185 return;
21186 }
21187 let query = format!(
21188 "SELECT blob_id, byte_length FROM _syncular_blobs
21189 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
21190 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
21191 AND blob_id NOT IN ({})
21192 ORDER BY created_at_ms ASC, blob_id ASC",
21193 self.visible_blob_ids_sql(),
21194 );
21195 let candidates: Vec<(String, i64)> = {
21196 let Ok(mut stmt) = self.conn.prepare(&query) else {
21197 return;
21198 };
21199 let Ok(rows) = stmt.query_map([], |row| {
21200 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
21201 }) else {
21202 return;
21203 };
21204 rows.filter_map(Result::ok).collect()
21205 };
21206 for (blob_id, byte_length) in candidates {
21207 if total <= max_bytes {
21208 break;
21209 }
21210 let _ = self.conn.execute(
21211 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
21212 rusqlite::params![blob_id],
21213 );
21214 total -= byte_length;
21215 }
21216 }
21217
21218 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
21219 #[cfg(feature = "bench-internals")]
21220 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
21221 if !self.schema_has_blobs() {
21222 return Ok(());
21223 }
21224 self.rebuild_overlay_if_dirty()?;
21225 self.conn
21226 .execute(
21227 &format!(
21228 "DELETE FROM _syncular_blobs
21229 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
21230 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
21231 AND blob_id NOT IN ({})",
21232 self.visible_blob_ids_sql(),
21233 ),
21234 [],
21235 )
21236 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21237 Ok(())
21238 }
21239
21240 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
21245 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
21246 return sql.clone();
21247 }
21248 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
21249 columns.push(quote_ident("_syncular_version"));
21250 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
21251 let primary_key = quote_ident(&table.primary_key);
21252 let updates = columns
21253 .iter()
21254 .filter(|column| **column != primary_key)
21255 .map(|column| format!("{column}=excluded.{column}"))
21256 .collect::<Vec<_>>()
21257 .join(", ");
21258 let sql = format!(
21259 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
21260 columns.join(", "),
21261 placeholders.join(", ")
21262 );
21263 self.insert_sql
21264 .borrow_mut()
21265 .insert(full_table.to_owned(), sql.clone());
21266 sql
21267 }
21268
21269 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
21270 self.overlay_dirty.table(table_name);
21271 self.write_row(&base_table(table_name), table_name, row, version)
21272 }
21273
21274 fn write_row(
21275 &self,
21276 full_table: &str,
21277 table_name: &str,
21278 row: &Row,
21279 version: i64,
21280 ) -> Result<(), String> {
21281 #[cfg(feature = "bench-internals")]
21282 let _phase = self.benchmark_phases.start(Phase::RowWrite);
21283 let table = self
21284 .schema
21285 .table(table_name)
21286 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
21287 let sql = self.insert_row_sql(full_table, table);
21288 let mut stmt = self
21289 .conn
21290 .prepare_cached(&sql)
21291 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21292 let params = row
21293 .iter()
21294 .map(RowParam::Cell)
21295 .chain(std::iter::once(RowParam::Version(version)));
21296 stmt.execute(rusqlite::params_from_iter(params))
21297 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21298 Ok(())
21299 }
21300
21301 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
21302 let table = self
21303 .schema
21304 .table(table_name)
21305 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
21306 self.overlay_dirty.table(table_name);
21307 let sql = format!(
21308 "DELETE FROM {} WHERE {}",
21309 base_table(table_name),
21310 row_id_predicate(table)
21311 );
21312 let mut stmt = self
21313 .conn
21314 .prepare_cached(&sql)
21315 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21316 stmt.execute(rusqlite::params![row_id])
21317 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21318 Ok(())
21319 }
21320
21321 fn has_acknowledged_rows(&self) -> bool {
21322 self.conn
21323 .query_row(
21324 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
21325 [],
21326 |row| row.get(0),
21327 )
21328 .expect("read acknowledged intent")
21329 }
21330
21331 fn acknowledged_operations(
21332 &self,
21333 rows: Option<&HashSet<(&str, &str)>>,
21334 tables: Option<&BTreeSet<&str>>,
21335 ) -> Result<Vec<OutboxOp>, String> {
21336 let keys = rows.map(|rows| {
21337 serde_json::to_string(
21338 &rows
21339 .iter()
21340 .map(|(table, id)| serde_json::to_string(&(table, id)).expect("row key"))
21341 .collect::<Vec<_>>(),
21342 )
21343 .expect("row keys")
21344 });
21345 let mut statement = self
21346 .conn
21347 .prepare("SELECT op_json,intent_json FROM _syncular_acked_rows WHERE (?1 IS NULL OR json_array(tbl,id) IN(SELECT value FROM json_each(?1))) AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx")
21348 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21349 let rows = statement
21350 .query_map(
21351 rusqlite::params![
21352 keys,
21353 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
21354 ],
21355 |row| {
21356 #[cfg(test)]
21357 self.acknowledged_replay_count
21358 .set(self.acknowledged_replay_count.get() + 1);
21359 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
21360 },
21361 )
21362 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21363 rows.map(|row| {
21364 let (op, intent) =
21365 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21366 let op: CommitOperation =
21367 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
21368 let table = self
21369 .schema
21370 .table(&op.table)
21371 .ok_or_else(|| "sync.unknown_table".to_owned())?;
21372 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
21373 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
21374 } else {
21375 op.values
21376 };
21377 Ok(OutboxOp {
21378 table: op.table,
21379 row_id: op.row_id,
21380 upsert: op.op == "upsert",
21381 base_version: op.base_version,
21382 values,
21383 insert: op.insert,
21384 })
21385 })
21386 .collect()
21387 }
21388
21389 fn drop_acknowledged_rows(
21390 &self,
21391 matches: impl Fn(&str, &Map<String, Value>) -> bool,
21392 ) -> Result<Vec<OutboxCommit>, String> {
21393 let mut statement = self
21394 .conn
21395 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
21396 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21397 let rows = statement
21398 .query_map([], |row| {
21399 Ok((
21400 row.get::<_, String>(0)?,
21401 row.get::<_, String>(1)?,
21402 row.get::<_, String>(2)?,
21403 row.get::<_, String>(3)?,
21404 ))
21405 })
21406 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21407 let mut doomed = HashSet::new();
21408 for row in rows {
21409 let (id, table, intent, row_id) =
21410 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21411 let values =
21412 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
21413 let base_matches = self
21414 .schema
21415 .table(&table)
21416 .map(|schema| self.stored_row(schema, &row_id, true))
21417 .transpose()?
21418 .flatten()
21419 .is_some_and(|(base, _)| {
21420 let schema = self.schema.table(&table).expect("acknowledged table");
21421 let values = schema
21422 .columns
21423 .iter()
21424 .zip(base.iter())
21425 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
21426 .collect();
21427 matches(&table, &values)
21428 });
21429 if matches(&table, &values) || base_matches {
21430 doomed.insert(id);
21431 }
21432 }
21433 let mut dropped = Vec::new();
21434 for id in doomed {
21435 let mut statement = self
21436 .conn
21437 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
21438 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21439 let operations = statement
21440 .query_map([&id], |row| row.get::<_, String>(0))
21441 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21442 let mut ops = Vec::new();
21443 for operation in operations {
21444 let json = operation
21445 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21446 let op: CommitOperation =
21447 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
21448 self.overlay_dirty.table(&op.table);
21449 ops.push(OutboxOp {
21450 table: op.table,
21451 row_id: op.row_id,
21452 upsert: op.op == "upsert",
21453 base_version: op.base_version,
21454 values: op.values,
21455 insert: op.insert,
21456 });
21457 }
21458 dropped.push(OutboxCommit {
21459 client_commit_id: id.clone(),
21460 ops,
21461 });
21462 self.conn
21463 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
21464 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21465 }
21466 Ok(dropped)
21467 }
21468
21469 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
21472 if self.overlay_dirty.get() {
21473 self.rebuild_overlay()?;
21474 }
21475 Ok(())
21476 }
21477
21478 fn rebuild_overlay(&mut self) -> Result<(), String> {
21482 #[cfg(feature = "bench-internals")]
21483 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
21484 #[cfg(test)]
21485 self.overlay_rebuild_count
21486 .set(self.overlay_rebuild_count.get() + 1);
21487 let dirty = self.overlay_dirty.snapshot();
21488 let tables: Vec<_> = self
21489 .schema
21490 .tables
21491 .iter()
21492 .filter(|table| {
21493 dirty
21494 .as_ref()
21495 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
21496 })
21497 .cloned()
21498 .collect();
21499 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
21500 self.begin_observation("syncular_overlay")?;
21501 let rebuilt = (|| {
21502 for table in &tables {
21503 for index in &table.fts_indexes {
21504 self.drop_fts_triggers(index)?;
21505 }
21506 let visible = visible_table(&table.name);
21507 let base = base_table(&table.name);
21508 self.conn
21509 .execute_batch(&format!(
21510 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
21511 ))
21512 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21513 }
21514 let acknowledged = self.acknowledged_operations(None, Some(&names))?;
21515 self.apply_outbox_ops(
21516 acknowledged
21517 .iter()
21518 .filter(|op| names.contains(op.table.as_str())),
21519 )?;
21520 for (commit, initial) in &self.failed_commits {
21521 for operation in &commit.ops {
21522 if !names.contains(operation.table.as_str()) || !operation.upsert {
21523 continue;
21524 }
21525 if let Some(table) = self.schema.table(&operation.table) {
21526 if self.visible_row(table, &operation.row_id)?.is_none()
21527 && operation.presented_values(table).is_some_and(|values| {
21528 table
21529 .columns
21530 .iter()
21531 .all(|column| values.contains_key(&column.name))
21532 })
21533 {
21534 let key = serde_json::to_string(&(
21535 operation.table.clone(),
21536 operation.row_id.clone(),
21537 ))
21538 .expect("row key serialization");
21539 if let Some(Value::Object(values)) = initial.get(&key) {
21540 self.apply_outbox_ops(&[OutboxOp {
21541 values: Some(values.clone()),
21542 ..operation.clone()
21543 }])?;
21544 }
21545 }
21546 }
21547 }
21548 self.apply_outbox_ops(
21549 commit
21550 .ops
21551 .iter()
21552 .filter(|op| names.contains(op.table.as_str())),
21553 )?;
21554 }
21555 for commit in &self.outbox {
21556 self.apply_outbox_ops(
21557 commit
21558 .ops
21559 .iter()
21560 .filter(|op| names.contains(op.table.as_str())),
21561 )?;
21562 }
21563 for table in &tables {
21564 for index in &table.fts_indexes {
21565 self.rebuild_fts_projection(table, index)?;
21566 self.create_fts_triggers(table, index)?;
21567 }
21568 }
21569 self.conn
21570 .execute_batch("RELEASE syncular_overlay")
21571 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21572 Ok(())
21573 })();
21574 if rebuilt.is_err() {
21575 self.conn
21576 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
21577 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21578 } else {
21579 self.overlay_dirty.set(false);
21580 }
21581 rebuilt
21582 }
21583
21584 fn apply_outbox_ops<'a>(
21589 &self,
21590 ops: impl IntoIterator<Item = &'a OutboxOp>,
21591 ) -> Result<(), String> {
21592 #[cfg(feature = "bench-internals")]
21593 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
21594 for op in ops {
21595 #[cfg(feature = "bench-internals")]
21596 if let Some(phase) = &mut phase {
21597 phase.unit();
21598 }
21599 if op.upsert {
21600 if let Some((table, values)) = self
21601 .schema
21602 .table(&op.table)
21603 .filter(|table| table.indexes.iter().any(|index| index.unique))
21604 .and_then(|table| Some((table, op.presented_values(table)?)))
21605 {
21606 let mut intended: Map<String, Value> = self
21607 .visible_row(table, &op.row_id)?
21608 .map(|(row, _)| {
21609 table
21610 .columns
21611 .iter()
21612 .zip(&row)
21613 .map(|(column, value)| {
21614 (column.name.clone(), column_value_to_json(value))
21615 })
21616 .collect()
21617 })
21618 .unwrap_or_default();
21619 intended.extend(values.into_owned());
21620 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
21621 continue;
21622 }
21623 }
21624 }
21625 self.apply_outbox_op(op)?;
21626 }
21627 Ok(())
21628 }
21629
21630 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
21631 let Some(table) = self.schema.table(&op.table) else {
21632 return Ok(());
21633 };
21634 let visible = visible_table(&table.name);
21635 if !op.upsert {
21636 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
21637 self.conn
21638 .prepare_cached(&sql)
21639 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
21640 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21641 return Ok(());
21642 }
21643 let Some(values) = op.presented_values(table) else {
21647 return Ok(());
21648 };
21649 let full = table
21655 .columns
21656 .iter()
21657 .all(|column| values.contains_key(&column.name));
21658 let local = self.visible_row(table, &op.row_id)?;
21659 if local.is_none() && !full {
21660 return Ok(());
21661 }
21662 let mut row: Row = Vec::with_capacity(table.columns.len());
21663 for (index, column) in table.columns.iter().enumerate() {
21664 let incoming = match values.get(&column.name) {
21665 Some(raw) => json_to_column_value(
21666 if raw.get("$bytes").is_some()
21667 && table
21668 .encrypted_columns
21669 .iter()
21670 .any(|encrypted| encrypted.index == index)
21671 {
21672 &table.wire_columns[index]
21673 } else {
21674 column
21675 },
21676 Some(raw),
21677 ),
21678 None => Ok(local
21679 .as_ref()
21680 .and_then(|(values, _)| values.get(index).cloned())
21681 .flatten()),
21682 };
21683 row.push(incoming.map_err(|_| {
21684 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
21685 })?);
21686 }
21687 let version = local.as_ref().map_or(-1, |(_, version)| *version);
21688 self.write_row(&visible, &table.name, &row, version)
21689 .map_err(|error| {
21690 if error.starts_with("UNIQUE constraint failed:") {
21691 "sync.constraint_violation: local write violates a unique constraint".to_owned()
21692 } else {
21693 error
21694 }
21695 })
21696 }
21697
21698 fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
21702 self.conn
21703 .query_row(
21704 &format!(
21705 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
21706 visible_table(&table.name),
21707 row_id_predicate(table)
21708 ),
21709 rusqlite::params![row_id],
21710 |row| row.get(0),
21711 )
21712 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
21713 }
21714
21715 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
21718 self.stored_row(table, row_id, false)
21719 }
21720
21721 fn stored_row(
21722 &self,
21723 table: &TableSchema,
21724 row_id: &str,
21725 base: bool,
21726 ) -> Result<Option<(Row, i64)>, String> {
21727 let sql = format!(
21728 "SELECT * FROM {} WHERE {}",
21729 if base {
21730 base_table(&table.name)
21731 } else {
21732 visible_table(&table.name)
21733 },
21734 row_id_predicate(table)
21735 );
21736 let mut stmt = self
21737 .conn
21738 .prepare_cached(&sql)
21739 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21740 let mut rows = stmt
21741 .query(rusqlite::params![row_id])
21742 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21743 let Some(row) = rows
21744 .next()
21745 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
21746 else {
21747 return Ok(None);
21748 };
21749 let mut values = Vec::with_capacity(table.columns.len());
21750 for (index, column) in table.columns.iter().enumerate() {
21751 let raw = row
21752 .get_ref(index)
21753 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21754 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
21755 if table
21756 .encrypted_columns
21757 .iter()
21758 .any(|encrypted| encrypted.index == index)
21759 {
21760 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
21761 continue;
21762 }
21763 }
21764 let json = sql_ref_to_json(column, raw);
21765 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
21766 "sync.local_corrupt: persisted local row value is invalid".to_owned()
21767 })?);
21768 }
21769 let version: i64 = row
21770 .get(table.columns.len())
21771 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
21772 Ok(Some((values, version)))
21773 }
21774
21775 #[must_use]
21779 pub fn realtime_policy(&self) -> RealtimePolicy {
21780 self.realtime_policy
21781 }
21782
21783 #[must_use]
21785 pub fn realtime_state(&self) -> RealtimeState {
21786 self.realtime_state
21787 }
21788
21789 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
21792 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
21793 return Err(
21794 "sync.invalid_request: disconnect realtime before changing the realtime policy"
21795 .to_owned(),
21796 );
21797 }
21798 self.realtime_policy = policy;
21799 match policy {
21800 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
21801 RealtimePolicy::Required | RealtimePolicy::Optional
21802 if self.realtime_state == RealtimeState::Disabled =>
21803 {
21804 self.set_realtime_state(RealtimeState::Disconnected, None);
21805 }
21806 RealtimePolicy::Required | RealtimePolicy::Optional => {}
21807 }
21808 Ok(())
21809 }
21810
21811 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
21814 self.realtime_state = state;
21815 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
21816 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
21817 self.realtime_reason_code = None;
21818 self.realtime_retry_delay_ms = None;
21819 }
21820 }
21821
21822 #[must_use]
21823 fn realtime_connected(&self) -> bool {
21824 self.realtime_state == RealtimeState::Connected
21825 }
21826
21827 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
21828 if !self.transport_enabled {
21829 return Err("sync.offline: transport gate is closed".into());
21830 }
21831 if self.realtime_policy == RealtimePolicy::Off {
21832 return Err(
21833 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
21834 );
21835 }
21836 if self.realtime_connected() {
21837 return Ok(());
21838 }
21839 self.realtime_state = RealtimeState::Connecting;
21840 match transport.realtime_connect_for_client(&self.client_id) {
21841 Ok(()) => {
21842 self.set_realtime_state(RealtimeState::Connected, None);
21843 Ok(())
21844 }
21845 Err(error) => {
21846 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
21847 Err(format!("{}: {}", error.code, error.message))
21848 }
21849 }
21850 }
21851
21852 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
21853 self.cancel_sync_round();
21854 if self.realtime_state == RealtimeState::Disconnected
21857 || self.realtime_state == RealtimeState::Disabled
21858 {
21859 return;
21860 }
21861 let _ = transport.realtime_close();
21862 self.set_realtime_state(
21863 if self.realtime_policy == RealtimePolicy::Off {
21864 RealtimeState::Disabled
21865 } else {
21866 RealtimeState::Disconnected
21867 },
21868 None,
21869 );
21870 self.presence.clear(); }
21872
21873 pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
21879 if self.realtime_state != RealtimeState::Connected {
21880 return;
21881 }
21882 self.cancel_sync_round();
21883 let _ = transport.realtime_close();
21884 self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
21885 self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
21886 self.presence.clear();
21887 }
21888
21889 pub fn set_presence(
21895 &mut self,
21896 transport: &mut dyn Transport,
21897 scope_key: &str,
21898 doc: Option<&Value>,
21899 ) -> Result<(), String> {
21900 if !self.transport_enabled {
21901 return Err("sync.offline: transport gate is closed".into());
21902 }
21903 if !self.realtime_connected() {
21904 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
21905 }
21906 let text = encode_presence_publish(scope_key, doc);
21907 transport
21908 .realtime_send(&text)
21909 .map_err(|e| format!("{}: {}", e.code, e.message))
21910 }
21911
21912 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
21914 self.presence
21915 .get(scope_key)
21916 .map(|peers| peers.values().cloned().collect())
21917 .unwrap_or_default()
21918 }
21919
21920 fn apply_presence(
21922 &mut self,
21923 scope_key: String,
21924 kind: Option<PresenceKind>,
21925 actor_id: Option<String>,
21926 client_id: Option<String>,
21927 doc: Option<Value>,
21928 error: Option<String>,
21929 ) {
21930 if error.is_some() {
21933 return;
21934 }
21935 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
21936 return;
21937 };
21938 let peer_key = format!("{actor_id} {client_id}");
21939 match kind {
21940 PresenceKind::Leave => {
21941 if let Some(peers) = self.presence.get_mut(&scope_key) {
21942 peers.remove(&peer_key);
21943 if peers.is_empty() {
21944 self.presence.remove(&scope_key);
21945 }
21946 }
21947 }
21948 _ => {
21949 let doc = match doc {
21950 Some(Value::Object(_)) => doc.unwrap(),
21951 _ => return,
21952 };
21953 self.presence.entry(scope_key).or_default().insert(
21954 peer_key,
21955 PresencePeer {
21956 actor_id,
21957 client_id,
21958 doc,
21959 },
21960 );
21961 }
21962 }
21963 }
21964
21965 pub fn on_realtime_text(&mut self, text: &str) {
21967 if self.stopped || !self.transport_enabled || self.security_preflight {
21968 return;
21969 }
21970 match parse_control(text) {
21971 Ok(ControlMessage::Hello { requires_sync, .. }) => {
21972 if requires_sync {
21973 self.set_sync_needed(true, true);
21975 }
21976 }
21977 Ok(ControlMessage::Presence {
21978 scope_key,
21979 kind,
21980 actor_id,
21981 client_id,
21982 doc,
21983 error,
21984 ..
21985 }) => {
21986 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
21987 }
21988 Ok(ControlMessage::Wake { .. }) => {
21989 self.set_sync_needed(true, true);
21991 }
21992 _ => {}
21993 }
21994 }
21995
21996 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
22000 let mut transport = crate::round::DownloadResults::default();
22001 self.on_realtime_binary(&mut transport, bytes);
22002 transport.controls
22003 }
22004
22005 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
22006 if self.stopped || !self.transport_enabled || self.security_preflight {
22007 return;
22008 }
22009 #[cfg(feature = "bench-internals")]
22010 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
22011 let message = match decode_message(bytes) {
22012 Ok(m) if m.msg_kind == MsgKind::Response => m,
22013 _ => {
22014 self.set_sync_needed(true, true);
22015 return;
22016 }
22017 };
22018 #[cfg(feature = "bench-internals")]
22019 drop(decode_phase);
22020 let coalesced_deletes =
22021 coalesced_delete_keys(&message.frames, |id, status, next_cursor| {
22022 if status != SubStatus::Active {
22023 return false;
22024 }
22025 let Some(sub) = self.subs.iter().find(|sub| sub.id == id) else {
22026 return false;
22027 };
22028 sub.state == SubState::Active
22029 && sub.bootstrap_state.is_none()
22030 && sub.synced_once
22031 && next_cursor.is_some_and(|cursor| cursor > sub.cursor)
22032 });
22033 let mut frames = message.frames.into_iter();
22034 let mut applied_cursor: Option<i64> = None;
22035 let mut any_covered = false;
22036 let mut dropped = self.begin_coalesced_apply().is_err();
22037 while let Some(frame) = frames.next() {
22038 let Frame::SubStart {
22039 id,
22040 status,
22041 effective_scopes,
22042 ..
22043 } = frame
22044 else {
22045 continue;
22046 };
22047 let mut body = Vec::new();
22048 let mut next_cursor: Option<i64> = None;
22049 for inner in frames.by_ref() {
22050 match inner {
22051 Frame::SubEnd {
22052 next_cursor: nc, ..
22053 } => {
22054 next_cursor = Some(nc);
22055 break;
22056 }
22057 Frame::Unknown { .. } => {}
22058 other => body.push(other),
22059 }
22060 }
22061 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
22062 dropped = true;
22063 continue;
22064 };
22065 let sub = &self.subs[sub_index];
22066 if status != SubStatus::Active
22069 || sub.state != SubState::Active
22070 || sub.bootstrap_state.is_some()
22071 || !sub.synced_once
22072 {
22073 dropped = true;
22074 continue;
22075 }
22076 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
22077 any_covered = true;
22079 continue;
22080 }
22081 let previous_effective = self.subs[sub_index].effective.clone();
22082 let previous_cursor = self.subs[sub_index].cursor;
22083 self.subs[sub_index].effective = Some(effective_scopes);
22084 let mut failed = false;
22085 for inner in body {
22086 match inner {
22087 Frame::Commit {
22088 tables,
22089 changes,
22090 commit_seq,
22091 ..
22092 } => {
22093 if self
22094 .apply_commit_frame_coalesced(
22095 &tables,
22096 changes,
22097 commit_seq,
22098 &coalesced_deletes,
22099 sub_index,
22100 AppliedServerCommitSource::Delta,
22101 )
22102 .is_err()
22103 {
22104 failed = true;
22105 break;
22106 }
22107 }
22108 _ => {
22109 failed = true;
22110 break;
22111 }
22112 }
22113 }
22114 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
22115 self.apply_sub_end(sub_index, cursor, None)
22116 .ok()
22117 .map(|()| cursor)
22118 });
22119 let Some(next_cursor) = committed else {
22120 self.subs[sub_index].effective = previous_effective;
22121 self.subs[sub_index].cursor = previous_cursor;
22122 dropped = true;
22123 continue;
22124 };
22125 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
22126 }
22127 if self.finish_coalesced_apply().is_err() {
22130 applied_cursor = None;
22131 dropped = true;
22132 }
22133 if let Some(cursor) = applied_cursor {
22134 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
22136 let _ = transport.realtime_send(&ack);
22137 } else if !any_covered || dropped {
22138 self.set_sync_needed(true, true);
22140 }
22141 }
22142
22143 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
22147 if !self.realtime_connected() {
22148 return;
22149 }
22150 let floor = self
22151 .subs
22152 .iter()
22153 .filter(|s| {
22154 s.state == SubState::Active
22155 && s.bootstrap_state.is_none()
22156 && s.synced_once
22157 && s.cursor >= 0
22158 })
22159 .map(|s| s.cursor)
22160 .min();
22161 if let Some(cursor) = floor {
22162 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
22163 let _ = transport.realtime_send(&ack);
22164 }
22165 }
22166}
22167
22168#[cfg(test)]
22169mod previous_version_wiring_tests {
22170 use super::*;
22171 use serde_json::{json, Value};
22172
22173 fn previous_version_schema(version: i32) -> Value {
22174 json!({
22175 "version": version,
22176 "tables": [{
22177 "name": "tasks",
22178 "primaryKey": "id",
22179 "columns": [
22180 { "name": "id", "type": "string", "nullable": false },
22181 { "name": "note", "type": "string", "nullable": true }
22182 ],
22183 "scopes": []
22184 }]
22185 })
22186 }
22187
22188 fn previous_version_limits() -> crate::api::ClientLimits {
22189 crate::api::ClientLimits {
22190 previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
22191 enabled: true,
22192 ..Default::default()
22193 }),
22194 ..Default::default()
22195 }
22196 }
22197
22198 fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
22200 previous_version_limits()
22201 .previous_version_context
22202 .expect("configured")
22203 }
22204
22205 fn recapture_previous_version(client: &SyncClient) {
22208 let replica_path = client
22209 .previous_version_replica_path()
22210 .expect("file-backed replica");
22211 let outcome = crate::previous_version::capture_previous_version_from_replica(
22212 &client.conn,
22213 &replica_path,
22214 2,
22215 2,
22216 &previous_version_config(),
22217 1,
22218 )
22219 .expect("capture");
22220 assert!(matches!(
22221 outcome,
22222 crate::previous_version::CaptureOutcome::Captured(_)
22223 ));
22224 }
22225
22226 fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
22227 Subscription {
22228 id: "wiring-sub".to_owned(),
22229 table: "tasks".to_owned(),
22230 requested: Vec::new(),
22231 params: None,
22232 cursor,
22233 bootstrap_state: bootstrap_state.map(str::to_owned),
22234 state,
22235 reason_code: None,
22236 effective: None,
22237 synced_once: false,
22238 }
22239 }
22240
22241 #[test]
22242 fn schema_downgrade_refuses_all_opens_and_recreate_without_mutation() {
22243 let path =
22244 std::env::temp_dir().join(format!("syncular-downgrade-{}.db", uuid::Uuid::new_v4()));
22245 let path_str = path.to_str().expect("temp path");
22246 let schema = previous_version_schema(3);
22247 let mut older = previous_version_schema(2);
22248 older["tables"][0]["columns"] =
22249 json!([{ "name": "id", "type": "string", "nullable": false }]);
22250 let mut client = SyncClient::open_path(
22251 "downgrade".into(),
22252 &previous_version_schema(1),
22253 previous_version_limits(),
22254 path_str,
22255 )
22256 .expect("fresh");
22257 client
22258 .mutate(vec![Mutation::Upsert {
22259 table: "tasks".into(),
22260 values: Map::from_iter([
22261 ("id".into(), json!("accepted")),
22262 ("note".into(), json!("old")),
22263 ]),
22264 base_version: None,
22265 }])
22266 .expect("v1 write");
22267 client
22268 .recreate_with_schema(&schema)
22269 .expect("upgrade captures context");
22270 let commit = client
22271 .mutate(vec![Mutation::Upsert {
22272 table: "tasks".into(),
22273 values: Map::from_iter([
22274 ("id".into(), json!("queued")),
22275 ("note".into(), json!("v3-only")),
22276 ]),
22277 base_version: None,
22278 }])
22279 .expect("queued v3 write");
22280 let changes = client.conn.total_changes();
22281 let outbox = client.pending_commit_ids();
22282 assert!(outbox.contains(&commit));
22283 let descriptor = client.get_meta("localSchemaDescriptor");
22284 let container = format!(
22285 "{path_str}{}",
22286 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
22287 );
22288 let context = std::fs::read(&container).expect("captured context");
22289 assert!(client
22290 .recreate_with_schema(&older)
22291 .expect_err("refuse recreate")
22292 .starts_with("client.schema_downgrade:"));
22293 assert_eq!(client.schema.version, 3);
22294 assert_eq!(client.conn.total_changes(), changes);
22295 assert_eq!(client.pending_commit_ids(), outbox);
22296 assert_eq!(client.get_meta("localSchemaDescriptor"), descriptor);
22297 assert_eq!(
22298 client
22299 .conn
22300 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
22301 row.get::<_, String>(0)
22302 })
22303 .expect("queued row"),
22304 "v3-only"
22305 );
22306 assert_eq!(std::fs::read(&container).expect("context remains"), context);
22307 drop(client);
22308 let before = std::fs::read(&path).expect("replica bytes");
22309 for entry in 0..3 {
22312 let error = match entry {
22313 0 => SyncClient::open_path(
22314 "downgrade".into(),
22315 &older,
22316 previous_version_limits(),
22317 path_str,
22318 ),
22319 1 => SyncClient::open_path_with_identity(
22320 None,
22321 &older,
22322 previous_version_limits(),
22323 path_str,
22324 ),
22325 _ => SyncClient::with_connection(
22326 "downgrade".into(),
22327 &older,
22328 previous_version_limits(),
22329 Connection::open(&path).expect("connection"),
22330 ),
22331 }
22332 .err()
22333 .expect("refuse open");
22334 assert!(error.starts_with("client.schema_downgrade:"));
22335 let conn = Connection::open(&path).expect("inspect");
22336 assert_eq!(
22337 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
22338 .expect("journal mode"),
22339 "delete"
22340 );
22341 drop(conn);
22342 assert_eq!(std::fs::read(&path).expect("replica unchanged"), before);
22343 assert_eq!(
22344 std::fs::read(&container).expect("context unchanged"),
22345 context
22346 );
22347 }
22348 let reopened = SyncClient::open_path(
22349 "downgrade".into(),
22350 &schema,
22351 previous_version_limits(),
22352 path_str,
22353 )
22354 .expect("compatible reopen");
22355 assert_eq!(reopened.pending_commit_ids(), outbox);
22356 assert_eq!(
22357 reopened
22358 .conn
22359 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
22360 row.get::<_, String>(0)
22361 })
22362 .expect("queued row remains"),
22363 "v3-only"
22364 );
22365 reopened.delete_meta(LOCAL_SCHEMA_VERSION_KEY);
22366 reopened.delete_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
22367 drop(reopened);
22368 let legacy = SyncClient::open_path(
22369 "downgrade".into(),
22370 &schema,
22371 previous_version_limits(),
22372 path_str,
22373 )
22374 .expect("legacy no-marker reopen");
22375 assert_eq!(legacy.pending_commit_ids(), outbox);
22376 assert!(!legacy.upgrading());
22377 assert_eq!(
22378 legacy
22379 .conn
22380 .query_row("SELECT note FROM tasks WHERE id = 'queued'", [], |row| {
22381 row.get::<_, String>(0)
22382 })
22383 .expect("legacy row remains"),
22384 "v3-only"
22385 );
22386 drop(legacy);
22387 std::fs::remove_file(path).expect("remove replica");
22388 let _ = std::fs::remove_file(container);
22389 }
22390
22391 #[test]
22392 fn schema_marker_errors_refuse_open_and_recreate_without_mutation() {
22393 for marker in [
22394 "''",
22395 "'bad'",
22396 "'0'",
22397 "'-1'",
22398 "'2.5'",
22399 "'03'",
22400 "'3\n'",
22401 "' 3'",
22402 "'+3'",
22403 "'3e0'",
22404 "'2147483648'",
22405 "NULL",
22406 "X'33'",
22407 ] {
22408 let conn = Connection::open_in_memory().expect("connection");
22409 conn.execute_batch(&format!("CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY, value); INSERT INTO _syncular_meta VALUES ('localSchemaVersion', {marker});")).expect("invalid marker fixture");
22410 let error = SyncClient::with_connection(
22411 "marker".into(),
22412 &previous_version_schema(3),
22413 ClientLimits::default(),
22414 conn,
22415 )
22416 .err()
22417 .expect("refuse marker");
22418 assert!(
22419 error.starts_with("sync.local_corrupt:"),
22420 "{marker}: {error}"
22421 );
22422 }
22423 for fault in [
22424 "UPDATE _syncular_meta SET value = 'bad' WHERE key = 'localSchemaVersion'",
22425 "ALTER TABLE _syncular_meta RENAME COLUMN value TO unreadable_value",
22426 ] {
22427 let mut client = SyncClient::new(
22428 "marker".into(),
22429 &previous_version_schema(3),
22430 ClientLimits::default(),
22431 )
22432 .expect("fresh");
22433 let commit = client
22434 .mutate(vec![Mutation::Upsert {
22435 table: "tasks".into(),
22436 values: Map::from_iter([("id".into(), json!("queued"))]),
22437 base_version: None,
22438 }])
22439 .expect("queued write");
22440 client.conn.execute_batch(fault).expect("marker fault");
22441 let changes = client.conn.total_changes();
22442 let error = client
22443 .recreate_with_schema(&previous_version_schema(4))
22444 .expect_err("refuse corrupt marker");
22445 assert!(error.starts_with("sync.local_corrupt:"));
22446 assert_eq!(client.conn.total_changes(), changes);
22447 assert_eq!(client.schema.version, 3);
22448 assert_eq!(client.pending_commit_ids(), vec![commit]);
22449 }
22450 for setup in [
22451 "CREATE TABLE _syncular_meta(key TEXT PRIMARY KEY)",
22452 "CREATE VIEW _syncular_meta AS SELECT 'localSchemaVersion' AS key, '3' AS value",
22453 ] {
22454 let path =
22455 std::env::temp_dir().join(format!("syncular-marker-{}.db", uuid::Uuid::new_v4()));
22456 let conn = Connection::open(&path).expect("connection");
22457 conn.execute_batch(setup)
22458 .expect("unreadable metadata fixture");
22459 drop(conn);
22460 let before = std::fs::read(&path).expect("before");
22461 let error = SyncClient::open_path_with_identity(
22462 None,
22463 &previous_version_schema(3),
22464 ClientLimits::default(),
22465 path.to_str().expect("path"),
22466 )
22467 .err()
22468 .expect("refuse unreadable marker");
22469 assert!(error.starts_with("sync.local_corrupt:"));
22470 assert_eq!(std::fs::read(&path).expect("after"), before);
22471 std::fs::remove_file(path).expect("remove fixture");
22472 }
22473 }
22474
22475 std::thread_local! {
22476 static CONCURRENT_UPGRADE: RefCell<Option<Connection>> =
22479 const { RefCell::new(None) };
22480 }
22481
22482 fn commit_concurrent_upgrade(attempts: i32) -> bool {
22483 CONCURRENT_UPGRADE.with(|slot| {
22484 if let Some(banner) = slot.borrow_mut().take() {
22485 banner
22486 .execute_batch(
22487 "UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion'; COMMIT",
22488 )
22489 .expect("commit the concurrent upgrade");
22490 }
22491 });
22492 attempts < 10
22493 }
22494
22495 fn journal_mode(conn: &Connection) -> String {
22496 conn.query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
22497 .expect("journal mode")
22498 }
22499
22500 fn temp_replica(label: &str) -> std::path::PathBuf {
22501 std::env::temp_dir().join(format!("syncular-{label}-{}.db", uuid::Uuid::new_v4()))
22502 }
22503
22504 #[test]
22505 fn file_identity_constructor_requires_the_returned_wal_mode() {
22506 for path in [":memory:", ""] {
22507 let error = SyncClient::open_path_with_identity(
22508 None,
22509 &previous_version_schema(1),
22510 ClientLimits::default(),
22511 path,
22512 )
22513 .err()
22514 .expect("temporary SQLite databases cannot enter WAL mode");
22515 assert_eq!(
22516 error,
22517 "sync.invalid_request: file-backed client requires WAL journal mode"
22518 );
22519 }
22520 let path = temp_replica("confirmed-wal");
22521 let client = SyncClient::open_path_with_identity(
22522 None,
22523 &previous_version_schema(1),
22524 ClientLimits::default(),
22525 path.to_str().unwrap(),
22526 )
22527 .unwrap();
22528 assert_eq!(journal_mode(&client.conn), "wal");
22529 drop(client);
22530 std::fs::remove_file(path).unwrap();
22531 }
22532
22533 #[test]
22534 fn non_wal_identity_open_is_accepted_when_wal_is_not_required() {
22535 for path in [":memory:", ""] {
22536 let client = SyncClient::open_path_with_identity_journal(
22537 None,
22538 &previous_version_schema(1),
22539 ClientLimits::default(),
22540 path,
22541 false,
22542 )
22543 .expect("a host without WAL can still open the identity constructor");
22544 assert_ne!(journal_mode(&client.conn), "wal");
22545 assert_eq!(
22546 client
22547 .conn
22548 .query_row("PRAGMA synchronous", [], |row| row.get::<_, i64>(0))
22549 .expect("synchronous"),
22550 2,
22551 "every identity open pins FULL durability"
22552 );
22553 }
22554 }
22555
22556 #[test]
22557 fn reset_replica_deletes_the_database_and_its_sidecars_through_the_vfs() {
22558 let path = temp_replica("reset");
22559 let path_str = path.to_str().expect("path");
22560 let client = SyncClient::open_path(
22561 "reset-replica".into(),
22562 &previous_version_schema(1),
22563 ClientLimits::default(),
22564 path_str,
22565 )
22566 .expect("replica");
22567 let original_id = client.client_id().to_owned();
22568 drop(client);
22569 for suffix in ["-journal", "-wal", "-shm"] {
22570 std::fs::write(format!("{path_str}{suffix}"), b"stale sidecar").expect("sidecar");
22571 }
22572
22573 SyncClient::reset_replica(path_str).expect("reset through the VFS");
22574
22575 for suffix in ["", "-journal", "-wal", "-shm"] {
22576 assert!(
22577 !std::path::Path::new(&format!("{path_str}{suffix}")).exists(),
22578 "reset removes the replica file {suffix:?}"
22579 );
22580 }
22581 let reopened = SyncClient::open_path_with_identity(
22582 None,
22583 &previous_version_schema(1),
22584 ClientLimits::default(),
22585 path_str,
22586 )
22587 .expect("fresh replica");
22588 assert_ne!(reopened.client_id(), original_id);
22589 assert!(reopened
22590 .query("SELECT id FROM tasks", &[])
22591 .expect("empty")
22592 .is_empty());
22593 drop(reopened);
22594 assert_eq!(
22595 SyncClient::reset_replica(":memory:").expect_err("memory has nothing to reset"),
22596 "sync.invalid_request: a memory replica has nothing to reset"
22597 );
22598 }
22599
22600 #[test]
22601 fn reset_replica_after_a_clean_close_without_sidecars_opens_an_empty_replica() {
22602 let path = temp_replica("reset-clean");
22603 let path_str = path.to_str().expect("path");
22604 let mut client = SyncClient::open_path(
22605 "reset-clean".into(),
22606 &previous_version_schema(1),
22607 ClientLimits::default(),
22608 path_str,
22609 )
22610 .expect("replica");
22611 client
22612 .mutate(vec![Mutation::Upsert {
22613 table: "tasks".into(),
22614 values: Map::from_iter([
22615 ("id".into(), Value::from("t1")),
22616 ("note".into(), Value::from("stale")),
22617 ]),
22618 base_version: None,
22619 }])
22620 .expect("stale row");
22621 assert_eq!(
22622 client
22623 .query("SELECT id FROM tasks", &[])
22624 .expect("row")
22625 .len(),
22626 1
22627 );
22628 drop(client);
22629 for suffix in ["-journal", "-wal", "-shm"] {
22630 assert!(
22631 !std::path::Path::new(&format!("{path_str}{suffix}")).exists(),
22632 "a clean close leaves no {suffix:?} sidecar"
22633 );
22634 }
22635
22636 SyncClient::reset_replica(path_str).expect("missing sidecars count as deleted");
22637
22638 assert!(!path.exists(), "reset removes the replica file");
22639 let reopened = SyncClient::open_path_with_identity(
22640 None,
22641 &previous_version_schema(1),
22642 ClientLimits::default(),
22643 path_str,
22644 )
22645 .expect("fresh replica");
22646 assert_ne!(reopened.client_id(), "reset-clean");
22647 assert!(reopened
22648 .query("SELECT id FROM tasks", &[])
22649 .expect("empty")
22650 .is_empty());
22651 drop(reopened);
22652 let _ = std::fs::remove_file(&path);
22653 }
22654
22655 #[test]
22656 fn non_positive_generated_schema_versions_are_refused_before_storage() {
22657 for version in [json!(0), json!(-1), json!(i32::MIN), json!(2147483648i64)] {
22658 let mut schema = previous_version_schema(1);
22659 schema["version"] = version.clone();
22660 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
22661 .err()
22662 .expect("refuse an out-of-range generated version");
22663 assert_eq!(
22664 error,
22665 "sync.invalid_request: generated schema version must be an integer in 1..=2147483647"
22666 );
22667 let path = temp_replica("bad-version");
22669 let path_str = path.to_str().expect("path");
22670 for identity in [false, true] {
22671 let error = if identity {
22672 SyncClient::open_path_with_identity(
22673 None,
22674 &schema,
22675 ClientLimits::default(),
22676 path_str,
22677 )
22678 } else {
22679 SyncClient::open_path(
22680 "bad-version".into(),
22681 &schema,
22682 ClientLimits::default(),
22683 path_str,
22684 )
22685 }
22686 .err()
22687 .expect("refuse before storage");
22688 assert!(
22689 error.starts_with("sync.invalid_request:"),
22690 "{version}: {error}"
22691 );
22692 assert!(
22693 !path.exists(),
22694 "{version}: a refused constructor creates no replica file"
22695 );
22696 }
22697 }
22698 let mut schema = previous_version_schema(1);
22700 schema
22701 .as_object_mut()
22702 .expect("schema object")
22703 .remove("version");
22704 let error = SyncClient::new("bad-version".into(), &schema, ClientLimits::default())
22705 .err()
22706 .expect("refuse a missing version");
22707 assert!(error.starts_with("sync.invalid_request:"), "{error}");
22708 }
22709
22710 #[test]
22711 fn a_duplicated_schema_marker_is_refused_without_a_reset() {
22712 let path = temp_replica("duplicate-marker");
22713 let path_str = path.to_str().expect("path");
22714 {
22715 let conn = Connection::open(path_str).expect("fixture");
22718 conn.execute_batch(
22719 "CREATE TABLE _syncular_meta(key TEXT, value);
22720 INSERT INTO _syncular_meta VALUES
22721 ('localSchemaVersion', '1'),
22722 ('localSchemaVersion', '2'),
22723 ('clientId', 'duplicate');",
22724 )
22725 .expect("duplicate marker fixture");
22726 }
22727 let error = SyncClient::open_path_with_identity(
22728 None,
22729 &previous_version_schema(3),
22730 ClientLimits::default(),
22731 path_str,
22732 )
22733 .err()
22734 .expect("refuse a duplicated marker");
22735 assert!(error.starts_with("sync.local_corrupt:"), "{error}");
22736 let probe = Connection::open(path_str).expect("probe");
22737 assert_eq!(
22738 probe
22739 .query_row(
22740 "SELECT COUNT(*) FROM _syncular_meta WHERE key = 'localSchemaVersion'",
22741 [],
22742 |row| row.get::<_, i64>(0),
22743 )
22744 .expect("marker rows"),
22745 2,
22746 "the duplicated rows are left for the operator, not collapsed"
22747 );
22748 assert_eq!(
22749 probe
22750 .query_row(
22751 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_syncular_%' AND name NOT LIKE 'sqlite_%'",
22752 [],
22753 |row| row.get::<_, i64>(0),
22754 )
22755 .expect("synced tables"),
22756 0,
22757 "a duplicated marker never drives the reset that recreates v3 tables"
22758 );
22759 assert_eq!(
22760 journal_mode(&probe),
22761 "delete",
22762 "the refusal leaves the journal mode alone"
22763 );
22764 drop(probe);
22765 std::fs::remove_file(path).expect("remove replica");
22766 }
22767
22768 #[test]
22769 fn a_refused_open_does_not_persist_wal_or_mutate_the_replica() {
22770 let path = temp_replica("refused-wal");
22771 let path_str = path.to_str().expect("path");
22772 drop(
22775 SyncClient::open_path(
22776 "owner".into(),
22777 &previous_version_schema(2),
22778 ClientLimits::default(),
22779 path_str,
22780 )
22781 .expect("v2 replica"),
22782 );
22783 let before = std::fs::read(&path).expect("before");
22784 let error = SyncClient::open_path_with_identity(
22785 Some("intruder".into()),
22786 &previous_version_schema(2),
22787 ClientLimits::default(),
22788 path_str,
22789 )
22790 .err()
22791 .expect("refuse the identity mismatch");
22792 assert!(error.starts_with("client.identity_mismatch:"), "{error}");
22793 let probe = Connection::open(path_str).expect("probe");
22794 assert_eq!(
22795 journal_mode(&probe),
22796 "delete",
22797 "a refused open must not persist WAL"
22798 );
22799 assert_eq!(std::fs::read(&path).expect("after"), before);
22800 assert_eq!(
22801 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
22802 Some("2")
22803 );
22804 drop(probe);
22805 std::fs::remove_file(path).expect("remove replica");
22806 }
22807
22808 #[test]
22809 fn a_concurrent_marker_upgrade_refuses_a_stale_open() {
22810 let path = temp_replica("marker-race");
22811 let path_str = path.to_str().expect("path");
22812 drop(
22813 SyncClient::open_path_with_identity(
22814 Some("stale".into()),
22815 &previous_version_schema(1),
22816 ClientLimits::default(),
22817 path_str,
22818 )
22819 .expect("v1 replica"),
22820 );
22821 let banner = Connection::open(path_str).expect("banner");
22825 banner
22826 .execute_batch(
22827 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
22828 )
22829 .expect("pending upgrade");
22830 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
22831 let mut conn = Connection::open(path_str).expect("stale connection");
22832 conn.busy_handler(Some(commit_concurrent_upgrade))
22833 .expect("busy handler");
22834 let error = SyncClient::with_connection(
22835 "stale".into(),
22836 &previous_version_schema(1),
22837 ClientLimits::default(),
22838 conn,
22839 )
22840 .err()
22841 .expect("a concurrent upgrade must refuse the stale open");
22842 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
22843 let probe = Connection::open(path_str).expect("probe");
22844 assert_eq!(
22845 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
22846 Some("2"),
22847 "the concurrent upgrade survives"
22848 );
22849 drop(probe);
22850 std::fs::remove_file(path).expect("remove replica");
22851 }
22852
22853 #[test]
22854 fn a_concurrent_marker_upgrade_refuses_a_stale_recreate() {
22855 let path = temp_replica("recreate-race");
22856 let path_str = path.to_str().expect("path");
22857 let mut client = SyncClient::open_path_with_identity(
22858 Some("stale".into()),
22859 &previous_version_schema(1),
22860 ClientLimits::default(),
22861 path_str,
22862 )
22863 .expect("v1 replica");
22864 client
22865 .mutate(vec![Mutation::Upsert {
22866 table: "tasks".into(),
22867 values: Map::from_iter([("id".into(), json!("queued"))]),
22868 base_version: None,
22869 }])
22870 .expect("queued write");
22871 let commit = client.pending_commit_ids();
22872 let banner = Connection::open(path_str).expect("banner");
22873 banner
22874 .execute_batch(
22875 "BEGIN IMMEDIATE; UPDATE _syncular_meta SET value = '2' WHERE key = 'localSchemaVersion';",
22876 )
22877 .expect("pending upgrade");
22878 CONCURRENT_UPGRADE.with(|slot| *slot.borrow_mut() = Some(banner));
22879 client
22880 .conn
22881 .busy_handler(Some(commit_concurrent_upgrade))
22882 .expect("busy handler");
22883 let error = client
22884 .recreate_with_schema(&previous_version_schema(1))
22885 .expect_err("a concurrent upgrade must refuse the stale recreate");
22886 assert!(error.starts_with("client.schema_downgrade:"), "{error}");
22887 assert_eq!(client.schema.version, 1, "the compiled schema is restored");
22888 assert_eq!(
22889 client.pending_commit_ids(),
22890 commit,
22891 "the outbox is preserved"
22892 );
22893 let probe = Connection::open(path_str).expect("probe");
22894 assert_eq!(
22895 meta_get(&probe, LOCAL_SCHEMA_VERSION_KEY).as_deref(),
22896 Some("2")
22897 );
22898 drop(probe);
22899 drop(client);
22900 std::fs::remove_file(path).expect("remove replica");
22901 }
22902
22903 #[test]
22904 fn failed_schema_record_writes_roll_back_recreation() {
22905 for key in [
22906 LOCAL_SCHEMA_VERSION_KEY,
22907 crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY,
22908 ] {
22909 let mut client = SyncClient::new(
22910 "schema-record".into(),
22911 &previous_version_schema(1),
22912 ClientLimits::default(),
22913 )
22914 .expect("v1");
22915 let marker = client.get_meta(LOCAL_SCHEMA_VERSION_KEY);
22916 let descriptor = client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY);
22917 client.conn.execute_batch(&format!("CREATE TRIGGER refuse_schema_record BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{key}' BEGIN SELECT RAISE(ABORT, 'refuse schema record'); END")).expect("fault");
22918 client
22919 .recreate_with_schema(&previous_version_schema(2))
22920 .expect_err("both record writes must propagate failure");
22921 assert_eq!(client.schema.version, 1);
22922 assert_eq!(client.get_meta(LOCAL_SCHEMA_VERSION_KEY), marker);
22923 assert_eq!(
22924 client.get_meta(crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY),
22925 descriptor
22926 );
22927 client
22928 .conn
22929 .execute_batch("DROP TRIGGER refuse_schema_record")
22930 .expect("remove fault");
22931 client
22932 .recreate_with_schema(&previous_version_schema(2))
22933 .expect("retry");
22934 assert_eq!(client.schema.version, 2);
22935 }
22936 }
22937
22938 #[test]
22939 fn a_failed_outer_commit_restores_every_reset_side_effect() {
22940 let path = temp_replica("outer-commit");
22941 let path_str = path.to_str().expect("path");
22942 let conn = Connection::open(path_str).expect("replica connection");
22945 let mut client = SyncClient::with_connection(
22946 "outer-commit".into(),
22947 &previous_version_schema(1),
22948 previous_version_limits(),
22949 conn,
22950 )
22951 .expect("v1 replica");
22952 assert_eq!(journal_mode(&client.conn), "delete");
22953 let commit = client
22954 .mutate(vec![Mutation::Upsert {
22955 table: "tasks".into(),
22956 values: Map::from_iter([
22957 ("id".into(), json!("queued")),
22958 ("note".into(), json!("dropped-by-v2")),
22959 ]),
22960 base_version: None,
22961 }])
22962 .expect("queued write");
22963 client.drain_change_batches();
22964 let mut dropped = previous_version_schema(2);
22966 dropped["tables"][0]["columns"] =
22967 json!([{ "name": "id", "type": "string", "nullable": false }]);
22968 let marker = meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY);
22971 let revision = client.local_revision();
22972 let rows = client.query("SELECT * FROM tasks", &[]).expect("rows");
22973 let columns: i64 = client
22974 .conn
22975 .query_row(
22976 "SELECT count(*) FROM pragma_table_info('tasks')",
22977 [],
22978 |row| row.get(0),
22979 )
22980 .expect("column count");
22981 let rebuilds = client.overlay_rebuild_count.get();
22982 let prior_subs = client.subs.len();
22983 let prior_changes = client.change_queue.len();
22984 let prior_intents = client.sync_intent_queue.len();
22985 let prior_last_change = client.last_change.clone();
22986 let prior_overlay_dirty = client.overlay_dirty.get();
22987 let prior_sync_needed = client.sync_needed;
22988 let prior_upgrading = client.upgrading;
22989 let prior_stopped = client.stopped;
22990 let prior_schema_floor = client.schema_floor.clone();
22991 let prior_active_round = client.active_round;
22992 let reader = Connection::open(path_str).expect("reader");
22995 reader
22996 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
22997 .expect("reader snapshot");
22998 client
22999 .conn
23000 .busy_timeout(std::time::Duration::ZERO)
23001 .expect("busy timeout");
23002 let error = client
23003 .recreate_with_schema(&dropped)
23004 .expect_err("a blocked COMMIT must refuse the recreation");
23005 assert!(error.starts_with("client.storage_busy:"), "{error}");
23006 assert!(
23007 client.overlay_rebuild_count.get() > rebuilds,
23008 "the reset ran to its overlay rebuild inside the transaction"
23009 );
23010 assert!(client.conn.is_autocommit(), "the transaction is released");
23011 assert_eq!(meta_get(&client.conn, LOCAL_SCHEMA_VERSION_KEY), marker);
23013 assert_eq!(client.local_revision(), revision);
23014 assert_eq!(
23015 client.query("SELECT * FROM tasks", &[]).expect("rows"),
23016 rows
23017 );
23018 assert_eq!(
23019 client
23020 .conn
23021 .query_row(
23022 "SELECT count(*) FROM pragma_table_info('tasks')",
23023 [],
23024 |row| { row.get::<_, i64>(0) }
23025 )
23026 .expect("column count"),
23027 columns,
23028 "the previous table layout survives the rollback"
23029 );
23030 assert_eq!(client.pending_commit_ids(), vec![commit.clone()]);
23031 assert_eq!(client.schema.version, 1, "compiled schema");
23034 assert_eq!(client.subs.len(), prior_subs);
23035 assert_eq!(client.change_queue.len(), prior_changes);
23036 assert_eq!(client.sync_intent_queue.len(), prior_intents);
23037 assert_eq!(client.last_change, prior_last_change);
23038 assert_eq!(client.overlay_dirty.get(), prior_overlay_dirty);
23039 assert_eq!(client.sync_needed, prior_sync_needed);
23040 assert_eq!(client.upgrading, prior_upgrading);
23041 assert_eq!(client.stopped, prior_stopped);
23042 assert_eq!(client.schema_floor, prior_schema_floor);
23043 assert_eq!(client.active_round, prior_active_round);
23044 assert!(
23045 client.rejections.is_empty(),
23046 "the dropped-column rejection is rolled back too"
23047 );
23048 let container = crate::previous_version::previous_version_container_path(path_str);
23051 assert!(std::path::Path::new(&container).exists());
23052 drop(reader);
23055 let snapshot = client
23056 .previous_version_snapshot(&PreviousVersionReadSpec {
23057 table: "tasks".into(),
23058 ..Default::default()
23059 })
23060 .expect("refuse stale capture");
23061 assert!(!snapshot.available);
23062 assert_eq!(snapshot.current_version, 1);
23063 assert!(!std::path::Path::new(&container).exists());
23064 client
23069 .conn
23070 .busy_timeout(std::time::Duration::from_secs(1))
23071 .expect("busy timeout");
23072 client
23073 .recreate_with_schema(&dropped)
23074 .expect("the recreation succeeds once the reader releases");
23075 assert_eq!(client.schema.version, 2);
23076 assert_eq!(
23077 client.pending_commit_ids(),
23078 vec![commit.clone()],
23079 "the reset kept the incompatible commit pending"
23080 );
23081 assert!(
23082 client.rejections.is_empty(),
23083 "no rejection exists before the send-time prepass"
23084 );
23085 client
23087 .prepare_sync_round(false)
23088 .expect("the send prepass runs during round preparation");
23089 assert!(client.pending_commit_ids().is_empty());
23090 assert!(
23091 client
23092 .rejections
23093 .iter()
23094 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE),
23095 "the dropped-column commit surfaced as a rejection at send time"
23096 );
23097 client.previous_version_discard();
23098 drop(client);
23099 std::fs::remove_file(path).expect("remove replica");
23100 }
23101
23102 #[test]
23103 fn the_send_time_incompatible_drop_is_atomic_and_publishes_once() {
23104 let v1 = previous_version_schema(1);
23105 let mut v2 = previous_version_schema(2);
23106 v2["tables"][0]["columns"] = json!([{ "name": "id", "type": "string", "nullable": false }]);
23108 let mut client = SyncClient::new("drop".to_string(), &v1, ClientLimits::default()).unwrap();
23109 let dropped = client
23110 .mutate(vec![Mutation::Upsert {
23111 table: "tasks".into(),
23112 values: Map::from_iter([
23113 ("id".into(), json!("dropped")),
23114 ("note".into(), json!("gone")),
23115 ]),
23116 base_version: None,
23117 }])
23118 .unwrap();
23119 let survivor = client
23123 .mutate(vec![Mutation::Delete {
23124 table: "tasks".into(),
23125 row_id: "gone".into(),
23126 base_version: None,
23127 }])
23128 .unwrap();
23129 client
23130 .recreate_with_schema(&v2)
23131 .expect("the reset preserves the outbox");
23132 assert_eq!(
23133 client.pending_commit_ids(),
23134 vec![dropped.clone(), survivor.clone()],
23135 "the reset preserves both commits"
23136 );
23137 assert!(client.rejections.is_empty(), "the reset classifies nothing");
23138 client.drain_change_batches();
23139 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
23140 assert_eq!(rows.len(), 1, "only the dropped upsert has a visible row");
23141
23142 for fault in [
23148 "outcome-insert",
23149 "outbox-delete",
23150 "overlay-rebuild",
23151 "publish-revision",
23152 ] {
23153 client.set_sync_needed(false, false);
23154 client.drain_change_batches();
23155 let revision = client.local_revision();
23156 let trigger = match fault {
23157 "outcome-insert" => {
23158 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23159 }
23160 "outbox-delete" => {
23161 "CREATE TRIGGER fail_drop BEFORE DELETE ON _syncular_outbox BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23162 }
23163 "publish-revision" => {
23168 "CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23169 }
23170 _ => {
23171 "CREATE TRIGGER fail_drop BEFORE DELETE ON tasks BEGIN SELECT RAISE(ABORT,'drop fault'); END"
23172 }
23173 };
23174 client.conn.execute_batch(trigger).unwrap();
23175 let outcome = client
23176 .prepare_sync_round(false)
23177 .err()
23178 .unwrap_or_else(|| panic!("{fault}: a fault in the drop must fail the round"));
23179 assert!(
23180 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "storage.failed"),
23181 "{fault}: {outcome:?}"
23182 );
23183 client.conn.execute_batch("DROP TRIGGER fail_drop").unwrap();
23184 assert!(client.conn.is_autocommit(), "{fault}: no leaked savepoint");
23185 assert_eq!(
23186 client.pending_commit_ids(),
23187 vec![dropped.clone(), survivor.clone()],
23188 "{fault}: the outbox is restored"
23189 );
23190 assert!(
23191 client.rejections.is_empty(),
23192 "{fault}: no in-memory rejection survives"
23193 );
23194 assert!(
23195 client.commit_outcome(&dropped).unwrap().is_none(),
23196 "{fault}: the durable outcome rolled back"
23197 );
23198 assert_eq!(client.local_revision(), revision, "{fault}: revision");
23199 assert_eq!(
23200 client.query("SELECT * FROM tasks", &[]).unwrap(),
23201 rows,
23202 "{fault}: the visible projection is unchanged"
23203 );
23204 assert!(
23205 client.drain_change_batches().is_empty(),
23206 "{fault}: nothing is published"
23207 );
23208 }
23209
23210 client
23213 .conn
23214 .execute_batch(
23215 "BEGIN; INSERT INTO _syncular_meta(key,value) VALUES ('outerProbe','kept');
23216 CREATE TRIGGER fail_drop BEFORE INSERT ON _syncular_commit_outcomes
23217 BEGIN SELECT RAISE(ABORT,'nested drop fault'); END;",
23218 )
23219 .unwrap();
23220 assert!(client.drop_incompatible_outbox().is_err());
23221 assert!(!client.conn.is_autocommit());
23222 assert_eq!(client.get_meta("outerProbe").as_deref(), Some("kept"));
23223 assert_eq!(
23224 client.pending_commit_ids(),
23225 vec![dropped.clone(), survivor.clone()]
23226 );
23227 assert!(client.commit_outcome(&dropped).unwrap().is_none());
23228 client
23229 .conn
23230 .execute_batch("DROP TRIGGER fail_drop; COMMIT")
23231 .unwrap();
23232
23233 client.set_meta(LOG_EPOCH_KEY, "epoch");
23238 client.set_sync_needed(false, false);
23239 client.drain_change_batches();
23240 let revision = client.local_revision();
23241 let prepared = client.prepare_sync_round(false).expect("the drop succeeds");
23242 assert_eq!(client.pending_commit_ids(), vec![survivor.clone()]);
23243 assert_eq!(
23244 prepared.meta.pushed_ids,
23245 vec![survivor.clone()],
23246 "the incompatible commit is classified before any operation is sent"
23247 );
23248 let rejection = client
23249 .rejections
23250 .iter()
23251 .find(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE)
23252 .expect("a sync.outbox_incompatible rejection");
23253 assert_eq!(rejection.client_commit_id, dropped);
23254 assert!(
23255 !rejection.retryable,
23256 "a schema-incompatible commit is final"
23257 );
23258 assert_eq!(
23259 client
23260 .commit_outcome(&dropped)
23261 .unwrap()
23262 .expect("a durable rejected outcome")
23263 .status,
23264 CommitOutcomeStatus::Rejected
23265 );
23266 assert_eq!(client.local_revision(), revision + 1, "one revision");
23267 let batches = client.drain_change_batches();
23268 assert_eq!(batches.len(), 1, "one change batch");
23269 assert!(
23270 batches[0].rejections_changed
23271 && batches[0].outcomes_changed
23272 && batches[0].status.is_some(),
23273 "{:?}",
23274 batches[0]
23275 );
23276 assert!(
23277 batches[0].tables.iter().any(|entry| entry.table == "tasks"),
23278 "{:?}",
23279 batches[0]
23280 );
23281 let visible = client.query("SELECT * FROM tasks", &[]).unwrap();
23282 assert!(
23283 visible.is_empty(),
23284 "the dropped projection is undone: {visible:?}"
23285 );
23286
23287 let path = temp_replica("outbox-drop-commit");
23291 let path_str = path.to_str().expect("path");
23292 let conn = Connection::open(path_str).expect("replica connection");
23293 let mut client =
23294 SyncClient::with_connection("drop-commit".into(), &v1, ClientLimits::default(), conn)
23295 .expect("v1 replica");
23296 let dropped = client
23297 .mutate(vec![Mutation::Upsert {
23298 table: "tasks".into(),
23299 values: Map::from_iter([
23300 ("id".into(), json!("dropped")),
23301 ("note".into(), json!("gone")),
23302 ]),
23303 base_version: None,
23304 }])
23305 .unwrap();
23306 client
23307 .recreate_with_schema(&v2)
23308 .expect("the reset preserves the outbox");
23309 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
23310 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
23311 assert_eq!(rows.len(), 1, "the optimistic row is visible");
23312 client.set_sync_needed(false, false);
23313 client.drain_change_batches();
23314 let revision = client.local_revision();
23315 let reader = Connection::open(path_str).expect("reader");
23316 reader
23317 .execute_batch("BEGIN; SELECT count(*) FROM _syncular_meta;")
23318 .expect("reader snapshot");
23319 client
23320 .conn
23321 .busy_timeout(std::time::Duration::ZERO)
23322 .expect("busy timeout");
23323 let outcome = client
23324 .prepare_sync_round(false)
23325 .err()
23326 .expect("a blocked COMMIT must fail the round");
23327 assert!(
23328 matches!(*outcome, SyncOutcome::Failed { ref error_code, .. } if error_code == "client.storage_busy"),
23329 "{outcome:?}"
23330 );
23331 assert!(client.conn.is_autocommit(), "the savepoint is released");
23332 assert_eq!(client.pending_commit_ids(), vec![dropped.clone()]);
23333 assert!(client.rejections.is_empty());
23334 assert!(client.commit_outcome(&dropped).unwrap().is_none());
23335 assert_eq!(client.local_revision(), revision);
23336 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
23337 assert!(client.drain_change_batches().is_empty());
23338 drop(reader);
23340 client
23341 .conn
23342 .busy_timeout(std::time::Duration::from_secs(1))
23343 .expect("busy timeout");
23344 client
23345 .prepare_sync_round(false)
23346 .expect("the drop succeeds after the reader releases");
23347 assert!(client.pending_commit_ids().is_empty());
23348 assert!(client
23349 .rejections
23350 .iter()
23351 .any(|rejection| rejection.code == OUTBOX_INCOMPATIBLE_CODE));
23352 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
23353 drop(client);
23354 std::fs::remove_file(path).expect("remove replica");
23355 }
23356
23357 #[test]
23358 fn previous_version_read_reasons_come_from_live_client_state() {
23359 use crate::previous_version::{
23360 PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
23361 };
23362 let path = std::env::temp_dir()
23363 .join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
23364 .to_string_lossy()
23365 .into_owned();
23366 let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
23367 {
23368 let mut setup = SyncClient::open_path(
23369 "wiring".to_owned(),
23370 &previous_version_schema(1),
23371 previous_version_limits(),
23372 &path,
23373 )
23374 .expect("v1 install");
23375 setup
23376 .mutate(vec![Mutation::Upsert {
23377 table: "tasks".to_owned(),
23378 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
23379 base_version: None,
23380 }])
23381 .expect("seed a row");
23382 }
23383 let mut client = SyncClient::open_path(
23384 "wiring".to_owned(),
23385 &previous_version_schema(2),
23386 previous_version_limits(),
23387 &path,
23388 )
23389 .expect("v2 bump captures");
23390 assert!(std::path::Path::new(&container).exists());
23391 let spec = PreviousVersionReadSpec {
23392 table: "tasks".to_owned(),
23393 ..Default::default()
23394 };
23395
23396 recapture_previous_version(&client);
23398 client.lease_state = Some(LeaseState {
23399 lease_id: None,
23400 expires_at_ms: None,
23401 error_code: Some("lease.expired".to_owned()),
23402 });
23403 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23404 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
23405 assert!(!snapshot.available);
23406 assert!(!std::path::Path::new(&container).exists());
23407
23408 recapture_previous_version(&client);
23410 client.set_now_ms(1_000);
23411 client.lease_state = Some(LeaseState {
23412 lease_id: Some("lease".to_owned()),
23413 expires_at_ms: Some(1_000),
23414 error_code: None,
23415 });
23416 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23417 assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
23418 assert!(!std::path::Path::new(&container).exists());
23419 client.lease_state = None;
23420
23421 recapture_previous_version(&client);
23423 client.subs.push(subscription(SubState::Revoked, 0, None));
23424 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23425 assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
23426 assert!(!std::path::Path::new(&container).exists());
23427 client.subs.clear();
23428
23429 recapture_previous_version(&client);
23432 client
23433 .subs
23434 .push(subscription(SubState::Active, 0, Some("resume")));
23435 assert!(!client.previous_version_coverage_complete());
23436 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23437 assert!(snapshot.available);
23438 assert_eq!(snapshot.previous_version, Some(2));
23439 assert_eq!(snapshot.rows.len(), 1);
23440
23441 recapture_previous_version(&client);
23442 client.subs[0].bootstrap_state = None;
23443 assert!(client.previous_version_coverage_complete());
23444 let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
23445 assert_eq!(
23446 snapshot.reason,
23447 Some(PreviousVersionReason::CoverageComplete)
23448 );
23449 assert!(!std::path::Path::new(&container).exists());
23450 client.subs.clear();
23451
23452 assert!(!client.previous_version_coverage_complete());
23454
23455 drop(client);
23456 let _ = std::fs::remove_file(&path);
23457 }
23458
23459 #[test]
23466 fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
23467 let path = std::env::temp_dir()
23468 .join(format!(
23469 "syncular-prev-lifetime-{}.db",
23470 uuid::Uuid::new_v4()
23471 ))
23472 .to_string_lossy()
23473 .into_owned();
23474 let container = format!(
23475 "{path}{}",
23476 crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
23477 );
23478 let exists = || std::path::Path::new(&container).exists();
23479 {
23480 let mut setup = SyncClient::open_path(
23481 "lifetime".to_owned(),
23482 &previous_version_schema(1),
23483 previous_version_limits(),
23484 &path,
23485 )
23486 .expect("v1 install");
23487 setup
23488 .mutate(vec![Mutation::Upsert {
23489 table: "tasks".to_owned(),
23490 values: Map::from_iter([("id".to_owned(), json!("t1"))]),
23491 base_version: None,
23492 }])
23493 .expect("seed a row");
23494 }
23495 let mut client = SyncClient::open_path(
23496 "lifetime".to_owned(),
23497 &previous_version_schema(2),
23498 previous_version_limits(),
23499 &path,
23500 )
23501 .expect("v2 bump captures");
23502 client.set_now_ms(1_000);
23505 assert!(exists());
23506
23507 client.lease_state = Some(LeaseState {
23509 lease_id: None,
23510 expires_at_ms: None,
23511 error_code: Some("lease.expired".to_owned()),
23512 });
23513 client.previous_version_lifetime_check();
23514 assert!(!exists());
23515 client.lease_state = None;
23516
23517 recapture_previous_version(&client);
23520 client.subs.push(subscription(SubState::Active, 0, None));
23521 client.previous_version_lifetime_check();
23522 assert!(!exists());
23523 client.subs.clear();
23524
23525 recapture_previous_version(&client);
23527 client.subs.push(subscription(SubState::Revoked, 0, None));
23528 client.previous_version_lifetime_check();
23529 assert!(!exists());
23530 client.subs.clear();
23531
23532 recapture_previous_version(&client);
23535 client
23536 .subs
23537 .push(subscription(SubState::Active, 0, Some("resume")));
23538 client.previous_version_lifetime_check();
23539 assert!(exists());
23540 client.subs.clear();
23541
23542 drop(client);
23543 let _ = std::fs::remove_file(&path);
23544 let _ = std::fs::remove_file(&container);
23545 let _ = std::fs::remove_file(format!("{container}-wal"));
23546 let _ = std::fs::remove_file(format!("{container}-shm"));
23547 }
23548}
23549
23550#[cfg(test)]
23551mod scope_move_delivery_tests {
23552 use super::*;
23553 use crate::native_transport::HostTransport;
23554 use crate::values::encode_row_json;
23555 use serde_json::json;
23556
23557 fn client() -> SyncClient {
23558 SyncClient::new(
23559 "scope-move".to_owned(),
23560 &json!({
23561 "version": 1,
23562 "tables": [{
23563 "name": "tasks",
23564 "primaryKey": "id",
23565 "columns": [
23566 { "name": "id", "type": "string", "nullable": false },
23567 { "name": "project_id", "type": "string", "nullable": false }
23568 ],
23569 "scopes": [{ "pattern": "project:{project_id}" }]
23570 }]
23571 }),
23572 ClientLimits::default(),
23573 )
23574 .expect("test client")
23575 }
23576
23577 fn upsert(client: &SyncClient, project: &str) -> ssp2::model::Change {
23578 ssp2::model::Change {
23579 table_index: 0,
23580 row_id: "t1".to_owned(),
23581 op: Op::Upsert,
23582 row_version: Some(1),
23583 scopes: vec![("project_id".to_owned(), project.to_owned())],
23584 row: Some(
23585 encode_row_json(
23586 client.schema.table("tasks").unwrap(),
23587 "t1",
23588 &Map::from_iter([
23589 ("id".to_owned(), Value::from("t1")),
23590 ("project_id".to_owned(), Value::from(project)),
23591 ]),
23592 &client.encryption,
23593 )
23594 .unwrap(),
23595 ),
23596 }
23597 }
23598
23599 fn delete(project: &str) -> ssp2::model::Change {
23600 ssp2::model::Change {
23601 table_index: 0,
23602 row_id: "t1".to_owned(),
23603 op: Op::Delete,
23604 row_version: None,
23605 scopes: vec![("project_id".to_owned(), project.to_owned())],
23606 row: None,
23607 }
23608 }
23609
23610 #[test]
23611 fn a_scope_move_keeps_the_row_in_either_section_order() {
23612 for order in [["old", "new"], ["new", "old"]] {
23613 let mut client = client();
23614 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
23615 client
23616 .subscribe(
23617 "old".into(),
23618 "tasks".into(),
23619 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
23620 None,
23621 )
23622 .unwrap();
23623 client
23624 .subscribe(
23625 "new".into(),
23626 "tasks".into(),
23627 vec![("project_id".to_owned(), vec!["p2".to_owned()])],
23628 None,
23629 )
23630 .unwrap();
23631 let seed = upsert(&client, "p1");
23632 client
23633 .apply_commit_frame(&["tasks".into()], &[seed], Some(1))
23634 .unwrap();
23635 assert_eq!(
23636 client.query("SELECT project_id FROM tasks", &[]).unwrap()[0]["project_id"],
23637 json!("p1")
23638 );
23639
23640 let mut frames = vec![Frame::RespHeader {
23641 required_schema_version: None,
23642 latest_schema_version: None,
23643 log_epoch: Some("epoch-1".to_owned()),
23644 reset_required: Some(false),
23645 }];
23646 for id in order {
23647 let project = if id == "old" { "p1" } else { "p2" };
23648 frames.push(Frame::SubStart {
23649 id: id.to_owned(),
23650 status: SubStatus::Active,
23651 reason_code: String::new(),
23652 effective_scopes: vec![("project_id".to_owned(), vec![project.to_owned()])],
23653 bootstrap: false,
23654 });
23655 frames.push(Frame::Commit {
23656 commit_seq: 2,
23657 created_at_ms: 0,
23658 actor_id: "actor".to_owned(),
23659 tables: vec!["tasks".to_owned()],
23660 changes: vec![if id == "old" {
23661 delete("p1")
23662 } else {
23663 upsert(&client, "p2")
23664 }],
23665 });
23666 frames.push(Frame::SubEnd {
23667 next_cursor: 2,
23668 bootstrap_state: None,
23669 });
23670 }
23671 let (_, meta) = client.build_request(false).unwrap();
23672 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
23673 let outcome = client.process_response(
23674 &mut transport,
23675 Message {
23676 wire_version: WIRE_VERSION,
23677 msg_kind: MsgKind::Response,
23678 frames,
23679 },
23680 &meta,
23681 );
23682 assert!(
23683 matches!(outcome, SyncOutcome::Ok(_)),
23684 "{order:?}: {outcome:?}"
23685 );
23686 let rows = client
23687 .query("SELECT id, project_id FROM tasks", &[])
23688 .unwrap();
23689 assert_eq!(rows.len(), 1, "{order:?}: the row was lost");
23690 assert_eq!(rows[0]["project_id"], json!("p2"), "{order:?}");
23691 }
23692 }
23693
23694 struct RefusalTransport;
23695
23696 impl Transport for RefusalTransport {
23697 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
23698 let mut error = TransportError::new(
23699 "sync.invalid_client_id",
23700 "clientId is bound to a different actor in this partition",
23701 );
23702 error.details = Some(json!({
23703 "causeKind": "status",
23704 "httpStatus": 400,
23705 "category": "invalid-request",
23706 "retryable": false,
23707 "recommendedAction": "resetClientId"
23708 }));
23709 Err(error)
23710 }
23711
23712 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
23713 self.sync(request)
23714 }
23715
23716 fn download_segment(
23717 &mut self,
23718 _request: &SegmentRequest,
23719 _on_progress: &mut dyn FnMut(u64),
23720 ) -> Result<Vec<u8>, TransportError> {
23721 Err(TransportError::new("sync.transport_failed", "offline"))
23722 }
23723
23724 fn realtime_connect(&mut self) -> Result<(), TransportError> {
23725 Ok(())
23726 }
23727
23728 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
23729 Ok(())
23730 }
23731
23732 fn realtime_close(&mut self) -> Result<(), TransportError> {
23733 Ok(())
23734 }
23735 }
23736
23737 #[test]
23738 fn an_http_refusal_carries_its_code_and_recommended_action() {
23739 let mut client = client();
23740 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
23741 let outcome = client.sync(&mut RefusalTransport);
23742 let SyncOutcome::Failed {
23743 error_code,
23744 recommended_action,
23745 details,
23746 ..
23747 } = outcome
23748 else {
23749 panic!("expected a failed round");
23750 };
23751 assert_eq!(error_code, "sync.invalid_client_id");
23752 assert_eq!(recommended_action.as_deref(), Some("resetClientId"));
23753 assert_eq!(
23754 details
23755 .as_ref()
23756 .and_then(|details| details.get("httpStatus")),
23757 Some(&json!(400))
23758 );
23759 }
23760
23761 #[test]
23762 fn reset_client_id_persists_and_changes_the_request_identity() {
23763 let mut client = client();
23764 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
23765 let previous = client.client_id().to_owned();
23766 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
23767 let next = client.reset_client_id(&mut transport);
23768 assert_ne!(next, previous);
23769 assert_eq!(client.client_id(), next);
23770 assert_eq!(client.get_meta(CLIENT_ID_KEY), Some(next.clone()));
23771
23772 client
23773 .subscribe(
23774 "tasks".into(),
23775 "tasks".into(),
23776 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
23777 None,
23778 )
23779 .unwrap();
23780 let (message, _) = client.build_request(false).unwrap();
23781 assert!(matches!(
23782 message.frames.first(),
23783 Some(Frame::ReqHeader { client_id, .. }) if client_id == &next
23784 ));
23785 }
23786}
23787
23788#[cfg(test)]
23789mod insert_intent_tests {
23790 use super::*;
23791 use serde_json::{json, Value};
23792
23793 fn tasks_schema(version: i32, extra: Value) -> Value {
23795 let mut columns = vec![
23796 json!({ "name": "id", "type": "string", "nullable": false }),
23797 json!({ "name": "title", "type": "string", "nullable": true }),
23798 ];
23799 columns.extend(extra.as_array().cloned().unwrap_or_default());
23800 json!({
23801 "version": version,
23802 "tables": [{ "name": "tasks", "primaryKey": "id", "columns": columns, "scopes": [] }]
23803 })
23804 }
23805
23806 fn with_note() -> Value {
23807 tasks_schema(
23808 2,
23809 json!([{ "name": "note", "type": "string", "nullable": true }]),
23810 )
23811 }
23812
23813 fn upsert(id: &str, title: &str) -> Mutation {
23814 Mutation::Upsert {
23815 table: "tasks".into(),
23816 values: Map::from_iter([("id".into(), json!(id)), ("title".into(), json!(title))]),
23817 base_version: None,
23818 }
23819 }
23820
23821 fn persisted_markers(client: &SyncClient) -> Vec<Vec<Option<Value>>> {
23822 let mut statement = client
23823 .conn
23824 .prepare("SELECT ops_json FROM _syncular_outbox ORDER BY seq")
23825 .unwrap();
23826 statement
23827 .query_map([], |row| row.get::<_, String>(0))
23828 .unwrap()
23829 .map(|raw| {
23830 serde_json::from_str::<Vec<Value>>(&raw.unwrap())
23831 .unwrap()
23832 .iter()
23833 .map(|entry| entry.get("insert").cloned())
23834 .collect()
23835 })
23836 .collect()
23837 }
23838
23839 fn queued_insert_after_bump(next: &Value, strip_marker: bool) -> SyncClient {
23843 let mut client = SyncClient::new(
23844 "insert-intent".into(),
23845 &tasks_schema(1, json!([])),
23846 ClientLimits::default(),
23847 )
23848 .unwrap();
23849 client.mutate(vec![upsert("q", "new")]).unwrap();
23850 if strip_marker {
23851 client
23852 .conn
23853 .execute_batch(
23854 "UPDATE _syncular_outbox SET ops_json = json_remove(ops_json, '$[0].insert')",
23855 )
23856 .unwrap();
23857 client.outbox[0].ops[0].insert = false;
23858 }
23859 client.recreate_with_schema(next).unwrap();
23860 client
23861 }
23862
23863 fn seed_base_row(client: &mut SyncClient, row: Row) {
23866 client
23867 .write_row(&base_table("tasks"), "tasks", &row, 1)
23868 .unwrap();
23869 client.overlay_dirty.set(true);
23870 client.rebuild_overlay().unwrap();
23871 }
23872
23873 fn sparse_payload(client: &SyncClient, values: Value) -> Vec<u8> {
23874 let table = client.schema.table("tasks").unwrap();
23875 encode_sparse_row_json(
23876 table,
23877 "q",
23878 values.as_object().unwrap(),
23879 &client.encryption,
23880 None,
23881 )
23882 .unwrap()
23883 }
23884
23885 #[test]
23886 fn mutate_records_insert_intent_only_for_rows_absent_at_call_time() {
23887 let mut client = SyncClient::new(
23888 "insert-intent".into(),
23889 &tasks_schema(1, json!([])),
23890 ClientLimits::default(),
23891 )
23892 .unwrap();
23893 client
23895 .mutate(vec![upsert("a", "first"), upsert("a", "second")])
23896 .unwrap();
23897 client.mutate(vec![upsert("a", "visible")]).unwrap();
23898 client
23899 .patch(
23900 "tasks",
23901 "a",
23902 Map::from_iter([("title".into(), json!("patched"))]),
23903 None,
23904 )
23905 .unwrap();
23906 client
23907 .mutate(vec![Mutation::Delete {
23908 table: "tasks".into(),
23909 row_id: "a".into(),
23910 base_version: None,
23911 }])
23912 .unwrap();
23913
23914 let marker = Some(Value::Bool(true));
23915 assert_eq!(
23916 persisted_markers(&client),
23917 vec![
23918 vec![marker.clone(), marker],
23919 vec![None],
23920 vec![None],
23921 vec![None]
23922 ]
23923 );
23924 let flags = |commits: &[OutboxCommit]| {
23925 commits
23926 .iter()
23927 .map(|commit| commit.ops.iter().map(|op| op.insert).collect::<Vec<_>>())
23928 .collect::<Vec<_>>()
23929 };
23930 let expected = vec![vec![true, true], vec![false], vec![false], vec![false]];
23931 assert_eq!(flags(&client.outbox), expected);
23932 assert_eq!(
23933 flags(&client.load_outbox().unwrap()),
23934 expected,
23935 "reload keeps it"
23936 );
23937
23938 let insert = serde_json::to_value(CommitOperation::from(&client.outbox[0].ops[0])).unwrap();
23940 assert_eq!(insert["insert"], json!(true));
23941 assert!(
23942 serde_json::from_value::<CommitOperation>(insert)
23943 .unwrap()
23944 .insert
23945 );
23946 let update = serde_json::to_value(CommitOperation::from(&client.outbox[1].ops[0])).unwrap();
23947 assert!(update.get("insert").is_none());
23948 }
23949
23950 #[test]
23951 fn insert_intent_replays_and_encodes_an_appended_nullable_column_as_null() {
23952 let mut client = queued_insert_after_bump(&with_note(), false);
23953 assert_eq!(
23954 client
23955 .query("SELECT id, title, note FROM tasks", &[])
23956 .unwrap(),
23957 vec![Map::from_iter([
23958 ("id".into(), json!("q")),
23959 ("title".into(), json!("new")),
23960 ("note".into(), Value::Null),
23961 ])],
23962 "the replayed insert is visible with the appended column NULL"
23963 );
23964 assert_eq!(
23965 client.pending_payloads(),
23966 vec![sparse_payload(
23967 &client,
23968 json!({ "id": "q", "title": "new", "note": null })
23969 )],
23970 "the insert presents the appended column as NULL"
23971 );
23972
23973 seed_base_row(
23975 &mut client,
23976 vec![
23977 Some(ColumnValue::String("q".into())),
23978 Some(ColumnValue::String("base".into())),
23979 Some(ColumnValue::String("server".into())),
23980 ],
23981 );
23982 assert_eq!(
23983 client.query("SELECT title, note FROM tasks", &[]).unwrap(),
23984 vec![Map::from_iter([
23985 ("title".into(), json!("new")),
23986 ("note".into(), Value::Null),
23987 ])]
23988 );
23989 }
23990
23991 #[test]
23992 fn an_operation_without_the_marker_keeps_its_presence_set() {
23993 let mut client = queued_insert_after_bump(&with_note(), true);
23994 assert!(
23995 client
23996 .query("SELECT id FROM tasks", &[])
23997 .unwrap()
23998 .is_empty(),
23999 "a partial operation over an absent row stays invisible"
24000 );
24001 assert_eq!(
24002 client.pending_payloads(),
24003 vec![sparse_payload(
24004 &client,
24005 json!({ "id": "q", "title": "new" })
24006 )],
24007 "the appended column stays absent"
24008 );
24009
24010 seed_base_row(
24011 &mut client,
24012 vec![
24013 Some(ColumnValue::String("q".into())),
24014 Some(ColumnValue::String("base".into())),
24015 Some(ColumnValue::String("server".into())),
24016 ],
24017 );
24018 assert_eq!(
24019 client.query("SELECT title, note FROM tasks", &[]).unwrap(),
24020 vec![Map::from_iter([
24021 ("title".into(), json!("new")),
24022 ("note".into(), json!("server")),
24023 ])],
24024 "the stored appended value survives"
24025 );
24026 }
24027
24028 #[test]
24029 fn insert_intent_missing_a_non_nullable_column_is_outbox_incompatible() {
24030 let with_rank = tasks_schema(
24031 2,
24032 json!([{ "name": "rank", "type": "integer", "nullable": false }]),
24033 );
24034 let mut client = SyncClient::new(
24035 "insert-intent".into(),
24036 &tasks_schema(1, json!([])),
24037 ClientLimits::default(),
24038 )
24039 .unwrap();
24040 let insert = client.mutate(vec![upsert("q", "new")]).unwrap();
24041 let update = client.mutate(vec![upsert("q", "edited")]).unwrap();
24044 client.recreate_with_schema(&with_rank).unwrap();
24045 assert_eq!(
24046 client.pending_commit_ids(),
24047 vec![insert.clone(), update.clone()],
24048 "the reset classifies nothing"
24049 );
24050 assert!(client.rejections.is_empty());
24051 assert!(
24052 client
24053 .query("SELECT id FROM tasks", &[])
24054 .unwrap()
24055 .is_empty(),
24056 "the unencodable insert stays invisible"
24057 );
24058
24059 assert!(client.drop_incompatible_outbox().unwrap());
24060 assert_eq!(client.pending_commit_ids(), vec![update]);
24061 assert_eq!(client.rejections.len(), 1);
24062 let rejection = &client.rejections[0];
24063 assert_eq!(rejection.client_commit_id, insert);
24064 assert_eq!(rejection.code, OUTBOX_INCOMPATIBLE_CODE);
24065 assert!(rejection.operation.as_ref().is_some_and(|op| op.insert));
24066 assert!(
24067 client.commit_outcome(&insert).unwrap().is_some(),
24068 "the journal accepts the marked operation"
24069 );
24070 }
24071
24072 #[test]
24073 fn the_stored_operation_validator_accepts_the_marker_only_on_upserts() {
24074 let upsert =
24075 json!({ "table": "tasks", "rowId": "q", "op": "upsert", "values": { "id": "q" } });
24076 let delete = json!({ "table": "tasks", "rowId": "q", "op": "delete" });
24077 let with = |base: &Value, insert: Value| {
24078 let mut value = base.clone();
24079 value["insert"] = insert;
24080 value
24081 };
24082 assert!(SyncClient::stored_operation_is_representable(&upsert));
24083 assert!(SyncClient::stored_operation_is_representable(&with(
24084 &upsert,
24085 json!(true)
24086 )));
24087 assert!(!SyncClient::stored_operation_is_representable(&with(
24088 &upsert,
24089 json!(false)
24090 )));
24091 assert!(!SyncClient::stored_operation_is_representable(&with(
24092 &upsert,
24093 json!("yes")
24094 )));
24095 assert!(SyncClient::stored_operation_is_representable(&delete));
24096 assert!(!SyncClient::stored_operation_is_representable(&with(
24097 &delete,
24098 json!(true)
24099 )));
24100 }
24101}