1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
27 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
28 ClientDiagnosticsStorage, ClientLimits, CommandEffects, CommitOperation,
29 CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution,
30 CommitOutcomeStatus, ConflictRecord, CoverageSnapshot, DiagnosticLastChange,
31 DiagnosticLastRound, DiagnosticRoundCounters, DiagnosticSubscription, LeaseState,
32 LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget, LocalDataRebootstrapInput,
33 LocalDataRebootstrapResult, Mutation, PresencePeer, QueryRow, QuerySnapshot, QueryValue,
34 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
35 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
36 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
37 CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
38};
39use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
40use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
41use crate::values::{
42 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
43 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
44 scope_map_to_json, sort_scope_map,
45};
46
47const DEFAULT_ACCEPT: u8 = 0b0111;
52const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
53const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
54const ACCEPT_SQLITE: u8 = 1 << 2;
55const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
56const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
57
58const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
60const LOCAL_REVISION_KEY: &str = "localRevision";
61const CLIENT_ID_KEY: &str = "clientId";
62const LEASE_STATE_KEY: &str = "leaseState";
63const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
64const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
67
68#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69enum SubState {
70 Active,
71 Revoked,
72 Failed,
73}
74
75#[cfg(test)]
76mod observation_tests {
77 use super::*;
78 use crate::native_transport::HostTransport;
79 use serde_json::json;
80
81 fn client() -> SyncClient {
82 SyncClient::new(
83 "retry-test".to_owned(),
84 &json!({
85 "version": 1,
86 "tables": [{
87 "name": "tasks",
88 "primaryKey": "id",
89 "columns": [
90 { "name": "id", "type": "string", "nullable": false },
91 { "name": "project_id", "type": "string", "nullable": false }
92 ],
93 "scopes": [{ "pattern": "project:{project_id}" }]
94 }]
95 }),
96 ClientLimits::default(),
97 )
98 .expect("test client")
99 }
100
101 #[test]
102 fn background_retry_deadlines_back_off_and_reset() {
103 let mut client = client();
104 client.schedule_background_retry();
105 assert!(matches!(
106 client.drain_sync_intents().as_slice(),
107 [SyncIntent::Background { delay_ms: 250 }]
108 ));
109 client.schedule_background_retry();
110 assert!(matches!(
111 client.drain_sync_intents().as_slice(),
112 [SyncIntent::Background { delay_ms: 500 }]
113 ));
114 client.reset_background_retry();
115 client.schedule_background_retry();
116 assert!(matches!(
117 client.drain_sync_intents().as_slice(),
118 [SyncIntent::Background { delay_ms: 250 }]
119 ));
120 }
121
122 #[test]
123 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
124 let mut client = client();
125 client
126 .subscribe(
127 "repair-tasks".to_owned(),
128 "tasks".to_owned(),
129 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
130 None,
131 )
132 .expect("subscribe");
133 {
134 let sub = client
135 .subs
136 .iter_mut()
137 .find(|sub| sub.id == "repair-tasks")
138 .expect("subscription");
139 sub.cursor = 42;
140 sub.synced_once = true;
141 let persisted = sub.clone();
142 client.persist_sub(&persisted);
143 }
144 for table in ["_syncular_base_tasks", "tasks"] {
145 client
146 .conn
147 .execute(
148 &format!(
149 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
150 ),
151 rusqlite::params!["server-row", "p1"],
152 )
153 .expect("seed server row");
154 }
155 let pending = client
156 .mutate(vec![Mutation::Upsert {
157 table: "tasks".to_owned(),
158 values: Map::from_iter([
159 ("id".to_owned(), Value::from("offline-row")),
160 ("project_id".to_owned(), Value::from("p1")),
161 ]),
162 base_version: None,
163 }])
164 .expect("queue offline work");
165 client.drain_change_batches();
166 client.drain_sync_intents();
167
168 assert_eq!(
169 client
170 .rebootstrap_local_data(&LocalDataRebootstrapInput {
171 rebootstrap_id: "support-case-001".to_owned(),
172 })
173 .expect("rebootstrap"),
174 LocalDataRebootstrapResult {
175 already_applied: false,
176 retained_commits: 1,
177 reset_subscriptions: 1,
178 }
179 );
180 let visible_ids = client
181 .conn
182 .prepare("SELECT id FROM tasks ORDER BY id")
183 .expect("prepare visible ids")
184 .query_map([], |row| row.get::<_, String>(0))
185 .expect("query visible ids")
186 .collect::<Result<Vec<_>, _>>()
187 .expect("collect visible ids");
188 assert_eq!(visible_ids, vec!["offline-row"]);
189 assert_eq!(client.pending_commit_ids(), vec![pending]);
190 assert_eq!(
191 client
192 .subscription_state("repair-tasks")
193 .expect("subscription")
194 .cursor,
195 -1
196 );
197 assert!(client.upgrading());
198 assert!(client.sync_needed());
199 assert!(matches!(
200 client.drain_sync_intents().as_slice(),
201 [SyncIntent::Interactive]
202 ));
203 assert_eq!(client.drain_change_batches().len(), 1);
204
205 assert_eq!(
206 client
207 .rebootstrap_local_data(&LocalDataRebootstrapInput {
208 rebootstrap_id: "support-case-001".to_owned(),
209 })
210 .expect("idempotent retry"),
211 LocalDataRebootstrapResult {
212 already_applied: true,
213 retained_commits: 0,
214 reset_subscriptions: 0,
215 }
216 );
217 assert!(client.drain_change_batches().is_empty());
218 }
219
220 #[test]
221 fn local_rebootstrap_cannot_bypass_schema_floor() {
222 let mut client = client();
223 client.set_schema_floor(Some(SchemaFloor {
224 required_schema_version: Some(2),
225 latest_schema_version: Some(2),
226 }));
227 let error = client
228 .rebootstrap_local_data(&LocalDataRebootstrapInput {
229 rebootstrap_id: "blocked-floor".to_owned(),
230 })
231 .expect_err("schema floor must block repair");
232 assert!(error.contains("cannot bypass an active schema-floor stop"));
233 }
234
235 #[test]
236 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
237 let mut client = client();
238 const COMMIT_COUNT: usize = 32;
239
240 for index in 0..COMMIT_COUNT {
241 client
242 .mutate(vec![Mutation::Upsert {
243 table: "tasks".to_owned(),
244 values: Map::from_iter([
245 ("id".to_owned(), Value::from(format!("task-{index}"))),
246 ("project_id".to_owned(), Value::from("project-1")),
247 ]),
248 base_version: None,
249 }])
250 .expect("queue commit");
251 }
252
253 let (_, request_meta) = client.build_request(false);
254 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
255 let mut frames = vec![Frame::RespHeader {
256 required_schema_version: None,
257 latest_schema_version: None,
258 }];
259 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
260 |(index, client_commit_id)| Frame::PushResult {
261 client_commit_id: client_commit_id.clone(),
262 status: PushStatus::Applied,
263 commit_seq: Some(index as i64 + 1),
264 results: vec![OpResult::Applied { op_index: 0 }],
265 },
266 ));
267 let response = Message {
268 msg_kind: MsgKind::Response,
269 frames,
270 };
271 let mut transport =
272 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
273
274 client.overlay_rebuild_count.set(0);
275 client.outcome_prune_count.set(0);
276 let outcome = client.process_response(&mut transport, response, &request_meta);
277 assert!(matches!(outcome, SyncOutcome::Ok(_)));
278 assert!(client.pending_commit_ids().is_empty());
279 assert_eq!(
280 client.overlay_rebuild_count.get(),
281 1,
282 "one response must reconcile its acknowledged commits with one overlay rebuild"
283 );
284 assert_eq!(
285 client.outcome_prune_count.get(),
286 1,
287 "one response must enforce outcome retention once"
288 );
289 }
290
291 #[test]
292 fn mixed_push_results_reconcile_and_prune_once_per_response() {
293 let mut client = client();
294 for index in 0..4 {
295 client
296 .mutate(vec![Mutation::Upsert {
297 table: "tasks".to_owned(),
298 values: Map::from_iter([
299 ("id".to_owned(), Value::from(format!("task-{index}"))),
300 ("project_id".to_owned(), Value::from("project-1")),
301 ]),
302 base_version: None,
303 }])
304 .expect("queue commit");
305 }
306
307 let (_, request_meta) = client.build_request(false);
308 let ids = &request_meta.pushed_ids;
309 assert_eq!(ids.len(), 4);
310 let response = Message {
311 msg_kind: MsgKind::Response,
312 frames: vec![
313 Frame::RespHeader {
314 required_schema_version: None,
315 latest_schema_version: None,
316 },
317 Frame::PushResult {
318 client_commit_id: ids[0].clone(),
319 status: PushStatus::Applied,
320 commit_seq: Some(1),
321 results: vec![OpResult::Applied { op_index: 0 }],
322 },
323 Frame::PushResult {
324 client_commit_id: ids[1].clone(),
325 status: PushStatus::Cached,
326 commit_seq: Some(2),
327 results: vec![OpResult::Applied { op_index: 0 }],
328 },
329 Frame::PushResult {
330 client_commit_id: ids[2].clone(),
331 status: PushStatus::Rejected,
332 commit_seq: None,
333 results: vec![OpResult::Error {
334 op_index: 0,
335 code: "sync.validation_failed".to_owned(),
336 message: "rejected".to_owned(),
337 retryable: false,
338 }],
339 },
340 Frame::PushResult {
341 client_commit_id: ids[3].clone(),
342 status: PushStatus::Rejected,
343 commit_seq: None,
344 results: vec![OpResult::Error {
345 op_index: 0,
346 code: "sync.idempotency_cache_miss".to_owned(),
347 message: "retry".to_owned(),
348 retryable: true,
349 }],
350 },
351 ],
352 };
353 let mut transport =
354 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
355
356 client.overlay_rebuild_count.set(0);
357 client.outcome_prune_count.set(0);
358 let outcome = client.process_response(&mut transport, response, &request_meta);
359 let SyncOutcome::Ok(report) = outcome else {
360 panic!("mixed push-result response failed");
361 };
362
363 assert_eq!(report.applied, ids[..2]);
364 assert_eq!(report.rejected, ids[2..3]);
365 assert_eq!(report.retryable, ids[3..4]);
366 assert_eq!(client.pending_commit_ids(), ids[3..4]);
367 assert_eq!(
368 client
369 .query("SELECT id FROM tasks ORDER BY id", &[])
370 .expect("query visible overlay"),
371 vec![Map::from_iter([("id".to_owned(), Value::from("task-3"))])]
372 );
373 assert_eq!(client.overlay_rebuild_count.get(), 1);
374 assert_eq!(client.outcome_prune_count.get(), 1);
375 }
376
377 #[test]
378 fn secondary_unique_collision_preserves_existing_synced_row() {
379 let client = SyncClient::new(
380 "unique-upsert-test".to_owned(),
381 &json!({
382 "version": 1,
383 "tables": [{
384 "name": "tasks",
385 "primaryKey": "id",
386 "columns": [
387 { "name": "id", "type": "string", "nullable": false },
388 { "name": "project_id", "type": "string", "nullable": false },
389 { "name": "title", "type": "string", "nullable": false }
390 ],
391 "scopes": [{ "pattern": "project:{project_id}" }],
392 "indexes": [{
393 "name": "tasks_by_project_title",
394 "columns": ["project_id", "title"],
395 "unique": true
396 }]
397 }]
398 }),
399 ClientLimits::default(),
400 )
401 .expect("test client");
402 let table = client.schema.table("tasks").expect("tasks table");
403 let sql = client.insert_row_sql(&base_table("tasks"), table);
404
405 client
406 .conn
407 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
408 .expect("insert first row");
409 client
410 .conn
411 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
412 .expect("update same primary key");
413 client
414 .conn
415 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
416 .expect("insert second row");
417 assert!(client
418 .conn
419 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
420 .is_err());
421
422 let rows = client
423 .conn
424 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
425 .expect("prepare rows")
426 .query_map([], |row| {
427 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
428 })
429 .expect("query rows")
430 .collect::<Result<Vec<_>, _>>()
431 .expect("collect rows");
432 assert_eq!(
433 rows,
434 vec![
435 ("t1".to_owned(), "updated".to_owned()),
436 ("t2".to_owned(), "original".to_owned())
437 ]
438 );
439 }
440
441 #[test]
442 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
443 let path = std::env::temp_dir().join(format!(
444 "syncular-startup-intent-{}.db",
445 uuid::Uuid::new_v4()
446 ));
447 let schema = json!({
448 "version": 1,
449 "tables": [{
450 "name": "tasks",
451 "primaryKey": "id",
452 "columns": [
453 { "name": "id", "type": "string", "nullable": false },
454 { "name": "project_id", "type": "string", "nullable": false }
455 ],
456 "scopes": [{ "pattern": "project:{project_id}" }]
457 }]
458 });
459
460 {
461 let mut first = SyncClient::open_path_with_identity(
462 None,
463 &schema,
464 ClientLimits::default(),
465 path.to_str().expect("UTF-8 temp path"),
466 )
467 .expect("first open");
468 first
469 .set_window(
470 &WindowBase {
471 table: "tasks".to_owned(),
472 variable: "project_id".to_owned(),
473 fixed_scopes: Vec::new(),
474 params: None,
475 },
476 &["persisted".to_owned()],
477 )
478 .expect("persist window");
479 }
480
481 let mut reopened = SyncClient::open_path_with_identity(
482 None,
483 &schema,
484 ClientLimits::default(),
485 path.to_str().expect("UTF-8 temp path"),
486 )
487 .expect("reopen");
488 assert!(reopened.sync_needed());
489 assert!(matches!(
490 reopened.drain_sync_intents().as_slice(),
491 [SyncIntent::Interactive]
492 ));
493 drop(reopened);
494 std::fs::remove_file(path).expect("remove temp database");
495 }
496
497 #[test]
498 fn reopening_preserves_immutable_subscription_identity_and_progress() {
499 let path = std::env::temp_dir().join(format!(
500 "syncular-subscription-identity-{}.db",
501 uuid::Uuid::new_v4()
502 ));
503 let schema = json!({
504 "version": 1,
505 "tables": [
506 {
507 "name": "tasks",
508 "primaryKey": "id",
509 "columns": [
510 { "name": "id", "type": "string", "nullable": false },
511 { "name": "project_id", "type": "string", "nullable": false }
512 ],
513 "scopes": [{ "pattern": "project:{project_id}" }]
514 },
515 {
516 "name": "docs",
517 "primaryKey": "id",
518 "columns": [
519 { "name": "id", "type": "string", "nullable": false },
520 { "name": "org_id", "type": "string", "nullable": false },
521 { "name": "project_id", "type": "string", "nullable": false }
522 ],
523 "scopes": [
524 { "pattern": "org:{org_id}" },
525 { "pattern": "project:{projectId}", "column": "project_id" }
526 ]
527 }
528 ]
529 });
530
531 {
532 let mut first = SyncClient::open_path_with_identity(
533 None,
534 &schema,
535 ClientLimits::default(),
536 path.to_str().expect("UTF-8 temp path"),
537 )
538 .expect("first open");
539 first
540 .subscribe(
541 "stable-subscription".to_owned(),
542 "tasks".to_owned(),
543 vec![(
544 "project_id".to_owned(),
545 vec!["p2".to_owned(), "p1".to_owned()],
546 )],
547 Some(r#"{"view":"v1"}"#.to_owned()),
548 )
549 .expect("persist subscription");
550 let persisted = {
551 let subscription = first
552 .subs
553 .iter_mut()
554 .find(|subscription| subscription.id == "stable-subscription")
555 .expect("subscription");
556 subscription.cursor = 41;
557 subscription.bootstrap_state = Some("resume-token".to_owned());
558 subscription.effective = Some(vec![(
559 "project_id".to_owned(),
560 vec!["p1".to_owned(), "p2".to_owned()],
561 )]);
562 subscription.synced_once = true;
563 subscription.clone()
564 };
565 first.persist_sub(&persisted);
566 }
567
568 let mut reopened = SyncClient::open_path_with_identity(
569 None,
570 &schema,
571 ClientLimits::default(),
572 path.to_str().expect("UTF-8 temp path"),
573 )
574 .expect("reopen");
575 let progress = reopened
576 .subscription_state("stable-subscription")
577 .expect("persisted state");
578 assert_eq!(progress.cursor, 41);
579 assert!(progress.has_resume_token);
580
581 reopened
582 .subscribe(
583 "stable-subscription".to_owned(),
584 "tasks".to_owned(),
585 vec![(
586 "project_id".to_owned(),
587 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
588 )],
589 Some(r#"{"view":"v1"}"#.to_owned()),
590 )
591 .expect("canonical intent is idempotent");
592 assert_eq!(
593 reopened
594 .subscription_state("stable-subscription")
595 .expect("unchanged state")
596 .cursor,
597 progress.cursor
598 );
599
600 for (table, scopes, params) in [
601 (
602 "tasks",
603 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
604 Some(r#"{"view":"v1"}"#.to_owned()),
605 ),
606 (
607 "tasks",
608 vec![(
609 "project_id".to_owned(),
610 vec!["p2".to_owned(), "p1".to_owned()],
611 )],
612 Some(r#"{"view":"v2"}"#.to_owned()),
613 ),
614 (
615 "docs",
616 vec![
617 ("org_id".to_owned(), vec!["o1".to_owned()]),
618 ("projectId".to_owned(), vec!["p1".to_owned()]),
619 ],
620 Some(r#"{"view":"v1"}"#.to_owned()),
621 ),
622 ] {
623 let error = reopened
624 .subscribe(
625 "stable-subscription".to_owned(),
626 table.to_owned(),
627 scopes,
628 params,
629 )
630 .expect_err("identity rebind must fail");
631 assert!(error.starts_with("client.subscription_intent_mismatch:"));
632 assert_eq!(
633 reopened
634 .subscription_state("stable-subscription")
635 .expect("unchanged state")
636 .cursor,
637 progress.cursor
638 );
639 }
640
641 drop(reopened);
642 std::fs::remove_file(path).expect("remove temp database");
643 }
644
645 #[test]
646 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
647 let path = std::env::temp_dir().join(format!(
648 "syncular-satisfied-schema-floor-{}.db",
649 uuid::Uuid::new_v4()
650 ));
651 let schema = json!({
652 "version": 23,
653 "tables": [{
654 "name": "tasks",
655 "primaryKey": "id",
656 "columns": [
657 { "name": "id", "type": "string", "nullable": false },
658 { "name": "project_id", "type": "string", "nullable": false }
659 ],
660 "scopes": [{ "pattern": "project:{project_id}" }]
661 }]
662 });
663
664 {
665 let mut first = SyncClient::open_path_with_identity(
666 None,
667 &schema,
668 ClientLimits::default(),
669 path.to_str().expect("UTF-8 temp path"),
670 )
671 .expect("first open");
672 first
673 .subscribe(
674 "tasks".to_owned(),
675 "tasks".to_owned(),
676 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
677 None,
678 )
679 .expect("persist subscription");
680 first.set_schema_floor(Some(SchemaFloor {
681 required_schema_version: Some(22),
682 latest_schema_version: Some(22),
683 }));
684 }
685
686 let mut reopened = SyncClient::open_path_with_identity(
687 None,
688 &schema,
689 ClientLimits::default(),
690 path.to_str().expect("UTF-8 temp path"),
691 )
692 .expect("reopen");
693 assert!(reopened.schema_floor().is_none());
694 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
695 assert!(reopened.sync_needed());
696 assert!(matches!(
697 reopened.drain_sync_intents().as_slice(),
698 [SyncIntent::Interactive]
699 ));
700 drop(reopened);
701 std::fs::remove_file(path).expect("remove temp database");
702 }
703
704 #[test]
705 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
706 let path = std::env::temp_dir().join(format!(
707 "syncular-unsatisfied-schema-floor-{}.db",
708 uuid::Uuid::new_v4()
709 ));
710 let schema = json!({
711 "version": 1,
712 "tables": [{
713 "name": "tasks",
714 "primaryKey": "id",
715 "columns": [
716 { "name": "id", "type": "string", "nullable": false },
717 { "name": "project_id", "type": "string", "nullable": false }
718 ],
719 "scopes": [{ "pattern": "project:{project_id}" }]
720 }]
721 });
722
723 {
724 let mut first = SyncClient::open_path_with_identity(
725 None,
726 &schema,
727 ClientLimits::default(),
728 path.to_str().expect("UTF-8 temp path"),
729 )
730 .expect("first open");
731 first
732 .subscribe(
733 "tasks".to_owned(),
734 "tasks".to_owned(),
735 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
736 None,
737 )
738 .expect("persist subscription");
739 first.set_schema_floor(Some(SchemaFloor {
740 required_schema_version: Some(2),
741 latest_schema_version: Some(2),
742 }));
743 }
744
745 let reopened = SyncClient::open_path_with_identity(
746 None,
747 &schema,
748 ClientLimits::default(),
749 path.to_str().expect("UTF-8 temp path"),
750 )
751 .expect("reopen");
752 assert_eq!(
753 reopened.schema_floor(),
754 Some(&SchemaFloor {
755 required_schema_version: Some(2),
756 latest_schema_version: Some(2),
757 })
758 );
759 assert!(!reopened.sync_needed());
760 drop(reopened);
761 std::fs::remove_file(path).expect("remove temp database");
762 }
763
764 #[test]
765 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
766 let path = std::env::temp_dir().join(format!(
767 "syncular-indexed-column-bump-{}.db",
768 uuid::Uuid::new_v4()
769 ));
770 let old_schema = json!({
771 "version": 1,
772 "tables": [
773 {
774 "name": "tasks",
775 "primaryKey": "id",
776 "columns": [
777 { "name": "id", "type": "string", "nullable": false },
778 { "name": "project_id", "type": "string", "nullable": false }
779 ],
780 "scopes": [{ "pattern": "project:{project_id}" }]
781 },
782 {
783 "name": "legacy",
784 "primaryKey": "id",
785 "columns": [
786 { "name": "id", "type": "string", "nullable": false },
787 { "name": "project_id", "type": "string", "nullable": false }
788 ],
789 "scopes": [{ "pattern": "project:{project_id}" }]
790 }
791 ]
792 });
793 {
794 let mut first = SyncClient::open_path_with_identity(
795 None,
796 &old_schema,
797 ClientLimits::default(),
798 path.to_str().expect("UTF-8 temp path"),
799 )
800 .expect("open old schema");
801 first
802 .subscribe(
803 "legacy-sub".to_owned(),
804 "legacy".to_owned(),
805 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
806 None,
807 )
808 .expect("persist legacy subscription");
809 }
810
811 let new_schema = json!({
812 "version": 2,
813 "tables": [{
814 "name": "tasks",
815 "primaryKey": "id",
816 "columns": [
817 { "name": "id", "type": "string", "nullable": false },
818 { "name": "project_id", "type": "string", "nullable": false },
819 { "name": "facility_membership_id", "type": "string", "nullable": true }
820 ],
821 "scopes": [{ "pattern": "project:{project_id}" }],
822 "indexes": [{
823 "name": "tasks_by_membership",
824 "columns": ["project_id", "facility_membership_id"],
825 "unique": false
826 }]
827 }]
828 });
829 let upgraded = SyncClient::open_path_with_identity(
830 None,
831 &new_schema,
832 ClientLimits::default(),
833 path.to_str().expect("UTF-8 temp path"),
834 )
835 .expect("open upgraded schema");
836 let columns = upgraded
837 .conn
838 .prepare("PRAGMA table_info(tasks)")
839 .expect("prepare columns")
840 .query_map([], |row| row.get::<_, String>(1))
841 .expect("query columns")
842 .collect::<Result<Vec<_>, _>>()
843 .expect("collect columns");
844 assert!(columns.contains(&"facility_membership_id".to_owned()));
845 let index_count: i64 = upgraded
846 .conn
847 .query_row(
848 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
849 [],
850 |row| row.get(0),
851 )
852 .expect("query index");
853 assert_eq!(index_count, 1);
854 assert!(upgraded.subscription_state("legacy-sub").is_none());
855 drop(upgraded);
856 std::fs::remove_file(path).expect("remove temp database");
857 }
858
859 #[test]
860 fn migrates_pre_envelope_outcome_journal_additively() {
861 let path = std::env::temp_dir().join(format!(
862 "syncular-outcome-migration-{}.db",
863 uuid::Uuid::new_v4()
864 ));
865 let conn = Connection::open(&path).expect("open legacy database");
866 conn.execute_batch(
867 "CREATE TABLE _syncular_commit_outcomes (
868 seq INTEGER PRIMARY KEY AUTOINCREMENT,
869 client_commit_id TEXT NOT NULL UNIQUE,
870 status TEXT NOT NULL,
871 recorded_at_ms INTEGER NOT NULL,
872 results_json TEXT NOT NULL,
873 resolution TEXT NOT NULL DEFAULT 'active',
874 resolved_at_ms INTEGER,
875 replacement_client_commit_id TEXT);",
876 )
877 .expect("create legacy outcome journal");
878 drop(conn);
879 let schema = json!({
880 "version": 1,
881 "tables": [{
882 "name": "tasks",
883 "primaryKey": "id",
884 "columns": [
885 { "name": "id", "type": "string", "nullable": false },
886 { "name": "project_id", "type": "string", "nullable": false }
887 ],
888 "scopes": [{ "pattern": "project:{project_id}" }]
889 }]
890 });
891 let client = SyncClient::open_path(
892 "migration-native".to_owned(),
893 &schema,
894 ClientLimits::default(),
895 path.to_str().expect("UTF-8 temp path"),
896 )
897 .expect("migrate database");
898 let has_operations = client
899 .conn
900 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
901 .expect("prepare table info")
902 .query_map([], |row| row.get::<_, String>(1))
903 .expect("query table info")
904 .filter_map(Result::ok)
905 .any(|column| column == "operations_json");
906 assert!(has_operations);
907 drop(client);
908 std::fs::remove_file(path).expect("remove temp database");
909 }
910
911 #[test]
912 fn durable_conflict_outcome_and_resolution_survive_reopen() {
913 let path = std::env::temp_dir().join(format!(
914 "syncular-durable-outcome-{}.db",
915 uuid::Uuid::new_v4()
916 ));
917 let schema = json!({
918 "version": 1,
919 "tables": [{
920 "name": "tasks",
921 "primaryKey": "id",
922 "columns": [
923 { "name": "id", "type": "string", "nullable": false },
924 { "name": "project_id", "type": "string", "nullable": false }
925 ],
926 "scopes": [{ "pattern": "project:{project_id}" }]
927 }]
928 });
929 let conflict = ConflictRecord {
930 client_commit_id: "losing-commit".to_owned(),
931 op_index: 0,
932 table: "tasks".to_owned(),
933 row_id: "t1".to_owned(),
934 code: "sync.version_conflict".to_owned(),
935 message: "stale base version".to_owned(),
936 server_version: 2,
937 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
938 operation: Some(CommitOperation {
939 table: "tasks".to_owned(),
940 row_id: "t1".to_owned(),
941 op: "upsert".to_owned(),
942 base_version: Some(1),
943 values: None,
944 changed_fields: None,
945 }),
946 };
947 let failed_operations = vec![
948 OutboxOp {
949 upsert: true,
950 table: "tasks".to_owned(),
951 row_id: "t1".to_owned(),
952 base_version: Some(1),
953 values: None,
954 changed_fields: None,
955 },
956 OutboxOp {
957 upsert: true,
958 table: "tasks".to_owned(),
959 row_id: "status-event-1".to_owned(),
960 base_version: Some(0),
961 values: None,
962 changed_fields: None,
963 },
964 ];
965
966 {
967 let mut first = SyncClient::open_path(
968 "durable-native".to_owned(),
969 &schema,
970 ClientLimits::default(),
971 path.to_str().expect("UTF-8 temp path"),
972 )
973 .expect("first open");
974 first
975 .begin_observation("test_outcome")
976 .expect("begin outcome");
977 first
978 .persist_commit_outcome(
979 "losing-commit",
980 CommitOutcomeStatus::Conflict,
981 &[CommitOperationOutcome::Conflict {
982 conflict: conflict.clone(),
983 }],
984 Some(&failed_operations),
985 )
986 .expect("persist outcome");
987 first.conflicts.push(conflict);
988 first
989 .finish_observation(
990 "test_outcome",
991 ChangeAccumulator {
992 conflicts: true,
993 outcomes: true,
994 ..ChangeAccumulator::default()
995 },
996 )
997 .expect("commit outcome");
998 }
999
1000 {
1001 let mut reopened = SyncClient::open_path(
1002 "durable-native".to_owned(),
1003 &schema,
1004 ClientLimits::default(),
1005 path.to_str().expect("UTF-8 temp path"),
1006 )
1007 .expect("reopen");
1008 assert_eq!(reopened.conflicts().len(), 1);
1009 let outcome = reopened
1010 .commit_outcome("losing-commit")
1011 .expect("read outcome")
1012 .expect("outcome");
1013 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
1014 let operations = outcome.operations.expect("aggregate envelope");
1015 assert_eq!(operations.len(), 2);
1016 assert_eq!(operations[1].row_id, "status-event-1");
1017 let resolved = reopened
1018 .resolve_commit_outcome(ResolveCommitOutcomeInput {
1019 client_commit_id: "losing-commit".to_owned(),
1020 resolution: CommitOutcomeResolution::ResolvedKeepServer,
1021 replacement_client_commit_id: None,
1022 })
1023 .expect("resolve");
1024 assert_eq!(
1025 resolved.resolution,
1026 CommitOutcomeResolution::ResolvedKeepServer
1027 );
1028 assert!(reopened.conflicts().is_empty());
1029 }
1030
1031 let reopened = SyncClient::open_path(
1032 "durable-native".to_owned(),
1033 &schema,
1034 ClientLimits::default(),
1035 path.to_str().expect("UTF-8 temp path"),
1036 )
1037 .expect("second reopen");
1038 assert!(reopened.conflicts().is_empty());
1039 assert_eq!(
1040 reopened
1041 .commit_outcome("losing-commit")
1042 .expect("read outcome")
1043 .expect("outcome")
1044 .resolution,
1045 CommitOutcomeResolution::ResolvedKeepServer
1046 );
1047 drop(reopened);
1048 std::fs::remove_file(path).expect("remove temp database");
1049 }
1050
1051 #[test]
1052 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
1053 let path =
1054 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
1055 let schema = json!({
1056 "version": 1,
1057 "tables": [{
1058 "name": "tasks",
1059 "primaryKey": "id",
1060 "columns": [
1061 { "name": "id", "type": "string", "nullable": false },
1062 { "name": "project_id", "type": "string", "nullable": false }
1063 ],
1064 "scopes": [{ "pattern": "project:{project_id}" }]
1065 }]
1066 });
1067 let mut client = SyncClient::open_path_with_identity(
1068 Some("sidecar-client".to_owned()),
1069 &schema,
1070 ClientLimits::default(),
1071 path.to_str().expect("UTF-8 temp path"),
1072 )
1073 .expect("open owner");
1074 let base = WindowBase {
1075 table: "tasks".to_owned(),
1076 variable: "project_id".to_owned(),
1077 fixed_scopes: Vec::new(),
1078 params: None,
1079 };
1080 client
1081 .set_window(&base, &["one".to_owned()])
1082 .expect("set window");
1083 client
1084 .mutate(vec![Mutation::Upsert {
1085 table: "tasks".to_owned(),
1086 values: Map::from_iter([
1087 ("id".to_owned(), Value::from("t1")),
1088 ("project_id".to_owned(), Value::from("one")),
1089 ]),
1090 base_version: None,
1091 }])
1092 .expect("local mutate");
1093
1094 let coverage = [WindowCoverage {
1095 base,
1096 units: vec!["one".to_owned(), "missing".to_owned()],
1097 }];
1098 let owner = client
1099 .query_snapshot(
1100 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
1101 &[],
1102 &coverage,
1103 )
1104 .expect("owner snapshot");
1105 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
1106 let sidecar = reader
1107 .query_snapshot(
1108 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
1109 &[],
1110 &coverage,
1111 )
1112 .expect("sidecar snapshot");
1113
1114 assert_eq!(sidecar.revision, owner.revision);
1115 assert_eq!(sidecar.rows, owner.rows);
1116 assert_eq!(
1117 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
1118 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
1119 );
1120 assert_eq!(sidecar.revision, "2");
1121 assert_eq!(sidecar.rows[0]["id"], "t1");
1122 assert_eq!(sidecar.rows[0]["server_version"], -1);
1123 assert!(!sidecar.coverage.complete);
1124 assert_eq!(sidecar.coverage.pending.len(), 1);
1125 assert_eq!(sidecar.coverage.missing.len(), 1);
1126
1127 drop(reader);
1128 drop(client);
1129 std::fs::remove_file(path).expect("remove temp database");
1130 }
1131
1132 #[test]
1133 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
1134 let schema = json!({
1135 "version": 1,
1136 "tables": [{
1137 "name": "catalogue_codes",
1138 "primaryKey": "id",
1139 "columns": [
1140 { "name": "id", "type": "string", "nullable": false },
1141 { "name": "release_id", "type": "string", "nullable": false },
1142 { "name": "code", "type": "string", "nullable": false },
1143 { "name": "title", "type": "string", "nullable": false }
1144 ],
1145 "scopes": [{ "pattern": "release:{release_id}" }],
1146 "ftsIndexes": [{
1147 "name": "catalogue_codes_fts",
1148 "columns": ["code", "title"],
1149 "tokenize": "unicode61 remove_diacritics 2"
1150 }]
1151 }]
1152 });
1153 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
1154 .expect("FTS5 client");
1155 let search = |client: &SyncClient, query: &str| {
1156 client
1157 .query(
1158 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
1159 &[Value::from(query)],
1160 )
1161 .expect("FTS query")
1162 };
1163
1164 client
1165 .mutate(vec![Mutation::Upsert {
1166 table: "catalogue_codes".to_owned(),
1167 values: Map::from_iter([
1168 ("id".to_owned(), Value::from("c1")),
1169 ("release_id".to_owned(), Value::from("r1")),
1170 ("code".to_owned(), Value::from("A01")),
1171 ("title".to_owned(), Value::from("Cholera")),
1172 ]),
1173 base_version: None,
1174 }])
1175 .expect("insert code");
1176 assert_eq!(search(&client, "cholera").len(), 1);
1177
1178 client
1179 .mutate(vec![Mutation::Upsert {
1180 table: "catalogue_codes".to_owned(),
1181 values: Map::from_iter([
1182 ("id".to_owned(), Value::from("c1")),
1183 ("release_id".to_owned(), Value::from("r1")),
1184 ("code".to_owned(), Value::from("A01")),
1185 ("title".to_owned(), Value::from("Enteric infection")),
1186 ]),
1187 base_version: None,
1188 }])
1189 .expect("update code");
1190 assert!(search(&client, "cholera").is_empty());
1191 assert_eq!(search(&client, "enteric").len(), 1);
1192
1193 client
1194 .mutate(vec![Mutation::Delete {
1195 table: "catalogue_codes".to_owned(),
1196 row_id: "c1".to_owned(),
1197 base_version: None,
1198 }])
1199 .expect("delete code");
1200 assert!(search(&client, "enteric").is_empty());
1201 }
1202
1203 #[test]
1204 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
1205 let schema = json!({
1206 "version": 1,
1207 "tables": [{
1208 "name": "patient_notes",
1209 "primaryKey": "id",
1210 "columns": [
1211 { "name": "id", "type": "string", "nullable": false },
1212 { "name": "practice_id", "type": "string", "nullable": false },
1213 { "name": "encryption_key_id", "type": "string", "nullable": false },
1214 { "name": "title", "type": "string", "nullable": false }
1215 ],
1216 "scopes": [{ "pattern": "practice:{practice_id}" }],
1217 "ftsIndexes": [{
1218 "name": "patient_notes_fts",
1219 "columns": ["title"],
1220 "tokenize": "unicode61 remove_diacritics 2"
1221 }]
1222 }]
1223 });
1224 let mut client = SyncClient::new(
1225 "local-purge-test".to_owned(),
1226 &schema,
1227 ClientLimits::default(),
1228 )
1229 .expect("local purge client");
1230 client
1231 .conn
1232 .execute(
1233 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
1234 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
1235 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
1236 [],
1237 )
1238 .expect("seed base rows");
1239 client.overlay_dirty.set(true);
1240 client.rebuild_overlay_if_dirty();
1241
1242 let note = |id: &str, key_id: &str, title: &str| {
1243 Map::from_iter([
1244 ("id".to_owned(), Value::from(id)),
1245 ("practice_id".to_owned(), Value::from("practice-1")),
1246 ("encryption_key_id".to_owned(), Value::from(key_id)),
1247 ("title".to_owned(), Value::from(title)),
1248 ])
1249 };
1250 let doomed = client
1251 .mutate(vec![
1252 Mutation::Upsert {
1253 table: "patient_notes".to_owned(),
1254 values: note("target", "key-revoked", "Target changed"),
1255 base_version: None,
1256 },
1257 Mutation::Upsert {
1258 table: "patient_notes".to_owned(),
1259 values: note("unrelated", "key-held", "Unrelated changed"),
1260 base_version: None,
1261 },
1262 ])
1263 .expect("doomed commit");
1264 let kept = client
1265 .mutate(vec![Mutation::Upsert {
1266 table: "patient_notes".to_owned(),
1267 values: note("kept", "key-held", "Kept optimistic"),
1268 base_version: None,
1269 }])
1270 .expect("kept commit");
1271 client.drain_change_batches();
1272
1273 let input = LocalDataPurgeInput {
1274 purge_id: "purge-001".to_owned(),
1275 targets: vec![LocalDataPurgeTarget {
1276 table: "patient_notes".to_owned(),
1277 selectors: BTreeMap::from([(
1278 "encryption_key_id".to_owned(),
1279 vec!["key-revoked".to_owned()],
1280 )]),
1281 }],
1282 };
1283 assert_eq!(
1284 client.purge_local_data(&input).expect("apply purge"),
1285 LocalDataPurgeResult {
1286 already_applied: false,
1287 purged_rows: 1,
1288 dropped_commits: 1,
1289 }
1290 );
1291 let rows = client
1292 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
1293 .expect("visible rows");
1294 assert_eq!(rows.len(), 2);
1295 assert_eq!(rows[0]["id"], "kept");
1296 assert_eq!(rows[1]["id"], "unrelated");
1297 assert_eq!(rows[1]["title"], "Unrelated original");
1298 let fts = client
1299 .query(
1300 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
1301 &[],
1302 )
1303 .expect("fts query");
1304 assert!(fts.is_empty());
1305 assert_eq!(client.outbox.len(), 1);
1306 assert_eq!(client.outbox[0].client_commit_id, kept);
1307 let outcome = client
1308 .commit_outcome(&doomed)
1309 .expect("read doomed outcome")
1310 .expect("doomed outcome");
1311 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
1312 match &outcome.results[0] {
1313 CommitOperationOutcome::Error { rejection } => {
1314 assert_eq!(rejection.code, "client.local_data_purged");
1315 assert_eq!(rejection.client_commit_id, doomed);
1316 }
1317 other => panic!("expected local purge rejection, got {other:?}"),
1318 }
1319 assert_eq!(client.drain_change_batches().len(), 1);
1320 assert_eq!(
1321 client.purge_local_data(&input).expect("retry purge"),
1322 LocalDataPurgeResult {
1323 already_applied: true,
1324 purged_rows: 0,
1325 dropped_commits: 0,
1326 }
1327 );
1328 let conflicting = LocalDataPurgeInput {
1329 purge_id: input.purge_id.clone(),
1330 targets: vec![LocalDataPurgeTarget {
1331 table: "patient_notes".to_owned(),
1332 selectors: BTreeMap::from([(
1333 "encryption_key_id".to_owned(),
1334 vec!["key-held".to_owned()],
1335 )]),
1336 }],
1337 };
1338 assert!(client
1339 .purge_local_data(&conflicting)
1340 .expect_err("id collision must fail")
1341 .contains("already used with a different plan"));
1342 }
1343}
1344
1345impl SubState {
1346 fn name(self) -> &'static str {
1347 match self {
1348 SubState::Active => "active",
1349 SubState::Revoked => "revoked",
1350 SubState::Failed => "failed",
1351 }
1352 }
1353
1354 fn parse(value: &str) -> Self {
1355 match value {
1356 "revoked" => Self::Revoked,
1357 "failed" => Self::Failed,
1358 _ => Self::Active,
1359 }
1360 }
1361}
1362
1363#[derive(Debug, Clone)]
1364struct Subscription {
1365 id: String,
1366 table: String,
1367 requested: Vec<(String, Vec<String>)>,
1368 params: Option<String>,
1369 cursor: i64,
1370 bootstrap_state: Option<String>,
1372 state: SubState,
1373 reason_code: Option<String>,
1374 effective: Option<Vec<(String, Vec<String>)>>,
1377 synced_once: bool,
1378}
1379
1380#[derive(Debug, Clone)]
1381struct OutboxOp {
1382 upsert: bool,
1383 table: String,
1384 row_id: String,
1385 base_version: Option<i64>,
1386 values: Option<Map<String, Value>>,
1389 changed_fields: Option<Vec<String>>,
1391}
1392
1393impl From<&OutboxOp> for CommitOperation {
1394 fn from(operation: &OutboxOp) -> Self {
1395 Self {
1396 table: operation.table.clone(),
1397 row_id: operation.row_id.clone(),
1398 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
1399 base_version: operation.base_version,
1400 values: operation.values.clone(),
1401 changed_fields: operation.changed_fields.clone(),
1402 }
1403 }
1404}
1405
1406#[derive(Debug, Clone)]
1407struct OutboxCommit {
1408 client_commit_id: String,
1409 ops: Vec<OutboxOp>,
1410}
1411
1412#[derive(Debug, Clone)]
1413struct CompiledLocalDataPurgeTarget {
1414 table: String,
1415 selectors: Vec<(String, Vec<String>)>,
1416}
1417
1418struct StoredCommitOutcomeRow {
1419 sequence: i64,
1420 client_commit_id: String,
1421 status: String,
1422 recorded_at_ms: i64,
1423 results_json: String,
1424 operations_json: Option<String>,
1425 resolution: String,
1426 resolved_at_ms: Option<i64>,
1427 replacement_client_commit_id: Option<String>,
1428}
1429
1430enum SectionError {
1433 FailClosed,
1434 Abort(String, String),
1435}
1436
1437struct RequestMeta {
1438 pushed_ids: Vec<String>,
1439 fresh: Vec<(String, bool)>,
1442 accept: u8,
1443 deferred_commits: usize,
1447}
1448
1449#[derive(Default)]
1450struct ChangeAccumulator {
1451 tables: BTreeMap<String, Option<BTreeSet<String>>>,
1453 windows: BTreeMap<(String, String), BTreeSet<String>>,
1454 status: bool,
1455 conflicts: bool,
1456 rejections: bool,
1457 outcomes: bool,
1458}
1459
1460impl ChangeAccumulator {
1461 fn table(&mut self, table: &str) {
1462 self.tables.insert(table.to_owned(), None);
1463 }
1464
1465 fn scope(&mut self, table: &str, key: String) {
1466 match self.tables.get_mut(table) {
1467 Some(None) => {}
1468 Some(Some(keys)) => {
1469 keys.insert(key);
1470 }
1471 None => {
1472 self.tables
1473 .insert(table.to_owned(), Some(BTreeSet::from([key])));
1474 }
1475 }
1476 }
1477
1478 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
1479 self.windows
1480 .entry((base_key.to_owned(), table.to_owned()))
1481 .or_default()
1482 .insert(unit.to_owned());
1483 }
1484
1485 fn touched(&self) -> bool {
1486 !self.tables.is_empty()
1487 || !self.windows.is_empty()
1488 || self.status
1489 || self.conflicts
1490 || self.rejections
1491 || self.outcomes
1492 }
1493}
1494
1495const PUSH_OPS_PER_REQUEST: usize = 500;
1501const MAX_LOCAL_PURGE_TARGETS: usize = 64;
1502const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
1503const MAX_LOCAL_PURGE_VALUES: usize = 128;
1504const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
1505pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
1506
1507pub struct SyncClient {
1508 conn: Connection,
1509 schema: ClientSchema,
1510 client_id: String,
1511 limits: ClientLimits,
1512 subs: Vec<Subscription>,
1513 outbox: Vec<OutboxCommit>,
1514 conflicts: Vec<ConflictRecord>,
1515 rejections: Vec<RejectionRecord>,
1516 schema_floor: Option<SchemaFloor>,
1517 lease_state: Option<LeaseState>,
1519 stopped: bool,
1521 upgrading: bool,
1523 sync_needed: bool,
1525 realtime_connected: bool,
1526 presence: HashMap<String, HashMap<String, PresencePeer>>,
1528 now_ms: Option<i64>,
1531 encryption: crate::values::EncryptionConfig,
1535 security_preflight: bool,
1538 insert_sql: RefCell<HashMap<String, String>>,
1544 overlay_dirty: Cell<bool>,
1548 #[cfg(test)]
1551 overlay_rebuild_count: Cell<usize>,
1552 #[cfg(test)]
1555 outcome_prune_count: Cell<usize>,
1556 change_queue: VecDeque<ClientChangeBatch>,
1558 sync_intent_queue: VecDeque<SyncIntent>,
1559 retry_delay_ms: u64,
1561 last_round: Option<DiagnosticLastRound>,
1562 last_change: Option<DiagnosticLastChange>,
1563}
1564
1565fn quote_ident(name: &str) -> String {
1566 format!("\"{}\"", name.replace('"', "\"\""))
1567}
1568
1569fn is_local_operation_code_like(value: &str) -> bool {
1570 let bytes = value.as_bytes();
1571 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
1572 && bytes
1573 .iter()
1574 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
1575}
1576
1577fn base_table(name: &str) -> String {
1578 quote_ident(&format!("_syncular_base_{name}"))
1579}
1580
1581type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
1586
1587fn window_base_key(base: &WindowBase) -> String {
1588 format!(
1589 "{}\0{}\0{}",
1590 base.table,
1591 base.variable,
1592 canonical_scope_json(&base.fixed_scopes)
1593 )
1594}
1595
1596fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
1598 let mut scopes = base.fixed_scopes.clone();
1599 scopes.retain(|(k, _)| k != &base.variable);
1600 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
1601 scopes
1602}
1603
1604fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
1608 let canonical = canonical_scope_json(&unit_scopes(base, unit));
1609 let digest = Sha256::digest(canonical.as_bytes());
1610 let hex = bytes_to_hex(&digest);
1611 format!("w:{}:{}", base.table, &hex[..16])
1612}
1613
1614fn visible_table(name: &str) -> String {
1615 quote_ident(name)
1616}
1617
1618const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1619
1620fn is_synced_table_name(name: &str) -> bool {
1623 if name.starts_with("sqlite_") {
1624 return false;
1625 }
1626 if name.starts_with("_syncular_base_") {
1627 return true; }
1629 !name.starts_with("_syncular_")
1631}
1632
1633fn blob_id_for(bytes: &[u8]) -> String {
1635 let digest = Sha256::digest(bytes);
1636 format!("sha256:{}", bytes_to_hex(&digest))
1637}
1638
1639enum RowParam<'a> {
1643 Cell(&'a Option<ColumnValue>),
1644 Version(i64),
1645}
1646
1647impl rusqlite::ToSql for RowParam<'_> {
1648 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1649 Ok(match self {
1650 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1651 RowParam::Cell(cell) => match cell {
1652 None => ToSqlOutput::Owned(SqlValue::Null),
1653 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1654 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1655 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1656 Some(ColumnValue::Boolean(b)) => {
1657 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1658 }
1659 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1660 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1661 }
1662 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1664 ToSqlOutput::Borrowed(ValueRef::Blob(b))
1665 }
1666 },
1667 })
1668 }
1669}
1670
1671fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1679 use ssp2::segment::ColumnType;
1680 let mismatch = || {
1681 Err(format!(
1682 "image column {:?} holds a value of the wrong type",
1683 column.name
1684 ))
1685 };
1686 match value {
1687 ValueRef::Null => {
1688 if !column.nullable {
1689 return Err(format!(
1690 "image column {:?} is NULL but not nullable",
1691 column.name
1692 ));
1693 }
1694 Ok(ToSqlOutput::Owned(SqlValue::Null))
1695 }
1696 ValueRef::Integer(i) => match column.ty {
1697 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1698 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1699 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1700 _ => mismatch(),
1701 },
1702 ValueRef::Real(_) => match column.ty {
1703 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1704 _ => mismatch(),
1705 },
1706 ValueRef::Text(t) => {
1707 std::str::from_utf8(t)
1708 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1709 match column.ty {
1710 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1711 Ok(ToSqlOutput::Borrowed(value))
1712 }
1713 _ => mismatch(),
1714 }
1715 }
1716 ValueRef::Blob(_) => match column.ty {
1717 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1719 _ => mismatch(),
1720 },
1721 }
1722}
1723
1724fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1725 use rusqlite::types::ValueRef;
1726 match value {
1727 ValueRef::Null => Value::Null,
1728 ValueRef::Integer(i) => match column.ty {
1729 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1730 ssp2::segment::ColumnType::Float => {
1731 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1732 }
1733 _ => Value::from(i),
1734 },
1735 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1736 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1737 ValueRef::Blob(b) => {
1738 let mut map = Map::new();
1739 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1740 Value::Object(map)
1741 }
1742 }
1743}
1744
1745fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1749 Ok(match value {
1750 Value::Null => SqlValue::Null,
1751 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1752 Value::Number(n) => {
1753 if let Some(i) = n.as_i64() {
1754 SqlValue::Integer(i)
1755 } else if let Some(f) = n.as_f64() {
1756 SqlValue::Real(f)
1757 } else {
1758 return Err(format!("query param number {n} is out of range"));
1759 }
1760 }
1761 Value::String(s) => SqlValue::Text(s.clone()),
1762 Value::Object(_) => {
1763 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1764 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1765 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1766 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1767 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1768 })?)
1769 } else {
1770 return Err(
1771 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1772 .to_owned(),
1773 );
1774 }
1775 }
1776 Value::Array(_) => return Err("query array params are not supported".to_owned()),
1777 })
1778}
1779
1780fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1785 use rusqlite::types::ValueRef;
1786 match value {
1787 ValueRef::Null => Value::Null,
1788 ValueRef::Integer(i) => {
1789 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1793 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1794 Value::from(i)
1795 } else {
1796 let mut map = Map::new();
1797 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1798 Value::Object(map)
1799 }
1800 }
1801 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1802 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1803 ValueRef::Blob(b) => {
1804 let mut map = Map::new();
1805 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1806 Value::Object(map)
1807 }
1808 }
1809}
1810
1811fn query_connection(
1812 conn: &Connection,
1813 sql: &str,
1814 params: &[QueryValue],
1815) -> Result<Vec<QueryRow>, String> {
1816 crate::query_guard::assert_read_only_query(sql)?;
1817 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1818 let bound: Vec<SqlValue> = params
1819 .iter()
1820 .map(json_param_to_sql)
1821 .collect::<Result<_, _>>()?;
1822 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
1823 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1824 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1825 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1826 let mut sql_rows = stmt
1827 .query(bound_refs.as_slice())
1828 .map_err(|e| e.to_string())?;
1829 let mut out = Vec::new();
1830 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1831 let mut record = Map::new();
1832 for (i, name) in column_names.iter().enumerate() {
1833 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1834 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1835 }
1836 out.push(record);
1837 }
1838 Ok(out)
1839}
1840
1841fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1842 let mut stmt = conn
1843 .prepare(
1844 "SELECT windows.unit, subscriptions.state_json
1845 FROM _syncular_windows AS windows
1846 JOIN _syncular_subscriptions AS subscriptions
1847 ON subscriptions.id = windows.sub_id
1848 WHERE windows.base = ?1
1849 ORDER BY windows.unit ASC",
1850 )
1851 .map_err(|error| error.to_string())?;
1852 let rows = stmt
1853 .query_map(rusqlite::params![window_base_key(base)], |row| {
1854 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1855 })
1856 .map_err(|error| error.to_string())?;
1857 let mut units = Vec::new();
1858 let mut pending = Vec::new();
1859 for row in rows {
1860 let (unit, raw) = row.map_err(|error| error.to_string())?;
1861 let state: Value = serde_json::from_str(&raw)
1862 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1863 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1864 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1865 || state
1866 .get("bootstrapState")
1867 .is_some_and(|value| !value.is_null());
1868 if is_pending {
1869 pending.push(unit.clone());
1870 }
1871 units.push(unit);
1872 }
1873 Ok(WindowState { units, pending })
1874}
1875
1876fn snapshot_connection(
1877 conn: &Connection,
1878 sql: &str,
1879 params: &[Value],
1880 coverage: &[WindowCoverage],
1881) -> Result<QuerySnapshot, String> {
1882 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1883 .map_err(|error| error.to_string())?;
1884 let result = (|| {
1885 let revision = conn
1886 .query_row(
1887 "SELECT value FROM _syncular_meta WHERE key = ?1",
1888 rusqlite::params![LOCAL_REVISION_KEY],
1889 |row| row.get::<_, String>(0),
1890 )
1891 .ok()
1892 .and_then(|value| value.parse::<u64>().ok())
1893 .unwrap_or(0);
1894 let rows = query_connection(conn, sql, params)?;
1895 let mut pending = Vec::new();
1896 let mut missing = Vec::new();
1897 for requested in coverage {
1898 let base_key = window_base_key(&requested.base);
1899 let state = persisted_window_state(conn, &requested.base)?;
1900 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1901 let reference = WindowUnitRef {
1902 base_key: base_key.clone(),
1903 unit: unit.clone(),
1904 };
1905 if !state.units.iter().any(|held| held == &unit) {
1906 missing.push(reference);
1907 } else if state.pending.iter().any(|held| held == &unit) {
1908 pending.push(reference);
1909 }
1910 }
1911 }
1912 Ok(QuerySnapshot {
1913 revision: revision.to_string(),
1914 rows,
1915 coverage: CoverageSnapshot {
1916 complete: pending.is_empty() && missing.is_empty(),
1917 pending,
1918 missing,
1919 },
1920 })
1921 })();
1922 match result {
1923 Ok(snapshot) => {
1924 conn.execute_batch("RELEASE syncular_snapshot_read")
1925 .map_err(|error| error.to_string())?;
1926 Ok(snapshot)
1927 }
1928 Err(error) => {
1929 let _ = conn.execute_batch(
1930 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1931 );
1932 Err(error)
1933 }
1934 }
1935}
1936
1937pub struct FileQuerySnapshotReader {
1942 path: String,
1943 conn: Option<Connection>,
1944}
1945
1946impl FileQuerySnapshotReader {
1947 #[must_use]
1948 pub fn new(path: impl Into<String>) -> Self {
1949 Self {
1950 path: path.into(),
1951 conn: None,
1952 }
1953 }
1954
1955 fn connection(&mut self) -> Result<&Connection, String> {
1956 if self.conn.is_none() {
1957 let conn = Connection::open_with_flags(
1958 &self.path,
1959 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1960 )
1961 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1962 conn.busy_timeout(std::time::Duration::from_millis(250))
1963 .map_err(|error| error.to_string())?;
1964 self.conn = Some(conn);
1965 }
1966 self.conn
1967 .as_ref()
1968 .ok_or_else(|| "read sidecar connection missing".to_owned())
1969 }
1970
1971 pub fn query_snapshot(
1972 &mut self,
1973 sql: &str,
1974 params: &[Value],
1975 coverage: &[WindowCoverage],
1976 ) -> Result<QuerySnapshot, String> {
1977 snapshot_connection(self.connection()?, sql, params, coverage)
1978 }
1979}
1980
1981impl SyncClient {
1982 pub fn new_with_identity(
1983 client_id: Option<String>,
1984 schema_json: &Value,
1985 limits: ClientLimits,
1986 ) -> Result<Self, String> {
1987 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1988 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1989 Self::with_connection(resolved, schema_json, limits, conn)
1990 }
1991
1992 pub fn new(
1993 client_id: String,
1994 schema_json: &Value,
1995 limits: ClientLimits,
1996 ) -> Result<Self, String> {
1997 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1998 Self::with_connection(client_id, schema_json, limits, conn)
1999 }
2000
2001 pub fn open_path(
2007 client_id: String,
2008 schema_json: &Value,
2009 limits: ClientLimits,
2010 path: &str,
2011 ) -> Result<Self, String> {
2012 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
2013 Self::with_connection(client_id, schema_json, limits, conn)
2014 }
2015
2016 pub fn open_path_with_identity(
2017 client_id: Option<String>,
2018 schema_json: &Value,
2019 limits: ClientLimits,
2020 path: &str,
2021 ) -> Result<Self, String> {
2022 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
2023 conn.busy_timeout(std::time::Duration::from_millis(250))
2029 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
2030 conn.pragma_update(None, "journal_mode", "WAL")
2031 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
2032 let persisted = conn
2033 .query_row(
2034 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
2035 [],
2036 |row| row.get::<_, String>(0),
2037 )
2038 .ok();
2039 let resolved = persisted
2040 .clone()
2041 .or(client_id.clone())
2042 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
2043 if let (Some(existing), Some(requested)) = (persisted, client_id) {
2044 if existing != requested {
2045 return Err(format!(
2046 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2047 ));
2048 }
2049 }
2050 Self::with_connection(resolved, schema_json, limits, conn)
2051 }
2052
2053 #[must_use]
2054 pub fn client_id(&self) -> &str {
2055 &self.client_id
2056 }
2057
2058 pub fn with_connection(
2065 client_id: String,
2066 schema_json: &Value,
2067 limits: ClientLimits,
2068 conn: Connection,
2069 ) -> Result<Self, String> {
2070 if limits.outcome_retention_max_entries == Some(0) {
2071 return Err(
2072 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
2073 );
2074 }
2075 let schema = parse_schema_json(schema_json)?;
2076 let mut client = SyncClient {
2077 conn,
2078 schema,
2079 client_id,
2080 limits,
2081 subs: Vec::new(),
2082 outbox: Vec::new(),
2083 conflicts: Vec::new(),
2084 rejections: Vec::new(),
2085 schema_floor: None,
2086 lease_state: None,
2087 stopped: false,
2088 upgrading: false,
2089 sync_needed: false,
2090 realtime_connected: false,
2091 presence: HashMap::new(),
2092 now_ms: None,
2093 encryption: crate::values::EncryptionConfig::default(),
2094 security_preflight: false,
2095 insert_sql: RefCell::new(HashMap::new()),
2096 overlay_dirty: Cell::new(false),
2097 #[cfg(test)]
2098 overlay_rebuild_count: Cell::new(0),
2099 #[cfg(test)]
2100 outcome_prune_count: Cell::new(0),
2101 change_queue: VecDeque::new(),
2102 sync_intent_queue: VecDeque::new(),
2103 retry_delay_ms: 250,
2104 last_round: None,
2105 last_change: None,
2106 };
2107 client
2111 .conn
2112 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2113 client.create_bookkeeping_tables()?;
2117 match client.get_meta(CLIENT_ID_KEY) {
2118 Some(existing) if existing != client.client_id => {
2119 return Err(format!(
2120 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
2121 client.client_id
2122 ));
2123 }
2124 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
2125 _ => {}
2126 }
2127 client.restore_persisted_state()?;
2128 let marker = client
2129 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
2130 .and_then(|value| value.parse::<i32>().ok());
2131 match marker {
2132 None => {
2133 client.create_synced_tables()?;
2134 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
2135 }
2136 Some(version) if version == client.schema.version => {
2137 client.create_synced_tables()?;
2138 }
2139 Some(_) => client.run_schema_reset()?,
2140 }
2141 client.clear_satisfied_persisted_schema_floor();
2142 client.prune_unknown_subscriptions()?;
2143 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
2144 client.overlay_dirty.set(true);
2147 client.rebuild_overlay();
2148 }
2149 client.enqueue_startup_sync_if_needed();
2155 Ok(client)
2156 }
2157
2158 pub fn set_now_ms(&mut self, now_ms: i64) {
2161 self.now_ms = Some(now_ms);
2162 }
2163
2164 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2168 self.encryption = encryption;
2169 }
2170
2171 #[must_use]
2172 pub fn security_lifecycle(&self) -> &'static str {
2173 if self.security_preflight {
2174 "preflight"
2175 } else {
2176 "active"
2177 }
2178 }
2179
2180 #[must_use]
2181 pub fn security_preflight(&self) -> bool {
2182 self.security_preflight
2183 }
2184
2185 pub fn begin_security_preflight(&mut self) {
2187 self.security_preflight = true;
2188 self.encryption = crate::values::EncryptionConfig::default();
2189 self.sync_intent_queue.clear();
2190 }
2191
2192 pub fn activate_security(
2194 &mut self,
2195 encryption: crate::values::EncryptionConfig,
2196 ) -> Result<(), String> {
2197 if !self.security_preflight {
2198 return Err(
2199 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2200 );
2201 }
2202 self.encryption = encryption;
2203 self.security_preflight = false;
2204 self.enqueue_startup_sync_if_needed();
2205 Ok(())
2206 }
2207
2208 fn clock_now_ms(&self) -> i64 {
2209 self.now_ms.unwrap_or_else(|| {
2210 std::time::SystemTime::now()
2211 .duration_since(std::time::UNIX_EPOCH)
2212 .map(|d| d.as_millis() as i64)
2213 .unwrap_or(0)
2214 })
2215 }
2216
2217 fn create_bookkeeping_tables(&self) -> Result<(), String> {
2218 self.conn
2220 .execute_batch(
2221 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2222 seq INTEGER PRIMARY KEY AUTOINCREMENT,
2223 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2224 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2225 seq INTEGER PRIMARY KEY AUTOINCREMENT,
2226 client_commit_id TEXT NOT NULL UNIQUE,
2227 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2228 recorded_at_ms INTEGER NOT NULL,
2229 results_json TEXT NOT NULL,
2230 operations_json TEXT,
2231 resolution TEXT NOT NULL DEFAULT 'active'
2232 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2233 resolved_at_ms INTEGER,
2234 replacement_client_commit_id TEXT);
2235 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2236 ON _syncular_commit_outcomes(resolution, seq);
2237 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
2238 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
2239 CREATE TABLE IF NOT EXISTS _syncular_meta (
2240 key TEXT PRIMARY KEY, value TEXT NOT NULL);
2241 CREATE TABLE IF NOT EXISTS _syncular_windows (
2242 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2243 PRIMARY KEY (base, unit));
2244 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
2245 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
2246 effective_scopes TEXT NOT NULL);",
2247 )
2248 .map_err(|e| e.to_string())?;
2249 let _ = self
2252 .conn
2253 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2254 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
2255 self.set_meta(LOCAL_REVISION_KEY, "0");
2256 }
2257 if self.schema_has_blobs() {
2262 self.conn
2263 .execute_batch(
2264 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
2265 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
2266 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
2267 created_at_ms INTEGER NOT NULL,
2268 last_used_ms INTEGER NOT NULL DEFAULT 0);
2269 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
2270 media_type TEXT, created_at_ms INTEGER NOT NULL);",
2271 )
2272 .map_err(|e| e.to_string())?;
2273 let _ = self.conn.execute_batch(
2277 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
2278 );
2279 }
2280 Ok(())
2281 }
2282
2283 fn schema_has_blobs(&self) -> bool {
2285 self.schema
2286 .tables
2287 .iter()
2288 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2289 }
2290
2291 fn get_meta(&self, key: &str) -> Option<String> {
2294 self.conn
2295 .query_row(
2296 "SELECT value FROM _syncular_meta WHERE key = ?1",
2297 rusqlite::params![key],
2298 |row| row.get::<_, String>(0),
2299 )
2300 .ok()
2301 }
2302
2303 fn set_meta(&self, key: &str, value: &str) {
2304 let _ = self.conn.execute(
2305 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
2306 rusqlite::params![key, value],
2307 );
2308 }
2309
2310 fn delete_meta(&self, key: &str) {
2311 let _ = self.conn.execute(
2312 "DELETE FROM _syncular_meta WHERE key = ?1",
2313 rusqlite::params![key],
2314 );
2315 }
2316
2317 fn restore_persisted_state(&mut self) -> Result<(), String> {
2318 self.subs = {
2319 let mut stmt = self
2320 .conn
2321 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
2322 .map_err(|error| error.to_string())?;
2323 let rows = stmt
2324 .query_map([], |row| {
2325 Ok((
2326 row.get::<_, String>(0)?,
2327 row.get::<_, String>(1)?,
2328 row.get::<_, String>(2)?,
2329 ))
2330 })
2331 .map_err(|error| error.to_string())?;
2332 let mut subscriptions = Vec::new();
2333 for row in rows {
2334 let (id, table, raw) = row.map_err(|error| error.to_string())?;
2335 let state: Value = serde_json::from_str(&raw)
2336 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
2337 let requested = json_to_scope_map(
2338 state.get("requested").unwrap_or(&Value::Object(Map::new())),
2339 )?;
2340 let effective = state
2341 .get("effectiveScopes")
2342 .filter(|value| !value.is_null())
2343 .map(json_to_scope_map)
2344 .transpose()?;
2345 subscriptions.push(Subscription {
2346 id,
2347 table,
2348 requested,
2349 params: state
2350 .get("params")
2351 .and_then(Value::as_str)
2352 .map(str::to_owned),
2353 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
2354 bootstrap_state: state
2355 .get("bootstrapState")
2356 .and_then(Value::as_str)
2357 .map(str::to_owned),
2358 state: SubState::parse(
2359 state
2360 .get("status")
2361 .and_then(Value::as_str)
2362 .unwrap_or("active"),
2363 ),
2364 reason_code: state
2365 .get("reasonCode")
2366 .and_then(Value::as_str)
2367 .map(str::to_owned),
2368 effective,
2369 synced_once: state
2370 .get("syncedOnce")
2371 .and_then(Value::as_bool)
2372 .unwrap_or(false),
2373 });
2374 }
2375 subscriptions
2376 };
2377
2378 self.outbox = {
2379 let mut stmt = self
2380 .conn
2381 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2382 .map_err(|error| error.to_string())?;
2383 let rows = stmt
2384 .query_map([], |row| {
2385 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2386 })
2387 .map_err(|error| error.to_string())?;
2388 let mut commits = Vec::new();
2389 for row in rows {
2390 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2391 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2392 format!("invalid persisted outbox {client_commit_id:?}: {error}")
2393 })?;
2394 let mut ops = Vec::with_capacity(entries.len());
2395 for entry in entries {
2396 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2397 ops.push(OutboxOp {
2398 upsert: op == "upsert",
2399 table: entry
2400 .get("table")
2401 .and_then(Value::as_str)
2402 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2403 .to_owned(),
2404 row_id: entry
2405 .get("rowId")
2406 .and_then(Value::as_str)
2407 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2408 .to_owned(),
2409 base_version: entry.get("baseVersion").and_then(Value::as_i64),
2410 values: entry.get("values").and_then(Value::as_object).cloned(),
2411 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
2412 |values| {
2413 values
2414 .iter()
2415 .filter_map(Value::as_str)
2416 .map(str::to_owned)
2417 .collect()
2418 },
2419 ),
2420 });
2421 }
2422 commits.push(OutboxCommit {
2423 client_commit_id,
2424 ops,
2425 });
2426 }
2427 commits
2428 };
2429
2430 self.prune_commit_outcomes()?;
2431 let active = self.commit_outcomes(CommitOutcomeQuery {
2432 active_only: true,
2433 ..CommitOutcomeQuery::default()
2434 })?;
2435 self.conflicts = active
2436 .iter()
2437 .flat_map(|outcome| outcome.results.iter())
2438 .filter_map(|result| match result {
2439 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2440 _ => None,
2441 })
2442 .collect();
2443 self.rejections = active
2444 .iter()
2445 .flat_map(|outcome| outcome.results.iter())
2446 .filter_map(|result| match result {
2447 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2448 _ => None,
2449 })
2450 .collect();
2451
2452 self.lease_state = self
2453 .get_meta(LEASE_STATE_KEY)
2454 .map(|raw| serde_json::from_str(&raw))
2455 .transpose()
2456 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2457 self.schema_floor = self
2458 .get_meta(SCHEMA_FLOOR_KEY)
2459 .map(|raw| serde_json::from_str(&raw))
2460 .transpose()
2461 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2462 self.stopped = self.schema_floor.is_some();
2463 Ok(())
2464 }
2465
2466 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
2469 let valid_tables: BTreeSet<String> = self
2470 .schema
2471 .tables
2472 .iter()
2473 .map(|table| table.name.clone())
2474 .collect();
2475 let stale_ids: Vec<String> = self
2476 .subs
2477 .iter()
2478 .filter(|sub| !valid_tables.contains(&sub.table))
2479 .map(|sub| sub.id.clone())
2480 .collect();
2481 for id in &stale_ids {
2482 self.conn
2483 .execute(
2484 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2485 rusqlite::params![id],
2486 )
2487 .map_err(|error| error.to_string())?;
2488 self.conn
2489 .execute(
2490 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2491 rusqlite::params![id],
2492 )
2493 .map_err(|error| error.to_string())?;
2494 self.conn
2495 .execute(
2496 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2497 rusqlite::params![id],
2498 )
2499 .map_err(|error| error.to_string())?;
2500 }
2501 self.subs.retain(|sub| valid_tables.contains(&sub.table));
2502 Ok(())
2503 }
2504
2505 #[must_use]
2506 pub fn local_revision(&self) -> u64 {
2507 self.get_meta(LOCAL_REVISION_KEY)
2508 .and_then(|value| value.parse().ok())
2509 .unwrap_or(0)
2510 }
2511
2512 #[must_use]
2513 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2514 SyncStatusSnapshot {
2515 current_schema_version: self.schema.version,
2516 outbox: self.outbox.len(),
2517 upgrading: self.upgrading,
2518 lease_state: self.lease_state.clone(),
2519 schema_floor: self.schema_floor.clone(),
2520 sync_needed: self.sync_needed,
2521 }
2522 }
2523
2524 pub fn diagnostics_snapshot(
2525 &self,
2526 request: &ClientDiagnosticsRequest,
2527 ) -> Result<ClientDiagnosticsSnapshot, String> {
2528 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
2529 return Err(format!(
2530 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
2531 ));
2532 }
2533 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
2534 for sub in &self.subs {
2535 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
2536 let complete =
2537 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
2538 let state = match sub.state {
2539 SubState::Revoked => "revoked",
2540 SubState::Failed => "failed",
2541 SubState::Active if reset => "reset",
2542 SubState::Active if complete => "complete",
2543 SubState::Active => "bootstrapping",
2544 };
2545 subscriptions.insert(
2546 sub.id.clone(),
2547 DiagnosticSubscription {
2548 id: sub.id.clone(),
2549 table: sub.table.clone(),
2550 state: state.to_owned(),
2551 complete,
2552 cursor: Some(sub.cursor),
2553 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
2554 },
2555 );
2556 }
2557 for expected in &request.expected_subscriptions {
2558 if expected.id.is_empty() || expected.table.is_empty() {
2559 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
2560 }
2561 if subscriptions
2562 .get(&expected.id)
2563 .is_some_and(|registered| registered.table != expected.table)
2564 {
2565 subscriptions.insert(
2566 expected.id.clone(),
2567 DiagnosticSubscription {
2568 id: expected.id.clone(),
2569 table: expected.table.clone(),
2570 state: "failed".to_owned(),
2571 complete: false,
2572 cursor: None,
2573 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
2574 },
2575 );
2576 } else {
2577 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
2578 DiagnosticSubscription {
2579 id: expected.id.clone(),
2580 table: expected.table.clone(),
2581 state: "unregistered".to_owned(),
2582 complete: false,
2583 cursor: None,
2584 reason_code: None,
2585 }
2586 });
2587 }
2588 }
2589 let mut ordered_subscriptions = Vec::new();
2590 let mut included = BTreeSet::new();
2591 for expected in &request.expected_subscriptions {
2592 if included.insert(expected.id.clone()) {
2593 if let Some(subscription) = subscriptions.get(&expected.id) {
2594 ordered_subscriptions.push(subscription.clone());
2595 }
2596 }
2597 }
2598 for (id, subscription) in &subscriptions {
2599 if included.insert(id.clone()) {
2600 ordered_subscriptions.push(subscription.clone());
2601 }
2602 }
2603 let subscriptions_truncated =
2604 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
2605 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
2606 let captured_at_ms = self.clock_now_ms();
2607 let lease = if let Some(error_code) = self
2608 .lease_state
2609 .as_ref()
2610 .and_then(|state| state.error_code.clone())
2611 {
2612 ClientDiagnosticsLease {
2613 state: "stopped".to_owned(),
2614 expires_at_ms: self
2615 .lease_state
2616 .as_ref()
2617 .and_then(|state| state.expires_at_ms),
2618 error_code: Some(Self::diagnostic_code(&error_code)),
2619 }
2620 } else if let Some(expires_at_ms) = self
2621 .lease_state
2622 .as_ref()
2623 .and_then(|state| state.expires_at_ms)
2624 {
2625 ClientDiagnosticsLease {
2626 state: if expires_at_ms <= captured_at_ms {
2627 "expired".to_owned()
2628 } else {
2629 "active".to_owned()
2630 },
2631 expires_at_ms: Some(expires_at_ms),
2632 error_code: None,
2633 }
2634 } else {
2635 ClientDiagnosticsLease {
2636 state: "none".to_owned(),
2637 expires_at_ms: None,
2638 error_code: None,
2639 }
2640 };
2641 let connectivity = match self.last_round.as_ref() {
2642 Some(round) if round.status == "succeeded" => "online",
2643 Some(round)
2644 if round.status == "failed"
2645 && round
2646 .error_code
2647 .as_deref()
2648 .is_some_and(Self::retryable_transport_code) =>
2649 {
2650 "offline"
2651 }
2652 _ => "unknown",
2653 };
2654 Ok(ClientDiagnosticsSnapshot {
2655 version: CLIENT_DIAGNOSTICS_VERSION,
2656 captured_at_ms,
2657 host: ClientDiagnosticsHost {
2658 kind: "direct".to_owned(),
2659 role: "single".to_owned(),
2660 connectivity: connectivity.to_owned(),
2661 realtime: if self.realtime_connected {
2662 "connected".to_owned()
2663 } else {
2664 "disconnected".to_owned()
2665 },
2666 },
2667 security_lifecycle: self.security_lifecycle().to_owned(),
2668 schema: ClientDiagnosticsSchema {
2669 current_version: self.schema.version,
2670 upgrading: self.upgrading,
2671 required_version: self
2672 .schema_floor
2673 .as_ref()
2674 .and_then(|floor| floor.required_schema_version),
2675 latest_version: self
2676 .schema_floor
2677 .as_ref()
2678 .and_then(|floor| floor.latest_schema_version),
2679 },
2680 replica: ClientDiagnosticsReplica {
2681 local_revision: self.local_revision().to_string(),
2682 sync_needed: self.sync_needed,
2683 pending_outbox: self.outbox.len(),
2684 },
2685 lease,
2686 subscriptions: ordered_subscriptions,
2687 subscriptions_truncated,
2688 last_round: self.last_round.clone(),
2689 last_change: self.last_change.clone(),
2690 storage: self.diagnostics_storage(),
2691 })
2692 }
2693
2694 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
2695 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
2696 let page_count: i64 = self
2697 .conn
2698 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
2699 let page_size: i64 = self
2700 .conn
2701 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
2702 let outbox_bytes: i64 = self.conn.query_row(
2703 "SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox",
2704 [],
2705 |row| row.get(0),
2706 )?;
2707 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.query_row(
2708 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
2709 [],
2710 |row| Ok((row.get(0)?, row.get(1)?)),
2711 )?;
2712 let blob_bytes = if self.schema_has_blobs() {
2713 self.conn.query_row(
2714 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
2715 [],
2716 |row| row.get(0),
2717 )?
2718 } else {
2719 0
2720 };
2721 let pressure = self
2722 .limits
2723 .blob_cache_max_bytes
2724 .is_some_and(|limit| blob_bytes > limit);
2725 Ok(ClientDiagnosticsStorage {
2726 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
2727 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
2728 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
2729 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
2730 retained_outcome_entries: Some(outcome_entries.max(0)),
2731 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
2732 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
2733 })
2734 };
2735 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
2736 status: "unreadable".to_owned(),
2737 database_bytes_approx: None,
2738 pending_outbox_bytes_approx: None,
2739 retained_outcome_bytes_approx: None,
2740 retained_outcome_entries: None,
2741 blob_cache_bytes_approx: None,
2742 pressure_reason_code: None,
2743 })
2744 }
2745
2746 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
2747 self.change_queue.drain(..).collect()
2748 }
2749
2750 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
2751 self.sync_intent_queue.drain(..).collect()
2752 }
2753
2754 fn schedule_background_retry(&mut self) {
2755 self.sync_intent_queue.push_back(SyncIntent::Background {
2756 delay_ms: self.retry_delay_ms,
2757 });
2758 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
2759 }
2760
2761 fn reset_background_retry(&mut self) {
2762 self.retry_delay_ms = 250;
2763 }
2764
2765 fn retryable_transport_code(code: &str) -> bool {
2766 code == "transport.failed"
2767 || code == "transport.unavailable"
2768 || code == "sync.transport_failed"
2769 }
2770
2771 fn diagnostic_code(code: &str) -> String {
2772 let valid = !code.is_empty()
2773 && code.len() <= 96
2774 && code.contains('.')
2775 && code
2776 .bytes()
2777 .next()
2778 .is_some_and(|byte| byte.is_ascii_lowercase())
2779 && code.bytes().all(|byte| {
2780 byte.is_ascii_lowercase()
2781 || byte.is_ascii_digit()
2782 || matches!(byte, b'.' | b'_' | b'-')
2783 });
2784 if valid {
2785 code.to_owned()
2786 } else {
2787 "client.unknown_failure".to_owned()
2788 }
2789 }
2790
2791 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
2792 if self.sync_needed != value {
2793 if self.begin_observation("syncular_status").is_ok() {
2794 self.sync_needed = value;
2795 let batch = ChangeAccumulator {
2796 status: true,
2797 ..ChangeAccumulator::default()
2798 };
2799 if self.finish_observation("syncular_status", batch).is_err() {
2800 self.rollback_observation("syncular_status");
2801 }
2802 } else {
2803 self.sync_needed = value;
2804 }
2805 }
2806 if value && interactive {
2807 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2808 }
2809 }
2810
2811 fn begin_observation(&self, name: &str) -> Result<(), String> {
2812 self.conn
2813 .execute_batch(&format!("SAVEPOINT {name}"))
2814 .map_err(|error| error.to_string())
2815 }
2816
2817 fn rollback_observation(&self, name: &str) {
2818 let _ = self
2819 .conn
2820 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
2821 }
2822
2823 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
2824 if !batch.touched() {
2825 self.conn
2826 .execute_batch(&format!("RELEASE {name}"))
2827 .map_err(|error| error.to_string())?;
2828 return Ok(());
2829 }
2830 let revision = self
2831 .local_revision()
2832 .checked_add(1)
2833 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
2834 self.conn
2835 .execute(
2836 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
2837 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
2838 )
2839 .map_err(|error| error.to_string())?;
2840 let status = batch.status.then(|| self.status_snapshot());
2841 let event = ClientChangeBatch {
2842 revision: revision.to_string(),
2843 tables: batch
2844 .tables
2845 .into_iter()
2846 .map(|(table, scope_keys)| TableChange {
2847 table,
2848 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
2849 })
2850 .collect(),
2851 windows: batch
2852 .windows
2853 .into_iter()
2854 .map(|((base_key, table), units)| WindowChange {
2855 base_key,
2856 table,
2857 units: units.into_iter().collect(),
2858 })
2859 .collect(),
2860 status,
2861 conflicts_changed: batch.conflicts,
2862 rejections_changed: batch.rejections,
2863 outcomes_changed: batch.outcomes,
2864 };
2865 self.conn
2866 .execute_batch(&format!("RELEASE {name}"))
2867 .map_err(|error| error.to_string())?;
2868 let mut diagnostic_tables = event
2869 .tables
2870 .iter()
2871 .map(|entry| entry.table.clone())
2872 .collect::<BTreeSet<_>>()
2873 .into_iter()
2874 .collect::<Vec<_>>();
2875 let mut diagnostic_windows = event
2876 .windows
2877 .iter()
2878 .map(|entry| entry.table.clone())
2879 .collect::<BTreeSet<_>>()
2880 .into_iter()
2881 .collect::<Vec<_>>();
2882 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
2883 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
2884 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
2885 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
2886 self.last_change = Some(DiagnosticLastChange {
2887 revision: event.revision.clone(),
2888 recorded_at_ms: self.clock_now_ms(),
2889 tables: diagnostic_tables,
2890 windows: diagnostic_windows,
2891 domains_truncated,
2892 status_changed: event.status.is_some(),
2893 conflicts_changed: event.conflicts_changed,
2894 rejections_changed: event.rejections_changed,
2895 outcomes_changed: event.outcomes_changed,
2896 });
2897 self.change_queue.push_back(event);
2898 Ok(())
2899 }
2900
2901 fn record_scope_map(
2902 &self,
2903 batch: &mut ChangeAccumulator,
2904 table_name: &str,
2905 scopes: &[(String, Vec<String>)],
2906 ) {
2907 let Some(table) = self.schema.table(table_name) else {
2908 return;
2909 };
2910 for (variable, values) in scopes {
2911 let Some(scope) = table
2912 .scope_variables
2913 .iter()
2914 .find(|scope| &scope.variable == variable)
2915 else {
2916 continue;
2917 };
2918 for value in values {
2919 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2920 }
2921 }
2922 }
2923
2924 fn record_row_scopes(
2926 &self,
2927 batch: &mut ChangeAccumulator,
2928 table_name: &str,
2929 row_id: &str,
2930 base: bool,
2931 ) -> bool {
2932 let Some(table) = self.schema.table(table_name) else {
2933 return false;
2934 };
2935 if table.scope_variables.is_empty() {
2936 return false;
2937 }
2938 let columns = table
2939 .scope_variables
2940 .iter()
2941 .map(|scope| quote_ident(&scope.column))
2942 .collect::<Vec<_>>()
2943 .join(", ");
2944 let full_table = if base {
2945 base_table(table_name)
2946 } else {
2947 visible_table(table_name)
2948 };
2949 let sql = format!(
2950 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
2951 quote_ident(&table.primary_key)
2952 );
2953 let Ok(mut stmt) = self.conn.prepare(&sql) else {
2954 return false;
2955 };
2956 let values = stmt.query_row(rusqlite::params![row_id], |row| {
2957 let mut values = Vec::with_capacity(table.scope_variables.len());
2958 for index in 0..table.scope_variables.len() {
2959 values.push(row.get::<_, Option<String>>(index)?);
2960 }
2961 Ok(values)
2962 });
2963 let Ok(values) = values else {
2964 return false;
2965 };
2966 let mut recorded = false;
2967 for (scope, value) in table.scope_variables.iter().zip(values) {
2968 if let Some(value) = value {
2969 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2970 recorded = true;
2971 }
2972 }
2973 recorded
2974 }
2975
2976 fn record_commit_changes(
2977 &self,
2978 batch: &mut ChangeAccumulator,
2979 tables: &[String],
2980 changes: &[ssp2::model::Change],
2981 ) {
2982 for change in changes {
2983 let Some(table_name) = tables.get(change.table_index as usize) else {
2984 continue;
2985 };
2986 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
2987 if let Some(table) = self.schema.table(table_name) {
2988 for (variable, value) in &change.scopes {
2989 if let Some(scope) = table
2990 .scope_variables
2991 .iter()
2992 .find(|scope| &scope.variable == variable)
2993 {
2994 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2995 precise = true;
2996 }
2997 }
2998 }
2999 if !precise {
3000 batch.table(table_name);
3001 }
3002 }
3003 }
3004
3005 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
3006 if effective.is_empty() {
3007 return false;
3008 }
3009 let Some(table) = self.schema.table(table_name) else {
3010 return false;
3011 };
3012 let mut clauses = Vec::new();
3013 let mut params = Vec::new();
3014 for (variable, values) in effective {
3015 let Some(column) = table.scope_column(variable) else {
3016 return false;
3017 };
3018 if values.is_empty() {
3019 return false;
3020 }
3021 let placeholders = values
3022 .iter()
3023 .map(|value| {
3024 params.push(SqlValue::Text(value.clone()));
3025 "?"
3026 })
3027 .collect::<Vec<_>>()
3028 .join(", ");
3029 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
3030 }
3031 let sql = format!(
3032 "SELECT 1 FROM {} WHERE {} LIMIT 1",
3033 base_table(table_name),
3034 clauses.join(" AND ")
3035 );
3036 self.conn
3037 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
3038 .is_ok()
3039 }
3040
3041 pub fn upgrading(&self) -> bool {
3043 self.upgrading
3044 }
3045
3046 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3052 let new_schema = parse_schema_json(schema_json)?;
3053 let marker: Option<i32> = self
3054 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
3055 .and_then(|v| v.parse().ok());
3056 self.schema = new_schema;
3057 if marker != Some(self.schema.version) {
3058 self.run_schema_reset()?;
3059 }
3060 self.prune_unknown_subscriptions()?;
3061 self.enqueue_startup_sync_if_needed();
3065 Ok(())
3066 }
3067
3068 fn enqueue_startup_sync_if_needed(&mut self) {
3069 let startup_work = !self.stopped
3070 && (!self.outbox.is_empty()
3071 || self.subs.iter().any(|sub| sub.state == SubState::Active));
3072 if startup_work {
3073 self.sync_needed = true;
3074 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3075 }
3076 }
3077
3078 fn clear_satisfied_persisted_schema_floor(&mut self) {
3085 let satisfied = self
3086 .schema_floor
3087 .as_ref()
3088 .and_then(|floor| floor.required_schema_version)
3089 .is_some_and(|required| self.schema.version >= required);
3090 if !satisfied {
3091 return;
3092 }
3093 self.schema_floor = None;
3094 self.stopped = false;
3095 self.delete_meta(SCHEMA_FLOOR_KEY);
3096 }
3097
3098 fn run_schema_reset(&mut self) -> Result<(), String> {
3104 self.begin_observation("syncular_schema_reset")?;
3105 let mut batch = ChangeAccumulator::default();
3106 let result = self.run_schema_reset_observed(&mut batch, true);
3107 if let Err(error) = result {
3108 self.rollback_observation("syncular_schema_reset");
3109 return Err(error);
3110 }
3111 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
3112 self.rollback_observation("syncular_schema_reset");
3113 return Err(error);
3114 }
3115 Ok(())
3116 }
3117
3118 fn run_schema_reset_observed(
3119 &mut self,
3120 batch: &mut ChangeAccumulator,
3121 drop_incompatible: bool,
3122 ) -> Result<(), String> {
3123 self.upgrading = true;
3124 batch.status = true;
3125 for table in &self.schema.tables {
3126 batch.table(&table.name);
3127 }
3128 for (base_key, unit, table) in self.load_registered_window_units() {
3129 batch.window(&base_key, &table, &unit);
3130 }
3131 self.insert_sql.borrow_mut().clear();
3133 self.overlay_dirty.set(true);
3134 let virtual_tables: Vec<String> = {
3142 let mut stmt = self
3143 .conn
3144 .prepare(
3145 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
3146 )
3147 .map_err(|e| e.to_string())?;
3148 let rows = stmt
3149 .query_map([], |row| row.get::<_, String>(0))
3150 .map_err(|e| e.to_string())?;
3151 rows.filter_map(Result::ok)
3152 .filter(|name| is_synced_table_name(name))
3153 .collect()
3154 };
3155 for name in virtual_tables {
3156 self.conn
3157 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3158 .map_err(|e| e.to_string())?;
3159 }
3160 let existing: Vec<String> = {
3161 let mut stmt = self
3162 .conn
3163 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
3164 .map_err(|e| e.to_string())?;
3165 let rows = stmt
3166 .query_map([], |row| row.get::<_, String>(0))
3167 .map_err(|e| e.to_string())?;
3168 rows.filter_map(Result::ok)
3169 .filter(|name| is_synced_table_name(name))
3170 .collect()
3171 };
3172 for name in &existing {
3173 if let Some(table) = name.strip_prefix("_syncular_base_") {
3174 batch.table(table);
3175 } else if !name.starts_with("_syncular_") {
3176 batch.table(name);
3177 }
3178 }
3179 for name in existing {
3180 self.conn
3181 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3182 .map_err(|e| e.to_string())?;
3183 }
3184 self.create_synced_tables()?;
3186 for sub in &mut self.subs {
3188 sub.cursor = -1;
3189 sub.bootstrap_state = None;
3190 sub.effective = None;
3191 sub.state = SubState::Active;
3192 sub.reason_code = None;
3193 sub.synced_once = false;
3194 }
3195 let subs = self.subs.clone();
3196 for sub in &subs {
3197 self.persist_sub(sub);
3198 }
3199 self.stopped = false;
3201 self.schema_floor = None;
3202 self.delete_meta(SCHEMA_FLOOR_KEY);
3203 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
3205 if drop_incompatible && self.drop_incompatible_outbox()? {
3209 batch.rejections = true;
3210 batch.status = true;
3211 batch.outcomes = true;
3212 }
3213 self.rebuild_overlay();
3215 Ok(())
3216 }
3217
3218 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
3222 let schema = &self.schema;
3223 let incompatible = self
3224 .outbox
3225 .iter()
3226 .filter(|commit| {
3227 commit.ops.iter().any(|op| {
3228 if !op.upsert {
3229 return false;
3230 }
3231 match schema.table(&op.table) {
3232 None => true,
3233 Some(table) => op.values.as_ref().is_some_and(|values| {
3234 values
3235 .keys()
3236 .any(|key| !table.columns.iter().any(|c| &c.name == key))
3237 }),
3238 }
3239 })
3240 })
3241 .cloned()
3242 .collect::<Vec<_>>();
3243 if incompatible.is_empty() {
3244 return Ok(false);
3245 }
3246 let mut rejections = Vec::new();
3247 for commit in &incompatible {
3248 let results = commit
3249 .ops
3250 .iter()
3251 .enumerate()
3252 .map(|(op_index, operation)| {
3253 let rejection = RejectionRecord {
3254 client_commit_id: commit.client_commit_id.clone(),
3255 op_index: op_index as i32,
3256 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
3257 message: "the persisted commit cannot encode under the current schema"
3258 .to_owned(),
3259 retryable: false,
3260 details: None,
3261 operation: Some(CommitOperation::from(operation)),
3262 };
3263 rejections.push(rejection.clone());
3264 CommitOperationOutcome::Error { rejection }
3265 })
3266 .collect::<Vec<_>>();
3267 self.persist_commit_outcome(
3268 &commit.client_commit_id,
3269 CommitOutcomeStatus::Rejected,
3270 &results,
3271 Some(&commit.ops),
3272 )?;
3273 self.delete_outbox_persisted(&commit.client_commit_id)?;
3274 }
3275 self.prune_commit_outcomes()?;
3276 let incompatible_ids = incompatible
3277 .iter()
3278 .map(|commit| commit.client_commit_id.as_str())
3279 .collect::<BTreeSet<_>>();
3280 self.outbox
3281 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
3282 self.rejections.extend(rejections);
3283 Ok(true)
3284 }
3285
3286 fn create_synced_tables(&self) -> Result<(), String> {
3289 for table in &self.schema.tables {
3290 for (full, index_prefix) in [
3294 (base_table(&table.name), "_syncular_base_"),
3295 (visible_table(&table.name), ""),
3296 ] {
3297 let mut cols: Vec<String> =
3298 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
3299 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
3300 let sql = format!(
3301 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
3302 cols.join(", "),
3303 quote_ident(&table.primary_key)
3304 );
3305 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
3306 for index in &table.indexes {
3310 let unique = if index.unique { "UNIQUE " } else { "" };
3311 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
3312 let cols_sql = index
3313 .columns
3314 .iter()
3315 .map(|c| quote_ident(c))
3316 .collect::<Vec<_>>()
3317 .join(", ");
3318 let index_sql = format!(
3319 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
3320 );
3321 self.conn
3322 .execute(&index_sql, [])
3323 .map_err(|e| e.to_string())?;
3324 }
3325 }
3326 }
3327 for table in &self.schema.tables {
3330 for index in &table.fts_indexes {
3331 self.create_fts_projection(table, index)?;
3332 }
3333 }
3334 Ok(())
3335 }
3336
3337 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
3338 let count: i64 = self
3339 .conn
3340 .query_row(
3341 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
3342 rusqlite::params![index.name],
3343 |row| row.get(0),
3344 )
3345 .map_err(|e| e.to_string())?;
3346 Ok(count > 0)
3347 }
3348
3349 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
3350 for suffix in ["ai", "ad", "au"] {
3351 self.conn
3352 .execute(
3353 &format!(
3354 "DROP TRIGGER IF EXISTS {}",
3355 quote_ident(&format!("{}_{suffix}", index.name))
3356 ),
3357 [],
3358 )
3359 .map_err(|e| e.to_string())?;
3360 }
3361 Ok(())
3362 }
3363
3364 fn create_fts_triggers(
3365 &self,
3366 table: &TableSchema,
3367 index: &FtsIndexSchema,
3368 ) -> Result<(), String> {
3369 let fts = quote_ident(&index.name);
3370 let source = visible_table(&table.name);
3371 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
3372 let pk = quote_ident(&table.primary_key);
3373 let projection_columns = std::iter::once(source_id.clone())
3374 .chain(index.columns.iter().map(|column| quote_ident(column)))
3375 .collect::<Vec<_>>()
3376 .join(", ");
3377 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
3378 .chain(
3379 index
3380 .columns
3381 .iter()
3382 .map(|column| format!("new.{}", quote_ident(column))),
3383 )
3384 .collect::<Vec<_>>()
3385 .join(", ");
3386 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
3387 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
3388 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
3389 let sql = format!(
3390 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
3391 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
3392 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
3393 ai = quote_ident(&format!("{}_ai", index.name)),
3394 ad = quote_ident(&format!("{}_ad", index.name)),
3395 au = quote_ident(&format!("{}_au", index.name)),
3396 );
3397 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
3398 }
3399
3400 fn rebuild_fts_projection(
3401 &self,
3402 table: &TableSchema,
3403 index: &FtsIndexSchema,
3404 ) -> Result<(), String> {
3405 let fts = quote_ident(&index.name);
3406 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
3407 let pk = quote_ident(&table.primary_key);
3408 let indexed_columns = index
3409 .columns
3410 .iter()
3411 .map(|column| quote_ident(column))
3412 .collect::<Vec<_>>();
3413 let projection_columns = std::iter::once(source_id)
3414 .chain(indexed_columns.iter().cloned())
3415 .collect::<Vec<_>>()
3416 .join(", ");
3417 let sql = format!(
3418 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
3419 columns = indexed_columns.join(", "),
3420 source = visible_table(&table.name),
3421 );
3422 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
3423 }
3424
3425 fn create_fts_projection(
3426 &self,
3427 table: &TableSchema,
3428 index: &FtsIndexSchema,
3429 ) -> Result<(), String> {
3430 let existed = self.fts_projection_exists(index)?;
3431 let tokenizer = index.tokenize.replace('\'', "''");
3432 let columns = index
3433 .columns
3434 .iter()
3435 .map(|column| quote_ident(column))
3436 .collect::<Vec<_>>()
3437 .join(", ");
3438 let sql = format!(
3439 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
3440 fts = quote_ident(&index.name),
3441 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
3442 );
3443 self.conn.execute(&sql, []).map_err(|error| {
3444 format!(
3445 "cannot create local FTS5 projection {:?}: {error}",
3446 index.name
3447 )
3448 })?;
3449 self.create_fts_triggers(table, index)?;
3450 if !existed {
3451 self.rebuild_fts_projection(table, index)?;
3452 }
3453 Ok(())
3454 }
3455
3456 fn persist_sub(&self, sub: &Subscription) {
3459 let state = serde_json::json!({
3460 "requested": scope_map_to_json(&sub.requested),
3461 "params": sub.params,
3462 "cursor": sub.cursor,
3463 "bootstrapState": sub.bootstrap_state,
3464 "status": sub.state.name(),
3465 "reasonCode": sub.reason_code,
3466 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3467 "syncedOnce": sub.synced_once,
3468 });
3469 let _ = self.conn.execute(
3470 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
3471 rusqlite::params![sub.id, sub.table, state.to_string()],
3472 );
3473 }
3474
3475 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
3476 let ops: Vec<Value> = commit
3477 .ops
3478 .iter()
3479 .map(|op| {
3480 serde_json::json!({
3481 "op": if op.upsert { "upsert" } else { "delete" },
3482 "table": op.table,
3483 "rowId": op.row_id,
3484 "baseVersion": op.base_version,
3485 "values": op.values.clone().map(Value::Object),
3486 "changedFields": op.changed_fields,
3487 })
3488 })
3489 .collect();
3490 let _ = self.conn.execute(
3491 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
3492 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
3493 );
3494 }
3495
3496 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
3497 self.conn
3498 .execute(
3499 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
3500 rusqlite::params![client_commit_id],
3501 )
3502 .map(|_| ())
3503 .map_err(|error| error.to_string())
3504 }
3505
3506 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
3507 match status {
3508 CommitOutcomeStatus::Applied => "applied",
3509 CommitOutcomeStatus::Cached => "cached",
3510 CommitOutcomeStatus::Conflict => "conflict",
3511 CommitOutcomeStatus::Rejected => "rejected",
3512 }
3513 }
3514
3515 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
3516 match resolution {
3517 CommitOutcomeResolution::Active => "active",
3518 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
3519 CommitOutcomeResolution::Superseded => "superseded",
3520 CommitOutcomeResolution::Dismissed => "dismissed",
3521 }
3522 }
3523
3524 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
3525 match value {
3526 "applied" => Ok(CommitOutcomeStatus::Applied),
3527 "cached" => Ok(CommitOutcomeStatus::Cached),
3528 "conflict" => Ok(CommitOutcomeStatus::Conflict),
3529 "rejected" => Ok(CommitOutcomeStatus::Rejected),
3530 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
3531 }
3532 }
3533
3534 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
3535 match value {
3536 "active" => Ok(CommitOutcomeResolution::Active),
3537 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
3538 "superseded" => Ok(CommitOutcomeResolution::Superseded),
3539 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
3540 _ => Err(format!(
3541 "invalid persisted commit outcome resolution {value:?}"
3542 )),
3543 }
3544 }
3545
3546 fn persist_commit_outcome(
3547 &self,
3548 client_commit_id: &str,
3549 status: CommitOutcomeStatus,
3550 results: &[CommitOperationOutcome],
3551 operations: Option<&[OutboxOp]>,
3552 ) -> Result<(), String> {
3553 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
3554 let operations_json = operations
3555 .map(|items| {
3556 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
3557 })
3558 .transpose()
3559 .map_err(|error| error.to_string())?;
3560 self.conn
3561 .execute(
3562 "INSERT INTO _syncular_commit_outcomes (
3563 client_commit_id, status, recorded_at_ms, results_json,
3564 operations_json, resolution
3565 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
3566 rusqlite::params![
3567 client_commit_id,
3568 Self::outcome_status_name(status),
3569 self.clock_now_ms(),
3570 results_json,
3571 operations_json
3572 ],
3573 )
3574 .map(|_| ())
3575 .map_err(|error| error.to_string())
3576 }
3577
3578 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
3579 let StoredCommitOutcomeRow {
3580 sequence,
3581 client_commit_id,
3582 status,
3583 recorded_at_ms,
3584 results_json,
3585 operations_json,
3586 resolution,
3587 resolved_at_ms,
3588 replacement_client_commit_id,
3589 } = row;
3590 Ok(CommitOutcome {
3591 sequence,
3592 client_commit_id,
3593 status: Self::parse_outcome_status(&status)?,
3594 recorded_at_ms,
3595 results: serde_json::from_str(&results_json)
3596 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
3597 operations: operations_json
3598 .map(|value| {
3599 serde_json::from_str(&value).map_err(|error| {
3600 format!("invalid persisted commit outcome operations: {error}")
3601 })
3602 })
3603 .transpose()?,
3604 resolution: Self::parse_outcome_resolution(&resolution)?,
3605 resolved_at_ms,
3606 replacement_client_commit_id,
3607 })
3608 }
3609
3610 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
3611 let row = self
3612 .conn
3613 .query_row(
3614 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
3615 resolution, resolved_at_ms, replacement_client_commit_id
3616 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
3617 rusqlite::params![client_commit_id],
3618 |row| {
3619 Ok(StoredCommitOutcomeRow {
3620 sequence: row.get(0)?,
3621 client_commit_id: row.get(1)?,
3622 status: row.get(2)?,
3623 recorded_at_ms: row.get(3)?,
3624 results_json: row.get(4)?,
3625 operations_json: row.get(5)?,
3626 resolution: row.get(6)?,
3627 resolved_at_ms: row.get(7)?,
3628 replacement_client_commit_id: row.get(8)?,
3629 })
3630 },
3631 )
3632 .optional()
3633 .map_err(|error| error.to_string())?;
3634 row.map(Self::outcome_from_row).transpose()
3635 }
3636
3637 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
3638 if query.limit == Some(0) {
3639 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
3640 }
3641 let mut sql = String::from(
3642 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
3643 resolution, resolved_at_ms, replacement_client_commit_id
3644 FROM _syncular_commit_outcomes",
3645 );
3646 if query.active_only {
3647 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
3648 }
3649 sql.push_str(" ORDER BY seq DESC");
3650 if let Some(limit) = query.limit {
3651 sql.push_str(&format!(" LIMIT {limit}"));
3652 }
3653 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
3654 let rows = stmt
3655 .query_map([], |row| {
3656 Ok(StoredCommitOutcomeRow {
3657 sequence: row.get(0)?,
3658 client_commit_id: row.get(1)?,
3659 status: row.get(2)?,
3660 recorded_at_ms: row.get(3)?,
3661 results_json: row.get(4)?,
3662 operations_json: row.get(5)?,
3663 resolution: row.get(6)?,
3664 resolved_at_ms: row.get(7)?,
3665 replacement_client_commit_id: row.get(8)?,
3666 })
3667 })
3668 .map_err(|error| error.to_string())?;
3669 let mut outcomes = Vec::new();
3670 for row in rows {
3671 outcomes.push(Self::outcome_from_row(
3672 row.map_err(|error| error.to_string())?,
3673 )?);
3674 }
3675 Ok(outcomes)
3676 }
3677
3678 fn prune_commit_outcomes(&self) -> Result<(), String> {
3679 #[cfg(test)]
3680 self.outcome_prune_count
3681 .set(self.outcome_prune_count.get() + 1);
3682 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
3683 let count = self
3684 .conn
3685 .query_row(
3686 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
3687 [],
3688 |row| row.get::<_, i64>(0),
3689 )
3690 .map_err(|error| error.to_string())? as usize;
3691 let excess = count.saturating_sub(max_entries);
3692 if excess == 0 {
3693 return Ok(());
3694 }
3695 let mut stmt = self
3696 .conn
3697 .prepare(
3698 "SELECT seq FROM _syncular_commit_outcomes
3699 WHERE status IN ('applied', 'cached') OR resolution != 'active'
3700 ORDER BY seq ASC LIMIT ?1",
3701 )
3702 .map_err(|error| error.to_string())?;
3703 let rows = stmt
3704 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
3705 .map_err(|error| error.to_string())?;
3706 let sequences = rows
3707 .collect::<Result<Vec<_>, _>>()
3708 .map_err(|error| error.to_string())?;
3709 drop(stmt);
3710 for sequence in sequences {
3711 self.conn
3712 .execute(
3713 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
3714 rusqlite::params![sequence],
3715 )
3716 .map_err(|error| error.to_string())?;
3717 }
3718 Ok(())
3719 }
3720
3721 pub fn subscribe(
3724 &mut self,
3725 id: String,
3726 table: String,
3727 scopes: Vec<(String, Vec<String>)>,
3728 params: Option<String>,
3729 ) -> Result<(), String> {
3730 if self.schema.table(&table).is_none() {
3731 return Err(format!("unknown table {table:?}"));
3732 }
3733 if let Some(existing) = self.subs.iter().find(|subscription| subscription.id == id) {
3734 let same_intent = existing.table == table
3735 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
3736 && existing.params == params;
3737 if same_intent {
3738 return Ok(());
3739 }
3740 return Err(
3741 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
3742 .to_owned(),
3743 );
3744 }
3745 let sub = Subscription {
3746 id: id.clone(),
3747 table,
3748 requested: scopes,
3749 params,
3750 cursor: -1,
3751 bootstrap_state: None,
3752 state: SubState::Active,
3753 reason_code: None,
3754 effective: None,
3755 synced_once: false,
3756 };
3757 self.persist_sub(&sub);
3758 self.subs.push(sub);
3759 Ok(())
3760 }
3761
3762 pub fn unsubscribe(&mut self, id: &str) {
3763 self.subs.retain(|s| s.id != id);
3764 let _ = self.conn.execute(
3765 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
3766 rusqlite::params![id],
3767 );
3768 }
3769
3770 pub fn set_window(
3778 &mut self,
3779 base: &WindowBase,
3780 units: &[String],
3781 ) -> Result<CommandEffects, String> {
3782 let table = self
3783 .schema
3784 .table(&base.table)
3785 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
3786 if table.scope_column(&base.variable).is_none() {
3787 return Err(format!(
3788 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
3789 base.table, base.variable
3790 ));
3791 }
3792 let base_key = window_base_key(base);
3793 let wanted: std::collections::HashSet<&String> = units.iter().collect();
3794 let live = self.load_window_units(&base_key);
3795 self.begin_observation("syncular_window")?;
3796 let mut batch = ChangeAccumulator::default();
3797 let mut changed = false;
3798
3799 for unit in units {
3801 if live.iter().any(|(u, _)| u == unit) {
3802 continue;
3803 }
3804 let sub_id = derive_sub_id(base, unit);
3805 self.delete_pending_evict(&sub_id);
3806 self.insert_window_unit(&base_key, unit, &sub_id);
3807 self.subscribe(
3808 sub_id,
3809 base.table.clone(),
3810 unit_scopes(base, unit),
3811 base.params.clone(),
3812 )?;
3813 batch.window(&base_key, &base.table, unit);
3814 changed = true;
3815 }
3816
3817 for (unit, sub_id) in live {
3819 if wanted.contains(&unit) {
3820 continue;
3821 }
3822 let effective = self
3823 .subs
3824 .iter()
3825 .find(|sub| sub.id == sub_id)
3826 .and_then(|sub| sub.effective.clone())
3827 .unwrap_or_else(|| unit_scopes(base, &unit));
3828 self.record_scope_map(&mut batch, &base.table, &effective);
3829 batch.window(&base_key, &base.table, &unit);
3830 self.evict_unit(&base_key, base, &unit, &sub_id);
3831 changed = true;
3832 }
3833 if let Err(error) = self.finish_observation("syncular_window", batch) {
3834 self.rollback_observation("syncular_window");
3835 return Err(error);
3836 }
3837 Ok(if changed {
3838 CommandEffects::interactive()
3839 } else {
3840 CommandEffects::none()
3841 })
3842 }
3843
3844 pub fn window_state(&self, base: &WindowBase) -> WindowState {
3849 let mut units = Vec::new();
3850 let mut pending = Vec::new();
3851 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
3852 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
3853 Some(sub) => {
3854 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
3855 }
3856 None => true,
3857 };
3858 if is_pending {
3859 pending.push(unit.clone());
3860 }
3861 units.push(unit);
3862 }
3863 WindowState { units, pending }
3864 }
3865
3866 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
3867 let mut stmt = match self
3868 .conn
3869 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
3870 {
3871 Ok(stmt) => stmt,
3872 Err(_) => return Vec::new(),
3873 };
3874 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
3875 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
3876 });
3877 match rows {
3878 Ok(rows) => rows.filter_map(Result::ok).collect(),
3879 Err(_) => Vec::new(),
3880 }
3881 }
3882
3883 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
3884 let mut stmt = match self.conn.prepare(
3885 "SELECT windows.base, windows.unit, subscriptions.tbl
3886 FROM _syncular_windows AS windows
3887 JOIN _syncular_subscriptions AS subscriptions
3888 ON subscriptions.id = windows.sub_id
3889 ORDER BY windows.base, windows.unit",
3890 ) {
3891 Ok(stmt) => stmt,
3892 Err(_) => return Vec::new(),
3893 };
3894 let rows = stmt.query_map([], |row| {
3895 Ok((
3896 row.get::<_, String>(0)?,
3897 row.get::<_, String>(1)?,
3898 row.get::<_, String>(2)?,
3899 ))
3900 });
3901 match rows {
3902 Ok(rows) => rows.filter_map(Result::ok).collect(),
3903 Err(_) => Vec::new(),
3904 }
3905 }
3906
3907 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
3908 self.conn
3909 .query_row(
3910 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
3911 rusqlite::params![sub_id],
3912 |row| Ok((row.get(0)?, row.get(1)?)),
3913 )
3914 .ok()
3915 }
3916
3917 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
3918 let _ = self.conn.execute(
3919 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
3920 rusqlite::params![base_key, unit, sub_id],
3921 );
3922 }
3923
3924 fn delete_window_unit(&self, base_key: &str, unit: &str) {
3925 let _ = self.conn.execute(
3926 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
3927 rusqlite::params![base_key, unit],
3928 );
3929 }
3930
3931 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
3938 let effective = self
3939 .subs
3940 .iter()
3941 .find(|s| s.id == sub_id)
3942 .and_then(|s| s.effective.clone())
3943 .unwrap_or_else(|| unit_scopes(base, unit));
3944 let pinned = self.pinned_row_ids(&base.table);
3945 let deferred = self
3946 .evict_scope_rows(&base.table, &effective, &pinned)
3947 .unwrap_or(false);
3948 self.delete_window_unit(base_key, unit);
3949 self.unsubscribe(sub_id);
3950 if deferred {
3951 self.save_pending_evict(sub_id, &base.table, &effective);
3952 } else {
3953 self.delete_pending_evict(sub_id);
3954 }
3955 self.rebuild_overlay();
3956 }
3957
3958 fn evict_scope_rows(
3962 &mut self,
3963 table_name: &str,
3964 effective: &[(String, Vec<String>)],
3965 pinned: &std::collections::HashSet<String>,
3966 ) -> Result<bool, ()> {
3967 if effective.is_empty() {
3968 return Ok(false);
3969 }
3970 let table = self.schema.table(table_name).ok_or(())?.clone();
3971 let mut clauses = Vec::new();
3972 let mut params: Vec<SqlValue> = Vec::new();
3973 for (variable, values) in effective {
3974 let column = table.scope_column(variable).ok_or(())?;
3975 let placeholders: Vec<String> = values
3976 .iter()
3977 .map(|v| {
3978 params.push(SqlValue::Text(v.clone()));
3979 "?".to_owned()
3980 })
3981 .collect();
3982 clauses.push(format!(
3983 "{} IN ({})",
3984 quote_ident(column),
3985 placeholders.join(", ")
3986 ));
3987 }
3988 let mut sql = format!(
3989 "DELETE FROM {} WHERE {}",
3990 base_table(table_name),
3991 clauses.join(" AND ")
3992 );
3993 if !pinned.is_empty() {
3994 let pk = quote_ident(&table.primary_key);
3995 let holes: Vec<String> = pinned
3996 .iter()
3997 .map(|id| {
3998 params.push(SqlValue::Text(id.clone()));
3999 "?".to_owned()
4000 })
4001 .collect();
4002 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
4003 }
4004 self.overlay_dirty.set(true);
4005 self.conn
4006 .execute(&sql, rusqlite::params_from_iter(params))
4007 .map_err(|_| ())?;
4008 if pinned.is_empty() {
4009 return Ok(false);
4010 }
4011 let mut where_params: Vec<SqlValue> = Vec::new();
4014 let mut where_clauses = Vec::new();
4015 for (variable, values) in effective {
4016 let column = table.scope_column(variable).ok_or(())?;
4017 let placeholders: Vec<String> = values
4018 .iter()
4019 .map(|v| {
4020 where_params.push(SqlValue::Text(v.clone()));
4021 "?".to_owned()
4022 })
4023 .collect();
4024 where_clauses.push(format!(
4025 "{} IN ({})",
4026 quote_ident(column),
4027 placeholders.join(", ")
4028 ));
4029 }
4030 let pk = quote_ident(&table.primary_key);
4031 let select = format!(
4032 "SELECT {} FROM {} WHERE {}",
4033 pk,
4034 base_table(table_name),
4035 where_clauses.join(" AND ")
4036 );
4037 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
4038 let survivors: Vec<String> = stmt
4039 .query_map(rusqlite::params_from_iter(where_params), |row| {
4040 row.get::<_, String>(0)
4041 })
4042 .map_err(|_| ())?
4043 .filter_map(Result::ok)
4044 .collect();
4045 Ok(survivors.iter().any(|id| pinned.contains(id)))
4046 }
4047
4048 fn drain_pending_evictions(&mut self) {
4052 let pending = self.load_pending_evictions();
4053 if pending.is_empty() {
4054 return;
4055 }
4056 for (sub_id, table_name, effective) in pending {
4057 if self.schema.table(&table_name).is_none() {
4058 self.delete_pending_evict(&sub_id);
4059 continue;
4060 }
4061 let pinned = self.pinned_row_ids(&table_name);
4062 let deferred = self
4063 .evict_scope_rows(&table_name, &effective, &pinned)
4064 .unwrap_or(false);
4065 if !deferred {
4066 self.delete_pending_evict(&sub_id);
4067 }
4068 }
4069 self.rebuild_overlay_if_dirty();
4070 }
4071
4072 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
4075 let mut pinned = std::collections::HashSet::new();
4076 for commit in &self.outbox {
4077 for op in &commit.ops {
4078 if op.table == table {
4079 pinned.insert(op.row_id.clone());
4080 }
4081 }
4082 }
4083 pinned
4084 }
4085
4086 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
4087 let _ = self.conn.execute(
4088 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
4089 VALUES (?1, ?2, ?3)",
4090 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
4091 );
4092 }
4093
4094 fn delete_pending_evict(&self, sub_id: &str) {
4095 let _ = self.conn.execute(
4096 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
4097 rusqlite::params![sub_id],
4098 );
4099 }
4100
4101 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
4102 let mut stmt = match self
4103 .conn
4104 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
4105 {
4106 Ok(stmt) => stmt,
4107 Err(_) => return Vec::new(),
4108 };
4109 let rows = stmt.query_map([], |row| {
4110 Ok((
4111 row.get::<_, String>(0)?,
4112 row.get::<_, String>(1)?,
4113 row.get::<_, String>(2)?,
4114 ))
4115 });
4116 let mut out = Vec::new();
4117 if let Ok(rows) = rows {
4118 for entry in rows.filter_map(Result::ok) {
4119 let (sub_id, table, json) = entry;
4120 if let Ok(value) = serde_json::from_str::<Value>(&json) {
4121 if let Ok(effective) = json_to_scope_map(&value) {
4122 out.push((sub_id, table, effective));
4123 }
4124 }
4125 }
4126 }
4127 out
4128 }
4129
4130 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
4132 if mutations.is_empty() {
4133 return Err("a commit must contain at least one operation (§6.1)".to_owned());
4134 }
4135 let mut ops = Vec::with_capacity(mutations.len());
4136 for mutation in mutations {
4137 match mutation {
4138 Mutation::Upsert {
4139 table,
4140 values,
4141 base_version,
4142 } => {
4143 let schema_table = self
4144 .schema
4145 .table(&table)
4146 .ok_or_else(|| format!("unknown table {table:?}"))?;
4147 let values = normalize_values_casing(schema_table, values)?;
4151 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
4152 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
4155 ops.push(OutboxOp {
4156 upsert: true,
4157 table,
4158 row_id,
4159 base_version,
4160 values: Some(values),
4161 changed_fields: None,
4162 });
4163 }
4164 Mutation::Delete {
4165 table,
4166 row_id,
4167 base_version,
4168 } => {
4169 if self.schema.table(&table).is_none() {
4170 return Err(format!("unknown table {table:?}"));
4171 }
4172 ops.push(OutboxOp {
4173 upsert: false,
4174 table,
4175 row_id,
4176 base_version,
4177 values: None,
4178 changed_fields: None,
4179 });
4180 }
4181 }
4182 }
4183 self.record_outbox_commit(ops)
4184 }
4185
4186 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
4187 let commit = OutboxCommit {
4188 client_commit_id: uuid::Uuid::new_v4().to_string(),
4189 ops,
4190 };
4191 self.begin_observation("syncular_mutation")?;
4192 let mut batch = ChangeAccumulator::default();
4193 for op in &commit.ops {
4194 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
4195 if let Some(values) = &op.values {
4196 if let Some(table) = self.schema.table(&op.table) {
4197 for scope in &table.scope_variables {
4198 if let Some(Value::String(value)) = values.get(&scope.column) {
4199 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
4200 precise = true;
4201 }
4202 }
4203 }
4204 }
4205 if !precise {
4206 batch.table(&op.table);
4207 }
4208 }
4209 self.persist_outbox_insert(&commit);
4210 let id = commit.client_commit_id.clone();
4211 self.outbox.push(commit);
4212 self.overlay_dirty.set(true);
4213 self.rebuild_overlay();
4214 batch.status = true;
4215 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
4216 self.rollback_observation("syncular_mutation");
4217 return Err(error);
4218 }
4219 Ok(id)
4220 }
4221
4222 pub fn patch(
4227 &mut self,
4228 table: &str,
4229 row_id: &str,
4230 partial: Map<String, Value>,
4231 base_version: Option<i64>,
4232 ) -> Result<String, String> {
4233 let schema_table = self
4234 .schema
4235 .table(table)
4236 .ok_or_else(|| format!("unknown table {table:?}"))?;
4237 let partial = normalize_values_casing(schema_table, partial)?;
4238 let mut values = self
4239 .read_rows(table)?
4240 .into_iter()
4241 .find(|row| row.row_id == row_id)
4242 .map(|row| row.values)
4243 .ok_or_else(|| {
4244 format!(
4245 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
4246 )
4247 })?;
4248 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
4249 changed_fields.sort();
4250 values.extend(partial);
4251 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
4252 if row_id_from_values != row_id {
4253 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
4254 }
4255 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
4256 self.record_outbox_commit(vec![OutboxOp {
4257 upsert: true,
4258 table: table.to_owned(),
4259 row_id: row_id.to_owned(),
4260 base_version,
4261 values: Some(values),
4262 changed_fields: Some(changed_fields),
4263 }])
4264 }
4265
4266 pub fn pending_commit_ids(&self) -> Vec<String> {
4267 self.outbox
4268 .iter()
4269 .map(|c| c.client_commit_id.clone())
4270 .collect()
4271 }
4272
4273 pub fn conflicts(&self) -> &[ConflictRecord] {
4274 &self.conflicts
4275 }
4276
4277 pub fn rejections(&self) -> &[RejectionRecord] {
4278 &self.rejections
4279 }
4280
4281 pub fn resolve_commit_outcome(
4282 &mut self,
4283 input: ResolveCommitOutcomeInput,
4284 ) -> Result<CommitOutcome, String> {
4285 let current = self
4286 .commit_outcome(&input.client_commit_id)?
4287 .ok_or_else(|| {
4288 format!(
4289 "sync.outcome_not_found: no durable outcome exists for {:?}",
4290 input.client_commit_id
4291 )
4292 })?;
4293 if current.resolution != CommitOutcomeResolution::Active {
4294 return Ok(current);
4295 }
4296 if input.resolution == CommitOutcomeResolution::Active {
4297 return Err("sync.invalid_request: resolution must leave active state".to_owned());
4298 }
4299 match input.resolution {
4300 CommitOutcomeResolution::Superseded => {
4301 let replacement = input
4302 .replacement_client_commit_id
4303 .as_deref()
4304 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
4305 .ok_or_else(|| {
4306 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
4307 .to_owned()
4308 })?;
4309 let _ = replacement;
4310 }
4311 _ if input.replacement_client_commit_id.is_some() => {
4312 return Err(
4313 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
4314 .to_owned(),
4315 );
4316 }
4317 _ => {}
4318 }
4319 let allowed = match current.status {
4320 CommitOutcomeStatus::Conflict => matches!(
4321 input.resolution,
4322 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
4323 ),
4324 CommitOutcomeStatus::Rejected => {
4325 input.resolution == CommitOutcomeResolution::Superseded
4326 }
4327 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
4328 input.resolution == CommitOutcomeResolution::Dismissed
4329 }
4330 };
4331 if !allowed {
4332 return Err(format!(
4333 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
4334 input.resolution, current.status
4335 ));
4336 }
4337
4338 self.begin_observation("syncular_outcome_resolution")?;
4339 let result = (|| {
4340 self.conn
4341 .execute(
4342 "UPDATE _syncular_commit_outcomes
4343 SET resolution = ?1, resolved_at_ms = ?2,
4344 replacement_client_commit_id = ?3
4345 WHERE client_commit_id = ?4 AND resolution = 'active'",
4346 rusqlite::params![
4347 Self::outcome_resolution_name(input.resolution),
4348 self.clock_now_ms(),
4349 input.replacement_client_commit_id,
4350 input.client_commit_id
4351 ],
4352 )
4353 .map_err(|error| error.to_string())?;
4354 let resolved = self
4355 .commit_outcome(¤t.client_commit_id)?
4356 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
4357 self.prune_commit_outcomes()?;
4358 Ok(resolved)
4359 })();
4360 let resolved = match result {
4361 Ok(outcome) => outcome,
4362 Err(error) => {
4363 self.rollback_observation("syncular_outcome_resolution");
4364 return Err(error);
4365 }
4366 };
4367 self.conflicts
4368 .retain(|record| record.client_commit_id != current.client_commit_id);
4369 self.rejections
4370 .retain(|record| record.client_commit_id != current.client_commit_id);
4371 let batch = ChangeAccumulator {
4372 conflicts: current.status == CommitOutcomeStatus::Conflict,
4373 rejections: current.status == CommitOutcomeStatus::Rejected,
4374 outcomes: true,
4375 ..ChangeAccumulator::default()
4376 };
4377 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
4378 self.rollback_observation("syncular_outcome_resolution");
4379 return Err(error);
4380 }
4381 Ok(resolved)
4382 }
4383
4384 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
4385 self.schema_floor.as_ref()
4386 }
4387
4388 pub fn lease_state(&self) -> Option<&LeaseState> {
4390 self.lease_state.as_ref()
4391 }
4392
4393 fn record_lease_error(&mut self, code: &str) {
4396 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
4397 return;
4398 }
4399 let mut next = self.lease_state.clone().unwrap_or_default();
4400 next.error_code = Some(code.to_owned());
4401 self.set_lease_state(Some(next));
4402 }
4403
4404 fn set_lease_state(&mut self, next: Option<LeaseState>) {
4405 if self.lease_state == next {
4406 return;
4407 }
4408 if self.begin_observation("syncular_lease").is_err() {
4409 return;
4410 }
4411 self.lease_state = next;
4412 if let Some(lease) = &self.lease_state {
4413 if let Ok(json) = serde_json::to_string(lease) {
4414 self.set_meta(LEASE_STATE_KEY, &json);
4415 }
4416 } else {
4417 self.delete_meta(LEASE_STATE_KEY);
4418 }
4419 let batch = ChangeAccumulator {
4420 status: true,
4421 ..ChangeAccumulator::default()
4422 };
4423 if self.finish_observation("syncular_lease", batch).is_err() {
4424 self.rollback_observation("syncular_lease");
4425 }
4426 }
4427
4428 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
4429 if self.schema_floor == next {
4430 return;
4431 }
4432 if self.begin_observation("syncular_schema_floor").is_err() {
4433 return;
4434 }
4435 self.schema_floor = next;
4436 self.stopped = self.schema_floor.is_some();
4437 if let Some(floor) = &self.schema_floor {
4438 if let Ok(json) = serde_json::to_string(floor) {
4439 self.set_meta(SCHEMA_FLOOR_KEY, &json);
4440 }
4441 } else {
4442 self.delete_meta(SCHEMA_FLOOR_KEY);
4443 }
4444 let batch = ChangeAccumulator {
4445 status: true,
4446 ..ChangeAccumulator::default()
4447 };
4448 if self
4449 .finish_observation("syncular_schema_floor", batch)
4450 .is_err()
4451 {
4452 self.rollback_observation("syncular_schema_floor");
4453 }
4454 }
4455
4456 fn set_upgrading(&mut self, value: bool) {
4457 if self.upgrading == value {
4458 return;
4459 }
4460 if self.begin_observation("syncular_upgrading").is_err() {
4461 return;
4462 }
4463 self.upgrading = value;
4464 let batch = ChangeAccumulator {
4465 status: true,
4466 ..ChangeAccumulator::default()
4467 };
4468 if self
4469 .finish_observation("syncular_upgrading", batch)
4470 .is_err()
4471 {
4472 self.rollback_observation("syncular_upgrading");
4473 }
4474 }
4475
4476 pub fn sync_needed(&self) -> bool {
4477 self.sync_needed
4478 }
4479
4480 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
4481 let sub = self.subs.iter().find(|s| s.id == id)?;
4482 Some(SubscriptionStateView {
4483 id: sub.id.clone(),
4484 table: sub.table.clone(),
4485 status: sub.state.name().to_owned(),
4486 cursor: sub.cursor,
4487 has_resume_token: sub.bootstrap_state.is_some(),
4488 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
4489 reason_code: sub.reason_code.clone(),
4490 })
4491 }
4492
4493 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
4494 let schema_table = self
4495 .schema
4496 .table(table)
4497 .ok_or_else(|| format!("unknown table {table:?}"))?;
4498 let sql = format!(
4499 "SELECT * FROM {} ORDER BY {} ASC",
4500 visible_table(table),
4501 quote_ident(&schema_table.primary_key)
4502 );
4503 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
4504 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
4505 let mut out = Vec::new();
4506 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
4507 let mut values = Map::new();
4508 for (i, column) in schema_table.columns.iter().enumerate() {
4509 let value = row.get_ref(i).map_err(|e| e.to_string())?;
4510 values.insert(column.name.clone(), sql_ref_to_json(column, value));
4511 }
4512 let version: i64 = row
4513 .get(schema_table.columns.len())
4514 .map_err(|e| e.to_string())?;
4515 let row_id = match values.get(&schema_table.primary_key) {
4516 Some(Value::String(s)) => s.clone(),
4517 Some(Value::Number(n)) => n.to_string(),
4518 Some(Value::Bool(b)) => b.to_string(),
4519 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
4520 };
4521 out.push(RowState {
4522 row_id,
4523 version,
4524 values,
4525 });
4526 }
4527 Ok(out)
4528 }
4529
4530 #[cfg(feature = "crdt-yjs")]
4546 fn crdt_column_bytes(
4547 &self,
4548 table: &str,
4549 row_id: &str,
4550 column: &str,
4551 ) -> Result<Option<Vec<u8>>, String> {
4552 let schema_table = self
4553 .schema
4554 .table(table)
4555 .ok_or_else(|| format!("unknown table {table:?}"))?;
4556 let col = schema_table
4557 .columns
4558 .iter()
4559 .find(|c| c.name == column)
4560 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
4561 if col.ty != ColumnType::Crdt {
4562 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
4563 }
4564 let sql = format!(
4565 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
4566 quote_ident(column),
4567 visible_table(table),
4568 quote_ident(&schema_table.primary_key)
4569 );
4570 let bytes: Option<Vec<u8>> = self
4571 .conn
4572 .query_row(&sql, rusqlite::params![row_id], |row| {
4573 row.get::<_, Option<Vec<u8>>>(0)
4574 })
4575 .map_err(|e| match e {
4576 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
4577 other => other.to_string(),
4578 })?;
4579 Ok(bytes)
4580 }
4581
4582 #[cfg(feature = "crdt-yjs")]
4586 pub fn crdt_text(
4587 &self,
4588 table: &str,
4589 row_id: &str,
4590 column: &str,
4591 name: &str,
4592 ) -> Result<String, String> {
4593 let bytes = self
4594 .crdt_column_bytes(table, row_id, column)?
4595 .unwrap_or_default();
4596 crate::crdt::text(&bytes, name)
4597 }
4598
4599 #[cfg(feature = "crdt-yjs")]
4603 pub fn crdt_insert_text(
4604 &mut self,
4605 table: &str,
4606 row_id: &str,
4607 column: &str,
4608 name: &str,
4609 index: u32,
4610 value: &str,
4611 ) -> Result<String, String> {
4612 let current = self
4613 .crdt_column_bytes(table, row_id, column)?
4614 .unwrap_or_default();
4615 let update = crate::crdt::insert_text(¤t, name, index, value)?;
4616 self.crdt_push_update(table, row_id, column, &update)
4617 }
4618
4619 #[cfg(feature = "crdt-yjs")]
4622 pub fn crdt_delete_text(
4623 &mut self,
4624 table: &str,
4625 row_id: &str,
4626 column: &str,
4627 name: &str,
4628 index: u32,
4629 len: u32,
4630 ) -> Result<String, String> {
4631 let current = self
4632 .crdt_column_bytes(table, row_id, column)?
4633 .unwrap_or_default();
4634 let update = crate::crdt::delete_text(¤t, name, index, len)?;
4635 self.crdt_push_update(table, row_id, column, &update)
4636 }
4637
4638 #[cfg(feature = "crdt-yjs")]
4643 pub fn crdt_apply_update(
4644 &mut self,
4645 table: &str,
4646 row_id: &str,
4647 column: &str,
4648 update: &[u8],
4649 ) -> Result<String, String> {
4650 let current = self
4651 .crdt_column_bytes(table, row_id, column)?
4652 .unwrap_or_default();
4653 let next = crate::crdt::apply_update(¤t, update)?;
4654 self.crdt_push_update(table, row_id, column, &next)
4655 }
4656
4657 #[cfg(feature = "crdt-yjs")]
4663 fn crdt_push_update(
4664 &mut self,
4665 table: &str,
4666 row_id: &str,
4667 column: &str,
4668 crdt_bytes: &[u8],
4669 ) -> Result<String, String> {
4670 let schema_table = self
4671 .schema
4672 .table(table)
4673 .ok_or_else(|| format!("unknown table {table:?}"))?
4674 .clone();
4675 let mut values: Map<String, Value> = self
4678 .read_rows(table)?
4679 .into_iter()
4680 .find(|r| r.row_id == row_id)
4681 .map(|r| r.values)
4682 .unwrap_or_else(|| {
4683 let mut map = Map::new();
4684 map.insert(
4685 schema_table.primary_key.clone(),
4686 Value::from(row_id.to_owned()),
4687 );
4688 map
4689 });
4690 let mut bytes_obj = Map::new();
4692 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
4693 values.insert(column.to_owned(), Value::Object(bytes_obj));
4694 self.mutate(vec![Mutation::Upsert {
4695 table: table.to_owned(),
4696 values,
4697 base_version: None,
4698 }])
4699 }
4700
4701 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
4716 query_connection(&self.conn, sql, params)
4717 }
4718
4719 pub fn query_snapshot(
4721 &mut self,
4722 sql: &str,
4723 params: &[QueryValue],
4724 coverage: &[WindowCoverage],
4725 ) -> Result<QuerySnapshot, String> {
4726 snapshot_connection(&self.conn, sql, params, coverage)
4727 }
4728
4729 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
4732 let mut frames = vec![Frame::ReqHeader {
4733 client_id: self.client_id.clone(),
4734 schema_version: self.schema.version,
4735 }];
4736 let mut pushed_ids = Vec::new();
4737 let mut ops_in_request = 0usize;
4738 let mut deferred_commits = 0usize;
4739 for (index, commit) in self.outbox.iter().enumerate() {
4740 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
4745 deferred_commits = self.outbox.len() - index;
4746 break;
4747 }
4748 ops_in_request += commit.ops.len();
4749 let operations = commit
4750 .ops
4751 .iter()
4752 .map(|op| {
4753 let payload = op.values.as_ref().and_then(|values| {
4754 let table = self.schema.table(&op.table)?;
4755 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
4759 });
4760 ssp2::model::Operation {
4761 table: op.table.clone(),
4762 row_id: op.row_id.clone(),
4763 op: if op.upsert { Op::Upsert } else { Op::Delete },
4764 base_version: op.base_version,
4765 payload,
4766 }
4767 })
4768 .collect();
4769 frames.push(Frame::PushCommit {
4770 client_commit_id: commit.client_commit_id.clone(),
4771 operations,
4772 });
4773 pushed_ids.push(commit.client_commit_id.clone());
4774 }
4775 let accept = self.limits.accept.unwrap_or(if url_capable {
4778 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
4779 } else {
4780 DEFAULT_ACCEPT
4781 });
4782 frames.push(Frame::PullHeader {
4783 limit_commits: self.limits.limit_commits.unwrap_or(0),
4784 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
4785 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
4786 accept,
4787 });
4788 let mut fresh = Vec::new();
4789 for sub in &self.subs {
4790 if sub.state != SubState::Active {
4791 continue;
4792 }
4793 let mut scopes = sub.requested.clone();
4794 sort_scope_map(&mut scopes);
4795 frames.push(Frame::Subscription {
4796 id: sub.id.clone(),
4797 table: sub.table.clone(),
4798 scopes,
4799 params: sub.params.clone().map(RawJson),
4800 cursor: sub.cursor,
4801 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
4802 });
4803 fresh.push((
4804 sub.id.clone(),
4805 sub.cursor < 0 && sub.bootstrap_state.is_none(),
4806 ));
4807 }
4808 let message = Message {
4809 msg_kind: MsgKind::Request,
4810 frames,
4811 };
4812 (
4813 message,
4814 RequestMeta {
4815 pushed_ids,
4816 fresh,
4817 accept,
4818 deferred_commits,
4819 },
4820 )
4821 }
4822
4823 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
4826 let started_at_ms = self.clock_now_ms();
4827 let outcome = self.sync_inner(transport);
4828 let completed_at_ms = self.clock_now_ms();
4829 self.last_round = Some(match &outcome {
4830 SyncOutcome::Ok(report) => DiagnosticLastRound {
4831 status: "succeeded".to_owned(),
4832 started_at_ms,
4833 completed_at_ms,
4834 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
4835 counters: Some(DiagnosticRoundCounters {
4836 pushed: report.pushed,
4837 applied: report.applied.len(),
4838 rejected: report.rejected.len(),
4839 retryable: report.retryable.len(),
4840 conflicts: report.conflicts,
4841 commits_applied: report.commits_applied,
4842 segment_rows_applied: report.segment_rows_applied,
4843 bootstrapping: report.bootstrapping.len(),
4844 resets: report.resets.len(),
4845 revoked: report.revoked.len(),
4846 failed: report.failed.len(),
4847 deferred_commits: report.deferred_commits,
4848 }),
4849 error_code: None,
4850 },
4851 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
4852 status: "failed".to_owned(),
4853 started_at_ms,
4854 completed_at_ms,
4855 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
4856 counters: None,
4857 error_code: Some(Self::diagnostic_code(error_code)),
4858 },
4859 });
4860 outcome
4861 }
4862
4863 fn sync_inner(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
4864 if self.stopped {
4865 return SyncOutcome::Ok(SyncReport {
4868 schema_floor: self.schema_floor.clone(),
4869 ..SyncReport::default()
4870 });
4871 }
4872 self.set_sync_needed(false, false);
4875 if self.schema_has_blobs() {
4878 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
4879 if Self::retryable_transport_code(&code) {
4880 self.schedule_background_retry();
4881 }
4882 return SyncOutcome::Failed {
4883 error_code: code,
4884 message,
4885 };
4886 }
4887 }
4888 let (message, meta) = self.build_request(transport.supports_url_fetch());
4889 let request_bytes = encode_message(&message);
4890 let round = if self.realtime_connected {
4894 transport.realtime_sync(&request_bytes)
4895 } else {
4896 transport.sync(&request_bytes)
4897 };
4898 let response_bytes = match round {
4899 Ok(bytes) => bytes,
4900 Err(TransportError { code, message }) => {
4901 self.record_lease_error(&code);
4904 if Self::retryable_transport_code(&code) {
4905 self.schedule_background_retry();
4906 }
4907 return SyncOutcome::Failed {
4908 error_code: code,
4909 message,
4910 };
4911 }
4912 };
4913 let response = match decode_message(&response_bytes) {
4914 Ok(message) => message,
4915 Err(error) => {
4916 return SyncOutcome::Failed {
4919 error_code: error.code.as_str().to_owned(),
4920 message: error.detail,
4921 };
4922 }
4923 };
4924 if response.msg_kind != MsgKind::Response {
4925 return SyncOutcome::Failed {
4926 error_code: "sync.invalid_request".to_owned(),
4927 message: "expected a response message".to_owned(),
4928 };
4929 }
4930 let mut outcome = self.process_response(transport, response, &meta);
4931 if let SyncOutcome::Ok(report) = &mut outcome {
4932 report.deferred_commits = meta.deferred_commits;
4933 }
4934 match &outcome {
4935 SyncOutcome::Ok(_) => self.reset_background_retry(),
4936 SyncOutcome::Failed { error_code, .. }
4937 if Self::retryable_transport_code(error_code) =>
4938 {
4939 self.schedule_background_retry();
4940 }
4941 SyncOutcome::Failed { .. } => {}
4942 }
4943 if meta.deferred_commits > 0 {
4944 self.set_sync_needed(true, true);
4947 }
4948 outcome
4949 }
4950
4951 pub fn sync_until_idle(
4952 &mut self,
4953 transport: &mut dyn Transport,
4954 max_rounds: Option<u32>,
4955 ) -> SyncOutcome {
4956 let rounds = max_rounds.unwrap_or(12).max(1);
4957 let mut aggregate = SyncReport::default();
4958 for _ in 0..rounds {
4959 match self.sync(transport) {
4960 SyncOutcome::Failed {
4961 error_code,
4962 message,
4963 } => {
4964 return SyncOutcome::Failed {
4965 error_code,
4966 message,
4967 };
4968 }
4969 SyncOutcome::Ok(report) => {
4970 aggregate.pushed += report.pushed;
4971 aggregate.applied.extend(report.applied.iter().cloned());
4972 aggregate.rejected.extend(report.rejected.iter().cloned());
4973 aggregate.retryable.extend(report.retryable.iter().cloned());
4974 aggregate.conflicts += report.conflicts;
4975 aggregate.commits_applied += report.commits_applied;
4976 aggregate.segment_rows_applied += report.segment_rows_applied;
4977 aggregate.bootstrapping = report.bootstrapping.clone();
4978 aggregate.resets.extend(report.resets.iter().cloned());
4979 aggregate.revoked.extend(report.revoked.iter().cloned());
4980 aggregate.failed.extend(report.failed.iter().cloned());
4981 aggregate.deferred_commits = report.deferred_commits;
4982 if report.schema_floor.is_some() {
4983 aggregate.schema_floor = report.schema_floor.clone();
4984 }
4985 let more = !report.bootstrapping.is_empty()
4991 || report.commits_applied > 0
4992 || report.segment_rows_applied > 0
4993 || !report.resets.is_empty()
4994 || self.sync_needed;
4995 if !more {
4996 break;
4997 }
4998 }
4999 }
5000 }
5001 SyncOutcome::Ok(aggregate)
5002 }
5003
5004 fn process_response(
5005 &mut self,
5006 transport: &mut dyn Transport,
5007 response: Message,
5008 meta: &RequestMeta,
5009 ) -> SyncOutcome {
5010 let mut report = SyncReport {
5011 pushed: meta.pushed_ids.len() as u32,
5012 ..SyncReport::default()
5013 };
5014 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
5015 HashMap::new();
5016 let pushed_ids = meta
5017 .pushed_ids
5018 .iter()
5019 .map(String::as_str)
5020 .collect::<HashSet<_>>();
5021 let mut last_final_push_result_id: Option<String> = None;
5022 for frame in &response.frames {
5023 match frame {
5024 Frame::PushResultDetails {
5025 client_commit_id,
5026 entries,
5027 } => {
5028 let details = rejection_details_by_commit
5029 .entry(client_commit_id.clone())
5030 .or_default();
5031 for entry in entries {
5032 let parsed = match RejectionDetails::parse(&entry.details.0) {
5033 Ok(value) => value,
5034 Err(message) => {
5035 return SyncOutcome::Failed {
5036 error_code: "sync.invalid_request".to_owned(),
5037 message,
5038 };
5039 }
5040 };
5041 details.insert(entry.op_index, parsed);
5042 }
5043 }
5044 Frame::PushResult {
5045 client_commit_id,
5046 status,
5047 results,
5048 ..
5049 } if pushed_ids.contains(client_commit_id.as_str())
5050 && Self::push_result_is_final(*status, results) =>
5051 {
5052 last_final_push_result_id = Some(client_commit_id.clone());
5053 }
5054 _ => {}
5055 }
5056 }
5057 let mut frames = response.frames.into_iter();
5058 match frames.next() {
5059 Some(Frame::RespHeader {
5060 required_schema_version,
5061 latest_schema_version,
5062 }) => {
5063 if let Some(required) = required_schema_version {
5064 let floor = SchemaFloor {
5067 required_schema_version: Some(required),
5068 latest_schema_version,
5069 };
5070 self.set_schema_floor(Some(floor.clone()));
5071 report.schema_floor = Some(floor);
5072 return SyncOutcome::Ok(report);
5073 }
5074 }
5075 _ => {
5076 return SyncOutcome::Failed {
5077 error_code: "sync.invalid_request".to_owned(),
5078 message: "response does not start with RESP_HEADER".to_owned(),
5079 };
5080 }
5081 }
5082
5083 let mut failure: Option<(String, String)> = None;
5084 while let Some(frame) = frames.next() {
5085 match frame {
5086 Frame::PushResult {
5087 client_commit_id,
5088 status,
5089 commit_seq: _,
5090 results,
5091 } => {
5092 let prune_outcomes =
5093 last_final_push_result_id.as_deref() == Some(&client_commit_id);
5094 self.handle_push_result(
5095 &client_commit_id,
5096 status,
5097 &results,
5098 rejection_details_by_commit.get(&client_commit_id),
5099 &mut report,
5100 prune_outcomes,
5101 );
5102 }
5103 Frame::PushResultDetails { .. } => {}
5104 Frame::SubStart {
5105 id,
5106 status,
5107 reason_code,
5108 effective_scopes,
5109 bootstrap: _,
5110 } => {
5111 let mut body = Vec::new();
5112 let mut sub_end: Option<(i64, Option<String>)> = None;
5113 for inner in frames.by_ref() {
5114 match inner {
5115 Frame::SubEnd {
5116 next_cursor,
5117 bootstrap_state,
5118 } => {
5119 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
5120 break;
5121 }
5122 Frame::Unknown { .. } => {}
5123 other => body.push(other),
5124 }
5125 }
5126 let Some((next_cursor, bootstrap_state)) = sub_end else {
5127 failure = Some((
5128 "sync.invalid_request".to_owned(),
5129 "subscription section without SUB_END".to_owned(),
5130 ));
5131 break;
5132 };
5133 if let Err(SectionError::Abort(code, message)) = self.process_section(
5134 transport,
5135 &id,
5136 status,
5137 &reason_code,
5138 effective_scopes,
5139 body,
5140 next_cursor,
5141 bootstrap_state,
5142 meta,
5143 &mut report,
5144 ) {
5145 failure = Some((code, message));
5146 break;
5147 }
5148 }
5149 Frame::Lease {
5150 lease_id,
5151 expires_at_ms,
5152 } => {
5153 self.set_lease_state(Some(LeaseState {
5156 lease_id: Some(lease_id),
5157 expires_at_ms: Some(expires_at_ms),
5158 error_code: None,
5159 }));
5160 }
5161 Frame::Error { code, message, .. } => {
5162 failure = Some((code, message));
5165 break;
5166 }
5167 Frame::Unknown { .. } => {}
5168 _ => {
5169 failure = Some((
5170 "sync.invalid_request".to_owned(),
5171 "unexpected frame in response".to_owned(),
5172 ));
5173 break;
5174 }
5175 }
5176 }
5177
5178 if self.overlay_dirty.get() {
5183 self.rebuild_overlay();
5184 }
5185 self.reconcile_blob_refcounts(false);
5192
5193 if let Some((error_code, message)) = failure {
5194 return SyncOutcome::Failed {
5195 error_code,
5196 message,
5197 };
5198 }
5199 self.drain_pending_evictions();
5202 self.ack_after_pull(transport);
5203 if self.upgrading && report.bootstrapping.is_empty() {
5206 self.set_upgrading(false);
5207 }
5208 SyncOutcome::Ok(report)
5209 }
5210
5211 fn handle_push_result(
5214 &mut self,
5215 client_commit_id: &str,
5216 status: PushStatus,
5217 results: &[OpResult],
5218 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
5219 report: &mut SyncReport,
5220 prune_outcomes: bool,
5221 ) {
5222 let Some(index) = self
5223 .outbox
5224 .iter()
5225 .position(|c| c.client_commit_id == client_commit_id)
5226 else {
5227 return;
5228 };
5229 if self.begin_observation("syncular_push_result").is_err() {
5230 return;
5231 }
5232 let mut batch = ChangeAccumulator::default();
5233 let operations = self.outbox[index].ops.clone();
5234 match status {
5235 PushStatus::Applied | PushStatus::Cached => {
5236 let journal_results = results
5239 .iter()
5240 .map(|result| {
5241 let op_index = match result {
5242 OpResult::Applied { op_index }
5243 | OpResult::Conflict { op_index, .. }
5244 | OpResult::Error { op_index, .. } => *op_index,
5245 };
5246 CommitOperationOutcome::Applied { op_index }
5247 })
5248 .collect::<Vec<_>>();
5249 let outcome_status = if status == PushStatus::Applied {
5250 CommitOutcomeStatus::Applied
5251 } else {
5252 CommitOutcomeStatus::Cached
5253 };
5254 let persisted = self
5255 .persist_commit_outcome(
5256 client_commit_id,
5257 outcome_status,
5258 &journal_results,
5259 None,
5260 )
5261 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
5262 let persisted = if prune_outcomes {
5263 persisted.and_then(|()| self.prune_commit_outcomes())
5264 } else {
5265 persisted
5266 };
5267 if persisted.is_err() {
5268 self.rollback_observation("syncular_push_result");
5269 return;
5270 }
5271 report.applied.push(client_commit_id.to_owned());
5272 self.outbox.remove(index);
5273 self.overlay_dirty.set(true);
5274 batch.status = true;
5275 batch.outcomes = true;
5276 }
5277 PushStatus::Rejected => {
5278 if results.iter().any(|result| {
5279 matches!(
5280 result,
5281 OpResult::Error {
5282 code,
5283 retryable: true,
5284 ..
5285 } if code == "sync.idempotency_cache_miss"
5286 )
5287 }) {
5288 report.retryable.push(client_commit_id.to_owned());
5291 if self
5292 .finish_observation("syncular_push_result", batch)
5293 .is_err()
5294 {
5295 self.rollback_observation("syncular_push_result");
5296 }
5297 return;
5298 }
5299
5300 let mut journal_results = Vec::with_capacity(results.len());
5301 let mut conflicts = Vec::new();
5302 let mut rejections = Vec::new();
5303 for result in results {
5304 match result {
5305 OpResult::Applied { op_index } => {
5306 journal_results.push(CommitOperationOutcome::Applied {
5307 op_index: *op_index,
5308 });
5309 }
5310 OpResult::Conflict {
5311 op_index,
5312 code,
5313 message,
5314 server_version,
5315 server_row,
5316 } => {
5317 let operation = operations
5318 .get(*op_index as usize)
5319 .map(CommitOperation::from);
5320 let (table, row_id) = operation
5321 .as_ref()
5322 .map(|op| (op.table.clone(), op.row_id.clone()))
5323 .unwrap_or_default();
5324 let server_row_json = self
5325 .schema
5326 .table(&table)
5327 .and_then(|t| {
5328 decode_row_bytes(t, server_row, &self.encryption)
5329 .ok()
5330 .map(|row| (t, row))
5331 })
5332 .map(|(t, row)| {
5333 let mut map = Map::new();
5334 for (i, column) in t.columns.iter().enumerate() {
5335 map.insert(
5336 column.name.clone(),
5337 column_value_to_json(row.get(i).unwrap_or(&None)),
5338 );
5339 }
5340 map
5341 })
5342 .unwrap_or_default();
5343 let conflict = ConflictRecord {
5344 client_commit_id: client_commit_id.to_owned(),
5345 op_index: *op_index,
5346 table,
5347 row_id,
5348 code: code.clone(),
5349 message: message.clone(),
5350 server_version: *server_version,
5351 server_row: server_row_json,
5352 operation,
5353 };
5354 journal_results.push(CommitOperationOutcome::Conflict {
5355 conflict: conflict.clone(),
5356 });
5357 conflicts.push(conflict);
5358 }
5359 OpResult::Error {
5360 op_index,
5361 code,
5362 message,
5363 retryable,
5364 } => {
5365 let rejection = RejectionRecord {
5366 client_commit_id: client_commit_id.to_owned(),
5367 op_index: *op_index,
5368 code: code.clone(),
5369 message: message.clone(),
5370 retryable: *retryable,
5371 details: rejection_details
5372 .and_then(|details| details.get(op_index))
5373 .cloned(),
5374 operation: operations
5375 .get(*op_index as usize)
5376 .map(CommitOperation::from),
5377 };
5378 journal_results.push(CommitOperationOutcome::Error {
5379 rejection: rejection.clone(),
5380 });
5381 rejections.push(rejection);
5382 }
5383 }
5384 }
5385 let outcome_status = if conflicts.is_empty() {
5386 CommitOutcomeStatus::Rejected
5387 } else {
5388 CommitOutcomeStatus::Conflict
5389 };
5390 let persisted = self
5391 .persist_commit_outcome(
5392 client_commit_id,
5393 outcome_status,
5394 &journal_results,
5395 Some(&operations),
5396 )
5397 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
5398 let persisted = if prune_outcomes {
5399 persisted.and_then(|()| self.prune_commit_outcomes())
5400 } else {
5401 persisted
5402 };
5403 if persisted.is_err() {
5404 self.rollback_observation("syncular_push_result");
5405 return;
5406 }
5407 report.conflicts += conflicts.len() as u32;
5408 report.rejected.push(client_commit_id.to_owned());
5409 batch.conflicts = !conflicts.is_empty();
5410 batch.rejections = !rejections.is_empty();
5411 batch.status = true;
5412 batch.outcomes = true;
5413 self.conflicts.extend(conflicts);
5414 self.rejections.extend(rejections);
5415 self.outbox.remove(index);
5416 self.overlay_dirty.set(true);
5417 }
5418 }
5419 if batch.status {
5420 for operation in &operations {
5421 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
5422 batch.table(&operation.table);
5423 }
5424 }
5425 }
5426 if self
5427 .finish_observation("syncular_push_result", batch)
5428 .is_err()
5429 {
5430 self.rollback_observation("syncular_push_result");
5431 }
5432 }
5433
5434 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
5435 status != PushStatus::Rejected
5436 || !results.iter().any(|result| {
5437 matches!(
5438 result,
5439 OpResult::Error {
5440 code,
5441 retryable: true,
5442 ..
5443 } if code == "sync.idempotency_cache_miss"
5444 )
5445 })
5446 }
5447
5448 #[allow(clippy::too_many_arguments)]
5451 fn process_section(
5452 &mut self,
5453 transport: &mut dyn Transport,
5454 id: &str,
5455 status: SubStatus,
5456 reason_code: &str,
5457 effective_scopes: Vec<(String, Vec<String>)>,
5458 body: Vec<Frame>,
5459 next_cursor: i64,
5460 bootstrap_state: Option<String>,
5461 meta: &RequestMeta,
5462 report: &mut SyncReport,
5463 ) -> Result<(), SectionError> {
5464 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
5465 return Ok(()); };
5467 match status {
5468 SubStatus::Revoked => {
5469 self.begin_observation("syncular_revocation")
5470 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5471 let mut batch = ChangeAccumulator::default();
5472 let registered = self.window_unit_by_sub_id(id);
5473 let (table, effective) = {
5475 let sub = &self.subs[sub_index];
5476 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5477 };
5478 let purged = self.purge_scope_rows(&table, &effective);
5479 match purged {
5480 Ok(()) => {
5481 self.record_scope_map(&mut batch, &table, &effective);
5482 let sub = &mut self.subs[sub_index];
5483 sub.state = SubState::Revoked;
5484 sub.reason_code = Some(if reason_code.is_empty() {
5485 "sync.scope_revoked".to_owned()
5486 } else {
5487 reason_code.to_owned()
5488 });
5489 report.revoked.push(id.to_owned());
5490 let doomed_effective = effective;
5491 let sub_table = table;
5492 self.persist_sub(&self.subs[sub_index].clone());
5493 let dropped = self
5494 .drop_doomed_outbox(&sub_table, &doomed_effective)
5495 .map_err(|message| {
5496 SectionError::Abort("storage.failed".to_owned(), message)
5497 })?;
5498 if dropped {
5499 batch.status = true;
5500 batch.rejections = true;
5501 batch.outcomes = true;
5502 }
5503 self.reconcile_blob_refcounts(true);
5506 }
5507 Err(()) => {
5508 let sub = &mut self.subs[sub_index];
5511 sub.state = SubState::Failed;
5512 sub.reason_code = Some("sync.scope_revoked".to_owned());
5513 report.failed.push(id.to_owned());
5514 self.persist_sub(&self.subs[sub_index].clone());
5515 }
5516 }
5517 if let Some((base_key, unit)) = registered {
5518 batch.window(&base_key, &self.subs[sub_index].table, &unit);
5519 }
5520 self.rebuild_overlay_if_dirty();
5521 self.finish_observation("syncular_revocation", batch)
5522 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5523 Ok(())
5524 }
5525 SubStatus::Reset => {
5526 self.begin_observation("syncular_reset")
5527 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5528 let mut batch = ChangeAccumulator::default();
5529 let registered = self.window_unit_by_sub_id(id);
5530 let sub = &mut self.subs[sub_index];
5533 sub.cursor = -1;
5534 sub.bootstrap_state = None;
5535 report.resets.push(id.to_owned());
5536 self.persist_sub(&self.subs[sub_index].clone());
5537 if let Some((base_key, unit)) = registered {
5538 batch.window(&base_key, &self.subs[sub_index].table, &unit);
5539 }
5540 self.finish_observation("syncular_reset", batch)
5541 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5542 Ok(())
5543 }
5544 SubStatus::Active => {
5545 let fresh = meta
5546 .fresh
5547 .iter()
5548 .find(|(fid, _)| fid == id)
5549 .map(|(_, f)| *f)
5550 .unwrap_or(false);
5551 let was_pending = self.subs[sub_index].cursor < 0
5552 || self.subs[sub_index].bootstrap_state.is_some();
5553 let registered = self.window_unit_by_sub_id(id);
5554 self.subs[sub_index].effective = Some(effective_scopes);
5556 self.begin_observation("syncular_section")
5557 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5558 let mut batch = ChangeAccumulator::default();
5559 let outcome = self.apply_section_body(
5560 transport, sub_index, body, fresh, meta, report, &mut batch,
5561 );
5562 match outcome {
5563 Ok(()) => {
5564 let sub = &mut self.subs[sub_index];
5565 sub.cursor = next_cursor;
5568 sub.bootstrap_state = bootstrap_state;
5569 sub.synced_once = true;
5570 if sub.bootstrap_state.is_some() {
5571 report.bootstrapping.push(id.to_owned());
5572 }
5573 let completed =
5574 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
5575 self.persist_sub(&self.subs[sub_index].clone());
5576 if completed {
5577 if let Some((base_key, unit)) = registered.clone() {
5578 batch.window(&base_key, &self.subs[sub_index].table, &unit);
5579 }
5580 }
5581 self.rebuild_overlay_if_dirty();
5582 self.finish_observation("syncular_section", batch)
5583 .map_err(|message| {
5584 SectionError::Abort("storage.failed".to_owned(), message)
5585 })?;
5586 Ok(())
5587 }
5588 Err(SectionError::FailClosed) => {
5589 self.rollback_observation("syncular_section");
5592 self.begin_observation("syncular_section_failure")
5593 .map_err(|message| {
5594 SectionError::Abort("storage.failed".to_owned(), message)
5595 })?;
5596 let mut failure_batch = ChangeAccumulator::default();
5597 let sub = &mut self.subs[sub_index];
5598 sub.state = SubState::Failed;
5599 sub.reason_code = Some("sync.scope_revoked".to_owned());
5600 report.failed.push(id.to_owned());
5601 self.persist_sub(&self.subs[sub_index].clone());
5602 if let Some((base_key, unit)) = registered {
5603 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
5604 }
5605 self.finish_observation("syncular_section_failure", failure_batch)
5606 .map_err(|message| {
5607 SectionError::Abort("storage.failed".to_owned(), message)
5608 })?;
5609 Ok(())
5610 }
5611 Err(SectionError::Abort(code, message)) => {
5612 self.rollback_observation("syncular_section");
5615 Err(SectionError::Abort(code, message))
5616 }
5617 }
5618 }
5619 }
5620 }
5621
5622 #[allow(clippy::too_many_arguments)]
5626 fn apply_section_body(
5627 &mut self,
5628 transport: &mut dyn Transport,
5629 sub_index: usize,
5630 body: Vec<Frame>,
5631 fresh: bool,
5632 meta: &RequestMeta,
5633 report: &mut SyncReport,
5634 batch: &mut ChangeAccumulator,
5635 ) -> Result<(), SectionError> {
5636 let mut saw_segment = false;
5637 for frame in body {
5638 match frame {
5639 Frame::Commit {
5640 tables, changes, ..
5641 } => {
5642 self.record_commit_changes(batch, &tables, &changes);
5643 self.apply_commit_changes(&tables, &changes)
5644 .map_err(|(c, m)| SectionError::Abort(c, m))?;
5645 report.commits_applied += 1;
5646 }
5647 Frame::SegmentInline { payload } => {
5648 let segment = decode_rows_segment(&payload)
5649 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
5650 let first = !saw_segment;
5651 saw_segment = true;
5652 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
5653 let cleared =
5654 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
5655 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
5656 if applied > 0 || cleared {
5657 batch.table(&segment.table);
5658 }
5659 report.segment_rows_applied += applied;
5660 }
5661 Frame::SegmentRef {
5662 segment_id,
5663 media_type,
5664 table,
5665 row_count,
5666 as_of_commit_seq,
5667 scope_digest,
5668 row_cursor,
5669 next_row_cursor,
5670 url,
5671 url_expires_at_ms,
5672 ..
5673 } => {
5674 let advertised = match media_type {
5677 MediaType::Rows => {
5678 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
5679 || meta.accept & ACCEPT_INLINE_ROWS != 0
5680 }
5681 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
5682 };
5683 if !advertised {
5684 return Err(SectionError::Abort(
5685 "sync.invalid_request".to_owned(),
5686 format!(
5687 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
5688 media_type.name()
5689 ),
5690 ));
5691 }
5692 let bytes = if let Some(url) = url {
5693 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
5698 return Err(SectionError::Abort(
5699 "sync.invalid_request".to_owned(),
5700 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
5701 .to_owned(),
5702 ));
5703 }
5704 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5706 return Err(SectionError::Abort(
5707 "sync.segment_expired".to_owned(),
5708 format!(
5709 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
5710 ),
5711 ));
5712 }
5713 transport
5714 .fetch_url(&url)
5715 .map_err(|e| SectionError::Abort(e.code, e.message))?
5716 } else {
5717 let requested_scopes_json =
5718 canonical_scope_json(&self.subs[sub_index].requested);
5719 transport
5720 .download_segment(&SegmentRequest {
5721 segment_id: segment_id.clone(),
5722 table,
5723 requested_scopes_json,
5724 })
5725 .map_err(|e| SectionError::Abort(e.code, e.message))?
5726 };
5727 let digest = Sha256::digest(&bytes);
5729 let expected = segment_id
5730 .strip_prefix("sha256:")
5731 .unwrap_or(segment_id.as_str());
5732 if bytes_to_hex(&digest) != expected {
5733 return Err(SectionError::Abort(
5734 "sync.invalid_request".to_owned(),
5735 "segment bytes do not match the content address (§5.1)".to_owned(),
5736 ));
5737 }
5738 if media_type == MediaType::Sqlite {
5739 if row_cursor.is_some() || next_row_cursor.is_some() {
5742 return Err(SectionError::Abort(
5743 "sync.invalid_request".to_owned(),
5744 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
5745 .to_owned(),
5746 ));
5747 }
5748 let first = !saw_segment;
5749 saw_segment = true;
5750 let sub_table = self.subs[sub_index].table.clone();
5751 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
5752 let cleared =
5753 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
5754 let applied = self.apply_sqlite_segment(
5755 sub_index,
5756 &bytes,
5757 fresh && first,
5758 row_count,
5759 as_of_commit_seq,
5760 &scope_digest,
5761 )?;
5762 if applied > 0 || cleared {
5763 batch.table(&sub_table);
5764 }
5765 report.segment_rows_applied += applied;
5766 } else {
5767 let segment = decode_rows_segment(&bytes).map_err(|e| {
5768 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
5769 })?;
5770 let first = row_cursor.is_none();
5771 saw_segment = true;
5772 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
5773 let cleared =
5774 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
5775 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
5776 if applied > 0 || cleared {
5777 batch.table(&segment.table);
5778 }
5779 report.segment_rows_applied += applied;
5780 }
5781 }
5782 Frame::Unknown { .. } => {}
5783 _ => {
5784 return Err(SectionError::Abort(
5785 "sync.invalid_request".to_owned(),
5786 "unexpected frame inside a subscription section".to_owned(),
5787 ));
5788 }
5789 }
5790 }
5791 Ok(())
5792 }
5793
5794 fn apply_commit_changes(
5795 &mut self,
5796 tables: &[String],
5797 changes: &[ssp2::model::Change],
5798 ) -> Result<(), (String, String)> {
5799 for change in changes {
5800 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
5801 (
5802 "sync.invalid_request".to_owned(),
5803 "change tableIndex out of range".to_owned(),
5804 )
5805 })?;
5806 let table = self.schema.table(table_name).ok_or_else(|| {
5807 (
5808 "sync.schema_mismatch".to_owned(),
5809 format!("change targets unknown table {table_name:?}"),
5810 )
5811 })?;
5812 match change.op {
5813 Op::Upsert => {
5814 let payload = change.row.as_ref().ok_or_else(|| {
5815 (
5816 "sync.invalid_request".to_owned(),
5817 "upsert change without row payload".to_owned(),
5818 )
5819 })?;
5820 let row = decode_row_bytes(table, payload, &self.encryption)
5822 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
5823 let version = change.row_version.unwrap_or(0);
5824 let table_name = table.name.clone();
5825 self.write_base_row(&table_name, &row, version)
5826 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
5827 }
5828 Op::Delete => {
5829 self.delete_base_row(table_name, &change.row_id)
5830 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
5831 }
5832 }
5833 }
5834 Ok(())
5835 }
5836
5837 fn apply_segment(
5842 &mut self,
5843 sub_index: usize,
5844 segment: &RowsSegment,
5845 first_fresh_page: bool,
5846 ) -> Result<u32, SectionError> {
5847 let (sub_table, effective) = {
5848 let sub = &self.subs[sub_index];
5849 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5850 };
5851 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5852 SectionError::Abort(
5853 "sync.schema_mismatch".to_owned(),
5854 format!("subscription table {sub_table:?} is not in the client schema"),
5855 )
5856 })?;
5857 let matches = segment.table == table.name
5862 && segment.schema_version == self.schema.version
5863 && segment.columns.len() == table.wire_columns.len()
5864 && segment
5865 .columns
5866 .iter()
5867 .zip(table.wire_columns.iter())
5868 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
5869 if !matches {
5870 return Err(SectionError::Abort(
5871 "sync.schema_mismatch".to_owned(),
5872 "segment column table does not match the generated schema (§5.2)".to_owned(),
5873 ));
5874 }
5875 if first_fresh_page {
5876 self.purge_scope_rows(&table.name, &effective)
5880 .map_err(|()| SectionError::FailClosed)?;
5881 }
5882 let mut applied = 0u32;
5883 for block in &segment.blocks {
5884 for row in block {
5885 let decrypted;
5890 let values = if table.has_encrypted_columns() {
5891 let mut values = row.values.clone();
5892 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
5893 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
5894 decrypted = values;
5895 &decrypted
5896 } else {
5897 &row.values
5898 };
5899 self.write_base_row(&table.name, values, row.server_version)
5902 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
5903 applied += 1;
5904 }
5905 }
5906 Ok(applied)
5907 }
5908
5909 fn apply_sqlite_segment(
5917 &mut self,
5918 sub_index: usize,
5919 bytes: &[u8],
5920 first_fresh_page: bool,
5921 row_count: i64,
5922 as_of_commit_seq: i64,
5923 scope_digest: &str,
5924 ) -> Result<u32, SectionError> {
5925 let invalid = |detail: &str| {
5926 SectionError::Abort(
5927 "sync.invalid_request".to_owned(),
5928 format!("sqlite segment rejected: {detail} (§5.3)"),
5929 )
5930 };
5931 let (sub_table, effective) = {
5932 let sub = &self.subs[sub_index];
5933 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5934 };
5935 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5936 SectionError::Abort(
5937 "sync.schema_mismatch".to_owned(),
5938 format!("subscription table {sub_table:?} is not in the client schema"),
5939 )
5940 })?;
5941
5942 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
5943 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
5944 let img = match rusqlite::Connection::open_with_flags(
5945 &path,
5946 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
5947 ) {
5948 Ok(conn) => conn,
5949 Err(_) => {
5950 let _ = std::fs::remove_file(&path);
5951 return Err(invalid("bytes do not open as a SQLite database"));
5952 }
5953 };
5954 let outcome = self.apply_sqlite_image(
5955 &img,
5956 &table,
5957 first_fresh_page,
5958 &effective,
5959 row_count,
5960 as_of_commit_seq,
5961 scope_digest,
5962 );
5963 drop(img);
5964 let _ = std::fs::remove_file(&path);
5965 outcome
5966 }
5967
5968 #[allow(clippy::too_many_arguments)]
5969 fn apply_sqlite_image(
5970 &mut self,
5971 img: &rusqlite::Connection,
5972 table: &crate::schema::TableSchema,
5973 first_fresh_page: bool,
5974 effective: &[(String, Vec<String>)],
5975 row_count: i64,
5976 as_of_commit_seq: i64,
5977 scope_digest: &str,
5978 ) -> Result<u32, SectionError> {
5979 let invalid = |detail: String| {
5980 SectionError::Abort(
5981 "sync.invalid_request".to_owned(),
5982 format!("sqlite segment rejected: {detail} (§5.3)"),
5983 )
5984 };
5985
5986 type MetaRow = (i64, String, i64, i64, String, i64, i64);
5989 let meta: MetaRow = img
5990 .query_row(
5991 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
5992 \"scopeDigest\", \"rowCount\",
5993 (SELECT count(*) FROM _syncular_segment)
5994 FROM _syncular_segment",
5995 [],
5996 |row| {
5997 Ok((
5998 row.get(0)?,
5999 row.get(1)?,
6000 row.get(2)?,
6001 row.get(3)?,
6002 row.get(4)?,
6003 row.get(5)?,
6004 row.get(6)?,
6005 ))
6006 },
6007 )
6008 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
6009 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
6010 if meta_count != 1 {
6011 return Err(invalid(format!(
6012 "_syncular_segment must contain exactly one row, found {meta_count}"
6013 )));
6014 }
6015 if format != 1 {
6016 return Err(invalid(format!("format {format}")));
6017 }
6018 if meta_table != table.name {
6019 return Err(invalid(format!("image table {meta_table:?}")));
6020 }
6021 if schema_version != i64::from(self.schema.version) {
6022 return Err(invalid(format!("schemaVersion {schema_version}")));
6023 }
6024 if pin != as_of_commit_seq {
6025 return Err(invalid(format!("asOfCommitSeq {pin}")));
6026 }
6027 if digest != scope_digest {
6028 return Err(invalid("scopeDigest mismatch".to_owned()));
6029 }
6030 if meta_rows != row_count {
6031 return Err(invalid(format!("rowCount {meta_rows}")));
6032 }
6033
6034 let mut names: Vec<String> = Vec::new();
6036 {
6037 let mut stmt = img
6038 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
6039 .map_err(|_| invalid("image data table missing".to_owned()))?;
6040 let mut rows = stmt
6041 .query([])
6042 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
6043 while let Some(row) = rows
6044 .next()
6045 .map_err(|_| invalid("image data table unreadable".to_owned()))?
6046 {
6047 names.push(
6048 row.get::<_, String>(1)
6049 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
6050 );
6051 }
6052 }
6053 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
6054 expected.push("_syncular_version");
6055 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
6056 return Err(SectionError::Abort(
6057 "sync.schema_mismatch".to_owned(),
6058 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
6059 ));
6060 }
6061
6062 if first_fresh_page {
6065 self.purge_scope_rows(&table.name, effective)
6066 .map_err(|()| SectionError::FailClosed)?;
6067 }
6068 self.overlay_dirty.set(true);
6074 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
6082 table.indexes.iter().filter(|i| !i.unique).collect()
6083 } else {
6084 Vec::new()
6085 };
6086 for index in &bulk_indexes {
6087 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
6088 self.conn
6089 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
6090 .map_err(|e| invalid(e.to_string()))?;
6091 }
6092 let insert = self.insert_row_sql(&base_table(&table.name), table);
6093 let applied = {
6094 let mut ins = self
6095 .conn
6096 .prepare_cached(&insert)
6097 .map_err(|e| invalid(e.to_string()))?;
6098 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
6099 let mut stmt = img
6100 .prepare(&format!(
6101 "SELECT {} FROM {}",
6102 column_list.join(", "),
6103 quote_ident(&table.name)
6104 ))
6105 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
6106 let mut rows = stmt
6107 .query([])
6108 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
6109 let version_index = table.columns.len();
6110 let mut applied = 0u32;
6111 while let Some(row) = rows
6112 .next()
6113 .map_err(|_| invalid("image row unreadable".to_owned()))?
6114 {
6115 for (i, column) in table.columns.iter().enumerate() {
6116 let cell = row
6117 .get_ref(i)
6118 .map_err(|_| invalid("image row unreadable".to_owned()))?;
6119 let param = image_cell_param(column, cell).map_err(&invalid)?;
6120 ins.raw_bind_parameter(i + 1, param)
6121 .map_err(|e| invalid(e.to_string()))?;
6122 }
6123 let version: i64 = row
6124 .get(version_index)
6125 .map_err(|_| invalid("image row unreadable".to_owned()))?;
6126 if version < 1 {
6127 return Err(invalid(format!(
6128 "row _syncular_version must be >= 1, got {version}"
6129 )));
6130 }
6131 ins.raw_bind_parameter(version_index + 1, version)
6132 .map_err(|e| invalid(e.to_string()))?;
6133 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
6134 applied += 1;
6135 }
6136 applied
6137 };
6138 for index in &bulk_indexes {
6139 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
6140 let cols_sql = index
6141 .columns
6142 .iter()
6143 .map(|c| quote_ident(c))
6144 .collect::<Vec<_>>()
6145 .join(", ");
6146 self.conn
6147 .execute(
6148 &format!(
6149 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
6150 base_table(&table.name)
6151 ),
6152 [],
6153 )
6154 .map_err(|e| invalid(e.to_string()))?;
6155 }
6156 if i64::from(applied) != row_count {
6157 return Err(invalid(format!(
6158 "image holds {applied} rows, descriptor says {row_count}"
6159 )));
6160 }
6161 Ok(applied)
6162 }
6163
6164 fn compile_local_data_purge(
6167 &self,
6168 input: &LocalDataPurgeInput,
6169 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
6170 let invalid = |message: String| format!("sync.invalid_request: {message}");
6171 if input.purge_id.is_empty()
6172 || input.purge_id.len() > 128
6173 || !is_local_operation_code_like(&input.purge_id)
6174 {
6175 return Err(invalid(
6176 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
6177 ));
6178 }
6179 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
6180 return Err(invalid(format!(
6181 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
6182 )));
6183 }
6184
6185 let mut deduplicated: BTreeMap<
6186 String,
6187 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
6188 > = BTreeMap::new();
6189 for target in &input.targets {
6190 let table = self.schema.table(&target.table).ok_or_else(|| {
6191 invalid(format!(
6192 "local purge names unknown table {:?}",
6193 target.table
6194 ))
6195 })?;
6196 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
6197 return Err(invalid(format!(
6198 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
6199 target.table
6200 )));
6201 }
6202 let mut selectors = Vec::with_capacity(target.selectors.len());
6203 let mut canonical_selectors = BTreeMap::new();
6204 for (column_name, raw_values) in &target.selectors {
6205 let Some((column_index, column)) = table
6206 .columns
6207 .iter()
6208 .enumerate()
6209 .find(|(_, column)| column.name == *column_name)
6210 else {
6211 return Err(invalid(format!(
6212 "local purge target {:?} names unknown column {:?}",
6213 target.table, column_name
6214 )));
6215 };
6216 let encrypted = table
6217 .encrypted_columns
6218 .iter()
6219 .any(|candidate| candidate.index == column_index);
6220 if column.ty != ColumnType::String || encrypted {
6221 return Err(invalid(format!(
6222 "local purge selector {:?}.{:?} must be a plaintext string column",
6223 target.table, column_name
6224 )));
6225 }
6226 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
6227 return Err(invalid(format!(
6228 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
6229 target.table, column_name
6230 )));
6231 }
6232 let mut values = raw_values.clone();
6233 values.sort();
6234 values.dedup();
6235 if values.iter().any(|value| {
6236 value.is_empty()
6237 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
6238 || !is_local_operation_code_like(value)
6239 }) {
6240 return Err(invalid(format!(
6241 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
6242 )));
6243 }
6244 selectors.push((column_name.clone(), values.clone()));
6245 canonical_selectors.insert(column_name.clone(), values);
6246 }
6247 let canonical = LocalDataPurgeTarget {
6248 table: target.table.clone(),
6249 selectors: canonical_selectors,
6250 };
6251 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
6252 deduplicated.insert(
6253 key,
6254 (
6255 CompiledLocalDataPurgeTarget {
6256 table: target.table.clone(),
6257 selectors,
6258 },
6259 canonical,
6260 ),
6261 );
6262 }
6263 let targets = deduplicated
6264 .values()
6265 .map(|(compiled, _)| compiled.clone())
6266 .collect::<Vec<_>>();
6267 let canonical = deduplicated
6268 .values()
6269 .map(|(_, target)| target.clone())
6270 .collect::<Vec<_>>();
6271 let canonical_plan =
6272 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
6273 Ok((targets, canonical_plan))
6274 }
6275
6276 fn local_purge_base_row_ids(
6277 &self,
6278 targets: &[CompiledLocalDataPurgeTarget],
6279 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
6280 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
6281 for target in targets {
6282 let table = self
6283 .schema
6284 .table(&target.table)
6285 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
6286 let mut clauses = Vec::with_capacity(target.selectors.len());
6287 let mut params = Vec::new();
6288 for (column, values) in &target.selectors {
6289 clauses.push(format!(
6290 "{} IN ({})",
6291 quote_ident(column),
6292 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
6293 ));
6294 params.extend(values.iter().cloned().map(SqlValue::Text));
6295 }
6296 let sql = format!(
6297 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
6298 quote_ident(&table.primary_key),
6299 base_table(&target.table),
6300 clauses.join(" AND ")
6301 );
6302 let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
6303 let rows = statement
6304 .query_map(rusqlite::params_from_iter(params), |row| {
6305 row.get::<_, String>(0)
6306 })
6307 .map_err(|error| error.to_string())?;
6308 let ids = by_table.entry(target.table.clone()).or_default();
6309 for row in rows {
6310 ids.insert(row.map_err(|error| error.to_string())?);
6311 }
6312 }
6313 Ok(by_table)
6314 }
6315
6316 fn local_purge_values_match(
6317 target: &CompiledLocalDataPurgeTarget,
6318 values: &Map<String, Value>,
6319 ) -> bool {
6320 target.selectors.iter().all(|(column, allowed)| {
6321 values
6322 .get(column)
6323 .and_then(Value::as_str)
6324 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
6325 })
6326 }
6327
6328 pub fn purge_local_data(
6332 &mut self,
6333 input: &LocalDataPurgeInput,
6334 ) -> Result<LocalDataPurgeResult, String> {
6335 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
6336 let meta_key = format!("localPurge:{}", input.purge_id);
6337 let applied_plan = self
6338 .conn
6339 .query_row(
6340 "SELECT value FROM _syncular_meta WHERE key = ?1",
6341 rusqlite::params![meta_key],
6342 |row| row.get::<_, String>(0),
6343 )
6344 .optional()
6345 .map_err(|error| error.to_string())?;
6346 if let Some(applied_plan) = applied_plan {
6347 if applied_plan != canonical_plan {
6348 return Err(format!(
6349 "sync.invalid_request: local purge id {:?} was already used with a different plan",
6350 input.purge_id
6351 ));
6352 }
6353 return Ok(LocalDataPurgeResult {
6354 already_applied: true,
6355 purged_rows: 0,
6356 dropped_commits: 0,
6357 });
6358 }
6359
6360 let prior_outbox = self.outbox.clone();
6361 let prior_rejection_count = self.rejections.len();
6362 let prior_overlay_dirty = self.overlay_dirty.get();
6363 self.begin_observation("syncular_local_purge")?;
6364 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
6365 let row_ids = self.local_purge_base_row_ids(&targets)?;
6366 let doomed = self
6367 .outbox
6368 .iter()
6369 .filter(|commit| {
6370 commit.ops.iter().any(|operation| {
6371 let matching_targets = targets
6372 .iter()
6373 .filter(|target| target.table == operation.table)
6374 .collect::<Vec<_>>();
6375 if matching_targets.is_empty() {
6376 return false;
6377 }
6378 if row_ids
6379 .get(&operation.table)
6380 .is_some_and(|ids| ids.contains(&operation.row_id))
6381 {
6382 return true;
6383 }
6384 operation.values.as_ref().is_some_and(|values| {
6385 matching_targets
6386 .iter()
6387 .any(|target| Self::local_purge_values_match(target, values))
6388 })
6389 })
6390 })
6391 .cloned()
6392 .collect::<Vec<_>>();
6393 let doomed_ids = doomed
6394 .iter()
6395 .map(|commit| commit.client_commit_id.clone())
6396 .collect::<BTreeSet<_>>();
6397 let mut batch = ChangeAccumulator::default();
6398 let mut rejections = Vec::new();
6399 for commit in &doomed {
6400 for operation in &commit.ops {
6401 batch.table(&operation.table);
6402 }
6403 let results = commit
6404 .ops
6405 .iter()
6406 .enumerate()
6407 .map(|(op_index, operation)| {
6408 let rejection = RejectionRecord {
6409 client_commit_id: commit.client_commit_id.clone(),
6410 op_index: op_index as i32,
6411 code: "client.local_data_purged".to_owned(),
6412 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
6413 retryable: false,
6414 details: None,
6415 operation: Some(CommitOperation::from(operation)),
6416 };
6417 rejections.push(rejection.clone());
6418 CommitOperationOutcome::Error { rejection }
6419 })
6420 .collect::<Vec<_>>();
6421 self.persist_commit_outcome(
6422 &commit.client_commit_id,
6423 CommitOutcomeStatus::Rejected,
6424 &results,
6425 Some(&commit.ops),
6426 )?;
6427 self.delete_outbox_persisted(&commit.client_commit_id)?;
6428 }
6429 if !doomed.is_empty() {
6430 self.outbox
6431 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
6432 self.rejections.extend(rejections);
6433 self.prune_commit_outcomes()?;
6434 self.overlay_dirty.set(true);
6435 batch.status = true;
6436 batch.rejections = true;
6437 batch.outcomes = true;
6438 }
6439
6440 let mut purged_rows = 0usize;
6441 for (table, ids) in &row_ids {
6442 if ids.is_empty() {
6443 continue;
6444 }
6445 batch.table(table);
6446 for row_id in ids {
6447 self.delete_base_row(table, row_id)?;
6448 purged_rows += 1;
6449 }
6450 }
6451 self.rebuild_overlay_if_dirty();
6452 self.reconcile_blob_refcounts(true);
6453 self.conn
6454 .execute(
6455 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
6456 rusqlite::params![meta_key, canonical_plan],
6457 )
6458 .map_err(|error| error.to_string())?;
6459 Ok((
6460 batch,
6461 LocalDataPurgeResult {
6462 already_applied: false,
6463 purged_rows,
6464 dropped_commits: doomed.len(),
6465 },
6466 ))
6467 })();
6468 let (batch, result) = match applied {
6469 Ok(value) => value,
6470 Err(error) => {
6471 self.rollback_observation("syncular_local_purge");
6472 self.outbox = prior_outbox;
6473 self.rejections.truncate(prior_rejection_count);
6474 self.overlay_dirty.set(prior_overlay_dirty);
6475 return Err(error);
6476 }
6477 };
6478 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
6479 self.rollback_observation("syncular_local_purge");
6480 self.outbox = prior_outbox;
6481 self.rejections.truncate(prior_rejection_count);
6482 self.overlay_dirty.set(prior_overlay_dirty);
6483 return Err(error);
6484 }
6485 Ok(result)
6486 }
6487
6488 pub fn rebootstrap_local_data(
6494 &mut self,
6495 input: &LocalDataRebootstrapInput,
6496 ) -> Result<LocalDataRebootstrapResult, String> {
6497 if input.rebootstrap_id.is_empty()
6498 || input.rebootstrap_id.len() > 128
6499 || !is_local_operation_code_like(&input.rebootstrap_id)
6500 {
6501 return Err(
6502 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
6503 .to_owned(),
6504 );
6505 }
6506 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
6507 if self.get_meta(&meta_key).is_some() {
6508 return Ok(LocalDataRebootstrapResult {
6509 already_applied: true,
6510 retained_commits: 0,
6511 reset_subscriptions: 0,
6512 });
6513 }
6514 if self.stopped || self.schema_floor.is_some() {
6515 return Err(
6516 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
6517 .to_owned(),
6518 );
6519 }
6520
6521 let retained_commits = self.outbox.len();
6522 let reset_subscriptions = self.subs.len();
6523 let prior_subs = self.subs.clone();
6524 let prior_upgrading = self.upgrading;
6525 let prior_stopped = self.stopped;
6526 let prior_schema_floor = self.schema_floor.clone();
6527 let prior_overlay_dirty = self.overlay_dirty.get();
6528 let prior_sync_needed = self.sync_needed;
6529 let prior_sync_intents = self.sync_intent_queue.clone();
6530
6531 self.begin_observation("syncular_local_rebootstrap")?;
6532 let mut batch = ChangeAccumulator::default();
6533 let applied = (|| -> Result<(), String> {
6534 self.run_schema_reset_observed(&mut batch, false)?;
6535 self.conn
6536 .execute(
6537 "INSERT INTO _syncular_meta(key, value) VALUES (?1, 'v1')",
6538 rusqlite::params![meta_key],
6539 )
6540 .map_err(|error| error.to_string())?;
6541 self.sync_needed = true;
6542 self.sync_intent_queue.push_back(SyncIntent::Interactive);
6543 batch.status = true;
6544 Ok(())
6545 })();
6546 if let Err(error) = applied {
6547 self.rollback_observation("syncular_local_rebootstrap");
6548 self.subs = prior_subs;
6549 self.upgrading = prior_upgrading;
6550 self.stopped = prior_stopped;
6551 self.schema_floor = prior_schema_floor;
6552 self.overlay_dirty.set(prior_overlay_dirty);
6553 self.sync_needed = prior_sync_needed;
6554 self.sync_intent_queue = prior_sync_intents;
6555 return Err(error);
6556 }
6557 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
6558 self.rollback_observation("syncular_local_rebootstrap");
6559 self.subs = prior_subs;
6560 self.upgrading = prior_upgrading;
6561 self.stopped = prior_stopped;
6562 self.schema_floor = prior_schema_floor;
6563 self.overlay_dirty.set(prior_overlay_dirty);
6564 self.sync_needed = prior_sync_needed;
6565 self.sync_intent_queue = prior_sync_intents;
6566 return Err(error);
6567 }
6568
6569 Ok(LocalDataRebootstrapResult {
6570 already_applied: false,
6571 retained_commits,
6572 reset_subscriptions,
6573 })
6574 }
6575
6576 fn purge_scope_rows(
6581 &mut self,
6582 table_name: &str,
6583 effective: &[(String, Vec<String>)],
6584 ) -> Result<(), ()> {
6585 if effective.is_empty() {
6586 return Ok(());
6587 }
6588 let table = self.schema.table(table_name).ok_or(())?.clone();
6589 let mut clauses = Vec::new();
6590 let mut params: Vec<SqlValue> = Vec::new();
6591 for (variable, values) in effective {
6592 let column = table.scope_column(variable).ok_or(())?;
6593 let placeholders: Vec<String> = values
6594 .iter()
6595 .map(|v| {
6596 params.push(SqlValue::Text(v.clone()));
6597 "?".to_owned()
6598 })
6599 .collect();
6600 clauses.push(format!(
6601 "{} IN ({})",
6602 quote_ident(column),
6603 placeholders.join(", ")
6604 ));
6605 }
6606 let sql = format!(
6607 "DELETE FROM {} WHERE {}",
6608 base_table(table_name),
6609 clauses.join(" AND ")
6610 );
6611 self.overlay_dirty.set(true);
6612 self.conn
6613 .execute(&sql, rusqlite::params_from_iter(params))
6614 .map_err(|_| ())?;
6615 Ok(())
6616 }
6617
6618 fn drop_doomed_outbox(
6621 &mut self,
6622 table_name: &str,
6623 effective: &[(String, Vec<String>)],
6624 ) -> Result<bool, String> {
6625 if effective.is_empty() {
6626 return Ok(false);
6627 }
6628 let Some(table) = self.schema.table(table_name).cloned() else {
6629 return Ok(false);
6630 };
6631 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
6632 for (variable, values) in effective {
6633 match table.scope_column(variable) {
6634 Some(column) => mappings.push((column, values)),
6635 None => return Ok(false), }
6637 }
6638 let doomed: Vec<OutboxCommit> = self
6639 .outbox
6640 .iter()
6641 .filter(|commit| {
6642 commit.ops.iter().any(|op| {
6643 op.upsert
6644 && op.table == table_name
6645 && op.values.as_ref().is_some_and(|values| {
6646 mappings
6647 .iter()
6648 .all(|(column, allowed)| match values.get(*column) {
6649 Some(Value::String(s)) => allowed.contains(s),
6650 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
6651 _ => false,
6652 })
6653 })
6654 })
6655 })
6656 .cloned()
6657 .collect();
6658 if doomed.is_empty() {
6659 return Ok(false);
6660 }
6661 let mut rejections = Vec::new();
6662 for commit in &doomed {
6663 let results = commit
6664 .ops
6665 .iter()
6666 .enumerate()
6667 .map(|(op_index, operation)| {
6668 let rejection = RejectionRecord {
6669 client_commit_id: commit.client_commit_id.clone(),
6670 op_index: op_index as i32,
6671 code: "sync.scope_revoked".to_owned(),
6672 message: "the commit was dropped because its effective scope was revoked"
6673 .to_owned(),
6674 retryable: false,
6675 details: None,
6676 operation: Some(CommitOperation::from(operation)),
6677 };
6678 rejections.push(rejection.clone());
6679 CommitOperationOutcome::Error { rejection }
6680 })
6681 .collect::<Vec<_>>();
6682 self.persist_commit_outcome(
6683 &commit.client_commit_id,
6684 CommitOutcomeStatus::Rejected,
6685 &results,
6686 Some(&commit.ops),
6687 )?;
6688 self.delete_outbox_persisted(&commit.client_commit_id)?;
6689 }
6690 self.prune_commit_outcomes()?;
6691 let doomed_ids = doomed
6692 .iter()
6693 .map(|commit| commit.client_commit_id.as_str())
6694 .collect::<BTreeSet<_>>();
6695 self.outbox
6696 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
6697 self.rejections.extend(rejections);
6698 self.overlay_dirty.set(true);
6699 Ok(true)
6700 }
6701
6702 pub fn upload_blob(
6709 &mut self,
6710 bytes: &[u8],
6711 media_type: Option<String>,
6712 name: Option<String>,
6713 ) -> Result<Value, String> {
6714 let blob_id = blob_id_for(bytes);
6715 let now = self.clock_now_ms();
6716 self.conn
6717 .execute(
6718 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
6719 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
6720 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
6721 )
6722 .map_err(|e| e.to_string())?;
6723 self.conn
6724 .execute(
6725 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
6726 rusqlite::params![blob_id, media_type, now],
6727 )
6728 .map_err(|e| e.to_string())?;
6729 self.enforce_blob_cache_cap();
6732 let mut obj = Map::new();
6733 obj.insert("blobId".to_owned(), Value::from(blob_id));
6734 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
6735 if let Some(mt) = media_type {
6736 obj.insert("mediaType".to_owned(), Value::from(mt));
6737 }
6738 if let Some(n) = name {
6739 obj.insert("name".to_owned(), Value::from(n));
6740 }
6741 Ok(Value::Object(obj))
6742 }
6743
6744 pub fn fetch_blob(
6748 &mut self,
6749 transport: &mut dyn Transport,
6750 blob_id_or_ref: &str,
6751 ) -> Result<Value, (String, String)> {
6752 let simple = |m: String| ("client.failed".to_owned(), m);
6753 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
6754 blob_id_or_ref.to_owned()
6755 } else {
6756 let value: Value = serde_json::from_str(blob_id_or_ref)
6757 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
6758 value
6759 .get("blobId")
6760 .and_then(Value::as_str)
6761 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
6762 .to_owned()
6763 };
6764 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
6765 return Ok(cached);
6766 }
6767 let bytes = match transport
6773 .blob_download(&blob_id)
6774 .map_err(|e| (e.code, e.message))?
6775 {
6776 BlobDownload::Bytes(bytes) => bytes,
6777 BlobDownload::Url {
6778 url,
6779 url_expires_at_ms,
6780 } => {
6781 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
6783 return Err((
6784 "sync.segment_expired".to_owned(),
6785 format!(
6786 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
6787 ),
6788 ));
6789 }
6790 transport
6791 .fetch_blob_url(&url)
6792 .map_err(|e| (e.code, e.message))?
6793 }
6794 };
6795 if blob_id_for(&bytes) != blob_id {
6797 return Err(simple(format!(
6798 "blob content address mismatch for {blob_id}"
6799 )));
6800 }
6801 let now = self.clock_now_ms();
6802 self.conn
6803 .execute(
6804 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
6805 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
6806 )
6807 .map_err(|e| simple(e.to_string()))?;
6808 self.enforce_blob_cache_cap();
6809 self.get_cached_blob(&blob_id)
6810 .map_err(simple)?
6811 .ok_or_else(|| simple("blob cache write failed".to_owned()))
6812 }
6813
6814 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
6815 let _ = self.conn.execute(
6818 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
6819 rusqlite::params![self.clock_now_ms(), blob_id],
6820 );
6821 let mut stmt = self
6822 .conn
6823 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
6824 .map_err(|e| e.to_string())?;
6825 let mut rows = stmt
6826 .query(rusqlite::params![blob_id])
6827 .map_err(|e| e.to_string())?;
6828 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
6829 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
6830 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
6831 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
6832 let mut obj = Map::new();
6833 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
6834 obj.insert("byteLength".to_owned(), Value::from(byte_length));
6835 let mut bytes_obj = Map::new();
6836 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
6837 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
6838 if let Some(mt) = media_type {
6839 obj.insert("mediaType".to_owned(), Value::from(mt));
6840 }
6841 return Ok(Some(Value::Object(obj)));
6842 }
6843 Ok(None)
6844 }
6845
6846 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
6848 let pending: Vec<(String, Option<String>)> = {
6849 let mut stmt = self
6850 .conn
6851 .prepare(
6852 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
6853 )
6854 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
6855 let rows = stmt
6856 .query_map([], |row| {
6857 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
6858 })
6859 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
6860 rows.filter_map(Result::ok).collect()
6861 };
6862 for (blob_id, media_type) in pending {
6863 let bytes: Option<Vec<u8>> = self
6864 .conn
6865 .query_row(
6866 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
6867 rusqlite::params![blob_id],
6868 |row| row.get(0),
6869 )
6870 .ok();
6871 if let Some(bytes) = bytes {
6872 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
6873 }
6874 let _ = self.conn.execute(
6875 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
6876 rusqlite::params![blob_id],
6877 );
6878 }
6879 Ok(())
6880 }
6881
6882 fn upload_one(
6889 &self,
6890 transport: &mut dyn Transport,
6891 blob_id: &str,
6892 bytes: &[u8],
6893 media_type: Option<&str>,
6894 ) -> Result<(), TransportError> {
6895 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
6896 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
6898 url,
6899 url_expires_at_ms,
6900 } => {
6901 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
6902 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
6903 return Ok(());
6904 }
6905 }
6907 BlobUploadGrant::None => {
6908 }
6910 }
6911 transport.blob_upload(blob_id, bytes, media_type)
6912 }
6913
6914 fn enforce_blob_cache_cap(&self) {
6923 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
6924 return;
6925 };
6926 let mut total: i64 = self
6927 .conn
6928 .query_row(
6929 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
6930 [],
6931 |row| row.get(0),
6932 )
6933 .unwrap_or(0);
6934 if total <= max_bytes {
6935 return;
6936 }
6937 let candidates: Vec<(String, i64)> = {
6938 let Ok(mut stmt) = self.conn.prepare(
6939 "SELECT blob_id, byte_length FROM _syncular_blobs
6940 WHERE refcount = 0
6941 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
6942 ORDER BY last_used_ms ASC, created_at_ms ASC",
6943 ) else {
6944 return;
6945 };
6946 let Ok(rows) = stmt.query_map([], |row| {
6947 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
6948 }) else {
6949 return;
6950 };
6951 rows.filter_map(Result::ok).collect()
6952 };
6953 for (blob_id, byte_length) in candidates {
6954 if total <= max_bytes {
6955 break;
6956 }
6957 let _ = self.conn.execute(
6958 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
6959 rusqlite::params![blob_id],
6960 );
6961 total -= byte_length;
6962 }
6963 }
6964
6965 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
6969 if !self.schema_has_blobs() {
6970 return;
6971 }
6972 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
6973 for table in self.schema.tables.clone() {
6974 let blob_cols: Vec<String> = table
6975 .columns
6976 .iter()
6977 .filter(|c| c.ty == ColumnType::BlobRef)
6978 .map(|c| c.name.clone())
6979 .collect();
6980 for column in blob_cols {
6981 let sql = format!(
6982 "SELECT {} FROM {} WHERE {} IS NOT NULL",
6983 quote_ident(&column),
6984 base_table(&table.name),
6985 quote_ident(&column)
6986 );
6987 let Ok(mut stmt) = self.conn.prepare(&sql) else {
6988 continue;
6989 };
6990 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
6991 continue;
6992 };
6993 for raw in rows.flatten().flatten() {
6994 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
6995 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
6996 *counts.entry(id.to_owned()).or_insert(0) += 1;
6997 }
6998 }
6999 }
7000 }
7001 }
7002 let _ = self
7003 .conn
7004 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
7005 for (blob_id, count) in &counts {
7006 let _ = self.conn.execute(
7007 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
7008 rusqlite::params![count, blob_id],
7009 );
7010 }
7011 if delete_orphans {
7012 let _ = self.conn.execute(
7013 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
7014 [],
7015 );
7016 }
7017 }
7018
7019 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
7024 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
7025 return sql.clone();
7026 }
7027 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
7028 columns.push(quote_ident("_syncular_version"));
7029 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
7030 let primary_key = quote_ident(&table.primary_key);
7031 let updates = columns
7032 .iter()
7033 .filter(|column| **column != primary_key)
7034 .map(|column| format!("{column}=excluded.{column}"))
7035 .collect::<Vec<_>>()
7036 .join(", ");
7037 let sql = format!(
7038 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
7039 columns.join(", "),
7040 placeholders.join(", ")
7041 );
7042 self.insert_sql
7043 .borrow_mut()
7044 .insert(full_table.to_owned(), sql.clone());
7045 sql
7046 }
7047
7048 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
7049 self.overlay_dirty.set(true);
7050 self.write_row(&base_table(table_name), table_name, row, version)
7051 }
7052
7053 fn write_row(
7054 &self,
7055 full_table: &str,
7056 table_name: &str,
7057 row: &Row,
7058 version: i64,
7059 ) -> Result<(), String> {
7060 let table = self
7061 .schema
7062 .table(table_name)
7063 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
7064 let sql = self.insert_row_sql(full_table, table);
7065 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
7066 let params = row
7067 .iter()
7068 .map(RowParam::Cell)
7069 .chain(std::iter::once(RowParam::Version(version)));
7070 stmt.execute(rusqlite::params_from_iter(params))
7071 .map_err(|e| e.to_string())?;
7072 Ok(())
7073 }
7074
7075 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
7076 let table = self
7077 .schema
7078 .table(table_name)
7079 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
7080 self.overlay_dirty.set(true);
7081 let sql = format!(
7082 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
7083 base_table(table_name),
7084 quote_ident(&table.primary_key)
7085 );
7086 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
7087 stmt.execute(rusqlite::params![row_id])
7088 .map_err(|e| e.to_string())?;
7089 Ok(())
7090 }
7091
7092 fn rebuild_overlay_if_dirty(&mut self) {
7095 if self.overlay_dirty.get() {
7096 self.rebuild_overlay();
7097 }
7098 }
7099
7100 fn rebuild_overlay(&mut self) {
7104 #[cfg(test)]
7105 self.overlay_rebuild_count
7106 .set(self.overlay_rebuild_count.get() + 1);
7107 self.exec("SAVEPOINT syncular_overlay");
7108 for table in self.schema.tables.clone() {
7109 for index in &table.fts_indexes {
7110 let _ = self.drop_fts_triggers(index);
7111 }
7112 let visible = visible_table(&table.name);
7113 let base = base_table(&table.name);
7114 self.exec(&format!("DELETE FROM {visible}"));
7115 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
7116 }
7117 for commit in self.outbox.clone() {
7118 for op in &commit.ops {
7119 let Some(table) = self.schema.table(&op.table).cloned() else {
7120 continue;
7121 };
7122 if op.upsert {
7123 let Some(values) = op.values.as_ref() else {
7124 continue;
7125 };
7126 let mut row: Row = Vec::with_capacity(table.columns.len());
7127 let mut ok = true;
7128 for column in &table.columns {
7129 match json_to_column_value(column, values.get(&column.name)) {
7130 Ok(v) => row.push(v),
7131 Err(_) => {
7132 ok = false;
7133 break;
7134 }
7135 }
7136 }
7137 if ok {
7138 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
7139 }
7140 } else {
7141 let sql = format!(
7142 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
7143 visible_table(&table.name),
7144 quote_ident(&table.primary_key)
7145 );
7146 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
7147 }
7148 }
7149 }
7150 for table in self.schema.tables.clone() {
7151 for index in &table.fts_indexes {
7152 let _ = self.rebuild_fts_projection(&table, index);
7153 let _ = self.create_fts_triggers(&table, index);
7154 }
7155 }
7156 self.exec("RELEASE syncular_overlay");
7157 self.overlay_dirty.set(false);
7158 }
7159
7160 fn exec(&self, sql: &str) {
7161 let _ = self.conn.execute_batch(sql);
7162 }
7163
7164 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
7167 transport
7168 .realtime_connect_for_client(&self.client_id)
7169 .map_err(|e| format!("{}: {}", e.code, e.message))?;
7170 self.realtime_connected = true;
7171 Ok(())
7172 }
7173
7174 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
7175 let _ = transport.realtime_close();
7176 self.realtime_connected = false;
7177 self.presence.clear(); }
7179
7180 pub fn set_presence(
7186 &mut self,
7187 transport: &mut dyn Transport,
7188 scope_key: &str,
7189 doc: Option<&Value>,
7190 ) -> Result<(), String> {
7191 if !self.realtime_connected {
7192 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
7193 }
7194 let text = encode_presence_publish(scope_key, doc);
7195 transport
7196 .realtime_send(&text)
7197 .map_err(|e| format!("{}: {}", e.code, e.message))
7198 }
7199
7200 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
7202 self.presence
7203 .get(scope_key)
7204 .map(|peers| peers.values().cloned().collect())
7205 .unwrap_or_default()
7206 }
7207
7208 fn apply_presence(
7210 &mut self,
7211 scope_key: String,
7212 kind: Option<PresenceKind>,
7213 actor_id: Option<String>,
7214 client_id: Option<String>,
7215 doc: Option<Value>,
7216 error: Option<String>,
7217 ) {
7218 if error.is_some() {
7221 return;
7222 }
7223 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
7224 return;
7225 };
7226 let peer_key = format!("{actor_id} {client_id}");
7227 match kind {
7228 PresenceKind::Leave => {
7229 if let Some(peers) = self.presence.get_mut(&scope_key) {
7230 peers.remove(&peer_key);
7231 if peers.is_empty() {
7232 self.presence.remove(&scope_key);
7233 }
7234 }
7235 }
7236 _ => {
7237 let doc = match doc {
7238 Some(Value::Object(_)) => doc.unwrap(),
7239 _ => return,
7240 };
7241 self.presence.entry(scope_key).or_default().insert(
7242 peer_key,
7243 PresencePeer {
7244 actor_id,
7245 client_id,
7246 doc,
7247 },
7248 );
7249 }
7250 }
7251 }
7252
7253 pub fn on_realtime_text(&mut self, text: &str) {
7255 match parse_control(text) {
7256 Ok(ControlMessage::Hello { requires_sync, .. }) => {
7257 if requires_sync {
7258 self.set_sync_needed(true, true);
7260 }
7261 }
7262 Ok(ControlMessage::Presence {
7263 scope_key,
7264 kind,
7265 actor_id,
7266 client_id,
7267 doc,
7268 error,
7269 ..
7270 }) => {
7271 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
7272 }
7273 Ok(ControlMessage::Wake { .. }) => {
7274 self.set_sync_needed(true, true);
7276 }
7277 _ => {}
7278 }
7279 }
7280
7281 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
7284 if self.stopped {
7285 return;
7286 }
7287 let message = match decode_message(bytes) {
7288 Ok(m) if m.msg_kind == MsgKind::Response => m,
7289 _ => {
7290 self.set_sync_needed(true, true);
7291 return;
7292 }
7293 };
7294 let mut frames = message.frames.into_iter();
7295 let mut applied_cursor: Option<i64> = None;
7296 let mut any_covered = false;
7297 let mut dropped = false;
7298 while let Some(frame) = frames.next() {
7299 let Frame::SubStart {
7300 id,
7301 status,
7302 effective_scopes,
7303 ..
7304 } = frame
7305 else {
7306 continue;
7307 };
7308 let mut body = Vec::new();
7309 let mut next_cursor: Option<i64> = None;
7310 for inner in frames.by_ref() {
7311 match inner {
7312 Frame::SubEnd {
7313 next_cursor: nc, ..
7314 } => {
7315 next_cursor = Some(nc);
7316 break;
7317 }
7318 Frame::Unknown { .. } => {}
7319 other => body.push(other),
7320 }
7321 }
7322 let Some(next_cursor) = next_cursor else {
7323 dropped = true;
7324 break;
7325 };
7326 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
7327 dropped = true;
7328 continue;
7329 };
7330 let sub = &self.subs[sub_index];
7331 if status != SubStatus::Active
7334 || sub.state != SubState::Active
7335 || sub.bootstrap_state.is_some()
7336 || !sub.synced_once
7337 {
7338 dropped = true;
7339 continue;
7340 }
7341 if next_cursor <= sub.cursor {
7342 any_covered = true;
7344 continue;
7345 }
7346 let previous_effective = self.subs[sub_index].effective.clone();
7347 let previous_cursor = self.subs[sub_index].cursor;
7348 if self.begin_observation("syncular_delta").is_err() {
7349 dropped = true;
7350 continue;
7351 }
7352 self.subs[sub_index].effective = Some(effective_scopes);
7353 let mut batch = ChangeAccumulator::default();
7354 let mut failed = false;
7355 for inner in body {
7356 if let Frame::Commit {
7357 tables, changes, ..
7358 } = inner
7359 {
7360 self.record_commit_changes(&mut batch, &tables, &changes);
7361 if self.apply_commit_changes(&tables, &changes).is_err() {
7362 failed = true;
7363 break;
7364 }
7365 }
7366 }
7367 if failed {
7368 self.rollback_observation("syncular_delta");
7369 self.subs[sub_index].effective = previous_effective;
7370 self.subs[sub_index].cursor = previous_cursor;
7371 self.overlay_dirty.set(true);
7372 self.rebuild_overlay();
7373 dropped = true;
7374 continue;
7375 }
7376 let sub = &mut self.subs[sub_index];
7377 sub.cursor = next_cursor;
7378 self.persist_sub(&self.subs[sub_index].clone());
7379 self.rebuild_overlay_if_dirty();
7380 if self.finish_observation("syncular_delta", batch).is_err() {
7381 self.rollback_observation("syncular_delta");
7382 self.subs[sub_index].effective = previous_effective;
7383 self.subs[sub_index].cursor = previous_cursor;
7384 self.overlay_dirty.set(true);
7385 self.rebuild_overlay();
7386 dropped = true;
7387 continue;
7388 }
7389 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
7390 }
7391 if let Some(cursor) = applied_cursor {
7392 self.reconcile_blob_refcounts(false);
7393 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
7395 let _ = transport.realtime_send(&ack);
7396 } else if !any_covered || dropped {
7397 self.set_sync_needed(true, true);
7399 }
7400 }
7401
7402 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
7406 if !self.realtime_connected {
7407 return;
7408 }
7409 let floor = self
7410 .subs
7411 .iter()
7412 .filter(|s| {
7413 s.state == SubState::Active
7414 && s.bootstrap_state.is_none()
7415 && s.synced_once
7416 && s.cursor >= 0
7417 })
7418 .map(|s| s.cursor)
7419 .min();
7420 if let Some(cursor) = floor {
7421 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
7422 let _ = transport.realtime_send(&ack);
7423 }
7424 }
7425}