1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, CommitOperation, CommitOperationOutcome,
27 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
28 ConflictRecord, CoverageSnapshot, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
29 LocalDataPurgeTarget, Mutation, PresencePeer, QuerySnapshot, RejectionDetails, RejectionRecord,
30 ResolveCommitOutcomeInput, RowState, SchemaFloor, SubscriptionStateView, SyncIntent,
31 SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange, WindowBase, WindowChange,
32 WindowCoverage, WindowState, WindowUnitRef,
33};
34use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
35use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
36use crate::values::{
37 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
38 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
39 scope_map_to_json, sort_scope_map,
40};
41
42const DEFAULT_ACCEPT: u8 = 0b0111;
47const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
48const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
49const ACCEPT_SQLITE: u8 = 1 << 2;
50const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
51
52const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
54const LOCAL_REVISION_KEY: &str = "localRevision";
55const CLIENT_ID_KEY: &str = "clientId";
56const LEASE_STATE_KEY: &str = "leaseState";
57const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
58const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63enum SubState {
64 Active,
65 Revoked,
66 Failed,
67}
68
69#[cfg(test)]
70mod observation_tests {
71 use super::*;
72 use serde_json::json;
73
74 fn client() -> SyncClient {
75 SyncClient::new(
76 "retry-test".to_owned(),
77 &json!({
78 "version": 1,
79 "tables": [{
80 "name": "tasks",
81 "primaryKey": "id",
82 "columns": [
83 { "name": "id", "type": "string", "nullable": false },
84 { "name": "project_id", "type": "string", "nullable": false }
85 ],
86 "scopes": [{ "pattern": "project:{project_id}" }]
87 }]
88 }),
89 ClientLimits::default(),
90 )
91 .expect("test client")
92 }
93
94 #[test]
95 fn background_retry_deadlines_back_off_and_reset() {
96 let mut client = client();
97 client.schedule_background_retry();
98 assert!(matches!(
99 client.drain_sync_intents().as_slice(),
100 [SyncIntent::Background { delay_ms: 250 }]
101 ));
102 client.schedule_background_retry();
103 assert!(matches!(
104 client.drain_sync_intents().as_slice(),
105 [SyncIntent::Background { delay_ms: 500 }]
106 ));
107 client.reset_background_retry();
108 client.schedule_background_retry();
109 assert!(matches!(
110 client.drain_sync_intents().as_slice(),
111 [SyncIntent::Background { delay_ms: 250 }]
112 ));
113 }
114
115 #[test]
116 fn secondary_unique_collision_preserves_existing_synced_row() {
117 let client = SyncClient::new(
118 "unique-upsert-test".to_owned(),
119 &json!({
120 "version": 1,
121 "tables": [{
122 "name": "tasks",
123 "primaryKey": "id",
124 "columns": [
125 { "name": "id", "type": "string", "nullable": false },
126 { "name": "project_id", "type": "string", "nullable": false },
127 { "name": "title", "type": "string", "nullable": false }
128 ],
129 "scopes": [{ "pattern": "project:{project_id}" }],
130 "indexes": [{
131 "name": "tasks_by_project_title",
132 "columns": ["project_id", "title"],
133 "unique": true
134 }]
135 }]
136 }),
137 ClientLimits::default(),
138 )
139 .expect("test client");
140 let table = client.schema.table("tasks").expect("tasks table");
141 let sql = client.insert_row_sql(&base_table("tasks"), table);
142
143 client
144 .conn
145 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
146 .expect("insert first row");
147 client
148 .conn
149 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
150 .expect("update same primary key");
151 client
152 .conn
153 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
154 .expect("insert second row");
155 assert!(client
156 .conn
157 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
158 .is_err());
159
160 let rows = client
161 .conn
162 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
163 .expect("prepare rows")
164 .query_map([], |row| {
165 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
166 })
167 .expect("query rows")
168 .collect::<Result<Vec<_>, _>>()
169 .expect("collect rows");
170 assert_eq!(
171 rows,
172 vec![
173 ("t1".to_owned(), "updated".to_owned()),
174 ("t2".to_owned(), "original".to_owned())
175 ]
176 );
177 }
178
179 #[test]
180 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
181 let path = std::env::temp_dir().join(format!(
182 "syncular-startup-intent-{}.db",
183 uuid::Uuid::new_v4()
184 ));
185 let schema = json!({
186 "version": 1,
187 "tables": [{
188 "name": "tasks",
189 "primaryKey": "id",
190 "columns": [
191 { "name": "id", "type": "string", "nullable": false },
192 { "name": "project_id", "type": "string", "nullable": false }
193 ],
194 "scopes": [{ "pattern": "project:{project_id}" }]
195 }]
196 });
197
198 {
199 let mut first = SyncClient::open_path_with_identity(
200 None,
201 &schema,
202 ClientLimits::default(),
203 path.to_str().expect("UTF-8 temp path"),
204 )
205 .expect("first open");
206 first
207 .set_window(
208 &WindowBase {
209 table: "tasks".to_owned(),
210 variable: "project_id".to_owned(),
211 fixed_scopes: Vec::new(),
212 params: None,
213 },
214 &["persisted".to_owned()],
215 )
216 .expect("persist window");
217 }
218
219 let mut reopened = SyncClient::open_path_with_identity(
220 None,
221 &schema,
222 ClientLimits::default(),
223 path.to_str().expect("UTF-8 temp path"),
224 )
225 .expect("reopen");
226 assert!(reopened.sync_needed());
227 assert!(matches!(
228 reopened.drain_sync_intents().as_slice(),
229 [SyncIntent::Interactive]
230 ));
231 drop(reopened);
232 std::fs::remove_file(path).expect("remove temp database");
233 }
234
235 #[test]
236 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
237 let path = std::env::temp_dir().join(format!(
238 "syncular-satisfied-schema-floor-{}.db",
239 uuid::Uuid::new_v4()
240 ));
241 let schema = json!({
242 "version": 23,
243 "tables": [{
244 "name": "tasks",
245 "primaryKey": "id",
246 "columns": [
247 { "name": "id", "type": "string", "nullable": false },
248 { "name": "project_id", "type": "string", "nullable": false }
249 ],
250 "scopes": [{ "pattern": "project:{project_id}" }]
251 }]
252 });
253
254 {
255 let mut first = SyncClient::open_path_with_identity(
256 None,
257 &schema,
258 ClientLimits::default(),
259 path.to_str().expect("UTF-8 temp path"),
260 )
261 .expect("first open");
262 first
263 .subscribe(
264 "tasks".to_owned(),
265 "tasks".to_owned(),
266 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
267 None,
268 )
269 .expect("persist subscription");
270 first.set_schema_floor(Some(SchemaFloor {
271 required_schema_version: Some(22),
272 latest_schema_version: Some(22),
273 }));
274 }
275
276 let mut reopened = SyncClient::open_path_with_identity(
277 None,
278 &schema,
279 ClientLimits::default(),
280 path.to_str().expect("UTF-8 temp path"),
281 )
282 .expect("reopen");
283 assert!(reopened.schema_floor().is_none());
284 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
285 assert!(reopened.sync_needed());
286 assert!(matches!(
287 reopened.drain_sync_intents().as_slice(),
288 [SyncIntent::Interactive]
289 ));
290 drop(reopened);
291 std::fs::remove_file(path).expect("remove temp database");
292 }
293
294 #[test]
295 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
296 let path = std::env::temp_dir().join(format!(
297 "syncular-unsatisfied-schema-floor-{}.db",
298 uuid::Uuid::new_v4()
299 ));
300 let schema = json!({
301 "version": 1,
302 "tables": [{
303 "name": "tasks",
304 "primaryKey": "id",
305 "columns": [
306 { "name": "id", "type": "string", "nullable": false },
307 { "name": "project_id", "type": "string", "nullable": false }
308 ],
309 "scopes": [{ "pattern": "project:{project_id}" }]
310 }]
311 });
312
313 {
314 let mut first = SyncClient::open_path_with_identity(
315 None,
316 &schema,
317 ClientLimits::default(),
318 path.to_str().expect("UTF-8 temp path"),
319 )
320 .expect("first open");
321 first
322 .subscribe(
323 "tasks".to_owned(),
324 "tasks".to_owned(),
325 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
326 None,
327 )
328 .expect("persist subscription");
329 first.set_schema_floor(Some(SchemaFloor {
330 required_schema_version: Some(2),
331 latest_schema_version: Some(2),
332 }));
333 }
334
335 let reopened = SyncClient::open_path_with_identity(
336 None,
337 &schema,
338 ClientLimits::default(),
339 path.to_str().expect("UTF-8 temp path"),
340 )
341 .expect("reopen");
342 assert_eq!(
343 reopened.schema_floor(),
344 Some(&SchemaFloor {
345 required_schema_version: Some(2),
346 latest_schema_version: Some(2),
347 })
348 );
349 assert!(!reopened.sync_needed());
350 drop(reopened);
351 std::fs::remove_file(path).expect("remove temp database");
352 }
353
354 #[test]
355 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
356 let path = std::env::temp_dir().join(format!(
357 "syncular-indexed-column-bump-{}.db",
358 uuid::Uuid::new_v4()
359 ));
360 let old_schema = json!({
361 "version": 1,
362 "tables": [
363 {
364 "name": "tasks",
365 "primaryKey": "id",
366 "columns": [
367 { "name": "id", "type": "string", "nullable": false },
368 { "name": "project_id", "type": "string", "nullable": false }
369 ],
370 "scopes": [{ "pattern": "project:{project_id}" }]
371 },
372 {
373 "name": "legacy",
374 "primaryKey": "id",
375 "columns": [
376 { "name": "id", "type": "string", "nullable": false },
377 { "name": "project_id", "type": "string", "nullable": false }
378 ],
379 "scopes": [{ "pattern": "project:{project_id}" }]
380 }
381 ]
382 });
383 {
384 let mut first = SyncClient::open_path_with_identity(
385 None,
386 &old_schema,
387 ClientLimits::default(),
388 path.to_str().expect("UTF-8 temp path"),
389 )
390 .expect("open old schema");
391 first
392 .subscribe(
393 "legacy-sub".to_owned(),
394 "legacy".to_owned(),
395 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
396 None,
397 )
398 .expect("persist legacy subscription");
399 }
400
401 let new_schema = json!({
402 "version": 2,
403 "tables": [{
404 "name": "tasks",
405 "primaryKey": "id",
406 "columns": [
407 { "name": "id", "type": "string", "nullable": false },
408 { "name": "project_id", "type": "string", "nullable": false },
409 { "name": "facility_membership_id", "type": "string", "nullable": true }
410 ],
411 "scopes": [{ "pattern": "project:{project_id}" }],
412 "indexes": [{
413 "name": "tasks_by_membership",
414 "columns": ["project_id", "facility_membership_id"],
415 "unique": false
416 }]
417 }]
418 });
419 let upgraded = SyncClient::open_path_with_identity(
420 None,
421 &new_schema,
422 ClientLimits::default(),
423 path.to_str().expect("UTF-8 temp path"),
424 )
425 .expect("open upgraded schema");
426 let columns = upgraded
427 .conn
428 .prepare("PRAGMA table_info(tasks)")
429 .expect("prepare columns")
430 .query_map([], |row| row.get::<_, String>(1))
431 .expect("query columns")
432 .collect::<Result<Vec<_>, _>>()
433 .expect("collect columns");
434 assert!(columns.contains(&"facility_membership_id".to_owned()));
435 let index_count: i64 = upgraded
436 .conn
437 .query_row(
438 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
439 [],
440 |row| row.get(0),
441 )
442 .expect("query index");
443 assert_eq!(index_count, 1);
444 assert!(upgraded.subscription_state("legacy-sub").is_none());
445 drop(upgraded);
446 std::fs::remove_file(path).expect("remove temp database");
447 }
448
449 #[test]
450 fn migrates_pre_envelope_outcome_journal_additively() {
451 let path = std::env::temp_dir().join(format!(
452 "syncular-outcome-migration-{}.db",
453 uuid::Uuid::new_v4()
454 ));
455 let conn = Connection::open(&path).expect("open legacy database");
456 conn.execute_batch(
457 "CREATE TABLE _syncular_commit_outcomes (
458 seq INTEGER PRIMARY KEY AUTOINCREMENT,
459 client_commit_id TEXT NOT NULL UNIQUE,
460 status TEXT NOT NULL,
461 recorded_at_ms INTEGER NOT NULL,
462 results_json TEXT NOT NULL,
463 resolution TEXT NOT NULL DEFAULT 'active',
464 resolved_at_ms INTEGER,
465 replacement_client_commit_id TEXT);",
466 )
467 .expect("create legacy outcome journal");
468 drop(conn);
469 let schema = json!({
470 "version": 1,
471 "tables": [{
472 "name": "tasks",
473 "primaryKey": "id",
474 "columns": [
475 { "name": "id", "type": "string", "nullable": false },
476 { "name": "project_id", "type": "string", "nullable": false }
477 ],
478 "scopes": [{ "pattern": "project:{project_id}" }]
479 }]
480 });
481 let client = SyncClient::open_path(
482 "migration-native".to_owned(),
483 &schema,
484 ClientLimits::default(),
485 path.to_str().expect("UTF-8 temp path"),
486 )
487 .expect("migrate database");
488 let has_operations = client
489 .conn
490 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
491 .expect("prepare table info")
492 .query_map([], |row| row.get::<_, String>(1))
493 .expect("query table info")
494 .filter_map(Result::ok)
495 .any(|column| column == "operations_json");
496 assert!(has_operations);
497 drop(client);
498 std::fs::remove_file(path).expect("remove temp database");
499 }
500
501 #[test]
502 fn durable_conflict_outcome_and_resolution_survive_reopen() {
503 let path = std::env::temp_dir().join(format!(
504 "syncular-durable-outcome-{}.db",
505 uuid::Uuid::new_v4()
506 ));
507 let schema = json!({
508 "version": 1,
509 "tables": [{
510 "name": "tasks",
511 "primaryKey": "id",
512 "columns": [
513 { "name": "id", "type": "string", "nullable": false },
514 { "name": "project_id", "type": "string", "nullable": false }
515 ],
516 "scopes": [{ "pattern": "project:{project_id}" }]
517 }]
518 });
519 let conflict = ConflictRecord {
520 client_commit_id: "losing-commit".to_owned(),
521 op_index: 0,
522 table: "tasks".to_owned(),
523 row_id: "t1".to_owned(),
524 code: "sync.version_conflict".to_owned(),
525 message: "stale base version".to_owned(),
526 server_version: 2,
527 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
528 operation: Some(CommitOperation {
529 table: "tasks".to_owned(),
530 row_id: "t1".to_owned(),
531 op: "upsert".to_owned(),
532 base_version: Some(1),
533 values: None,
534 changed_fields: None,
535 }),
536 };
537 let failed_operations = vec![
538 OutboxOp {
539 upsert: true,
540 table: "tasks".to_owned(),
541 row_id: "t1".to_owned(),
542 base_version: Some(1),
543 values: None,
544 changed_fields: None,
545 },
546 OutboxOp {
547 upsert: true,
548 table: "tasks".to_owned(),
549 row_id: "status-event-1".to_owned(),
550 base_version: Some(0),
551 values: None,
552 changed_fields: None,
553 },
554 ];
555
556 {
557 let mut first = SyncClient::open_path(
558 "durable-native".to_owned(),
559 &schema,
560 ClientLimits::default(),
561 path.to_str().expect("UTF-8 temp path"),
562 )
563 .expect("first open");
564 first
565 .begin_observation("test_outcome")
566 .expect("begin outcome");
567 first
568 .persist_commit_outcome(
569 "losing-commit",
570 CommitOutcomeStatus::Conflict,
571 &[CommitOperationOutcome::Conflict {
572 conflict: conflict.clone(),
573 }],
574 Some(&failed_operations),
575 )
576 .expect("persist outcome");
577 first.conflicts.push(conflict);
578 first
579 .finish_observation(
580 "test_outcome",
581 ChangeAccumulator {
582 conflicts: true,
583 outcomes: true,
584 ..ChangeAccumulator::default()
585 },
586 )
587 .expect("commit outcome");
588 }
589
590 {
591 let mut reopened = SyncClient::open_path(
592 "durable-native".to_owned(),
593 &schema,
594 ClientLimits::default(),
595 path.to_str().expect("UTF-8 temp path"),
596 )
597 .expect("reopen");
598 assert_eq!(reopened.conflicts().len(), 1);
599 let outcome = reopened
600 .commit_outcome("losing-commit")
601 .expect("read outcome")
602 .expect("outcome");
603 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
604 let operations = outcome.operations.expect("aggregate envelope");
605 assert_eq!(operations.len(), 2);
606 assert_eq!(operations[1].row_id, "status-event-1");
607 let resolved = reopened
608 .resolve_commit_outcome(ResolveCommitOutcomeInput {
609 client_commit_id: "losing-commit".to_owned(),
610 resolution: CommitOutcomeResolution::ResolvedKeepServer,
611 replacement_client_commit_id: None,
612 })
613 .expect("resolve");
614 assert_eq!(
615 resolved.resolution,
616 CommitOutcomeResolution::ResolvedKeepServer
617 );
618 assert!(reopened.conflicts().is_empty());
619 }
620
621 let reopened = SyncClient::open_path(
622 "durable-native".to_owned(),
623 &schema,
624 ClientLimits::default(),
625 path.to_str().expect("UTF-8 temp path"),
626 )
627 .expect("second reopen");
628 assert!(reopened.conflicts().is_empty());
629 assert_eq!(
630 reopened
631 .commit_outcome("losing-commit")
632 .expect("read outcome")
633 .expect("outcome")
634 .resolution,
635 CommitOutcomeResolution::ResolvedKeepServer
636 );
637 drop(reopened);
638 std::fs::remove_file(path).expect("remove temp database");
639 }
640
641 #[test]
642 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
643 let path =
644 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
645 let schema = json!({
646 "version": 1,
647 "tables": [{
648 "name": "tasks",
649 "primaryKey": "id",
650 "columns": [
651 { "name": "id", "type": "string", "nullable": false },
652 { "name": "project_id", "type": "string", "nullable": false }
653 ],
654 "scopes": [{ "pattern": "project:{project_id}" }]
655 }]
656 });
657 let mut client = SyncClient::open_path_with_identity(
658 Some("sidecar-client".to_owned()),
659 &schema,
660 ClientLimits::default(),
661 path.to_str().expect("UTF-8 temp path"),
662 )
663 .expect("open owner");
664 let base = WindowBase {
665 table: "tasks".to_owned(),
666 variable: "project_id".to_owned(),
667 fixed_scopes: Vec::new(),
668 params: None,
669 };
670 client
671 .set_window(&base, &["one".to_owned()])
672 .expect("set window");
673 client
674 .mutate(vec![Mutation::Upsert {
675 table: "tasks".to_owned(),
676 values: Map::from_iter([
677 ("id".to_owned(), Value::from("t1")),
678 ("project_id".to_owned(), Value::from("one")),
679 ]),
680 base_version: None,
681 }])
682 .expect("local mutate");
683
684 let coverage = [WindowCoverage {
685 base,
686 units: vec!["one".to_owned(), "missing".to_owned()],
687 }];
688 let owner = client
689 .query_snapshot(
690 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
691 &[],
692 &coverage,
693 )
694 .expect("owner snapshot");
695 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
696 let sidecar = reader
697 .query_snapshot(
698 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
699 &[],
700 &coverage,
701 )
702 .expect("sidecar snapshot");
703
704 assert_eq!(sidecar.revision, owner.revision);
705 assert_eq!(sidecar.rows, owner.rows);
706 assert_eq!(
707 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
708 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
709 );
710 assert_eq!(sidecar.revision, "2");
711 assert_eq!(sidecar.rows[0]["id"], "t1");
712 assert_eq!(sidecar.rows[0]["server_version"], -1);
713 assert!(!sidecar.coverage.complete);
714 assert_eq!(sidecar.coverage.pending.len(), 1);
715 assert_eq!(sidecar.coverage.missing.len(), 1);
716
717 drop(reader);
718 drop(client);
719 std::fs::remove_file(path).expect("remove temp database");
720 }
721
722 #[test]
723 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
724 let schema = json!({
725 "version": 1,
726 "tables": [{
727 "name": "catalogue_codes",
728 "primaryKey": "id",
729 "columns": [
730 { "name": "id", "type": "string", "nullable": false },
731 { "name": "release_id", "type": "string", "nullable": false },
732 { "name": "code", "type": "string", "nullable": false },
733 { "name": "title", "type": "string", "nullable": false }
734 ],
735 "scopes": [{ "pattern": "release:{release_id}" }],
736 "ftsIndexes": [{
737 "name": "catalogue_codes_fts",
738 "columns": ["code", "title"],
739 "tokenize": "unicode61 remove_diacritics 2"
740 }]
741 }]
742 });
743 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
744 .expect("FTS5 client");
745 let search = |client: &SyncClient, query: &str| {
746 client
747 .query(
748 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
749 &[Value::from(query)],
750 )
751 .expect("FTS query")
752 };
753
754 client
755 .mutate(vec![Mutation::Upsert {
756 table: "catalogue_codes".to_owned(),
757 values: Map::from_iter([
758 ("id".to_owned(), Value::from("c1")),
759 ("release_id".to_owned(), Value::from("r1")),
760 ("code".to_owned(), Value::from("A01")),
761 ("title".to_owned(), Value::from("Cholera")),
762 ]),
763 base_version: None,
764 }])
765 .expect("insert code");
766 assert_eq!(search(&client, "cholera").len(), 1);
767
768 client
769 .mutate(vec![Mutation::Upsert {
770 table: "catalogue_codes".to_owned(),
771 values: Map::from_iter([
772 ("id".to_owned(), Value::from("c1")),
773 ("release_id".to_owned(), Value::from("r1")),
774 ("code".to_owned(), Value::from("A01")),
775 ("title".to_owned(), Value::from("Enteric infection")),
776 ]),
777 base_version: None,
778 }])
779 .expect("update code");
780 assert!(search(&client, "cholera").is_empty());
781 assert_eq!(search(&client, "enteric").len(), 1);
782
783 client
784 .mutate(vec![Mutation::Delete {
785 table: "catalogue_codes".to_owned(),
786 row_id: "c1".to_owned(),
787 base_version: None,
788 }])
789 .expect("delete code");
790 assert!(search(&client, "enteric").is_empty());
791 }
792
793 #[test]
794 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
795 let schema = json!({
796 "version": 1,
797 "tables": [{
798 "name": "patient_notes",
799 "primaryKey": "id",
800 "columns": [
801 { "name": "id", "type": "string", "nullable": false },
802 { "name": "practice_id", "type": "string", "nullable": false },
803 { "name": "encryption_key_id", "type": "string", "nullable": false },
804 { "name": "title", "type": "string", "nullable": false }
805 ],
806 "scopes": [{ "pattern": "practice:{practice_id}" }],
807 "ftsIndexes": [{
808 "name": "patient_notes_fts",
809 "columns": ["title"],
810 "tokenize": "unicode61 remove_diacritics 2"
811 }]
812 }]
813 });
814 let mut client = SyncClient::new(
815 "local-purge-test".to_owned(),
816 &schema,
817 ClientLimits::default(),
818 )
819 .expect("local purge client");
820 client
821 .conn
822 .execute(
823 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
824 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
825 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
826 [],
827 )
828 .expect("seed base rows");
829 client.overlay_dirty.set(true);
830 client.rebuild_overlay_if_dirty();
831
832 let note = |id: &str, key_id: &str, title: &str| {
833 Map::from_iter([
834 ("id".to_owned(), Value::from(id)),
835 ("practice_id".to_owned(), Value::from("practice-1")),
836 ("encryption_key_id".to_owned(), Value::from(key_id)),
837 ("title".to_owned(), Value::from(title)),
838 ])
839 };
840 let doomed = client
841 .mutate(vec![
842 Mutation::Upsert {
843 table: "patient_notes".to_owned(),
844 values: note("target", "key-revoked", "Target changed"),
845 base_version: None,
846 },
847 Mutation::Upsert {
848 table: "patient_notes".to_owned(),
849 values: note("unrelated", "key-held", "Unrelated changed"),
850 base_version: None,
851 },
852 ])
853 .expect("doomed commit");
854 let kept = client
855 .mutate(vec![Mutation::Upsert {
856 table: "patient_notes".to_owned(),
857 values: note("kept", "key-held", "Kept optimistic"),
858 base_version: None,
859 }])
860 .expect("kept commit");
861 client.drain_change_batches();
862
863 let input = LocalDataPurgeInput {
864 purge_id: "purge-001".to_owned(),
865 targets: vec![LocalDataPurgeTarget {
866 table: "patient_notes".to_owned(),
867 selectors: BTreeMap::from([(
868 "encryption_key_id".to_owned(),
869 vec!["key-revoked".to_owned()],
870 )]),
871 }],
872 };
873 assert_eq!(
874 client.purge_local_data(&input).expect("apply purge"),
875 LocalDataPurgeResult {
876 already_applied: false,
877 purged_rows: 1,
878 dropped_commits: 1,
879 }
880 );
881 let rows = client
882 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
883 .expect("visible rows");
884 assert_eq!(rows.len(), 2);
885 assert_eq!(rows[0]["id"], "kept");
886 assert_eq!(rows[1]["id"], "unrelated");
887 assert_eq!(rows[1]["title"], "Unrelated original");
888 let fts = client
889 .query(
890 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
891 &[],
892 )
893 .expect("fts query");
894 assert!(fts.is_empty());
895 assert_eq!(client.outbox.len(), 1);
896 assert_eq!(client.outbox[0].client_commit_id, kept);
897 let outcome = client
898 .commit_outcome(&doomed)
899 .expect("read doomed outcome")
900 .expect("doomed outcome");
901 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
902 match &outcome.results[0] {
903 CommitOperationOutcome::Error { rejection } => {
904 assert_eq!(rejection.code, "client.local_data_purged");
905 assert_eq!(rejection.client_commit_id, doomed);
906 }
907 other => panic!("expected local purge rejection, got {other:?}"),
908 }
909 assert_eq!(client.drain_change_batches().len(), 1);
910 assert_eq!(
911 client.purge_local_data(&input).expect("retry purge"),
912 LocalDataPurgeResult {
913 already_applied: true,
914 purged_rows: 0,
915 dropped_commits: 0,
916 }
917 );
918 let conflicting = LocalDataPurgeInput {
919 purge_id: input.purge_id.clone(),
920 targets: vec![LocalDataPurgeTarget {
921 table: "patient_notes".to_owned(),
922 selectors: BTreeMap::from([(
923 "encryption_key_id".to_owned(),
924 vec!["key-held".to_owned()],
925 )]),
926 }],
927 };
928 assert!(client
929 .purge_local_data(&conflicting)
930 .expect_err("id collision must fail")
931 .contains("already used with a different plan"));
932 }
933}
934
935impl SubState {
936 fn name(self) -> &'static str {
937 match self {
938 SubState::Active => "active",
939 SubState::Revoked => "revoked",
940 SubState::Failed => "failed",
941 }
942 }
943
944 fn parse(value: &str) -> Self {
945 match value {
946 "revoked" => Self::Revoked,
947 "failed" => Self::Failed,
948 _ => Self::Active,
949 }
950 }
951}
952
953#[derive(Debug, Clone)]
954struct Subscription {
955 id: String,
956 table: String,
957 requested: Vec<(String, Vec<String>)>,
958 params: Option<String>,
959 cursor: i64,
960 bootstrap_state: Option<String>,
962 state: SubState,
963 reason_code: Option<String>,
964 effective: Option<Vec<(String, Vec<String>)>>,
967 synced_once: bool,
968}
969
970#[derive(Debug, Clone)]
971struct OutboxOp {
972 upsert: bool,
973 table: String,
974 row_id: String,
975 base_version: Option<i64>,
976 values: Option<Map<String, Value>>,
979 changed_fields: Option<Vec<String>>,
981}
982
983impl From<&OutboxOp> for CommitOperation {
984 fn from(operation: &OutboxOp) -> Self {
985 Self {
986 table: operation.table.clone(),
987 row_id: operation.row_id.clone(),
988 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
989 base_version: operation.base_version,
990 values: operation.values.clone(),
991 changed_fields: operation.changed_fields.clone(),
992 }
993 }
994}
995
996#[derive(Debug, Clone)]
997struct OutboxCommit {
998 client_commit_id: String,
999 ops: Vec<OutboxOp>,
1000}
1001
1002#[derive(Debug, Clone)]
1003struct CompiledLocalDataPurgeTarget {
1004 table: String,
1005 selectors: Vec<(String, Vec<String>)>,
1006}
1007
1008struct StoredCommitOutcomeRow {
1009 sequence: i64,
1010 client_commit_id: String,
1011 status: String,
1012 recorded_at_ms: i64,
1013 results_json: String,
1014 operations_json: Option<String>,
1015 resolution: String,
1016 resolved_at_ms: Option<i64>,
1017 replacement_client_commit_id: Option<String>,
1018}
1019
1020enum SectionError {
1023 FailClosed,
1024 Abort(String, String),
1025}
1026
1027struct RequestMeta {
1028 pushed_ids: Vec<String>,
1029 fresh: Vec<(String, bool)>,
1032 accept: u8,
1033 deferred_commits: usize,
1037}
1038
1039#[derive(Default)]
1040struct ChangeAccumulator {
1041 tables: BTreeMap<String, Option<BTreeSet<String>>>,
1043 windows: BTreeMap<(String, String), BTreeSet<String>>,
1044 status: bool,
1045 conflicts: bool,
1046 rejections: bool,
1047 outcomes: bool,
1048}
1049
1050impl ChangeAccumulator {
1051 fn table(&mut self, table: &str) {
1052 self.tables.insert(table.to_owned(), None);
1053 }
1054
1055 fn scope(&mut self, table: &str, key: String) {
1056 match self.tables.get_mut(table) {
1057 Some(None) => {}
1058 Some(Some(keys)) => {
1059 keys.insert(key);
1060 }
1061 None => {
1062 self.tables
1063 .insert(table.to_owned(), Some(BTreeSet::from([key])));
1064 }
1065 }
1066 }
1067
1068 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
1069 self.windows
1070 .entry((base_key.to_owned(), table.to_owned()))
1071 .or_default()
1072 .insert(unit.to_owned());
1073 }
1074
1075 fn touched(&self) -> bool {
1076 !self.tables.is_empty()
1077 || !self.windows.is_empty()
1078 || self.status
1079 || self.conflicts
1080 || self.rejections
1081 || self.outcomes
1082 }
1083}
1084
1085const PUSH_OPS_PER_REQUEST: usize = 500;
1091const MAX_LOCAL_PURGE_TARGETS: usize = 64;
1092const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
1093const MAX_LOCAL_PURGE_VALUES: usize = 128;
1094const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
1095
1096pub struct SyncClient {
1097 conn: Connection,
1098 schema: ClientSchema,
1099 client_id: String,
1100 limits: ClientLimits,
1101 subs: Vec<Subscription>,
1102 outbox: Vec<OutboxCommit>,
1103 conflicts: Vec<ConflictRecord>,
1104 rejections: Vec<RejectionRecord>,
1105 schema_floor: Option<SchemaFloor>,
1106 lease_state: Option<LeaseState>,
1108 stopped: bool,
1110 upgrading: bool,
1112 sync_needed: bool,
1114 realtime_connected: bool,
1115 presence: HashMap<String, HashMap<String, PresencePeer>>,
1117 now_ms: Option<i64>,
1120 encryption: crate::values::EncryptionConfig,
1124 insert_sql: RefCell<HashMap<String, String>>,
1130 overlay_dirty: Cell<bool>,
1134 change_queue: VecDeque<ClientChangeBatch>,
1136 sync_intent_queue: VecDeque<SyncIntent>,
1137 retry_delay_ms: u64,
1139}
1140
1141fn quote_ident(name: &str) -> String {
1142 format!("\"{}\"", name.replace('"', "\"\""))
1143}
1144
1145fn is_local_purge_code_like(value: &str) -> bool {
1146 let bytes = value.as_bytes();
1147 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
1148 && bytes
1149 .iter()
1150 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
1151}
1152
1153fn base_table(name: &str) -> String {
1154 quote_ident(&format!("_syncular_base_{name}"))
1155}
1156
1157type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
1162
1163fn window_base_key(base: &WindowBase) -> String {
1164 format!(
1165 "{}\0{}\0{}",
1166 base.table,
1167 base.variable,
1168 canonical_scope_json(&base.fixed_scopes)
1169 )
1170}
1171
1172fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
1174 let mut scopes = base.fixed_scopes.clone();
1175 scopes.retain(|(k, _)| k != &base.variable);
1176 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
1177 scopes
1178}
1179
1180fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
1184 let canonical = canonical_scope_json(&unit_scopes(base, unit));
1185 let digest = Sha256::digest(canonical.as_bytes());
1186 let hex = bytes_to_hex(&digest);
1187 format!("w:{}:{}", base.table, &hex[..16])
1188}
1189
1190fn visible_table(name: &str) -> String {
1191 quote_ident(name)
1192}
1193
1194const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1195
1196fn is_synced_table_name(name: &str) -> bool {
1199 if name.starts_with("sqlite_") {
1200 return false;
1201 }
1202 if name.starts_with("_syncular_base_") {
1203 return true; }
1205 !name.starts_with("_syncular_")
1207}
1208
1209fn blob_id_for(bytes: &[u8]) -> String {
1211 let digest = Sha256::digest(bytes);
1212 format!("sha256:{}", bytes_to_hex(&digest))
1213}
1214
1215enum RowParam<'a> {
1219 Cell(&'a Option<ColumnValue>),
1220 Version(i64),
1221}
1222
1223impl rusqlite::ToSql for RowParam<'_> {
1224 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1225 Ok(match self {
1226 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1227 RowParam::Cell(cell) => match cell {
1228 None => ToSqlOutput::Owned(SqlValue::Null),
1229 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1230 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1231 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1232 Some(ColumnValue::Boolean(b)) => {
1233 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1234 }
1235 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1236 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1237 }
1238 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1240 ToSqlOutput::Borrowed(ValueRef::Blob(b))
1241 }
1242 },
1243 })
1244 }
1245}
1246
1247fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1255 use ssp2::segment::ColumnType;
1256 let mismatch = || {
1257 Err(format!(
1258 "image column {:?} holds a value of the wrong type",
1259 column.name
1260 ))
1261 };
1262 match value {
1263 ValueRef::Null => {
1264 if !column.nullable {
1265 return Err(format!(
1266 "image column {:?} is NULL but not nullable",
1267 column.name
1268 ));
1269 }
1270 Ok(ToSqlOutput::Owned(SqlValue::Null))
1271 }
1272 ValueRef::Integer(i) => match column.ty {
1273 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1274 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1275 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1276 _ => mismatch(),
1277 },
1278 ValueRef::Real(_) => match column.ty {
1279 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1280 _ => mismatch(),
1281 },
1282 ValueRef::Text(t) => {
1283 std::str::from_utf8(t)
1284 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1285 match column.ty {
1286 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1287 Ok(ToSqlOutput::Borrowed(value))
1288 }
1289 _ => mismatch(),
1290 }
1291 }
1292 ValueRef::Blob(_) => match column.ty {
1293 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1295 _ => mismatch(),
1296 },
1297 }
1298}
1299
1300fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1301 use rusqlite::types::ValueRef;
1302 match value {
1303 ValueRef::Null => Value::Null,
1304 ValueRef::Integer(i) => match column.ty {
1305 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1306 ssp2::segment::ColumnType::Float => {
1307 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1308 }
1309 _ => Value::from(i),
1310 },
1311 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1312 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1313 ValueRef::Blob(b) => {
1314 let mut map = Map::new();
1315 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1316 Value::Object(map)
1317 }
1318 }
1319}
1320
1321fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1325 Ok(match value {
1326 Value::Null => SqlValue::Null,
1327 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1328 Value::Number(n) => {
1329 if let Some(i) = n.as_i64() {
1330 SqlValue::Integer(i)
1331 } else if let Some(f) = n.as_f64() {
1332 SqlValue::Real(f)
1333 } else {
1334 return Err(format!("query param number {n} is out of range"));
1335 }
1336 }
1337 Value::String(s) => SqlValue::Text(s.clone()),
1338 Value::Object(_) => {
1339 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1340 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1341 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1342 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1343 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1344 })?)
1345 } else {
1346 return Err(
1347 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1348 .to_owned(),
1349 );
1350 }
1351 }
1352 Value::Array(_) => return Err("query array params are not supported".to_owned()),
1353 })
1354}
1355
1356fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1361 use rusqlite::types::ValueRef;
1362 match value {
1363 ValueRef::Null => Value::Null,
1364 ValueRef::Integer(i) => {
1365 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1369 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1370 Value::from(i)
1371 } else {
1372 let mut map = Map::new();
1373 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1374 Value::Object(map)
1375 }
1376 }
1377 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1378 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1379 ValueRef::Blob(b) => {
1380 let mut map = Map::new();
1381 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1382 Value::Object(map)
1383 }
1384 }
1385}
1386
1387fn query_connection(
1388 conn: &Connection,
1389 sql: &str,
1390 params: &[Value],
1391) -> Result<Vec<Map<String, Value>>, String> {
1392 crate::query_guard::assert_read_only_query(sql)?;
1393 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1394 let bound: Vec<SqlValue> = params
1395 .iter()
1396 .map(json_param_to_sql)
1397 .collect::<Result<_, _>>()?;
1398 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
1399 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1400 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1401 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1402 let mut sql_rows = stmt
1403 .query(bound_refs.as_slice())
1404 .map_err(|e| e.to_string())?;
1405 let mut out = Vec::new();
1406 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1407 let mut record = Map::new();
1408 for (i, name) in column_names.iter().enumerate() {
1409 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1410 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1411 }
1412 out.push(record);
1413 }
1414 Ok(out)
1415}
1416
1417fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1418 let mut stmt = conn
1419 .prepare(
1420 "SELECT windows.unit, subscriptions.state_json
1421 FROM _syncular_windows AS windows
1422 JOIN _syncular_subscriptions AS subscriptions
1423 ON subscriptions.id = windows.sub_id
1424 WHERE windows.base = ?1
1425 ORDER BY windows.unit ASC",
1426 )
1427 .map_err(|error| error.to_string())?;
1428 let rows = stmt
1429 .query_map(rusqlite::params![window_base_key(base)], |row| {
1430 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1431 })
1432 .map_err(|error| error.to_string())?;
1433 let mut units = Vec::new();
1434 let mut pending = Vec::new();
1435 for row in rows {
1436 let (unit, raw) = row.map_err(|error| error.to_string())?;
1437 let state: Value = serde_json::from_str(&raw)
1438 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1439 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1440 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1441 || state
1442 .get("bootstrapState")
1443 .is_some_and(|value| !value.is_null());
1444 if is_pending {
1445 pending.push(unit.clone());
1446 }
1447 units.push(unit);
1448 }
1449 Ok(WindowState { units, pending })
1450}
1451
1452fn snapshot_connection(
1453 conn: &Connection,
1454 sql: &str,
1455 params: &[Value],
1456 coverage: &[WindowCoverage],
1457) -> Result<QuerySnapshot, String> {
1458 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1459 .map_err(|error| error.to_string())?;
1460 let result = (|| {
1461 let revision = conn
1462 .query_row(
1463 "SELECT value FROM _syncular_meta WHERE key = ?1",
1464 rusqlite::params![LOCAL_REVISION_KEY],
1465 |row| row.get::<_, String>(0),
1466 )
1467 .ok()
1468 .and_then(|value| value.parse::<u64>().ok())
1469 .unwrap_or(0);
1470 let rows = query_connection(conn, sql, params)?;
1471 let mut pending = Vec::new();
1472 let mut missing = Vec::new();
1473 for requested in coverage {
1474 let base_key = window_base_key(&requested.base);
1475 let state = persisted_window_state(conn, &requested.base)?;
1476 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1477 let reference = WindowUnitRef {
1478 base_key: base_key.clone(),
1479 unit: unit.clone(),
1480 };
1481 if !state.units.iter().any(|held| held == &unit) {
1482 missing.push(reference);
1483 } else if state.pending.iter().any(|held| held == &unit) {
1484 pending.push(reference);
1485 }
1486 }
1487 }
1488 Ok(QuerySnapshot {
1489 revision: revision.to_string(),
1490 rows,
1491 coverage: CoverageSnapshot {
1492 complete: pending.is_empty() && missing.is_empty(),
1493 pending,
1494 missing,
1495 },
1496 })
1497 })();
1498 match result {
1499 Ok(snapshot) => {
1500 conn.execute_batch("RELEASE syncular_snapshot_read")
1501 .map_err(|error| error.to_string())?;
1502 Ok(snapshot)
1503 }
1504 Err(error) => {
1505 let _ = conn.execute_batch(
1506 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1507 );
1508 Err(error)
1509 }
1510 }
1511}
1512
1513pub struct FileQuerySnapshotReader {
1518 path: String,
1519 conn: Option<Connection>,
1520}
1521
1522impl FileQuerySnapshotReader {
1523 #[must_use]
1524 pub fn new(path: impl Into<String>) -> Self {
1525 Self {
1526 path: path.into(),
1527 conn: None,
1528 }
1529 }
1530
1531 fn connection(&mut self) -> Result<&Connection, String> {
1532 if self.conn.is_none() {
1533 let conn = Connection::open_with_flags(
1534 &self.path,
1535 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1536 )
1537 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1538 conn.busy_timeout(std::time::Duration::from_millis(250))
1539 .map_err(|error| error.to_string())?;
1540 self.conn = Some(conn);
1541 }
1542 self.conn
1543 .as_ref()
1544 .ok_or_else(|| "read sidecar connection missing".to_owned())
1545 }
1546
1547 pub fn query_snapshot(
1548 &mut self,
1549 sql: &str,
1550 params: &[Value],
1551 coverage: &[WindowCoverage],
1552 ) -> Result<QuerySnapshot, String> {
1553 snapshot_connection(self.connection()?, sql, params, coverage)
1554 }
1555}
1556
1557impl SyncClient {
1558 pub fn new_with_identity(
1559 client_id: Option<String>,
1560 schema_json: &Value,
1561 limits: ClientLimits,
1562 ) -> Result<Self, String> {
1563 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1564 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1565 Self::with_connection(resolved, schema_json, limits, conn)
1566 }
1567
1568 pub fn new(
1569 client_id: String,
1570 schema_json: &Value,
1571 limits: ClientLimits,
1572 ) -> Result<Self, String> {
1573 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1574 Self::with_connection(client_id, schema_json, limits, conn)
1575 }
1576
1577 pub fn open_path(
1583 client_id: String,
1584 schema_json: &Value,
1585 limits: ClientLimits,
1586 path: &str,
1587 ) -> Result<Self, String> {
1588 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1589 Self::with_connection(client_id, schema_json, limits, conn)
1590 }
1591
1592 pub fn open_path_with_identity(
1593 client_id: Option<String>,
1594 schema_json: &Value,
1595 limits: ClientLimits,
1596 path: &str,
1597 ) -> Result<Self, String> {
1598 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1599 conn.busy_timeout(std::time::Duration::from_millis(250))
1605 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1606 conn.pragma_update(None, "journal_mode", "WAL")
1607 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
1608 let persisted = conn
1609 .query_row(
1610 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
1611 [],
1612 |row| row.get::<_, String>(0),
1613 )
1614 .ok();
1615 let resolved = persisted
1616 .clone()
1617 .or(client_id.clone())
1618 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1619 if let (Some(existing), Some(requested)) = (persisted, client_id) {
1620 if existing != requested {
1621 return Err(format!(
1622 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
1623 ));
1624 }
1625 }
1626 Self::with_connection(resolved, schema_json, limits, conn)
1627 }
1628
1629 #[must_use]
1630 pub fn client_id(&self) -> &str {
1631 &self.client_id
1632 }
1633
1634 pub fn with_connection(
1641 client_id: String,
1642 schema_json: &Value,
1643 limits: ClientLimits,
1644 conn: Connection,
1645 ) -> Result<Self, String> {
1646 if limits.outcome_retention_max_entries == Some(0) {
1647 return Err(
1648 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
1649 );
1650 }
1651 let schema = parse_schema_json(schema_json)?;
1652 let mut client = SyncClient {
1653 conn,
1654 schema,
1655 client_id,
1656 limits,
1657 subs: Vec::new(),
1658 outbox: Vec::new(),
1659 conflicts: Vec::new(),
1660 rejections: Vec::new(),
1661 schema_floor: None,
1662 lease_state: None,
1663 stopped: false,
1664 upgrading: false,
1665 sync_needed: false,
1666 realtime_connected: false,
1667 presence: HashMap::new(),
1668 now_ms: None,
1669 encryption: crate::values::EncryptionConfig::default(),
1670 insert_sql: RefCell::new(HashMap::new()),
1671 overlay_dirty: Cell::new(false),
1672 change_queue: VecDeque::new(),
1673 sync_intent_queue: VecDeque::new(),
1674 retry_delay_ms: 250,
1675 };
1676 client
1680 .conn
1681 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
1682 client.create_bookkeeping_tables()?;
1686 match client.get_meta(CLIENT_ID_KEY) {
1687 Some(existing) if existing != client.client_id => {
1688 return Err(format!(
1689 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
1690 client.client_id
1691 ));
1692 }
1693 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
1694 _ => {}
1695 }
1696 client.restore_persisted_state()?;
1697 let marker = client
1698 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1699 .and_then(|value| value.parse::<i32>().ok());
1700 match marker {
1701 None => {
1702 client.create_synced_tables()?;
1703 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
1704 }
1705 Some(version) if version == client.schema.version => {
1706 client.create_synced_tables()?;
1707 }
1708 Some(_) => client.run_schema_reset()?,
1709 }
1710 client.clear_satisfied_persisted_schema_floor();
1711 client.prune_unknown_subscriptions()?;
1712 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
1713 client.overlay_dirty.set(true);
1716 client.rebuild_overlay();
1717 }
1718 client.enqueue_startup_sync_if_needed();
1724 Ok(client)
1725 }
1726
1727 pub fn set_now_ms(&mut self, now_ms: i64) {
1730 self.now_ms = Some(now_ms);
1731 }
1732
1733 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
1737 self.encryption = encryption;
1738 }
1739
1740 fn clock_now_ms(&self) -> i64 {
1741 self.now_ms.unwrap_or_else(|| {
1742 std::time::SystemTime::now()
1743 .duration_since(std::time::UNIX_EPOCH)
1744 .map(|d| d.as_millis() as i64)
1745 .unwrap_or(0)
1746 })
1747 }
1748
1749 fn create_bookkeeping_tables(&self) -> Result<(), String> {
1750 self.conn
1752 .execute_batch(
1753 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
1754 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1755 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
1756 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
1757 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1758 client_commit_id TEXT NOT NULL UNIQUE,
1759 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
1760 recorded_at_ms INTEGER NOT NULL,
1761 results_json TEXT NOT NULL,
1762 operations_json TEXT,
1763 resolution TEXT NOT NULL DEFAULT 'active'
1764 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
1765 resolved_at_ms INTEGER,
1766 replacement_client_commit_id TEXT);
1767 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
1768 ON _syncular_commit_outcomes(resolution, seq);
1769 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
1770 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
1771 CREATE TABLE IF NOT EXISTS _syncular_meta (
1772 key TEXT PRIMARY KEY, value TEXT NOT NULL);
1773 CREATE TABLE IF NOT EXISTS _syncular_windows (
1774 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
1775 PRIMARY KEY (base, unit));
1776 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
1777 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
1778 effective_scopes TEXT NOT NULL);",
1779 )
1780 .map_err(|e| e.to_string())?;
1781 let _ = self
1784 .conn
1785 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
1786 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
1787 self.set_meta(LOCAL_REVISION_KEY, "0");
1788 }
1789 if self.schema_has_blobs() {
1794 self.conn
1795 .execute_batch(
1796 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
1797 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
1798 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
1799 created_at_ms INTEGER NOT NULL,
1800 last_used_ms INTEGER NOT NULL DEFAULT 0);
1801 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
1802 media_type TEXT, created_at_ms INTEGER NOT NULL);",
1803 )
1804 .map_err(|e| e.to_string())?;
1805 let _ = self.conn.execute_batch(
1809 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
1810 );
1811 }
1812 Ok(())
1813 }
1814
1815 fn schema_has_blobs(&self) -> bool {
1817 self.schema
1818 .tables
1819 .iter()
1820 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
1821 }
1822
1823 fn get_meta(&self, key: &str) -> Option<String> {
1826 self.conn
1827 .query_row(
1828 "SELECT value FROM _syncular_meta WHERE key = ?1",
1829 rusqlite::params![key],
1830 |row| row.get::<_, String>(0),
1831 )
1832 .ok()
1833 }
1834
1835 fn set_meta(&self, key: &str, value: &str) {
1836 let _ = self.conn.execute(
1837 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
1838 rusqlite::params![key, value],
1839 );
1840 }
1841
1842 fn delete_meta(&self, key: &str) {
1843 let _ = self.conn.execute(
1844 "DELETE FROM _syncular_meta WHERE key = ?1",
1845 rusqlite::params![key],
1846 );
1847 }
1848
1849 fn restore_persisted_state(&mut self) -> Result<(), String> {
1850 self.subs = {
1851 let mut stmt = self
1852 .conn
1853 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
1854 .map_err(|error| error.to_string())?;
1855 let rows = stmt
1856 .query_map([], |row| {
1857 Ok((
1858 row.get::<_, String>(0)?,
1859 row.get::<_, String>(1)?,
1860 row.get::<_, String>(2)?,
1861 ))
1862 })
1863 .map_err(|error| error.to_string())?;
1864 let mut subscriptions = Vec::new();
1865 for row in rows {
1866 let (id, table, raw) = row.map_err(|error| error.to_string())?;
1867 let state: Value = serde_json::from_str(&raw)
1868 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
1869 let requested = json_to_scope_map(
1870 state.get("requested").unwrap_or(&Value::Object(Map::new())),
1871 )?;
1872 let effective = state
1873 .get("effectiveScopes")
1874 .filter(|value| !value.is_null())
1875 .map(json_to_scope_map)
1876 .transpose()?;
1877 subscriptions.push(Subscription {
1878 id,
1879 table,
1880 requested,
1881 params: state
1882 .get("params")
1883 .and_then(Value::as_str)
1884 .map(str::to_owned),
1885 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
1886 bootstrap_state: state
1887 .get("bootstrapState")
1888 .and_then(Value::as_str)
1889 .map(str::to_owned),
1890 state: SubState::parse(
1891 state
1892 .get("status")
1893 .and_then(Value::as_str)
1894 .unwrap_or("active"),
1895 ),
1896 reason_code: state
1897 .get("reasonCode")
1898 .and_then(Value::as_str)
1899 .map(str::to_owned),
1900 effective,
1901 synced_once: state
1902 .get("syncedOnce")
1903 .and_then(Value::as_bool)
1904 .unwrap_or(false),
1905 });
1906 }
1907 subscriptions
1908 };
1909
1910 self.outbox = {
1911 let mut stmt = self
1912 .conn
1913 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
1914 .map_err(|error| error.to_string())?;
1915 let rows = stmt
1916 .query_map([], |row| {
1917 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1918 })
1919 .map_err(|error| error.to_string())?;
1920 let mut commits = Vec::new();
1921 for row in rows {
1922 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
1923 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
1924 format!("invalid persisted outbox {client_commit_id:?}: {error}")
1925 })?;
1926 let mut ops = Vec::with_capacity(entries.len());
1927 for entry in entries {
1928 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
1929 ops.push(OutboxOp {
1930 upsert: op == "upsert",
1931 table: entry
1932 .get("table")
1933 .and_then(Value::as_str)
1934 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
1935 .to_owned(),
1936 row_id: entry
1937 .get("rowId")
1938 .and_then(Value::as_str)
1939 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
1940 .to_owned(),
1941 base_version: entry.get("baseVersion").and_then(Value::as_i64),
1942 values: entry.get("values").and_then(Value::as_object).cloned(),
1943 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
1944 |values| {
1945 values
1946 .iter()
1947 .filter_map(Value::as_str)
1948 .map(str::to_owned)
1949 .collect()
1950 },
1951 ),
1952 });
1953 }
1954 commits.push(OutboxCommit {
1955 client_commit_id,
1956 ops,
1957 });
1958 }
1959 commits
1960 };
1961
1962 self.prune_commit_outcomes()?;
1963 let active = self.commit_outcomes(CommitOutcomeQuery {
1964 active_only: true,
1965 ..CommitOutcomeQuery::default()
1966 })?;
1967 self.conflicts = active
1968 .iter()
1969 .flat_map(|outcome| outcome.results.iter())
1970 .filter_map(|result| match result {
1971 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
1972 _ => None,
1973 })
1974 .collect();
1975 self.rejections = active
1976 .iter()
1977 .flat_map(|outcome| outcome.results.iter())
1978 .filter_map(|result| match result {
1979 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
1980 _ => None,
1981 })
1982 .collect();
1983
1984 self.lease_state = self
1985 .get_meta(LEASE_STATE_KEY)
1986 .map(|raw| serde_json::from_str(&raw))
1987 .transpose()
1988 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
1989 self.schema_floor = self
1990 .get_meta(SCHEMA_FLOOR_KEY)
1991 .map(|raw| serde_json::from_str(&raw))
1992 .transpose()
1993 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
1994 self.stopped = self.schema_floor.is_some();
1995 Ok(())
1996 }
1997
1998 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
2001 let valid_tables: BTreeSet<String> = self
2002 .schema
2003 .tables
2004 .iter()
2005 .map(|table| table.name.clone())
2006 .collect();
2007 let stale_ids: Vec<String> = self
2008 .subs
2009 .iter()
2010 .filter(|sub| !valid_tables.contains(&sub.table))
2011 .map(|sub| sub.id.clone())
2012 .collect();
2013 for id in &stale_ids {
2014 self.conn
2015 .execute(
2016 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2017 rusqlite::params![id],
2018 )
2019 .map_err(|error| error.to_string())?;
2020 self.conn
2021 .execute(
2022 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2023 rusqlite::params![id],
2024 )
2025 .map_err(|error| error.to_string())?;
2026 self.conn
2027 .execute(
2028 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2029 rusqlite::params![id],
2030 )
2031 .map_err(|error| error.to_string())?;
2032 }
2033 self.subs.retain(|sub| valid_tables.contains(&sub.table));
2034 Ok(())
2035 }
2036
2037 #[must_use]
2038 pub fn local_revision(&self) -> u64 {
2039 self.get_meta(LOCAL_REVISION_KEY)
2040 .and_then(|value| value.parse().ok())
2041 .unwrap_or(0)
2042 }
2043
2044 #[must_use]
2045 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2046 SyncStatusSnapshot {
2047 outbox: self.outbox.len(),
2048 upgrading: self.upgrading,
2049 lease_state: self.lease_state.clone(),
2050 schema_floor: self.schema_floor.clone(),
2051 sync_needed: self.sync_needed,
2052 }
2053 }
2054
2055 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
2056 self.change_queue.drain(..).collect()
2057 }
2058
2059 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
2060 self.sync_intent_queue.drain(..).collect()
2061 }
2062
2063 fn schedule_background_retry(&mut self) {
2064 self.sync_intent_queue.push_back(SyncIntent::Background {
2065 delay_ms: self.retry_delay_ms,
2066 });
2067 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
2068 }
2069
2070 fn reset_background_retry(&mut self) {
2071 self.retry_delay_ms = 250;
2072 }
2073
2074 fn retryable_transport_code(code: &str) -> bool {
2075 code == "transport.failed" || code == "sync.transport_failed"
2076 }
2077
2078 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
2079 if self.sync_needed != value {
2080 if self.begin_observation("syncular_status").is_ok() {
2081 self.sync_needed = value;
2082 let batch = ChangeAccumulator {
2083 status: true,
2084 ..ChangeAccumulator::default()
2085 };
2086 if self.finish_observation("syncular_status", batch).is_err() {
2087 self.rollback_observation("syncular_status");
2088 }
2089 } else {
2090 self.sync_needed = value;
2091 }
2092 }
2093 if value && interactive {
2094 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2095 }
2096 }
2097
2098 fn begin_observation(&self, name: &str) -> Result<(), String> {
2099 self.conn
2100 .execute_batch(&format!("SAVEPOINT {name}"))
2101 .map_err(|error| error.to_string())
2102 }
2103
2104 fn rollback_observation(&self, name: &str) {
2105 let _ = self
2106 .conn
2107 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
2108 }
2109
2110 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
2111 if !batch.touched() {
2112 self.conn
2113 .execute_batch(&format!("RELEASE {name}"))
2114 .map_err(|error| error.to_string())?;
2115 return Ok(());
2116 }
2117 let revision = self
2118 .local_revision()
2119 .checked_add(1)
2120 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
2121 self.conn
2122 .execute(
2123 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
2124 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
2125 )
2126 .map_err(|error| error.to_string())?;
2127 let status = batch.status.then(|| self.status_snapshot());
2128 let event = ClientChangeBatch {
2129 revision: revision.to_string(),
2130 tables: batch
2131 .tables
2132 .into_iter()
2133 .map(|(table, scope_keys)| TableChange {
2134 table,
2135 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
2136 })
2137 .collect(),
2138 windows: batch
2139 .windows
2140 .into_iter()
2141 .map(|((base_key, table), units)| WindowChange {
2142 base_key,
2143 table,
2144 units: units.into_iter().collect(),
2145 })
2146 .collect(),
2147 status,
2148 conflicts_changed: batch.conflicts,
2149 rejections_changed: batch.rejections,
2150 outcomes_changed: batch.outcomes,
2151 };
2152 self.conn
2153 .execute_batch(&format!("RELEASE {name}"))
2154 .map_err(|error| error.to_string())?;
2155 self.change_queue.push_back(event);
2156 Ok(())
2157 }
2158
2159 fn record_scope_map(
2160 &self,
2161 batch: &mut ChangeAccumulator,
2162 table_name: &str,
2163 scopes: &[(String, Vec<String>)],
2164 ) {
2165 let Some(table) = self.schema.table(table_name) else {
2166 return;
2167 };
2168 for (variable, values) in scopes {
2169 let Some(scope) = table
2170 .scope_variables
2171 .iter()
2172 .find(|scope| &scope.variable == variable)
2173 else {
2174 continue;
2175 };
2176 for value in values {
2177 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2178 }
2179 }
2180 }
2181
2182 fn record_row_scopes(
2184 &self,
2185 batch: &mut ChangeAccumulator,
2186 table_name: &str,
2187 row_id: &str,
2188 base: bool,
2189 ) -> bool {
2190 let Some(table) = self.schema.table(table_name) else {
2191 return false;
2192 };
2193 if table.scope_variables.is_empty() {
2194 return false;
2195 }
2196 let columns = table
2197 .scope_variables
2198 .iter()
2199 .map(|scope| quote_ident(&scope.column))
2200 .collect::<Vec<_>>()
2201 .join(", ");
2202 let full_table = if base {
2203 base_table(table_name)
2204 } else {
2205 visible_table(table_name)
2206 };
2207 let sql = format!(
2208 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
2209 quote_ident(&table.primary_key)
2210 );
2211 let Ok(mut stmt) = self.conn.prepare(&sql) else {
2212 return false;
2213 };
2214 let values = stmt.query_row(rusqlite::params![row_id], |row| {
2215 let mut values = Vec::with_capacity(table.scope_variables.len());
2216 for index in 0..table.scope_variables.len() {
2217 values.push(row.get::<_, Option<String>>(index)?);
2218 }
2219 Ok(values)
2220 });
2221 let Ok(values) = values else {
2222 return false;
2223 };
2224 let mut recorded = false;
2225 for (scope, value) in table.scope_variables.iter().zip(values) {
2226 if let Some(value) = value {
2227 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2228 recorded = true;
2229 }
2230 }
2231 recorded
2232 }
2233
2234 fn record_commit_changes(
2235 &self,
2236 batch: &mut ChangeAccumulator,
2237 tables: &[String],
2238 changes: &[ssp2::model::Change],
2239 ) {
2240 for change in changes {
2241 let Some(table_name) = tables.get(change.table_index as usize) else {
2242 continue;
2243 };
2244 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
2245 if let Some(table) = self.schema.table(table_name) {
2246 for (variable, value) in &change.scopes {
2247 if let Some(scope) = table
2248 .scope_variables
2249 .iter()
2250 .find(|scope| &scope.variable == variable)
2251 {
2252 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2253 precise = true;
2254 }
2255 }
2256 }
2257 if !precise {
2258 batch.table(table_name);
2259 }
2260 }
2261 }
2262
2263 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
2264 if effective.is_empty() {
2265 return false;
2266 }
2267 let Some(table) = self.schema.table(table_name) else {
2268 return false;
2269 };
2270 let mut clauses = Vec::new();
2271 let mut params = Vec::new();
2272 for (variable, values) in effective {
2273 let Some(column) = table.scope_column(variable) else {
2274 return false;
2275 };
2276 if values.is_empty() {
2277 return false;
2278 }
2279 let placeholders = values
2280 .iter()
2281 .map(|value| {
2282 params.push(SqlValue::Text(value.clone()));
2283 "?"
2284 })
2285 .collect::<Vec<_>>()
2286 .join(", ");
2287 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
2288 }
2289 let sql = format!(
2290 "SELECT 1 FROM {} WHERE {} LIMIT 1",
2291 base_table(table_name),
2292 clauses.join(" AND ")
2293 );
2294 self.conn
2295 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
2296 .is_ok()
2297 }
2298
2299 pub fn upgrading(&self) -> bool {
2301 self.upgrading
2302 }
2303
2304 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
2310 let new_schema = parse_schema_json(schema_json)?;
2311 let marker: Option<i32> = self
2312 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
2313 .and_then(|v| v.parse().ok());
2314 self.schema = new_schema;
2315 if marker != Some(self.schema.version) {
2316 self.run_schema_reset()?;
2317 }
2318 self.prune_unknown_subscriptions()?;
2319 self.enqueue_startup_sync_if_needed();
2323 Ok(())
2324 }
2325
2326 fn enqueue_startup_sync_if_needed(&mut self) {
2327 let startup_work = !self.stopped
2328 && (!self.outbox.is_empty()
2329 || self.subs.iter().any(|sub| sub.state == SubState::Active));
2330 if startup_work {
2331 self.sync_needed = true;
2332 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2333 }
2334 }
2335
2336 fn clear_satisfied_persisted_schema_floor(&mut self) {
2343 let satisfied = self
2344 .schema_floor
2345 .as_ref()
2346 .and_then(|floor| floor.required_schema_version)
2347 .is_some_and(|required| self.schema.version >= required);
2348 if !satisfied {
2349 return;
2350 }
2351 self.schema_floor = None;
2352 self.stopped = false;
2353 self.delete_meta(SCHEMA_FLOOR_KEY);
2354 }
2355
2356 fn run_schema_reset(&mut self) -> Result<(), String> {
2362 self.begin_observation("syncular_schema_reset")?;
2363 let mut batch = ChangeAccumulator::default();
2364 let result = self.run_schema_reset_observed(&mut batch);
2365 if let Err(error) = result {
2366 self.rollback_observation("syncular_schema_reset");
2367 return Err(error);
2368 }
2369 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
2370 self.rollback_observation("syncular_schema_reset");
2371 return Err(error);
2372 }
2373 Ok(())
2374 }
2375
2376 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
2377 self.upgrading = true;
2378 batch.status = true;
2379 for table in &self.schema.tables {
2380 batch.table(&table.name);
2381 }
2382 for (base_key, unit, table) in self.load_registered_window_units() {
2383 batch.window(&base_key, &table, &unit);
2384 }
2385 self.insert_sql.borrow_mut().clear();
2387 self.overlay_dirty.set(true);
2388 let virtual_tables: Vec<String> = {
2396 let mut stmt = self
2397 .conn
2398 .prepare(
2399 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
2400 )
2401 .map_err(|e| e.to_string())?;
2402 let rows = stmt
2403 .query_map([], |row| row.get::<_, String>(0))
2404 .map_err(|e| e.to_string())?;
2405 rows.filter_map(Result::ok)
2406 .filter(|name| is_synced_table_name(name))
2407 .collect()
2408 };
2409 for name in virtual_tables {
2410 self.conn
2411 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2412 .map_err(|e| e.to_string())?;
2413 }
2414 let existing: Vec<String> = {
2415 let mut stmt = self
2416 .conn
2417 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
2418 .map_err(|e| e.to_string())?;
2419 let rows = stmt
2420 .query_map([], |row| row.get::<_, String>(0))
2421 .map_err(|e| e.to_string())?;
2422 rows.filter_map(Result::ok)
2423 .filter(|name| is_synced_table_name(name))
2424 .collect()
2425 };
2426 for name in &existing {
2427 if let Some(table) = name.strip_prefix("_syncular_base_") {
2428 batch.table(table);
2429 } else if !name.starts_with("_syncular_") {
2430 batch.table(name);
2431 }
2432 }
2433 for name in existing {
2434 self.conn
2435 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2436 .map_err(|e| e.to_string())?;
2437 }
2438 self.create_synced_tables()?;
2440 for sub in &mut self.subs {
2442 sub.cursor = -1;
2443 sub.bootstrap_state = None;
2444 sub.effective = None;
2445 sub.state = SubState::Active;
2446 sub.reason_code = None;
2447 sub.synced_once = false;
2448 }
2449 let subs = self.subs.clone();
2450 for sub in &subs {
2451 self.persist_sub(sub);
2452 }
2453 self.stopped = false;
2455 self.schema_floor = None;
2456 self.delete_meta(SCHEMA_FLOOR_KEY);
2457 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
2459 if self.drop_incompatible_outbox()? {
2463 batch.rejections = true;
2464 batch.status = true;
2465 batch.outcomes = true;
2466 }
2467 self.rebuild_overlay();
2469 Ok(())
2470 }
2471
2472 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
2476 let schema = &self.schema;
2477 let incompatible = self
2478 .outbox
2479 .iter()
2480 .filter(|commit| {
2481 commit.ops.iter().any(|op| {
2482 if !op.upsert {
2483 return false;
2484 }
2485 match schema.table(&op.table) {
2486 None => true,
2487 Some(table) => op.values.as_ref().is_some_and(|values| {
2488 values
2489 .keys()
2490 .any(|key| !table.columns.iter().any(|c| &c.name == key))
2491 }),
2492 }
2493 })
2494 })
2495 .cloned()
2496 .collect::<Vec<_>>();
2497 if incompatible.is_empty() {
2498 return Ok(false);
2499 }
2500 let mut rejections = Vec::new();
2501 for commit in &incompatible {
2502 let results = commit
2503 .ops
2504 .iter()
2505 .enumerate()
2506 .map(|(op_index, operation)| {
2507 let rejection = RejectionRecord {
2508 client_commit_id: commit.client_commit_id.clone(),
2509 op_index: op_index as i32,
2510 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
2511 message: "the persisted commit cannot encode under the current schema"
2512 .to_owned(),
2513 retryable: false,
2514 details: None,
2515 operation: Some(CommitOperation::from(operation)),
2516 };
2517 rejections.push(rejection.clone());
2518 CommitOperationOutcome::Error { rejection }
2519 })
2520 .collect::<Vec<_>>();
2521 self.persist_commit_outcome(
2522 &commit.client_commit_id,
2523 CommitOutcomeStatus::Rejected,
2524 &results,
2525 Some(&commit.ops),
2526 )?;
2527 self.delete_outbox_persisted(&commit.client_commit_id)?;
2528 }
2529 self.prune_commit_outcomes()?;
2530 let incompatible_ids = incompatible
2531 .iter()
2532 .map(|commit| commit.client_commit_id.as_str())
2533 .collect::<BTreeSet<_>>();
2534 self.outbox
2535 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
2536 self.rejections.extend(rejections);
2537 Ok(true)
2538 }
2539
2540 fn create_synced_tables(&self) -> Result<(), String> {
2543 for table in &self.schema.tables {
2544 for (full, index_prefix) in [
2548 (base_table(&table.name), "_syncular_base_"),
2549 (visible_table(&table.name), ""),
2550 ] {
2551 let mut cols: Vec<String> =
2552 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
2553 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
2554 let sql = format!(
2555 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
2556 cols.join(", "),
2557 quote_ident(&table.primary_key)
2558 );
2559 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
2560 for index in &table.indexes {
2564 let unique = if index.unique { "UNIQUE " } else { "" };
2565 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
2566 let cols_sql = index
2567 .columns
2568 .iter()
2569 .map(|c| quote_ident(c))
2570 .collect::<Vec<_>>()
2571 .join(", ");
2572 let index_sql = format!(
2573 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
2574 );
2575 self.conn
2576 .execute(&index_sql, [])
2577 .map_err(|e| e.to_string())?;
2578 }
2579 }
2580 }
2581 for table in &self.schema.tables {
2584 for index in &table.fts_indexes {
2585 self.create_fts_projection(table, index)?;
2586 }
2587 }
2588 Ok(())
2589 }
2590
2591 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
2592 let count: i64 = self
2593 .conn
2594 .query_row(
2595 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
2596 rusqlite::params![index.name],
2597 |row| row.get(0),
2598 )
2599 .map_err(|e| e.to_string())?;
2600 Ok(count > 0)
2601 }
2602
2603 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
2604 for suffix in ["ai", "ad", "au"] {
2605 self.conn
2606 .execute(
2607 &format!(
2608 "DROP TRIGGER IF EXISTS {}",
2609 quote_ident(&format!("{}_{suffix}", index.name))
2610 ),
2611 [],
2612 )
2613 .map_err(|e| e.to_string())?;
2614 }
2615 Ok(())
2616 }
2617
2618 fn create_fts_triggers(
2619 &self,
2620 table: &TableSchema,
2621 index: &FtsIndexSchema,
2622 ) -> Result<(), String> {
2623 let fts = quote_ident(&index.name);
2624 let source = visible_table(&table.name);
2625 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2626 let pk = quote_ident(&table.primary_key);
2627 let projection_columns = std::iter::once(source_id.clone())
2628 .chain(index.columns.iter().map(|column| quote_ident(column)))
2629 .collect::<Vec<_>>()
2630 .join(", ");
2631 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
2632 .chain(
2633 index
2634 .columns
2635 .iter()
2636 .map(|column| format!("new.{}", quote_ident(column))),
2637 )
2638 .collect::<Vec<_>>()
2639 .join(", ");
2640 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
2641 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
2642 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
2643 let sql = format!(
2644 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
2645 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
2646 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
2647 ai = quote_ident(&format!("{}_ai", index.name)),
2648 ad = quote_ident(&format!("{}_ad", index.name)),
2649 au = quote_ident(&format!("{}_au", index.name)),
2650 );
2651 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2652 }
2653
2654 fn rebuild_fts_projection(
2655 &self,
2656 table: &TableSchema,
2657 index: &FtsIndexSchema,
2658 ) -> Result<(), String> {
2659 let fts = quote_ident(&index.name);
2660 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2661 let pk = quote_ident(&table.primary_key);
2662 let indexed_columns = index
2663 .columns
2664 .iter()
2665 .map(|column| quote_ident(column))
2666 .collect::<Vec<_>>();
2667 let projection_columns = std::iter::once(source_id)
2668 .chain(indexed_columns.iter().cloned())
2669 .collect::<Vec<_>>()
2670 .join(", ");
2671 let sql = format!(
2672 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
2673 columns = indexed_columns.join(", "),
2674 source = visible_table(&table.name),
2675 );
2676 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2677 }
2678
2679 fn create_fts_projection(
2680 &self,
2681 table: &TableSchema,
2682 index: &FtsIndexSchema,
2683 ) -> Result<(), String> {
2684 let existed = self.fts_projection_exists(index)?;
2685 let tokenizer = index.tokenize.replace('\'', "''");
2686 let columns = index
2687 .columns
2688 .iter()
2689 .map(|column| quote_ident(column))
2690 .collect::<Vec<_>>()
2691 .join(", ");
2692 let sql = format!(
2693 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
2694 fts = quote_ident(&index.name),
2695 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
2696 );
2697 self.conn.execute(&sql, []).map_err(|error| {
2698 format!(
2699 "cannot create local FTS5 projection {:?}: {error}",
2700 index.name
2701 )
2702 })?;
2703 self.create_fts_triggers(table, index)?;
2704 if !existed {
2705 self.rebuild_fts_projection(table, index)?;
2706 }
2707 Ok(())
2708 }
2709
2710 fn persist_sub(&self, sub: &Subscription) {
2713 let state = serde_json::json!({
2714 "requested": scope_map_to_json(&sub.requested),
2715 "params": sub.params,
2716 "cursor": sub.cursor,
2717 "bootstrapState": sub.bootstrap_state,
2718 "status": sub.state.name(),
2719 "reasonCode": sub.reason_code,
2720 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2721 "syncedOnce": sub.synced_once,
2722 });
2723 let _ = self.conn.execute(
2724 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
2725 rusqlite::params![sub.id, sub.table, state.to_string()],
2726 );
2727 }
2728
2729 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
2730 let ops: Vec<Value> = commit
2731 .ops
2732 .iter()
2733 .map(|op| {
2734 serde_json::json!({
2735 "op": if op.upsert { "upsert" } else { "delete" },
2736 "table": op.table,
2737 "rowId": op.row_id,
2738 "baseVersion": op.base_version,
2739 "values": op.values.clone().map(Value::Object),
2740 "changedFields": op.changed_fields,
2741 })
2742 })
2743 .collect();
2744 let _ = self.conn.execute(
2745 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
2746 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
2747 );
2748 }
2749
2750 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
2751 self.conn
2752 .execute(
2753 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
2754 rusqlite::params![client_commit_id],
2755 )
2756 .map(|_| ())
2757 .map_err(|error| error.to_string())
2758 }
2759
2760 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
2761 match status {
2762 CommitOutcomeStatus::Applied => "applied",
2763 CommitOutcomeStatus::Cached => "cached",
2764 CommitOutcomeStatus::Conflict => "conflict",
2765 CommitOutcomeStatus::Rejected => "rejected",
2766 }
2767 }
2768
2769 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
2770 match resolution {
2771 CommitOutcomeResolution::Active => "active",
2772 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
2773 CommitOutcomeResolution::Superseded => "superseded",
2774 CommitOutcomeResolution::Dismissed => "dismissed",
2775 }
2776 }
2777
2778 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
2779 match value {
2780 "applied" => Ok(CommitOutcomeStatus::Applied),
2781 "cached" => Ok(CommitOutcomeStatus::Cached),
2782 "conflict" => Ok(CommitOutcomeStatus::Conflict),
2783 "rejected" => Ok(CommitOutcomeStatus::Rejected),
2784 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
2785 }
2786 }
2787
2788 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
2789 match value {
2790 "active" => Ok(CommitOutcomeResolution::Active),
2791 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
2792 "superseded" => Ok(CommitOutcomeResolution::Superseded),
2793 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
2794 _ => Err(format!(
2795 "invalid persisted commit outcome resolution {value:?}"
2796 )),
2797 }
2798 }
2799
2800 fn persist_commit_outcome(
2801 &self,
2802 client_commit_id: &str,
2803 status: CommitOutcomeStatus,
2804 results: &[CommitOperationOutcome],
2805 operations: Option<&[OutboxOp]>,
2806 ) -> Result<(), String> {
2807 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
2808 let operations_json = operations
2809 .map(|items| {
2810 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
2811 })
2812 .transpose()
2813 .map_err(|error| error.to_string())?;
2814 self.conn
2815 .execute(
2816 "INSERT INTO _syncular_commit_outcomes (
2817 client_commit_id, status, recorded_at_ms, results_json,
2818 operations_json, resolution
2819 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
2820 rusqlite::params![
2821 client_commit_id,
2822 Self::outcome_status_name(status),
2823 self.clock_now_ms(),
2824 results_json,
2825 operations_json
2826 ],
2827 )
2828 .map(|_| ())
2829 .map_err(|error| error.to_string())
2830 }
2831
2832 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
2833 let StoredCommitOutcomeRow {
2834 sequence,
2835 client_commit_id,
2836 status,
2837 recorded_at_ms,
2838 results_json,
2839 operations_json,
2840 resolution,
2841 resolved_at_ms,
2842 replacement_client_commit_id,
2843 } = row;
2844 Ok(CommitOutcome {
2845 sequence,
2846 client_commit_id,
2847 status: Self::parse_outcome_status(&status)?,
2848 recorded_at_ms,
2849 results: serde_json::from_str(&results_json)
2850 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
2851 operations: operations_json
2852 .map(|value| {
2853 serde_json::from_str(&value).map_err(|error| {
2854 format!("invalid persisted commit outcome operations: {error}")
2855 })
2856 })
2857 .transpose()?,
2858 resolution: Self::parse_outcome_resolution(&resolution)?,
2859 resolved_at_ms,
2860 replacement_client_commit_id,
2861 })
2862 }
2863
2864 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
2865 let row = self
2866 .conn
2867 .query_row(
2868 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2869 resolution, resolved_at_ms, replacement_client_commit_id
2870 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
2871 rusqlite::params![client_commit_id],
2872 |row| {
2873 Ok(StoredCommitOutcomeRow {
2874 sequence: row.get(0)?,
2875 client_commit_id: row.get(1)?,
2876 status: row.get(2)?,
2877 recorded_at_ms: row.get(3)?,
2878 results_json: row.get(4)?,
2879 operations_json: row.get(5)?,
2880 resolution: row.get(6)?,
2881 resolved_at_ms: row.get(7)?,
2882 replacement_client_commit_id: row.get(8)?,
2883 })
2884 },
2885 )
2886 .optional()
2887 .map_err(|error| error.to_string())?;
2888 row.map(Self::outcome_from_row).transpose()
2889 }
2890
2891 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
2892 if query.limit == Some(0) {
2893 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
2894 }
2895 let mut sql = String::from(
2896 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2897 resolution, resolved_at_ms, replacement_client_commit_id
2898 FROM _syncular_commit_outcomes",
2899 );
2900 if query.active_only {
2901 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
2902 }
2903 sql.push_str(" ORDER BY seq DESC");
2904 if let Some(limit) = query.limit {
2905 sql.push_str(&format!(" LIMIT {limit}"));
2906 }
2907 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
2908 let rows = stmt
2909 .query_map([], |row| {
2910 Ok(StoredCommitOutcomeRow {
2911 sequence: row.get(0)?,
2912 client_commit_id: row.get(1)?,
2913 status: row.get(2)?,
2914 recorded_at_ms: row.get(3)?,
2915 results_json: row.get(4)?,
2916 operations_json: row.get(5)?,
2917 resolution: row.get(6)?,
2918 resolved_at_ms: row.get(7)?,
2919 replacement_client_commit_id: row.get(8)?,
2920 })
2921 })
2922 .map_err(|error| error.to_string())?;
2923 let mut outcomes = Vec::new();
2924 for row in rows {
2925 outcomes.push(Self::outcome_from_row(
2926 row.map_err(|error| error.to_string())?,
2927 )?);
2928 }
2929 Ok(outcomes)
2930 }
2931
2932 fn prune_commit_outcomes(&self) -> Result<(), String> {
2933 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
2934 let count = self
2935 .conn
2936 .query_row(
2937 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
2938 [],
2939 |row| row.get::<_, i64>(0),
2940 )
2941 .map_err(|error| error.to_string())? as usize;
2942 let excess = count.saturating_sub(max_entries);
2943 if excess == 0 {
2944 return Ok(());
2945 }
2946 let mut stmt = self
2947 .conn
2948 .prepare(
2949 "SELECT seq FROM _syncular_commit_outcomes
2950 WHERE status IN ('applied', 'cached') OR resolution != 'active'
2951 ORDER BY seq ASC LIMIT ?1",
2952 )
2953 .map_err(|error| error.to_string())?;
2954 let rows = stmt
2955 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
2956 .map_err(|error| error.to_string())?;
2957 let sequences = rows
2958 .collect::<Result<Vec<_>, _>>()
2959 .map_err(|error| error.to_string())?;
2960 drop(stmt);
2961 for sequence in sequences {
2962 self.conn
2963 .execute(
2964 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
2965 rusqlite::params![sequence],
2966 )
2967 .map_err(|error| error.to_string())?;
2968 }
2969 Ok(())
2970 }
2971
2972 pub fn subscribe(
2975 &mut self,
2976 id: String,
2977 table: String,
2978 scopes: Vec<(String, Vec<String>)>,
2979 params: Option<String>,
2980 ) -> Result<(), String> {
2981 if self.schema.table(&table).is_none() {
2982 return Err(format!("unknown table {table:?}"));
2983 }
2984 let sub = Subscription {
2985 id: id.clone(),
2986 table,
2987 requested: scopes,
2988 params,
2989 cursor: -1,
2990 bootstrap_state: None,
2991 state: SubState::Active,
2992 reason_code: None,
2993 effective: None,
2994 synced_once: false,
2995 };
2996 self.persist_sub(&sub);
2997 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
2998 *existing = sub;
2999 } else {
3000 self.subs.push(sub);
3001 }
3002 Ok(())
3003 }
3004
3005 pub fn unsubscribe(&mut self, id: &str) {
3006 self.subs.retain(|s| s.id != id);
3007 let _ = self.conn.execute(
3008 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
3009 rusqlite::params![id],
3010 );
3011 }
3012
3013 pub fn set_window(
3021 &mut self,
3022 base: &WindowBase,
3023 units: &[String],
3024 ) -> Result<CommandEffects, String> {
3025 let table = self
3026 .schema
3027 .table(&base.table)
3028 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
3029 if table.scope_column(&base.variable).is_none() {
3030 return Err(format!(
3031 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
3032 base.table, base.variable
3033 ));
3034 }
3035 let base_key = window_base_key(base);
3036 let wanted: std::collections::HashSet<&String> = units.iter().collect();
3037 let live = self.load_window_units(&base_key);
3038 self.begin_observation("syncular_window")?;
3039 let mut batch = ChangeAccumulator::default();
3040 let mut changed = false;
3041
3042 for unit in units {
3044 if live.iter().any(|(u, _)| u == unit) {
3045 continue;
3046 }
3047 let sub_id = derive_sub_id(base, unit);
3048 self.delete_pending_evict(&sub_id);
3049 self.insert_window_unit(&base_key, unit, &sub_id);
3050 self.subscribe(
3051 sub_id,
3052 base.table.clone(),
3053 unit_scopes(base, unit),
3054 base.params.clone(),
3055 )?;
3056 batch.window(&base_key, &base.table, unit);
3057 changed = true;
3058 }
3059
3060 for (unit, sub_id) in live {
3062 if wanted.contains(&unit) {
3063 continue;
3064 }
3065 let effective = self
3066 .subs
3067 .iter()
3068 .find(|sub| sub.id == sub_id)
3069 .and_then(|sub| sub.effective.clone())
3070 .unwrap_or_else(|| unit_scopes(base, &unit));
3071 self.record_scope_map(&mut batch, &base.table, &effective);
3072 batch.window(&base_key, &base.table, &unit);
3073 self.evict_unit(&base_key, base, &unit, &sub_id);
3074 changed = true;
3075 }
3076 if let Err(error) = self.finish_observation("syncular_window", batch) {
3077 self.rollback_observation("syncular_window");
3078 return Err(error);
3079 }
3080 Ok(if changed {
3081 CommandEffects::interactive()
3082 } else {
3083 CommandEffects::none()
3084 })
3085 }
3086
3087 pub fn window_state(&self, base: &WindowBase) -> WindowState {
3092 let mut units = Vec::new();
3093 let mut pending = Vec::new();
3094 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
3095 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
3096 Some(sub) => {
3097 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
3098 }
3099 None => true,
3100 };
3101 if is_pending {
3102 pending.push(unit.clone());
3103 }
3104 units.push(unit);
3105 }
3106 WindowState { units, pending }
3107 }
3108
3109 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
3110 let mut stmt = match self
3111 .conn
3112 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
3113 {
3114 Ok(stmt) => stmt,
3115 Err(_) => return Vec::new(),
3116 };
3117 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
3118 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
3119 });
3120 match rows {
3121 Ok(rows) => rows.filter_map(Result::ok).collect(),
3122 Err(_) => Vec::new(),
3123 }
3124 }
3125
3126 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
3127 let mut stmt = match self.conn.prepare(
3128 "SELECT windows.base, windows.unit, subscriptions.tbl
3129 FROM _syncular_windows AS windows
3130 JOIN _syncular_subscriptions AS subscriptions
3131 ON subscriptions.id = windows.sub_id
3132 ORDER BY windows.base, windows.unit",
3133 ) {
3134 Ok(stmt) => stmt,
3135 Err(_) => return Vec::new(),
3136 };
3137 let rows = stmt.query_map([], |row| {
3138 Ok((
3139 row.get::<_, String>(0)?,
3140 row.get::<_, String>(1)?,
3141 row.get::<_, String>(2)?,
3142 ))
3143 });
3144 match rows {
3145 Ok(rows) => rows.filter_map(Result::ok).collect(),
3146 Err(_) => Vec::new(),
3147 }
3148 }
3149
3150 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
3151 self.conn
3152 .query_row(
3153 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
3154 rusqlite::params![sub_id],
3155 |row| Ok((row.get(0)?, row.get(1)?)),
3156 )
3157 .ok()
3158 }
3159
3160 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
3161 let _ = self.conn.execute(
3162 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
3163 rusqlite::params![base_key, unit, sub_id],
3164 );
3165 }
3166
3167 fn delete_window_unit(&self, base_key: &str, unit: &str) {
3168 let _ = self.conn.execute(
3169 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
3170 rusqlite::params![base_key, unit],
3171 );
3172 }
3173
3174 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
3181 let effective = self
3182 .subs
3183 .iter()
3184 .find(|s| s.id == sub_id)
3185 .and_then(|s| s.effective.clone())
3186 .unwrap_or_else(|| unit_scopes(base, unit));
3187 let pinned = self.pinned_row_ids(&base.table);
3188 let deferred = self
3189 .evict_scope_rows(&base.table, &effective, &pinned)
3190 .unwrap_or(false);
3191 self.delete_window_unit(base_key, unit);
3192 self.unsubscribe(sub_id);
3193 if deferred {
3194 self.save_pending_evict(sub_id, &base.table, &effective);
3195 } else {
3196 self.delete_pending_evict(sub_id);
3197 }
3198 self.rebuild_overlay();
3199 }
3200
3201 fn evict_scope_rows(
3205 &mut self,
3206 table_name: &str,
3207 effective: &[(String, Vec<String>)],
3208 pinned: &std::collections::HashSet<String>,
3209 ) -> Result<bool, ()> {
3210 if effective.is_empty() {
3211 return Ok(false);
3212 }
3213 let table = self.schema.table(table_name).ok_or(())?.clone();
3214 let mut clauses = Vec::new();
3215 let mut params: Vec<SqlValue> = Vec::new();
3216 for (variable, values) in effective {
3217 let column = table.scope_column(variable).ok_or(())?;
3218 let placeholders: Vec<String> = values
3219 .iter()
3220 .map(|v| {
3221 params.push(SqlValue::Text(v.clone()));
3222 "?".to_owned()
3223 })
3224 .collect();
3225 clauses.push(format!(
3226 "{} IN ({})",
3227 quote_ident(column),
3228 placeholders.join(", ")
3229 ));
3230 }
3231 let mut sql = format!(
3232 "DELETE FROM {} WHERE {}",
3233 base_table(table_name),
3234 clauses.join(" AND ")
3235 );
3236 if !pinned.is_empty() {
3237 let pk = quote_ident(&table.primary_key);
3238 let holes: Vec<String> = pinned
3239 .iter()
3240 .map(|id| {
3241 params.push(SqlValue::Text(id.clone()));
3242 "?".to_owned()
3243 })
3244 .collect();
3245 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
3246 }
3247 self.overlay_dirty.set(true);
3248 self.conn
3249 .execute(&sql, rusqlite::params_from_iter(params))
3250 .map_err(|_| ())?;
3251 if pinned.is_empty() {
3252 return Ok(false);
3253 }
3254 let mut where_params: Vec<SqlValue> = Vec::new();
3257 let mut where_clauses = Vec::new();
3258 for (variable, values) in effective {
3259 let column = table.scope_column(variable).ok_or(())?;
3260 let placeholders: Vec<String> = values
3261 .iter()
3262 .map(|v| {
3263 where_params.push(SqlValue::Text(v.clone()));
3264 "?".to_owned()
3265 })
3266 .collect();
3267 where_clauses.push(format!(
3268 "{} IN ({})",
3269 quote_ident(column),
3270 placeholders.join(", ")
3271 ));
3272 }
3273 let pk = quote_ident(&table.primary_key);
3274 let select = format!(
3275 "SELECT {} FROM {} WHERE {}",
3276 pk,
3277 base_table(table_name),
3278 where_clauses.join(" AND ")
3279 );
3280 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
3281 let survivors: Vec<String> = stmt
3282 .query_map(rusqlite::params_from_iter(where_params), |row| {
3283 row.get::<_, String>(0)
3284 })
3285 .map_err(|_| ())?
3286 .filter_map(Result::ok)
3287 .collect();
3288 Ok(survivors.iter().any(|id| pinned.contains(id)))
3289 }
3290
3291 fn drain_pending_evictions(&mut self) {
3295 let pending = self.load_pending_evictions();
3296 if pending.is_empty() {
3297 return;
3298 }
3299 for (sub_id, table_name, effective) in pending {
3300 if self.schema.table(&table_name).is_none() {
3301 self.delete_pending_evict(&sub_id);
3302 continue;
3303 }
3304 let pinned = self.pinned_row_ids(&table_name);
3305 let deferred = self
3306 .evict_scope_rows(&table_name, &effective, &pinned)
3307 .unwrap_or(false);
3308 if !deferred {
3309 self.delete_pending_evict(&sub_id);
3310 }
3311 }
3312 self.rebuild_overlay_if_dirty();
3313 }
3314
3315 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
3318 let mut pinned = std::collections::HashSet::new();
3319 for commit in &self.outbox {
3320 for op in &commit.ops {
3321 if op.table == table {
3322 pinned.insert(op.row_id.clone());
3323 }
3324 }
3325 }
3326 pinned
3327 }
3328
3329 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
3330 let _ = self.conn.execute(
3331 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
3332 VALUES (?1, ?2, ?3)",
3333 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
3334 );
3335 }
3336
3337 fn delete_pending_evict(&self, sub_id: &str) {
3338 let _ = self.conn.execute(
3339 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
3340 rusqlite::params![sub_id],
3341 );
3342 }
3343
3344 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
3345 let mut stmt = match self
3346 .conn
3347 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
3348 {
3349 Ok(stmt) => stmt,
3350 Err(_) => return Vec::new(),
3351 };
3352 let rows = stmt.query_map([], |row| {
3353 Ok((
3354 row.get::<_, String>(0)?,
3355 row.get::<_, String>(1)?,
3356 row.get::<_, String>(2)?,
3357 ))
3358 });
3359 let mut out = Vec::new();
3360 if let Ok(rows) = rows {
3361 for entry in rows.filter_map(Result::ok) {
3362 let (sub_id, table, json) = entry;
3363 if let Ok(value) = serde_json::from_str::<Value>(&json) {
3364 if let Ok(effective) = json_to_scope_map(&value) {
3365 out.push((sub_id, table, effective));
3366 }
3367 }
3368 }
3369 }
3370 out
3371 }
3372
3373 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
3375 if mutations.is_empty() {
3376 return Err("a commit must contain at least one operation (§6.1)".to_owned());
3377 }
3378 let mut ops = Vec::with_capacity(mutations.len());
3379 for mutation in mutations {
3380 match mutation {
3381 Mutation::Upsert {
3382 table,
3383 values,
3384 base_version,
3385 } => {
3386 let schema_table = self
3387 .schema
3388 .table(&table)
3389 .ok_or_else(|| format!("unknown table {table:?}"))?;
3390 let values = normalize_values_casing(schema_table, values)?;
3394 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
3395 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
3398 ops.push(OutboxOp {
3399 upsert: true,
3400 table,
3401 row_id,
3402 base_version,
3403 values: Some(values),
3404 changed_fields: None,
3405 });
3406 }
3407 Mutation::Delete {
3408 table,
3409 row_id,
3410 base_version,
3411 } => {
3412 if self.schema.table(&table).is_none() {
3413 return Err(format!("unknown table {table:?}"));
3414 }
3415 ops.push(OutboxOp {
3416 upsert: false,
3417 table,
3418 row_id,
3419 base_version,
3420 values: None,
3421 changed_fields: None,
3422 });
3423 }
3424 }
3425 }
3426 self.record_outbox_commit(ops)
3427 }
3428
3429 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
3430 let commit = OutboxCommit {
3431 client_commit_id: uuid::Uuid::new_v4().to_string(),
3432 ops,
3433 };
3434 self.begin_observation("syncular_mutation")?;
3435 let mut batch = ChangeAccumulator::default();
3436 for op in &commit.ops {
3437 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
3438 if let Some(values) = &op.values {
3439 if let Some(table) = self.schema.table(&op.table) {
3440 for scope in &table.scope_variables {
3441 if let Some(Value::String(value)) = values.get(&scope.column) {
3442 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
3443 precise = true;
3444 }
3445 }
3446 }
3447 }
3448 if !precise {
3449 batch.table(&op.table);
3450 }
3451 }
3452 self.persist_outbox_insert(&commit);
3453 let id = commit.client_commit_id.clone();
3454 self.outbox.push(commit);
3455 self.overlay_dirty.set(true);
3456 self.rebuild_overlay();
3457 batch.status = true;
3458 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
3459 self.rollback_observation("syncular_mutation");
3460 return Err(error);
3461 }
3462 Ok(id)
3463 }
3464
3465 pub fn patch(
3470 &mut self,
3471 table: &str,
3472 row_id: &str,
3473 partial: Map<String, Value>,
3474 base_version: Option<i64>,
3475 ) -> Result<String, String> {
3476 let schema_table = self
3477 .schema
3478 .table(table)
3479 .ok_or_else(|| format!("unknown table {table:?}"))?;
3480 let partial = normalize_values_casing(schema_table, partial)?;
3481 let mut values = self
3482 .read_rows(table)?
3483 .into_iter()
3484 .find(|row| row.row_id == row_id)
3485 .map(|row| row.values)
3486 .ok_or_else(|| {
3487 format!(
3488 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
3489 )
3490 })?;
3491 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
3492 changed_fields.sort();
3493 values.extend(partial);
3494 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
3495 if row_id_from_values != row_id {
3496 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
3497 }
3498 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
3499 self.record_outbox_commit(vec![OutboxOp {
3500 upsert: true,
3501 table: table.to_owned(),
3502 row_id: row_id.to_owned(),
3503 base_version,
3504 values: Some(values),
3505 changed_fields: Some(changed_fields),
3506 }])
3507 }
3508
3509 pub fn pending_commit_ids(&self) -> Vec<String> {
3510 self.outbox
3511 .iter()
3512 .map(|c| c.client_commit_id.clone())
3513 .collect()
3514 }
3515
3516 pub fn conflicts(&self) -> &[ConflictRecord] {
3517 &self.conflicts
3518 }
3519
3520 pub fn rejections(&self) -> &[RejectionRecord] {
3521 &self.rejections
3522 }
3523
3524 pub fn resolve_commit_outcome(
3525 &mut self,
3526 input: ResolveCommitOutcomeInput,
3527 ) -> Result<CommitOutcome, String> {
3528 let current = self
3529 .commit_outcome(&input.client_commit_id)?
3530 .ok_or_else(|| {
3531 format!(
3532 "sync.outcome_not_found: no durable outcome exists for {:?}",
3533 input.client_commit_id
3534 )
3535 })?;
3536 if current.resolution != CommitOutcomeResolution::Active {
3537 return Ok(current);
3538 }
3539 if input.resolution == CommitOutcomeResolution::Active {
3540 return Err("sync.invalid_request: resolution must leave active state".to_owned());
3541 }
3542 match input.resolution {
3543 CommitOutcomeResolution::Superseded => {
3544 let replacement = input
3545 .replacement_client_commit_id
3546 .as_deref()
3547 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
3548 .ok_or_else(|| {
3549 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
3550 .to_owned()
3551 })?;
3552 let _ = replacement;
3553 }
3554 _ if input.replacement_client_commit_id.is_some() => {
3555 return Err(
3556 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
3557 .to_owned(),
3558 );
3559 }
3560 _ => {}
3561 }
3562 let allowed = match current.status {
3563 CommitOutcomeStatus::Conflict => matches!(
3564 input.resolution,
3565 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
3566 ),
3567 CommitOutcomeStatus::Rejected => {
3568 input.resolution == CommitOutcomeResolution::Superseded
3569 }
3570 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
3571 input.resolution == CommitOutcomeResolution::Dismissed
3572 }
3573 };
3574 if !allowed {
3575 return Err(format!(
3576 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
3577 input.resolution, current.status
3578 ));
3579 }
3580
3581 self.begin_observation("syncular_outcome_resolution")?;
3582 let result = (|| {
3583 self.conn
3584 .execute(
3585 "UPDATE _syncular_commit_outcomes
3586 SET resolution = ?1, resolved_at_ms = ?2,
3587 replacement_client_commit_id = ?3
3588 WHERE client_commit_id = ?4 AND resolution = 'active'",
3589 rusqlite::params![
3590 Self::outcome_resolution_name(input.resolution),
3591 self.clock_now_ms(),
3592 input.replacement_client_commit_id,
3593 input.client_commit_id
3594 ],
3595 )
3596 .map_err(|error| error.to_string())?;
3597 let resolved = self
3598 .commit_outcome(¤t.client_commit_id)?
3599 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
3600 self.prune_commit_outcomes()?;
3601 Ok(resolved)
3602 })();
3603 let resolved = match result {
3604 Ok(outcome) => outcome,
3605 Err(error) => {
3606 self.rollback_observation("syncular_outcome_resolution");
3607 return Err(error);
3608 }
3609 };
3610 self.conflicts
3611 .retain(|record| record.client_commit_id != current.client_commit_id);
3612 self.rejections
3613 .retain(|record| record.client_commit_id != current.client_commit_id);
3614 let batch = ChangeAccumulator {
3615 conflicts: current.status == CommitOutcomeStatus::Conflict,
3616 rejections: current.status == CommitOutcomeStatus::Rejected,
3617 outcomes: true,
3618 ..ChangeAccumulator::default()
3619 };
3620 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
3621 self.rollback_observation("syncular_outcome_resolution");
3622 return Err(error);
3623 }
3624 Ok(resolved)
3625 }
3626
3627 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
3628 self.schema_floor.as_ref()
3629 }
3630
3631 pub fn lease_state(&self) -> Option<&LeaseState> {
3633 self.lease_state.as_ref()
3634 }
3635
3636 fn record_lease_error(&mut self, code: &str) {
3639 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
3640 return;
3641 }
3642 let mut next = self.lease_state.clone().unwrap_or_default();
3643 next.error_code = Some(code.to_owned());
3644 self.set_lease_state(Some(next));
3645 }
3646
3647 fn set_lease_state(&mut self, next: Option<LeaseState>) {
3648 if self.lease_state == next {
3649 return;
3650 }
3651 if self.begin_observation("syncular_lease").is_err() {
3652 return;
3653 }
3654 self.lease_state = next;
3655 if let Some(lease) = &self.lease_state {
3656 if let Ok(json) = serde_json::to_string(lease) {
3657 self.set_meta(LEASE_STATE_KEY, &json);
3658 }
3659 } else {
3660 self.delete_meta(LEASE_STATE_KEY);
3661 }
3662 let batch = ChangeAccumulator {
3663 status: true,
3664 ..ChangeAccumulator::default()
3665 };
3666 if self.finish_observation("syncular_lease", batch).is_err() {
3667 self.rollback_observation("syncular_lease");
3668 }
3669 }
3670
3671 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
3672 if self.schema_floor == next {
3673 return;
3674 }
3675 if self.begin_observation("syncular_schema_floor").is_err() {
3676 return;
3677 }
3678 self.schema_floor = next;
3679 self.stopped = self.schema_floor.is_some();
3680 if let Some(floor) = &self.schema_floor {
3681 if let Ok(json) = serde_json::to_string(floor) {
3682 self.set_meta(SCHEMA_FLOOR_KEY, &json);
3683 }
3684 } else {
3685 self.delete_meta(SCHEMA_FLOOR_KEY);
3686 }
3687 let batch = ChangeAccumulator {
3688 status: true,
3689 ..ChangeAccumulator::default()
3690 };
3691 if self
3692 .finish_observation("syncular_schema_floor", batch)
3693 .is_err()
3694 {
3695 self.rollback_observation("syncular_schema_floor");
3696 }
3697 }
3698
3699 fn set_upgrading(&mut self, value: bool) {
3700 if self.upgrading == value {
3701 return;
3702 }
3703 if self.begin_observation("syncular_upgrading").is_err() {
3704 return;
3705 }
3706 self.upgrading = value;
3707 let batch = ChangeAccumulator {
3708 status: true,
3709 ..ChangeAccumulator::default()
3710 };
3711 if self
3712 .finish_observation("syncular_upgrading", batch)
3713 .is_err()
3714 {
3715 self.rollback_observation("syncular_upgrading");
3716 }
3717 }
3718
3719 pub fn sync_needed(&self) -> bool {
3720 self.sync_needed
3721 }
3722
3723 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
3724 let sub = self.subs.iter().find(|s| s.id == id)?;
3725 Some(SubscriptionStateView {
3726 id: sub.id.clone(),
3727 table: sub.table.clone(),
3728 status: sub.state.name().to_owned(),
3729 cursor: sub.cursor,
3730 has_resume_token: sub.bootstrap_state.is_some(),
3731 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3732 reason_code: sub.reason_code.clone(),
3733 })
3734 }
3735
3736 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
3737 let schema_table = self
3738 .schema
3739 .table(table)
3740 .ok_or_else(|| format!("unknown table {table:?}"))?;
3741 let sql = format!(
3742 "SELECT * FROM {} ORDER BY {} ASC",
3743 visible_table(table),
3744 quote_ident(&schema_table.primary_key)
3745 );
3746 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
3747 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
3748 let mut out = Vec::new();
3749 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
3750 let mut values = Map::new();
3751 for (i, column) in schema_table.columns.iter().enumerate() {
3752 let value = row.get_ref(i).map_err(|e| e.to_string())?;
3753 values.insert(column.name.clone(), sql_ref_to_json(column, value));
3754 }
3755 let version: i64 = row
3756 .get(schema_table.columns.len())
3757 .map_err(|e| e.to_string())?;
3758 let row_id = match values.get(&schema_table.primary_key) {
3759 Some(Value::String(s)) => s.clone(),
3760 Some(Value::Number(n)) => n.to_string(),
3761 Some(Value::Bool(b)) => b.to_string(),
3762 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
3763 };
3764 out.push(RowState {
3765 row_id,
3766 version,
3767 values,
3768 });
3769 }
3770 Ok(out)
3771 }
3772
3773 #[cfg(feature = "crdt-yjs")]
3789 fn crdt_column_bytes(
3790 &self,
3791 table: &str,
3792 row_id: &str,
3793 column: &str,
3794 ) -> Result<Option<Vec<u8>>, String> {
3795 let schema_table = self
3796 .schema
3797 .table(table)
3798 .ok_or_else(|| format!("unknown table {table:?}"))?;
3799 let col = schema_table
3800 .columns
3801 .iter()
3802 .find(|c| c.name == column)
3803 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
3804 if col.ty != ColumnType::Crdt {
3805 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
3806 }
3807 let sql = format!(
3808 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
3809 quote_ident(column),
3810 visible_table(table),
3811 quote_ident(&schema_table.primary_key)
3812 );
3813 let bytes: Option<Vec<u8>> = self
3814 .conn
3815 .query_row(&sql, rusqlite::params![row_id], |row| {
3816 row.get::<_, Option<Vec<u8>>>(0)
3817 })
3818 .map_err(|e| match e {
3819 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
3820 other => other.to_string(),
3821 })?;
3822 Ok(bytes)
3823 }
3824
3825 #[cfg(feature = "crdt-yjs")]
3829 pub fn crdt_text(
3830 &self,
3831 table: &str,
3832 row_id: &str,
3833 column: &str,
3834 name: &str,
3835 ) -> Result<String, String> {
3836 let bytes = self
3837 .crdt_column_bytes(table, row_id, column)?
3838 .unwrap_or_default();
3839 crate::crdt::text(&bytes, name)
3840 }
3841
3842 #[cfg(feature = "crdt-yjs")]
3846 pub fn crdt_insert_text(
3847 &mut self,
3848 table: &str,
3849 row_id: &str,
3850 column: &str,
3851 name: &str,
3852 index: u32,
3853 value: &str,
3854 ) -> Result<String, String> {
3855 let current = self
3856 .crdt_column_bytes(table, row_id, column)?
3857 .unwrap_or_default();
3858 let update = crate::crdt::insert_text(¤t, name, index, value)?;
3859 self.crdt_push_update(table, row_id, column, &update)
3860 }
3861
3862 #[cfg(feature = "crdt-yjs")]
3865 pub fn crdt_delete_text(
3866 &mut self,
3867 table: &str,
3868 row_id: &str,
3869 column: &str,
3870 name: &str,
3871 index: u32,
3872 len: u32,
3873 ) -> Result<String, String> {
3874 let current = self
3875 .crdt_column_bytes(table, row_id, column)?
3876 .unwrap_or_default();
3877 let update = crate::crdt::delete_text(¤t, name, index, len)?;
3878 self.crdt_push_update(table, row_id, column, &update)
3879 }
3880
3881 #[cfg(feature = "crdt-yjs")]
3886 pub fn crdt_apply_update(
3887 &mut self,
3888 table: &str,
3889 row_id: &str,
3890 column: &str,
3891 update: &[u8],
3892 ) -> Result<String, String> {
3893 let current = self
3894 .crdt_column_bytes(table, row_id, column)?
3895 .unwrap_or_default();
3896 let next = crate::crdt::apply_update(¤t, update)?;
3897 self.crdt_push_update(table, row_id, column, &next)
3898 }
3899
3900 #[cfg(feature = "crdt-yjs")]
3906 fn crdt_push_update(
3907 &mut self,
3908 table: &str,
3909 row_id: &str,
3910 column: &str,
3911 crdt_bytes: &[u8],
3912 ) -> Result<String, String> {
3913 let schema_table = self
3914 .schema
3915 .table(table)
3916 .ok_or_else(|| format!("unknown table {table:?}"))?
3917 .clone();
3918 let mut values: Map<String, Value> = self
3921 .read_rows(table)?
3922 .into_iter()
3923 .find(|r| r.row_id == row_id)
3924 .map(|r| r.values)
3925 .unwrap_or_else(|| {
3926 let mut map = Map::new();
3927 map.insert(
3928 schema_table.primary_key.clone(),
3929 Value::from(row_id.to_owned()),
3930 );
3931 map
3932 });
3933 let mut bytes_obj = Map::new();
3935 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
3936 values.insert(column.to_owned(), Value::Object(bytes_obj));
3937 self.mutate(vec![Mutation::Upsert {
3938 table: table.to_owned(),
3939 values,
3940 base_version: None,
3941 }])
3942 }
3943
3944 pub fn query(&self, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {
3959 query_connection(&self.conn, sql, params)
3960 }
3961
3962 pub fn query_snapshot(
3964 &mut self,
3965 sql: &str,
3966 params: &[Value],
3967 coverage: &[WindowCoverage],
3968 ) -> Result<QuerySnapshot, String> {
3969 snapshot_connection(&self.conn, sql, params, coverage)
3970 }
3971
3972 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
3975 let mut frames = vec![Frame::ReqHeader {
3976 client_id: self.client_id.clone(),
3977 schema_version: self.schema.version,
3978 }];
3979 let mut pushed_ids = Vec::new();
3980 let mut ops_in_request = 0usize;
3981 let mut deferred_commits = 0usize;
3982 for (index, commit) in self.outbox.iter().enumerate() {
3983 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
3988 deferred_commits = self.outbox.len() - index;
3989 break;
3990 }
3991 ops_in_request += commit.ops.len();
3992 let operations = commit
3993 .ops
3994 .iter()
3995 .map(|op| {
3996 let payload = op.values.as_ref().and_then(|values| {
3997 let table = self.schema.table(&op.table)?;
3998 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
4002 });
4003 ssp2::model::Operation {
4004 table: op.table.clone(),
4005 row_id: op.row_id.clone(),
4006 op: if op.upsert { Op::Upsert } else { Op::Delete },
4007 base_version: op.base_version,
4008 payload,
4009 }
4010 })
4011 .collect();
4012 frames.push(Frame::PushCommit {
4013 client_commit_id: commit.client_commit_id.clone(),
4014 operations,
4015 });
4016 pushed_ids.push(commit.client_commit_id.clone());
4017 }
4018 let accept = self.limits.accept.unwrap_or(if url_capable {
4021 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
4022 } else {
4023 DEFAULT_ACCEPT
4024 });
4025 frames.push(Frame::PullHeader {
4026 limit_commits: self.limits.limit_commits.unwrap_or(0),
4027 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
4028 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
4029 accept,
4030 });
4031 let mut fresh = Vec::new();
4032 for sub in &self.subs {
4033 if sub.state != SubState::Active {
4034 continue;
4035 }
4036 let mut scopes = sub.requested.clone();
4037 sort_scope_map(&mut scopes);
4038 frames.push(Frame::Subscription {
4039 id: sub.id.clone(),
4040 table: sub.table.clone(),
4041 scopes,
4042 params: sub.params.clone().map(RawJson),
4043 cursor: sub.cursor,
4044 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
4045 });
4046 fresh.push((
4047 sub.id.clone(),
4048 sub.cursor < 0 && sub.bootstrap_state.is_none(),
4049 ));
4050 }
4051 let message = Message {
4052 msg_kind: MsgKind::Request,
4053 frames,
4054 };
4055 (
4056 message,
4057 RequestMeta {
4058 pushed_ids,
4059 fresh,
4060 accept,
4061 deferred_commits,
4062 },
4063 )
4064 }
4065
4066 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
4069 if self.stopped {
4070 return SyncOutcome::Ok(SyncReport {
4073 schema_floor: self.schema_floor.clone(),
4074 ..SyncReport::default()
4075 });
4076 }
4077 self.set_sync_needed(false, false);
4080 if self.schema_has_blobs() {
4083 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
4084 if Self::retryable_transport_code(&code) {
4085 self.schedule_background_retry();
4086 }
4087 return SyncOutcome::Failed {
4088 error_code: code,
4089 message,
4090 };
4091 }
4092 }
4093 let (message, meta) = self.build_request(transport.supports_url_fetch());
4094 let request_bytes = encode_message(&message);
4095 let round = if self.realtime_connected {
4099 transport.realtime_sync(&request_bytes)
4100 } else {
4101 transport.sync(&request_bytes)
4102 };
4103 let response_bytes = match round {
4104 Ok(bytes) => bytes,
4105 Err(TransportError { code, message }) => {
4106 self.record_lease_error(&code);
4109 if Self::retryable_transport_code(&code) {
4110 self.schedule_background_retry();
4111 }
4112 return SyncOutcome::Failed {
4113 error_code: code,
4114 message,
4115 };
4116 }
4117 };
4118 let response = match decode_message(&response_bytes) {
4119 Ok(message) => message,
4120 Err(error) => {
4121 return SyncOutcome::Failed {
4124 error_code: error.code.as_str().to_owned(),
4125 message: error.detail,
4126 };
4127 }
4128 };
4129 if response.msg_kind != MsgKind::Response {
4130 return SyncOutcome::Failed {
4131 error_code: "sync.invalid_request".to_owned(),
4132 message: "expected a response message".to_owned(),
4133 };
4134 }
4135 let outcome = self.process_response(transport, response, &meta);
4136 match &outcome {
4137 SyncOutcome::Ok(_) => self.reset_background_retry(),
4138 SyncOutcome::Failed { error_code, .. }
4139 if Self::retryable_transport_code(error_code) =>
4140 {
4141 self.schedule_background_retry();
4142 }
4143 SyncOutcome::Failed { .. } => {}
4144 }
4145 if meta.deferred_commits > 0 {
4146 self.set_sync_needed(true, true);
4149 }
4150 outcome
4151 }
4152
4153 pub fn sync_until_idle(
4154 &mut self,
4155 transport: &mut dyn Transport,
4156 max_rounds: Option<u32>,
4157 ) -> SyncOutcome {
4158 let rounds = max_rounds.unwrap_or(12).max(1);
4159 let mut aggregate = SyncReport::default();
4160 for _ in 0..rounds {
4161 match self.sync(transport) {
4162 SyncOutcome::Failed {
4163 error_code,
4164 message,
4165 } => {
4166 return SyncOutcome::Failed {
4167 error_code,
4168 message,
4169 };
4170 }
4171 SyncOutcome::Ok(report) => {
4172 aggregate.pushed += report.pushed;
4173 aggregate.applied.extend(report.applied.iter().cloned());
4174 aggregate.rejected.extend(report.rejected.iter().cloned());
4175 aggregate.retryable.extend(report.retryable.iter().cloned());
4176 aggregate.conflicts += report.conflicts;
4177 aggregate.commits_applied += report.commits_applied;
4178 aggregate.segment_rows_applied += report.segment_rows_applied;
4179 aggregate.bootstrapping = report.bootstrapping.clone();
4180 aggregate.resets.extend(report.resets.iter().cloned());
4181 aggregate.revoked.extend(report.revoked.iter().cloned());
4182 aggregate.failed.extend(report.failed.iter().cloned());
4183 if report.schema_floor.is_some() {
4184 aggregate.schema_floor = report.schema_floor.clone();
4185 }
4186 let more = !report.bootstrapping.is_empty()
4192 || report.commits_applied > 0
4193 || report.segment_rows_applied > 0
4194 || !report.resets.is_empty()
4195 || self.sync_needed;
4196 if !more {
4197 break;
4198 }
4199 }
4200 }
4201 }
4202 SyncOutcome::Ok(aggregate)
4203 }
4204
4205 fn process_response(
4206 &mut self,
4207 transport: &mut dyn Transport,
4208 response: Message,
4209 meta: &RequestMeta,
4210 ) -> SyncOutcome {
4211 let mut report = SyncReport {
4212 pushed: meta.pushed_ids.len() as u32,
4213 ..SyncReport::default()
4214 };
4215 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
4216 HashMap::new();
4217 for frame in &response.frames {
4218 if let Frame::PushResultDetails {
4219 client_commit_id,
4220 entries,
4221 } = frame
4222 {
4223 let details = rejection_details_by_commit
4224 .entry(client_commit_id.clone())
4225 .or_default();
4226 for entry in entries {
4227 let parsed = match RejectionDetails::parse(&entry.details.0) {
4228 Ok(value) => value,
4229 Err(message) => {
4230 return SyncOutcome::Failed {
4231 error_code: "sync.invalid_request".to_owned(),
4232 message,
4233 };
4234 }
4235 };
4236 details.insert(entry.op_index, parsed);
4237 }
4238 }
4239 }
4240 let mut frames = response.frames.into_iter();
4241 match frames.next() {
4242 Some(Frame::RespHeader {
4243 required_schema_version,
4244 latest_schema_version,
4245 }) => {
4246 if let Some(required) = required_schema_version {
4247 let floor = SchemaFloor {
4250 required_schema_version: Some(required),
4251 latest_schema_version,
4252 };
4253 self.set_schema_floor(Some(floor.clone()));
4254 report.schema_floor = Some(floor);
4255 return SyncOutcome::Ok(report);
4256 }
4257 }
4258 _ => {
4259 return SyncOutcome::Failed {
4260 error_code: "sync.invalid_request".to_owned(),
4261 message: "response does not start with RESP_HEADER".to_owned(),
4262 };
4263 }
4264 }
4265
4266 let mut failure: Option<(String, String)> = None;
4267 while let Some(frame) = frames.next() {
4268 match frame {
4269 Frame::PushResult {
4270 client_commit_id,
4271 status,
4272 commit_seq: _,
4273 results,
4274 } => {
4275 self.handle_push_result(
4276 &client_commit_id,
4277 status,
4278 &results,
4279 rejection_details_by_commit.get(&client_commit_id),
4280 &mut report,
4281 );
4282 }
4283 Frame::PushResultDetails { .. } => {}
4284 Frame::SubStart {
4285 id,
4286 status,
4287 reason_code,
4288 effective_scopes,
4289 bootstrap: _,
4290 } => {
4291 let mut body = Vec::new();
4292 let mut sub_end: Option<(i64, Option<String>)> = None;
4293 for inner in frames.by_ref() {
4294 match inner {
4295 Frame::SubEnd {
4296 next_cursor,
4297 bootstrap_state,
4298 } => {
4299 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
4300 break;
4301 }
4302 Frame::Unknown { .. } => {}
4303 other => body.push(other),
4304 }
4305 }
4306 let Some((next_cursor, bootstrap_state)) = sub_end else {
4307 failure = Some((
4308 "sync.invalid_request".to_owned(),
4309 "subscription section without SUB_END".to_owned(),
4310 ));
4311 break;
4312 };
4313 if let Err(SectionError::Abort(code, message)) = self.process_section(
4314 transport,
4315 &id,
4316 status,
4317 &reason_code,
4318 effective_scopes,
4319 body,
4320 next_cursor,
4321 bootstrap_state,
4322 meta,
4323 &mut report,
4324 ) {
4325 failure = Some((code, message));
4326 break;
4327 }
4328 }
4329 Frame::Lease {
4330 lease_id,
4331 expires_at_ms,
4332 } => {
4333 self.set_lease_state(Some(LeaseState {
4336 lease_id: Some(lease_id),
4337 expires_at_ms: Some(expires_at_ms),
4338 error_code: None,
4339 }));
4340 }
4341 Frame::Error { code, message, .. } => {
4342 failure = Some((code, message));
4345 break;
4346 }
4347 Frame::Unknown { .. } => {}
4348 _ => {
4349 failure = Some((
4350 "sync.invalid_request".to_owned(),
4351 "unexpected frame in response".to_owned(),
4352 ));
4353 break;
4354 }
4355 }
4356 }
4357
4358 if self.overlay_dirty.get() {
4361 self.rebuild_overlay();
4362 }
4363 self.reconcile_blob_refcounts(false);
4370
4371 if let Some((error_code, message)) = failure {
4372 return SyncOutcome::Failed {
4373 error_code,
4374 message,
4375 };
4376 }
4377 self.drain_pending_evictions();
4380 self.ack_after_pull(transport);
4381 if self.upgrading && report.bootstrapping.is_empty() {
4384 self.set_upgrading(false);
4385 }
4386 SyncOutcome::Ok(report)
4387 }
4388
4389 fn handle_push_result(
4392 &mut self,
4393 client_commit_id: &str,
4394 status: PushStatus,
4395 results: &[OpResult],
4396 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
4397 report: &mut SyncReport,
4398 ) {
4399 let Some(index) = self
4400 .outbox
4401 .iter()
4402 .position(|c| c.client_commit_id == client_commit_id)
4403 else {
4404 return;
4405 };
4406 if self.begin_observation("syncular_push_result").is_err() {
4407 return;
4408 }
4409 let mut batch = ChangeAccumulator::default();
4410 let operations = self.outbox[index].ops.clone();
4411 match status {
4412 PushStatus::Applied | PushStatus::Cached => {
4413 let journal_results = results
4416 .iter()
4417 .map(|result| {
4418 let op_index = match result {
4419 OpResult::Applied { op_index }
4420 | OpResult::Conflict { op_index, .. }
4421 | OpResult::Error { op_index, .. } => *op_index,
4422 };
4423 CommitOperationOutcome::Applied { op_index }
4424 })
4425 .collect::<Vec<_>>();
4426 let outcome_status = if status == PushStatus::Applied {
4427 CommitOutcomeStatus::Applied
4428 } else {
4429 CommitOutcomeStatus::Cached
4430 };
4431 if self
4432 .persist_commit_outcome(
4433 client_commit_id,
4434 outcome_status,
4435 &journal_results,
4436 None,
4437 )
4438 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4439 .and_then(|()| self.prune_commit_outcomes())
4440 .is_err()
4441 {
4442 self.rollback_observation("syncular_push_result");
4443 return;
4444 }
4445 report.applied.push(client_commit_id.to_owned());
4446 self.outbox.remove(index);
4447 self.overlay_dirty.set(true);
4448 batch.status = true;
4449 batch.outcomes = true;
4450 }
4451 PushStatus::Rejected => {
4452 if results.iter().any(|result| {
4453 matches!(
4454 result,
4455 OpResult::Error {
4456 code,
4457 retryable: true,
4458 ..
4459 } if code == "sync.idempotency_cache_miss"
4460 )
4461 }) {
4462 report.retryable.push(client_commit_id.to_owned());
4465 if self
4466 .finish_observation("syncular_push_result", batch)
4467 .is_err()
4468 {
4469 self.rollback_observation("syncular_push_result");
4470 }
4471 return;
4472 }
4473
4474 let mut journal_results = Vec::with_capacity(results.len());
4475 let mut conflicts = Vec::new();
4476 let mut rejections = Vec::new();
4477 for result in results {
4478 match result {
4479 OpResult::Applied { op_index } => {
4480 journal_results.push(CommitOperationOutcome::Applied {
4481 op_index: *op_index,
4482 });
4483 }
4484 OpResult::Conflict {
4485 op_index,
4486 code,
4487 message,
4488 server_version,
4489 server_row,
4490 } => {
4491 let operation = operations
4492 .get(*op_index as usize)
4493 .map(CommitOperation::from);
4494 let (table, row_id) = operation
4495 .as_ref()
4496 .map(|op| (op.table.clone(), op.row_id.clone()))
4497 .unwrap_or_default();
4498 let server_row_json = self
4499 .schema
4500 .table(&table)
4501 .and_then(|t| {
4502 decode_row_bytes(t, server_row, &self.encryption)
4503 .ok()
4504 .map(|row| (t, row))
4505 })
4506 .map(|(t, row)| {
4507 let mut map = Map::new();
4508 for (i, column) in t.columns.iter().enumerate() {
4509 map.insert(
4510 column.name.clone(),
4511 column_value_to_json(row.get(i).unwrap_or(&None)),
4512 );
4513 }
4514 map
4515 })
4516 .unwrap_or_default();
4517 let conflict = ConflictRecord {
4518 client_commit_id: client_commit_id.to_owned(),
4519 op_index: *op_index,
4520 table,
4521 row_id,
4522 code: code.clone(),
4523 message: message.clone(),
4524 server_version: *server_version,
4525 server_row: server_row_json,
4526 operation,
4527 };
4528 journal_results.push(CommitOperationOutcome::Conflict {
4529 conflict: conflict.clone(),
4530 });
4531 conflicts.push(conflict);
4532 }
4533 OpResult::Error {
4534 op_index,
4535 code,
4536 message,
4537 retryable,
4538 } => {
4539 let rejection = RejectionRecord {
4540 client_commit_id: client_commit_id.to_owned(),
4541 op_index: *op_index,
4542 code: code.clone(),
4543 message: message.clone(),
4544 retryable: *retryable,
4545 details: rejection_details
4546 .and_then(|details| details.get(op_index))
4547 .cloned(),
4548 operation: operations
4549 .get(*op_index as usize)
4550 .map(CommitOperation::from),
4551 };
4552 journal_results.push(CommitOperationOutcome::Error {
4553 rejection: rejection.clone(),
4554 });
4555 rejections.push(rejection);
4556 }
4557 }
4558 }
4559 let outcome_status = if conflicts.is_empty() {
4560 CommitOutcomeStatus::Rejected
4561 } else {
4562 CommitOutcomeStatus::Conflict
4563 };
4564 if self
4565 .persist_commit_outcome(
4566 client_commit_id,
4567 outcome_status,
4568 &journal_results,
4569 Some(&operations),
4570 )
4571 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4572 .and_then(|()| self.prune_commit_outcomes())
4573 .is_err()
4574 {
4575 self.rollback_observation("syncular_push_result");
4576 return;
4577 }
4578 report.conflicts += conflicts.len() as u32;
4579 report.rejected.push(client_commit_id.to_owned());
4580 batch.conflicts = !conflicts.is_empty();
4581 batch.rejections = !rejections.is_empty();
4582 batch.status = true;
4583 batch.outcomes = true;
4584 self.conflicts.extend(conflicts);
4585 self.rejections.extend(rejections);
4586 self.outbox.remove(index);
4587 self.overlay_dirty.set(true);
4588 }
4589 }
4590 if batch.status {
4591 for operation in &operations {
4592 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
4593 batch.table(&operation.table);
4594 }
4595 }
4596 self.rebuild_overlay_if_dirty();
4597 }
4598 if self
4599 .finish_observation("syncular_push_result", batch)
4600 .is_err()
4601 {
4602 self.rollback_observation("syncular_push_result");
4603 }
4604 }
4605
4606 #[allow(clippy::too_many_arguments)]
4609 fn process_section(
4610 &mut self,
4611 transport: &mut dyn Transport,
4612 id: &str,
4613 status: SubStatus,
4614 reason_code: &str,
4615 effective_scopes: Vec<(String, Vec<String>)>,
4616 body: Vec<Frame>,
4617 next_cursor: i64,
4618 bootstrap_state: Option<String>,
4619 meta: &RequestMeta,
4620 report: &mut SyncReport,
4621 ) -> Result<(), SectionError> {
4622 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4623 return Ok(()); };
4625 match status {
4626 SubStatus::Revoked => {
4627 self.begin_observation("syncular_revocation")
4628 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4629 let mut batch = ChangeAccumulator::default();
4630 let registered = self.window_unit_by_sub_id(id);
4631 let (table, effective) = {
4633 let sub = &self.subs[sub_index];
4634 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4635 };
4636 let purged = self.purge_scope_rows(&table, &effective);
4637 match purged {
4638 Ok(()) => {
4639 self.record_scope_map(&mut batch, &table, &effective);
4640 let sub = &mut self.subs[sub_index];
4641 sub.state = SubState::Revoked;
4642 sub.reason_code = Some(if reason_code.is_empty() {
4643 "sync.scope_revoked".to_owned()
4644 } else {
4645 reason_code.to_owned()
4646 });
4647 report.revoked.push(id.to_owned());
4648 let doomed_effective = effective;
4649 let sub_table = table;
4650 self.persist_sub(&self.subs[sub_index].clone());
4651 let dropped = self
4652 .drop_doomed_outbox(&sub_table, &doomed_effective)
4653 .map_err(|message| {
4654 SectionError::Abort("storage.failed".to_owned(), message)
4655 })?;
4656 if dropped {
4657 batch.status = true;
4658 batch.rejections = true;
4659 batch.outcomes = true;
4660 }
4661 self.reconcile_blob_refcounts(true);
4664 }
4665 Err(()) => {
4666 let sub = &mut self.subs[sub_index];
4669 sub.state = SubState::Failed;
4670 sub.reason_code = Some("sync.scope_revoked".to_owned());
4671 report.failed.push(id.to_owned());
4672 self.persist_sub(&self.subs[sub_index].clone());
4673 }
4674 }
4675 if let Some((base_key, unit)) = registered {
4676 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4677 }
4678 self.rebuild_overlay_if_dirty();
4679 self.finish_observation("syncular_revocation", batch)
4680 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4681 Ok(())
4682 }
4683 SubStatus::Reset => {
4684 self.begin_observation("syncular_reset")
4685 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4686 let mut batch = ChangeAccumulator::default();
4687 let registered = self.window_unit_by_sub_id(id);
4688 let sub = &mut self.subs[sub_index];
4691 sub.cursor = -1;
4692 sub.bootstrap_state = None;
4693 report.resets.push(id.to_owned());
4694 self.persist_sub(&self.subs[sub_index].clone());
4695 if let Some((base_key, unit)) = registered {
4696 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4697 }
4698 self.finish_observation("syncular_reset", batch)
4699 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4700 Ok(())
4701 }
4702 SubStatus::Active => {
4703 let fresh = meta
4704 .fresh
4705 .iter()
4706 .find(|(fid, _)| fid == id)
4707 .map(|(_, f)| *f)
4708 .unwrap_or(false);
4709 let was_pending = self.subs[sub_index].cursor < 0
4710 || self.subs[sub_index].bootstrap_state.is_some();
4711 let registered = self.window_unit_by_sub_id(id);
4712 self.subs[sub_index].effective = Some(effective_scopes);
4714 self.begin_observation("syncular_section")
4715 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4716 let mut batch = ChangeAccumulator::default();
4717 let outcome = self.apply_section_body(
4718 transport, sub_index, body, fresh, meta, report, &mut batch,
4719 );
4720 match outcome {
4721 Ok(()) => {
4722 let sub = &mut self.subs[sub_index];
4723 sub.cursor = next_cursor;
4726 sub.bootstrap_state = bootstrap_state;
4727 sub.synced_once = true;
4728 if sub.bootstrap_state.is_some() {
4729 report.bootstrapping.push(id.to_owned());
4730 }
4731 let completed =
4732 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
4733 self.persist_sub(&self.subs[sub_index].clone());
4734 if completed {
4735 if let Some((base_key, unit)) = registered.clone() {
4736 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4737 }
4738 }
4739 self.rebuild_overlay_if_dirty();
4740 self.finish_observation("syncular_section", batch)
4741 .map_err(|message| {
4742 SectionError::Abort("storage.failed".to_owned(), message)
4743 })?;
4744 Ok(())
4745 }
4746 Err(SectionError::FailClosed) => {
4747 self.rollback_observation("syncular_section");
4750 self.begin_observation("syncular_section_failure")
4751 .map_err(|message| {
4752 SectionError::Abort("storage.failed".to_owned(), message)
4753 })?;
4754 let mut failure_batch = ChangeAccumulator::default();
4755 let sub = &mut self.subs[sub_index];
4756 sub.state = SubState::Failed;
4757 sub.reason_code = Some("sync.scope_revoked".to_owned());
4758 report.failed.push(id.to_owned());
4759 self.persist_sub(&self.subs[sub_index].clone());
4760 if let Some((base_key, unit)) = registered {
4761 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
4762 }
4763 self.finish_observation("syncular_section_failure", failure_batch)
4764 .map_err(|message| {
4765 SectionError::Abort("storage.failed".to_owned(), message)
4766 })?;
4767 Ok(())
4768 }
4769 Err(SectionError::Abort(code, message)) => {
4770 self.rollback_observation("syncular_section");
4773 Err(SectionError::Abort(code, message))
4774 }
4775 }
4776 }
4777 }
4778 }
4779
4780 #[allow(clippy::too_many_arguments)]
4784 fn apply_section_body(
4785 &mut self,
4786 transport: &mut dyn Transport,
4787 sub_index: usize,
4788 body: Vec<Frame>,
4789 fresh: bool,
4790 meta: &RequestMeta,
4791 report: &mut SyncReport,
4792 batch: &mut ChangeAccumulator,
4793 ) -> Result<(), SectionError> {
4794 let mut saw_segment = false;
4795 for frame in body {
4796 match frame {
4797 Frame::Commit {
4798 tables, changes, ..
4799 } => {
4800 self.record_commit_changes(batch, &tables, &changes);
4801 self.apply_commit_changes(&tables, &changes)
4802 .map_err(|(c, m)| SectionError::Abort(c, m))?;
4803 report.commits_applied += 1;
4804 }
4805 Frame::SegmentInline { payload } => {
4806 let segment = decode_rows_segment(&payload)
4807 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
4808 let first = !saw_segment;
4809 saw_segment = true;
4810 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4811 let cleared =
4812 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4813 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4814 if applied > 0 || cleared {
4815 batch.table(&segment.table);
4816 }
4817 report.segment_rows_applied += applied;
4818 }
4819 Frame::SegmentRef {
4820 segment_id,
4821 media_type,
4822 table,
4823 row_count,
4824 as_of_commit_seq,
4825 scope_digest,
4826 row_cursor,
4827 next_row_cursor,
4828 url,
4829 url_expires_at_ms,
4830 ..
4831 } => {
4832 let advertised = match media_type {
4835 MediaType::Rows => {
4836 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
4837 || meta.accept & ACCEPT_INLINE_ROWS != 0
4838 }
4839 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
4840 };
4841 if !advertised {
4842 return Err(SectionError::Abort(
4843 "sync.invalid_request".to_owned(),
4844 format!(
4845 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
4846 media_type.name()
4847 ),
4848 ));
4849 }
4850 let bytes = if let Some(url) = url {
4851 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
4856 return Err(SectionError::Abort(
4857 "sync.invalid_request".to_owned(),
4858 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
4859 .to_owned(),
4860 ));
4861 }
4862 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
4864 return Err(SectionError::Abort(
4865 "sync.segment_expired".to_owned(),
4866 format!(
4867 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
4868 ),
4869 ));
4870 }
4871 transport
4872 .fetch_url(&url)
4873 .map_err(|e| SectionError::Abort(e.code, e.message))?
4874 } else {
4875 let requested_scopes_json =
4876 canonical_scope_json(&self.subs[sub_index].requested);
4877 transport
4878 .download_segment(&SegmentRequest {
4879 segment_id: segment_id.clone(),
4880 table,
4881 requested_scopes_json,
4882 })
4883 .map_err(|e| SectionError::Abort(e.code, e.message))?
4884 };
4885 let digest = Sha256::digest(&bytes);
4887 let expected = segment_id
4888 .strip_prefix("sha256:")
4889 .unwrap_or(segment_id.as_str());
4890 if bytes_to_hex(&digest) != expected {
4891 return Err(SectionError::Abort(
4892 "sync.invalid_request".to_owned(),
4893 "segment bytes do not match the content address (§5.1)".to_owned(),
4894 ));
4895 }
4896 if media_type == MediaType::Sqlite {
4897 if row_cursor.is_some() || next_row_cursor.is_some() {
4900 return Err(SectionError::Abort(
4901 "sync.invalid_request".to_owned(),
4902 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
4903 .to_owned(),
4904 ));
4905 }
4906 let first = !saw_segment;
4907 saw_segment = true;
4908 let sub_table = self.subs[sub_index].table.clone();
4909 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4910 let cleared =
4911 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
4912 let applied = self.apply_sqlite_segment(
4913 sub_index,
4914 &bytes,
4915 fresh && first,
4916 row_count,
4917 as_of_commit_seq,
4918 &scope_digest,
4919 )?;
4920 if applied > 0 || cleared {
4921 batch.table(&sub_table);
4922 }
4923 report.segment_rows_applied += applied;
4924 } else {
4925 let segment = decode_rows_segment(&bytes).map_err(|e| {
4926 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
4927 })?;
4928 let first = row_cursor.is_none();
4929 saw_segment = true;
4930 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4931 let cleared =
4932 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4933 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4934 if applied > 0 || cleared {
4935 batch.table(&segment.table);
4936 }
4937 report.segment_rows_applied += applied;
4938 }
4939 }
4940 Frame::Unknown { .. } => {}
4941 _ => {
4942 return Err(SectionError::Abort(
4943 "sync.invalid_request".to_owned(),
4944 "unexpected frame inside a subscription section".to_owned(),
4945 ));
4946 }
4947 }
4948 }
4949 Ok(())
4950 }
4951
4952 fn apply_commit_changes(
4953 &mut self,
4954 tables: &[String],
4955 changes: &[ssp2::model::Change],
4956 ) -> Result<(), (String, String)> {
4957 for change in changes {
4958 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
4959 (
4960 "sync.invalid_request".to_owned(),
4961 "change tableIndex out of range".to_owned(),
4962 )
4963 })?;
4964 let table = self.schema.table(table_name).ok_or_else(|| {
4965 (
4966 "sync.schema_mismatch".to_owned(),
4967 format!("change targets unknown table {table_name:?}"),
4968 )
4969 })?;
4970 match change.op {
4971 Op::Upsert => {
4972 let payload = change.row.as_ref().ok_or_else(|| {
4973 (
4974 "sync.invalid_request".to_owned(),
4975 "upsert change without row payload".to_owned(),
4976 )
4977 })?;
4978 let row = decode_row_bytes(table, payload, &self.encryption)
4980 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4981 let version = change.row_version.unwrap_or(0);
4982 let table_name = table.name.clone();
4983 self.write_base_row(&table_name, &row, version)
4984 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4985 }
4986 Op::Delete => {
4987 self.delete_base_row(table_name, &change.row_id)
4988 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4989 }
4990 }
4991 }
4992 Ok(())
4993 }
4994
4995 fn apply_segment(
5000 &mut self,
5001 sub_index: usize,
5002 segment: &RowsSegment,
5003 first_fresh_page: bool,
5004 ) -> Result<u32, SectionError> {
5005 let (sub_table, effective) = {
5006 let sub = &self.subs[sub_index];
5007 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5008 };
5009 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5010 SectionError::Abort(
5011 "sync.schema_mismatch".to_owned(),
5012 format!("subscription table {sub_table:?} is not in the client schema"),
5013 )
5014 })?;
5015 let matches = segment.table == table.name
5020 && segment.schema_version == self.schema.version
5021 && segment.columns.len() == table.wire_columns.len()
5022 && segment
5023 .columns
5024 .iter()
5025 .zip(table.wire_columns.iter())
5026 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
5027 if !matches {
5028 return Err(SectionError::Abort(
5029 "sync.schema_mismatch".to_owned(),
5030 "segment column table does not match the generated schema (§5.2)".to_owned(),
5031 ));
5032 }
5033 if first_fresh_page {
5034 self.purge_scope_rows(&table.name, &effective)
5038 .map_err(|()| SectionError::FailClosed)?;
5039 }
5040 let mut applied = 0u32;
5041 for block in &segment.blocks {
5042 for row in block {
5043 let decrypted;
5048 let values = if table.has_encrypted_columns() {
5049 let mut values = row.values.clone();
5050 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
5051 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
5052 decrypted = values;
5053 &decrypted
5054 } else {
5055 &row.values
5056 };
5057 self.write_base_row(&table.name, values, row.server_version)
5060 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
5061 applied += 1;
5062 }
5063 }
5064 Ok(applied)
5065 }
5066
5067 fn apply_sqlite_segment(
5075 &mut self,
5076 sub_index: usize,
5077 bytes: &[u8],
5078 first_fresh_page: bool,
5079 row_count: i64,
5080 as_of_commit_seq: i64,
5081 scope_digest: &str,
5082 ) -> Result<u32, SectionError> {
5083 let invalid = |detail: &str| {
5084 SectionError::Abort(
5085 "sync.invalid_request".to_owned(),
5086 format!("sqlite segment rejected: {detail} (§5.3)"),
5087 )
5088 };
5089 let (sub_table, effective) = {
5090 let sub = &self.subs[sub_index];
5091 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5092 };
5093 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5094 SectionError::Abort(
5095 "sync.schema_mismatch".to_owned(),
5096 format!("subscription table {sub_table:?} is not in the client schema"),
5097 )
5098 })?;
5099
5100 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
5101 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
5102 let img = match rusqlite::Connection::open_with_flags(
5103 &path,
5104 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
5105 ) {
5106 Ok(conn) => conn,
5107 Err(_) => {
5108 let _ = std::fs::remove_file(&path);
5109 return Err(invalid("bytes do not open as a SQLite database"));
5110 }
5111 };
5112 let outcome = self.apply_sqlite_image(
5113 &img,
5114 &table,
5115 first_fresh_page,
5116 &effective,
5117 row_count,
5118 as_of_commit_seq,
5119 scope_digest,
5120 );
5121 drop(img);
5122 let _ = std::fs::remove_file(&path);
5123 outcome
5124 }
5125
5126 #[allow(clippy::too_many_arguments)]
5127 fn apply_sqlite_image(
5128 &mut self,
5129 img: &rusqlite::Connection,
5130 table: &crate::schema::TableSchema,
5131 first_fresh_page: bool,
5132 effective: &[(String, Vec<String>)],
5133 row_count: i64,
5134 as_of_commit_seq: i64,
5135 scope_digest: &str,
5136 ) -> Result<u32, SectionError> {
5137 let invalid = |detail: String| {
5138 SectionError::Abort(
5139 "sync.invalid_request".to_owned(),
5140 format!("sqlite segment rejected: {detail} (§5.3)"),
5141 )
5142 };
5143
5144 type MetaRow = (i64, String, i64, i64, String, i64, i64);
5147 let meta: MetaRow = img
5148 .query_row(
5149 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
5150 \"scopeDigest\", \"rowCount\",
5151 (SELECT count(*) FROM _syncular_segment)
5152 FROM _syncular_segment",
5153 [],
5154 |row| {
5155 Ok((
5156 row.get(0)?,
5157 row.get(1)?,
5158 row.get(2)?,
5159 row.get(3)?,
5160 row.get(4)?,
5161 row.get(5)?,
5162 row.get(6)?,
5163 ))
5164 },
5165 )
5166 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
5167 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
5168 if meta_count != 1 {
5169 return Err(invalid(format!(
5170 "_syncular_segment must contain exactly one row, found {meta_count}"
5171 )));
5172 }
5173 if format != 1 {
5174 return Err(invalid(format!("format {format}")));
5175 }
5176 if meta_table != table.name {
5177 return Err(invalid(format!("image table {meta_table:?}")));
5178 }
5179 if schema_version != i64::from(self.schema.version) {
5180 return Err(invalid(format!("schemaVersion {schema_version}")));
5181 }
5182 if pin != as_of_commit_seq {
5183 return Err(invalid(format!("asOfCommitSeq {pin}")));
5184 }
5185 if digest != scope_digest {
5186 return Err(invalid("scopeDigest mismatch".to_owned()));
5187 }
5188 if meta_rows != row_count {
5189 return Err(invalid(format!("rowCount {meta_rows}")));
5190 }
5191
5192 let mut names: Vec<String> = Vec::new();
5194 {
5195 let mut stmt = img
5196 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
5197 .map_err(|_| invalid("image data table missing".to_owned()))?;
5198 let mut rows = stmt
5199 .query([])
5200 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5201 while let Some(row) = rows
5202 .next()
5203 .map_err(|_| invalid("image data table unreadable".to_owned()))?
5204 {
5205 names.push(
5206 row.get::<_, String>(1)
5207 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
5208 );
5209 }
5210 }
5211 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
5212 expected.push("_syncular_version");
5213 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
5214 return Err(SectionError::Abort(
5215 "sync.schema_mismatch".to_owned(),
5216 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
5217 ));
5218 }
5219
5220 if first_fresh_page {
5223 self.purge_scope_rows(&table.name, effective)
5224 .map_err(|()| SectionError::FailClosed)?;
5225 }
5226 self.overlay_dirty.set(true);
5232 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
5240 table.indexes.iter().filter(|i| !i.unique).collect()
5241 } else {
5242 Vec::new()
5243 };
5244 for index in &bulk_indexes {
5245 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
5246 self.conn
5247 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
5248 .map_err(|e| invalid(e.to_string()))?;
5249 }
5250 let insert = self.insert_row_sql(&base_table(&table.name), table);
5251 let applied = {
5252 let mut ins = self
5253 .conn
5254 .prepare_cached(&insert)
5255 .map_err(|e| invalid(e.to_string()))?;
5256 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
5257 let mut stmt = img
5258 .prepare(&format!(
5259 "SELECT {} FROM {}",
5260 column_list.join(", "),
5261 quote_ident(&table.name)
5262 ))
5263 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5264 let mut rows = stmt
5265 .query([])
5266 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5267 let version_index = table.columns.len();
5268 let mut applied = 0u32;
5269 while let Some(row) = rows
5270 .next()
5271 .map_err(|_| invalid("image row unreadable".to_owned()))?
5272 {
5273 for (i, column) in table.columns.iter().enumerate() {
5274 let cell = row
5275 .get_ref(i)
5276 .map_err(|_| invalid("image row unreadable".to_owned()))?;
5277 let param = image_cell_param(column, cell).map_err(&invalid)?;
5278 ins.raw_bind_parameter(i + 1, param)
5279 .map_err(|e| invalid(e.to_string()))?;
5280 }
5281 let version: i64 = row
5282 .get(version_index)
5283 .map_err(|_| invalid("image row unreadable".to_owned()))?;
5284 if version < 1 {
5285 return Err(invalid(format!(
5286 "row _syncular_version must be >= 1, got {version}"
5287 )));
5288 }
5289 ins.raw_bind_parameter(version_index + 1, version)
5290 .map_err(|e| invalid(e.to_string()))?;
5291 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
5292 applied += 1;
5293 }
5294 applied
5295 };
5296 for index in &bulk_indexes {
5297 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
5298 let cols_sql = index
5299 .columns
5300 .iter()
5301 .map(|c| quote_ident(c))
5302 .collect::<Vec<_>>()
5303 .join(", ");
5304 self.conn
5305 .execute(
5306 &format!(
5307 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
5308 base_table(&table.name)
5309 ),
5310 [],
5311 )
5312 .map_err(|e| invalid(e.to_string()))?;
5313 }
5314 if i64::from(applied) != row_count {
5315 return Err(invalid(format!(
5316 "image holds {applied} rows, descriptor says {row_count}"
5317 )));
5318 }
5319 Ok(applied)
5320 }
5321
5322 fn compile_local_data_purge(
5325 &self,
5326 input: &LocalDataPurgeInput,
5327 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
5328 let invalid = |message: String| format!("sync.invalid_request: {message}");
5329 if input.purge_id.is_empty()
5330 || input.purge_id.len() > 128
5331 || !is_local_purge_code_like(&input.purge_id)
5332 {
5333 return Err(invalid(
5334 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
5335 ));
5336 }
5337 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
5338 return Err(invalid(format!(
5339 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
5340 )));
5341 }
5342
5343 let mut deduplicated: BTreeMap<
5344 String,
5345 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
5346 > = BTreeMap::new();
5347 for target in &input.targets {
5348 let table = self.schema.table(&target.table).ok_or_else(|| {
5349 invalid(format!(
5350 "local purge names unknown table {:?}",
5351 target.table
5352 ))
5353 })?;
5354 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
5355 return Err(invalid(format!(
5356 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
5357 target.table
5358 )));
5359 }
5360 let mut selectors = Vec::with_capacity(target.selectors.len());
5361 let mut canonical_selectors = BTreeMap::new();
5362 for (column_name, raw_values) in &target.selectors {
5363 let Some((column_index, column)) = table
5364 .columns
5365 .iter()
5366 .enumerate()
5367 .find(|(_, column)| column.name == *column_name)
5368 else {
5369 return Err(invalid(format!(
5370 "local purge target {:?} names unknown column {:?}",
5371 target.table, column_name
5372 )));
5373 };
5374 let encrypted = table
5375 .encrypted_columns
5376 .iter()
5377 .any(|candidate| candidate.index == column_index);
5378 if column.ty != ColumnType::String || encrypted {
5379 return Err(invalid(format!(
5380 "local purge selector {:?}.{:?} must be a plaintext string column",
5381 target.table, column_name
5382 )));
5383 }
5384 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
5385 return Err(invalid(format!(
5386 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
5387 target.table, column_name
5388 )));
5389 }
5390 let mut values = raw_values.clone();
5391 values.sort();
5392 values.dedup();
5393 if values.iter().any(|value| {
5394 value.is_empty()
5395 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
5396 || !is_local_purge_code_like(value)
5397 }) {
5398 return Err(invalid(format!(
5399 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
5400 )));
5401 }
5402 selectors.push((column_name.clone(), values.clone()));
5403 canonical_selectors.insert(column_name.clone(), values);
5404 }
5405 let canonical = LocalDataPurgeTarget {
5406 table: target.table.clone(),
5407 selectors: canonical_selectors,
5408 };
5409 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
5410 deduplicated.insert(
5411 key,
5412 (
5413 CompiledLocalDataPurgeTarget {
5414 table: target.table.clone(),
5415 selectors,
5416 },
5417 canonical,
5418 ),
5419 );
5420 }
5421 let targets = deduplicated
5422 .values()
5423 .map(|(compiled, _)| compiled.clone())
5424 .collect::<Vec<_>>();
5425 let canonical = deduplicated
5426 .values()
5427 .map(|(_, target)| target.clone())
5428 .collect::<Vec<_>>();
5429 let canonical_plan =
5430 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
5431 Ok((targets, canonical_plan))
5432 }
5433
5434 fn local_purge_base_row_ids(
5435 &self,
5436 targets: &[CompiledLocalDataPurgeTarget],
5437 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
5438 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
5439 for target in targets {
5440 let table = self
5441 .schema
5442 .table(&target.table)
5443 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
5444 let mut clauses = Vec::with_capacity(target.selectors.len());
5445 let mut params = Vec::new();
5446 for (column, values) in &target.selectors {
5447 clauses.push(format!(
5448 "{} IN ({})",
5449 quote_ident(column),
5450 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
5451 ));
5452 params.extend(values.iter().cloned().map(SqlValue::Text));
5453 }
5454 let sql = format!(
5455 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
5456 quote_ident(&table.primary_key),
5457 base_table(&target.table),
5458 clauses.join(" AND ")
5459 );
5460 let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
5461 let rows = statement
5462 .query_map(rusqlite::params_from_iter(params), |row| {
5463 row.get::<_, String>(0)
5464 })
5465 .map_err(|error| error.to_string())?;
5466 let ids = by_table.entry(target.table.clone()).or_default();
5467 for row in rows {
5468 ids.insert(row.map_err(|error| error.to_string())?);
5469 }
5470 }
5471 Ok(by_table)
5472 }
5473
5474 fn local_purge_values_match(
5475 target: &CompiledLocalDataPurgeTarget,
5476 values: &Map<String, Value>,
5477 ) -> bool {
5478 target.selectors.iter().all(|(column, allowed)| {
5479 values
5480 .get(column)
5481 .and_then(Value::as_str)
5482 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
5483 })
5484 }
5485
5486 pub fn purge_local_data(
5490 &mut self,
5491 input: &LocalDataPurgeInput,
5492 ) -> Result<LocalDataPurgeResult, String> {
5493 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
5494 let meta_key = format!("localPurge:{}", input.purge_id);
5495 let applied_plan = self
5496 .conn
5497 .query_row(
5498 "SELECT value FROM _syncular_meta WHERE key = ?1",
5499 rusqlite::params![meta_key],
5500 |row| row.get::<_, String>(0),
5501 )
5502 .optional()
5503 .map_err(|error| error.to_string())?;
5504 if let Some(applied_plan) = applied_plan {
5505 if applied_plan != canonical_plan {
5506 return Err(format!(
5507 "sync.invalid_request: local purge id {:?} was already used with a different plan",
5508 input.purge_id
5509 ));
5510 }
5511 return Ok(LocalDataPurgeResult {
5512 already_applied: true,
5513 purged_rows: 0,
5514 dropped_commits: 0,
5515 });
5516 }
5517
5518 let prior_outbox = self.outbox.clone();
5519 let prior_rejection_count = self.rejections.len();
5520 let prior_overlay_dirty = self.overlay_dirty.get();
5521 self.begin_observation("syncular_local_purge")?;
5522 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
5523 let row_ids = self.local_purge_base_row_ids(&targets)?;
5524 let doomed = self
5525 .outbox
5526 .iter()
5527 .filter(|commit| {
5528 commit.ops.iter().any(|operation| {
5529 let matching_targets = targets
5530 .iter()
5531 .filter(|target| target.table == operation.table)
5532 .collect::<Vec<_>>();
5533 if matching_targets.is_empty() {
5534 return false;
5535 }
5536 if row_ids
5537 .get(&operation.table)
5538 .is_some_and(|ids| ids.contains(&operation.row_id))
5539 {
5540 return true;
5541 }
5542 operation.values.as_ref().is_some_and(|values| {
5543 matching_targets
5544 .iter()
5545 .any(|target| Self::local_purge_values_match(target, values))
5546 })
5547 })
5548 })
5549 .cloned()
5550 .collect::<Vec<_>>();
5551 let doomed_ids = doomed
5552 .iter()
5553 .map(|commit| commit.client_commit_id.clone())
5554 .collect::<BTreeSet<_>>();
5555 let mut batch = ChangeAccumulator::default();
5556 let mut rejections = Vec::new();
5557 for commit in &doomed {
5558 for operation in &commit.ops {
5559 batch.table(&operation.table);
5560 }
5561 let results = commit
5562 .ops
5563 .iter()
5564 .enumerate()
5565 .map(|(op_index, operation)| {
5566 let rejection = RejectionRecord {
5567 client_commit_id: commit.client_commit_id.clone(),
5568 op_index: op_index as i32,
5569 code: "client.local_data_purged".to_owned(),
5570 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
5571 retryable: false,
5572 details: None,
5573 operation: Some(CommitOperation::from(operation)),
5574 };
5575 rejections.push(rejection.clone());
5576 CommitOperationOutcome::Error { rejection }
5577 })
5578 .collect::<Vec<_>>();
5579 self.persist_commit_outcome(
5580 &commit.client_commit_id,
5581 CommitOutcomeStatus::Rejected,
5582 &results,
5583 Some(&commit.ops),
5584 )?;
5585 self.delete_outbox_persisted(&commit.client_commit_id)?;
5586 }
5587 if !doomed.is_empty() {
5588 self.outbox
5589 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
5590 self.rejections.extend(rejections);
5591 self.prune_commit_outcomes()?;
5592 self.overlay_dirty.set(true);
5593 batch.status = true;
5594 batch.rejections = true;
5595 batch.outcomes = true;
5596 }
5597
5598 let mut purged_rows = 0usize;
5599 for (table, ids) in &row_ids {
5600 if ids.is_empty() {
5601 continue;
5602 }
5603 batch.table(table);
5604 for row_id in ids {
5605 self.delete_base_row(table, row_id)?;
5606 purged_rows += 1;
5607 }
5608 }
5609 self.rebuild_overlay_if_dirty();
5610 self.reconcile_blob_refcounts(true);
5611 self.conn
5612 .execute(
5613 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
5614 rusqlite::params![meta_key, canonical_plan],
5615 )
5616 .map_err(|error| error.to_string())?;
5617 Ok((
5618 batch,
5619 LocalDataPurgeResult {
5620 already_applied: false,
5621 purged_rows,
5622 dropped_commits: doomed.len(),
5623 },
5624 ))
5625 })();
5626 let (batch, result) = match applied {
5627 Ok(value) => value,
5628 Err(error) => {
5629 self.rollback_observation("syncular_local_purge");
5630 self.outbox = prior_outbox;
5631 self.rejections.truncate(prior_rejection_count);
5632 self.overlay_dirty.set(prior_overlay_dirty);
5633 return Err(error);
5634 }
5635 };
5636 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
5637 self.rollback_observation("syncular_local_purge");
5638 self.outbox = prior_outbox;
5639 self.rejections.truncate(prior_rejection_count);
5640 self.overlay_dirty.set(prior_overlay_dirty);
5641 return Err(error);
5642 }
5643 Ok(result)
5644 }
5645
5646 fn purge_scope_rows(
5651 &mut self,
5652 table_name: &str,
5653 effective: &[(String, Vec<String>)],
5654 ) -> Result<(), ()> {
5655 if effective.is_empty() {
5656 return Ok(());
5657 }
5658 let table = self.schema.table(table_name).ok_or(())?.clone();
5659 let mut clauses = Vec::new();
5660 let mut params: Vec<SqlValue> = Vec::new();
5661 for (variable, values) in effective {
5662 let column = table.scope_column(variable).ok_or(())?;
5663 let placeholders: Vec<String> = values
5664 .iter()
5665 .map(|v| {
5666 params.push(SqlValue::Text(v.clone()));
5667 "?".to_owned()
5668 })
5669 .collect();
5670 clauses.push(format!(
5671 "{} IN ({})",
5672 quote_ident(column),
5673 placeholders.join(", ")
5674 ));
5675 }
5676 let sql = format!(
5677 "DELETE FROM {} WHERE {}",
5678 base_table(table_name),
5679 clauses.join(" AND ")
5680 );
5681 self.overlay_dirty.set(true);
5682 self.conn
5683 .execute(&sql, rusqlite::params_from_iter(params))
5684 .map_err(|_| ())?;
5685 Ok(())
5686 }
5687
5688 fn drop_doomed_outbox(
5691 &mut self,
5692 table_name: &str,
5693 effective: &[(String, Vec<String>)],
5694 ) -> Result<bool, String> {
5695 if effective.is_empty() {
5696 return Ok(false);
5697 }
5698 let Some(table) = self.schema.table(table_name).cloned() else {
5699 return Ok(false);
5700 };
5701 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
5702 for (variable, values) in effective {
5703 match table.scope_column(variable) {
5704 Some(column) => mappings.push((column, values)),
5705 None => return Ok(false), }
5707 }
5708 let doomed: Vec<OutboxCommit> = self
5709 .outbox
5710 .iter()
5711 .filter(|commit| {
5712 commit.ops.iter().any(|op| {
5713 op.upsert
5714 && op.table == table_name
5715 && op.values.as_ref().is_some_and(|values| {
5716 mappings
5717 .iter()
5718 .all(|(column, allowed)| match values.get(*column) {
5719 Some(Value::String(s)) => allowed.contains(s),
5720 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
5721 _ => false,
5722 })
5723 })
5724 })
5725 })
5726 .cloned()
5727 .collect();
5728 if doomed.is_empty() {
5729 return Ok(false);
5730 }
5731 let mut rejections = Vec::new();
5732 for commit in &doomed {
5733 let results = commit
5734 .ops
5735 .iter()
5736 .enumerate()
5737 .map(|(op_index, operation)| {
5738 let rejection = RejectionRecord {
5739 client_commit_id: commit.client_commit_id.clone(),
5740 op_index: op_index as i32,
5741 code: "sync.scope_revoked".to_owned(),
5742 message: "the commit was dropped because its effective scope was revoked"
5743 .to_owned(),
5744 retryable: false,
5745 details: None,
5746 operation: Some(CommitOperation::from(operation)),
5747 };
5748 rejections.push(rejection.clone());
5749 CommitOperationOutcome::Error { rejection }
5750 })
5751 .collect::<Vec<_>>();
5752 self.persist_commit_outcome(
5753 &commit.client_commit_id,
5754 CommitOutcomeStatus::Rejected,
5755 &results,
5756 Some(&commit.ops),
5757 )?;
5758 self.delete_outbox_persisted(&commit.client_commit_id)?;
5759 }
5760 self.prune_commit_outcomes()?;
5761 let doomed_ids = doomed
5762 .iter()
5763 .map(|commit| commit.client_commit_id.as_str())
5764 .collect::<BTreeSet<_>>();
5765 self.outbox
5766 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
5767 self.rejections.extend(rejections);
5768 self.overlay_dirty.set(true);
5769 Ok(true)
5770 }
5771
5772 pub fn upload_blob(
5779 &mut self,
5780 bytes: &[u8],
5781 media_type: Option<String>,
5782 name: Option<String>,
5783 ) -> Result<Value, String> {
5784 let blob_id = blob_id_for(bytes);
5785 let now = self.clock_now_ms();
5786 self.conn
5787 .execute(
5788 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
5789 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
5790 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
5791 )
5792 .map_err(|e| e.to_string())?;
5793 self.conn
5794 .execute(
5795 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
5796 rusqlite::params![blob_id, media_type, now],
5797 )
5798 .map_err(|e| e.to_string())?;
5799 self.enforce_blob_cache_cap();
5802 let mut obj = Map::new();
5803 obj.insert("blobId".to_owned(), Value::from(blob_id));
5804 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
5805 if let Some(mt) = media_type {
5806 obj.insert("mediaType".to_owned(), Value::from(mt));
5807 }
5808 if let Some(n) = name {
5809 obj.insert("name".to_owned(), Value::from(n));
5810 }
5811 Ok(Value::Object(obj))
5812 }
5813
5814 pub fn fetch_blob(
5818 &mut self,
5819 transport: &mut dyn Transport,
5820 blob_id_or_ref: &str,
5821 ) -> Result<Value, (String, String)> {
5822 let simple = |m: String| ("client.failed".to_owned(), m);
5823 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
5824 blob_id_or_ref.to_owned()
5825 } else {
5826 let value: Value = serde_json::from_str(blob_id_or_ref)
5827 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
5828 value
5829 .get("blobId")
5830 .and_then(Value::as_str)
5831 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
5832 .to_owned()
5833 };
5834 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
5835 return Ok(cached);
5836 }
5837 let bytes = match transport
5843 .blob_download(&blob_id)
5844 .map_err(|e| (e.code, e.message))?
5845 {
5846 BlobDownload::Bytes(bytes) => bytes,
5847 BlobDownload::Url {
5848 url,
5849 url_expires_at_ms,
5850 } => {
5851 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5853 return Err((
5854 "sync.segment_expired".to_owned(),
5855 format!(
5856 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
5857 ),
5858 ));
5859 }
5860 transport
5861 .fetch_blob_url(&url)
5862 .map_err(|e| (e.code, e.message))?
5863 }
5864 };
5865 if blob_id_for(&bytes) != blob_id {
5867 return Err(simple(format!(
5868 "blob content address mismatch for {blob_id}"
5869 )));
5870 }
5871 let now = self.clock_now_ms();
5872 self.conn
5873 .execute(
5874 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
5875 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
5876 )
5877 .map_err(|e| simple(e.to_string()))?;
5878 self.enforce_blob_cache_cap();
5879 self.get_cached_blob(&blob_id)
5880 .map_err(simple)?
5881 .ok_or_else(|| simple("blob cache write failed".to_owned()))
5882 }
5883
5884 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
5885 let _ = self.conn.execute(
5888 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
5889 rusqlite::params![self.clock_now_ms(), blob_id],
5890 );
5891 let mut stmt = self
5892 .conn
5893 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
5894 .map_err(|e| e.to_string())?;
5895 let mut rows = stmt
5896 .query(rusqlite::params![blob_id])
5897 .map_err(|e| e.to_string())?;
5898 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
5899 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
5900 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
5901 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
5902 let mut obj = Map::new();
5903 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
5904 obj.insert("byteLength".to_owned(), Value::from(byte_length));
5905 let mut bytes_obj = Map::new();
5906 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
5907 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
5908 if let Some(mt) = media_type {
5909 obj.insert("mediaType".to_owned(), Value::from(mt));
5910 }
5911 return Ok(Some(Value::Object(obj)));
5912 }
5913 Ok(None)
5914 }
5915
5916 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
5918 let pending: Vec<(String, Option<String>)> = {
5919 let mut stmt = self
5920 .conn
5921 .prepare(
5922 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
5923 )
5924 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5925 let rows = stmt
5926 .query_map([], |row| {
5927 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
5928 })
5929 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5930 rows.filter_map(Result::ok).collect()
5931 };
5932 for (blob_id, media_type) in pending {
5933 let bytes: Option<Vec<u8>> = self
5934 .conn
5935 .query_row(
5936 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
5937 rusqlite::params![blob_id],
5938 |row| row.get(0),
5939 )
5940 .ok();
5941 if let Some(bytes) = bytes {
5942 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
5943 }
5944 let _ = self.conn.execute(
5945 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
5946 rusqlite::params![blob_id],
5947 );
5948 }
5949 Ok(())
5950 }
5951
5952 fn upload_one(
5959 &self,
5960 transport: &mut dyn Transport,
5961 blob_id: &str,
5962 bytes: &[u8],
5963 media_type: Option<&str>,
5964 ) -> Result<(), TransportError> {
5965 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
5966 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
5968 url,
5969 url_expires_at_ms,
5970 } => {
5971 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
5972 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
5973 return Ok(());
5974 }
5975 }
5977 BlobUploadGrant::None => {
5978 }
5980 }
5981 transport.blob_upload(blob_id, bytes, media_type)
5982 }
5983
5984 fn enforce_blob_cache_cap(&self) {
5993 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
5994 return;
5995 };
5996 let mut total: i64 = self
5997 .conn
5998 .query_row(
5999 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
6000 [],
6001 |row| row.get(0),
6002 )
6003 .unwrap_or(0);
6004 if total <= max_bytes {
6005 return;
6006 }
6007 let candidates: Vec<(String, i64)> = {
6008 let Ok(mut stmt) = self.conn.prepare(
6009 "SELECT blob_id, byte_length FROM _syncular_blobs
6010 WHERE refcount = 0
6011 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
6012 ORDER BY last_used_ms ASC, created_at_ms ASC",
6013 ) else {
6014 return;
6015 };
6016 let Ok(rows) = stmt.query_map([], |row| {
6017 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
6018 }) else {
6019 return;
6020 };
6021 rows.filter_map(Result::ok).collect()
6022 };
6023 for (blob_id, byte_length) in candidates {
6024 if total <= max_bytes {
6025 break;
6026 }
6027 let _ = self.conn.execute(
6028 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
6029 rusqlite::params![blob_id],
6030 );
6031 total -= byte_length;
6032 }
6033 }
6034
6035 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
6039 if !self.schema_has_blobs() {
6040 return;
6041 }
6042 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
6043 for table in self.schema.tables.clone() {
6044 let blob_cols: Vec<String> = table
6045 .columns
6046 .iter()
6047 .filter(|c| c.ty == ColumnType::BlobRef)
6048 .map(|c| c.name.clone())
6049 .collect();
6050 for column in blob_cols {
6051 let sql = format!(
6052 "SELECT {} FROM {} WHERE {} IS NOT NULL",
6053 quote_ident(&column),
6054 base_table(&table.name),
6055 quote_ident(&column)
6056 );
6057 let Ok(mut stmt) = self.conn.prepare(&sql) else {
6058 continue;
6059 };
6060 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
6061 continue;
6062 };
6063 for raw in rows.flatten().flatten() {
6064 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
6065 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
6066 *counts.entry(id.to_owned()).or_insert(0) += 1;
6067 }
6068 }
6069 }
6070 }
6071 }
6072 let _ = self
6073 .conn
6074 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
6075 for (blob_id, count) in &counts {
6076 let _ = self.conn.execute(
6077 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
6078 rusqlite::params![count, blob_id],
6079 );
6080 }
6081 if delete_orphans {
6082 let _ = self.conn.execute(
6083 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
6084 [],
6085 );
6086 }
6087 }
6088
6089 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
6094 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
6095 return sql.clone();
6096 }
6097 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
6098 columns.push(quote_ident("_syncular_version"));
6099 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
6100 let primary_key = quote_ident(&table.primary_key);
6101 let updates = columns
6102 .iter()
6103 .filter(|column| **column != primary_key)
6104 .map(|column| format!("{column}=excluded.{column}"))
6105 .collect::<Vec<_>>()
6106 .join(", ");
6107 let sql = format!(
6108 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
6109 columns.join(", "),
6110 placeholders.join(", ")
6111 );
6112 self.insert_sql
6113 .borrow_mut()
6114 .insert(full_table.to_owned(), sql.clone());
6115 sql
6116 }
6117
6118 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
6119 self.overlay_dirty.set(true);
6120 self.write_row(&base_table(table_name), table_name, row, version)
6121 }
6122
6123 fn write_row(
6124 &self,
6125 full_table: &str,
6126 table_name: &str,
6127 row: &Row,
6128 version: i64,
6129 ) -> Result<(), String> {
6130 let table = self
6131 .schema
6132 .table(table_name)
6133 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
6134 let sql = self.insert_row_sql(full_table, table);
6135 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
6136 let params = row
6137 .iter()
6138 .map(RowParam::Cell)
6139 .chain(std::iter::once(RowParam::Version(version)));
6140 stmt.execute(rusqlite::params_from_iter(params))
6141 .map_err(|e| e.to_string())?;
6142 Ok(())
6143 }
6144
6145 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
6146 let table = self
6147 .schema
6148 .table(table_name)
6149 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
6150 self.overlay_dirty.set(true);
6151 let sql = format!(
6152 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
6153 base_table(table_name),
6154 quote_ident(&table.primary_key)
6155 );
6156 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
6157 stmt.execute(rusqlite::params![row_id])
6158 .map_err(|e| e.to_string())?;
6159 Ok(())
6160 }
6161
6162 fn rebuild_overlay_if_dirty(&mut self) {
6165 if self.overlay_dirty.get() {
6166 self.rebuild_overlay();
6167 }
6168 }
6169
6170 fn rebuild_overlay(&mut self) {
6174 self.exec("SAVEPOINT syncular_overlay");
6175 for table in self.schema.tables.clone() {
6176 for index in &table.fts_indexes {
6177 let _ = self.drop_fts_triggers(index);
6178 }
6179 let visible = visible_table(&table.name);
6180 let base = base_table(&table.name);
6181 self.exec(&format!("DELETE FROM {visible}"));
6182 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
6183 }
6184 for commit in self.outbox.clone() {
6185 for op in &commit.ops {
6186 let Some(table) = self.schema.table(&op.table).cloned() else {
6187 continue;
6188 };
6189 if op.upsert {
6190 let Some(values) = op.values.as_ref() else {
6191 continue;
6192 };
6193 let mut row: Row = Vec::with_capacity(table.columns.len());
6194 let mut ok = true;
6195 for column in &table.columns {
6196 match json_to_column_value(column, values.get(&column.name)) {
6197 Ok(v) => row.push(v),
6198 Err(_) => {
6199 ok = false;
6200 break;
6201 }
6202 }
6203 }
6204 if ok {
6205 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
6206 }
6207 } else {
6208 let sql = format!(
6209 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
6210 visible_table(&table.name),
6211 quote_ident(&table.primary_key)
6212 );
6213 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
6214 }
6215 }
6216 }
6217 for table in self.schema.tables.clone() {
6218 for index in &table.fts_indexes {
6219 let _ = self.rebuild_fts_projection(&table, index);
6220 let _ = self.create_fts_triggers(&table, index);
6221 }
6222 }
6223 self.exec("RELEASE syncular_overlay");
6224 self.overlay_dirty.set(false);
6225 }
6226
6227 fn exec(&self, sql: &str) {
6228 let _ = self.conn.execute_batch(sql);
6229 }
6230
6231 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
6234 transport
6235 .realtime_connect_for_client(&self.client_id)
6236 .map_err(|e| format!("{}: {}", e.code, e.message))?;
6237 self.realtime_connected = true;
6238 Ok(())
6239 }
6240
6241 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
6242 let _ = transport.realtime_close();
6243 self.realtime_connected = false;
6244 self.presence.clear(); }
6246
6247 pub fn set_presence(
6253 &mut self,
6254 transport: &mut dyn Transport,
6255 scope_key: &str,
6256 doc: Option<&Value>,
6257 ) -> Result<(), String> {
6258 if !self.realtime_connected {
6259 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
6260 }
6261 let text = encode_presence_publish(scope_key, doc);
6262 transport
6263 .realtime_send(&text)
6264 .map_err(|e| format!("{}: {}", e.code, e.message))
6265 }
6266
6267 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
6269 self.presence
6270 .get(scope_key)
6271 .map(|peers| peers.values().cloned().collect())
6272 .unwrap_or_default()
6273 }
6274
6275 fn apply_presence(
6277 &mut self,
6278 scope_key: String,
6279 kind: Option<PresenceKind>,
6280 actor_id: Option<String>,
6281 client_id: Option<String>,
6282 doc: Option<Value>,
6283 error: Option<String>,
6284 ) {
6285 if error.is_some() {
6288 return;
6289 }
6290 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
6291 return;
6292 };
6293 let peer_key = format!("{actor_id} {client_id}");
6294 match kind {
6295 PresenceKind::Leave => {
6296 if let Some(peers) = self.presence.get_mut(&scope_key) {
6297 peers.remove(&peer_key);
6298 if peers.is_empty() {
6299 self.presence.remove(&scope_key);
6300 }
6301 }
6302 }
6303 _ => {
6304 let doc = match doc {
6305 Some(Value::Object(_)) => doc.unwrap(),
6306 _ => return,
6307 };
6308 self.presence.entry(scope_key).or_default().insert(
6309 peer_key,
6310 PresencePeer {
6311 actor_id,
6312 client_id,
6313 doc,
6314 },
6315 );
6316 }
6317 }
6318 }
6319
6320 pub fn on_realtime_text(&mut self, text: &str) {
6322 match parse_control(text) {
6323 Ok(ControlMessage::Hello { requires_sync, .. }) => {
6324 if requires_sync {
6325 self.set_sync_needed(true, true);
6327 }
6328 }
6329 Ok(ControlMessage::Presence {
6330 scope_key,
6331 kind,
6332 actor_id,
6333 client_id,
6334 doc,
6335 error,
6336 ..
6337 }) => {
6338 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
6339 }
6340 Ok(ControlMessage::Wake { .. }) => {
6341 self.set_sync_needed(true, true);
6343 }
6344 _ => {}
6345 }
6346 }
6347
6348 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
6351 if self.stopped {
6352 return;
6353 }
6354 let message = match decode_message(bytes) {
6355 Ok(m) if m.msg_kind == MsgKind::Response => m,
6356 _ => {
6357 self.set_sync_needed(true, true);
6358 return;
6359 }
6360 };
6361 let mut frames = message.frames.into_iter();
6362 let mut applied_cursor: Option<i64> = None;
6363 let mut any_covered = false;
6364 let mut dropped = false;
6365 while let Some(frame) = frames.next() {
6366 let Frame::SubStart {
6367 id,
6368 status,
6369 effective_scopes,
6370 ..
6371 } = frame
6372 else {
6373 continue;
6374 };
6375 let mut body = Vec::new();
6376 let mut next_cursor: Option<i64> = None;
6377 for inner in frames.by_ref() {
6378 match inner {
6379 Frame::SubEnd {
6380 next_cursor: nc, ..
6381 } => {
6382 next_cursor = Some(nc);
6383 break;
6384 }
6385 Frame::Unknown { .. } => {}
6386 other => body.push(other),
6387 }
6388 }
6389 let Some(next_cursor) = next_cursor else {
6390 dropped = true;
6391 break;
6392 };
6393 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
6394 dropped = true;
6395 continue;
6396 };
6397 let sub = &self.subs[sub_index];
6398 if status != SubStatus::Active
6401 || sub.state != SubState::Active
6402 || sub.bootstrap_state.is_some()
6403 || !sub.synced_once
6404 {
6405 dropped = true;
6406 continue;
6407 }
6408 if next_cursor <= sub.cursor {
6409 any_covered = true;
6411 continue;
6412 }
6413 let previous_effective = self.subs[sub_index].effective.clone();
6414 let previous_cursor = self.subs[sub_index].cursor;
6415 if self.begin_observation("syncular_delta").is_err() {
6416 dropped = true;
6417 continue;
6418 }
6419 self.subs[sub_index].effective = Some(effective_scopes);
6420 let mut batch = ChangeAccumulator::default();
6421 let mut failed = false;
6422 for inner in body {
6423 if let Frame::Commit {
6424 tables, changes, ..
6425 } = inner
6426 {
6427 self.record_commit_changes(&mut batch, &tables, &changes);
6428 if self.apply_commit_changes(&tables, &changes).is_err() {
6429 failed = true;
6430 break;
6431 }
6432 }
6433 }
6434 if failed {
6435 self.rollback_observation("syncular_delta");
6436 self.subs[sub_index].effective = previous_effective;
6437 self.subs[sub_index].cursor = previous_cursor;
6438 self.overlay_dirty.set(true);
6439 self.rebuild_overlay();
6440 dropped = true;
6441 continue;
6442 }
6443 let sub = &mut self.subs[sub_index];
6444 sub.cursor = next_cursor;
6445 self.persist_sub(&self.subs[sub_index].clone());
6446 self.rebuild_overlay_if_dirty();
6447 if self.finish_observation("syncular_delta", batch).is_err() {
6448 self.rollback_observation("syncular_delta");
6449 self.subs[sub_index].effective = previous_effective;
6450 self.subs[sub_index].cursor = previous_cursor;
6451 self.overlay_dirty.set(true);
6452 self.rebuild_overlay();
6453 dropped = true;
6454 continue;
6455 }
6456 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
6457 }
6458 if let Some(cursor) = applied_cursor {
6459 self.reconcile_blob_refcounts(false);
6460 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
6462 let _ = transport.realtime_send(&ack);
6463 } else if !any_covered || dropped {
6464 self.set_sync_needed(true, true);
6466 }
6467 }
6468
6469 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
6473 if !self.realtime_connected {
6474 return;
6475 }
6476 let floor = self
6477 .subs
6478 .iter()
6479 .filter(|s| {
6480 s.state == SubState::Active
6481 && s.bootstrap_state.is_none()
6482 && s.synced_once
6483 && s.cursor >= 0
6484 })
6485 .map(|s| s.cursor)
6486 .min();
6487 if let Some(cursor) = floor {
6488 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
6489 let _ = transport.realtime_send(&ack);
6490 }
6491 }
6492}