1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, CommitOperation, CommitOperationOutcome,
27 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
28 ConflictRecord, CoverageSnapshot, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
29 LocalDataPurgeTarget, Mutation, PresencePeer, QueryRow, QuerySnapshot, QueryValue,
30 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
31 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
32 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
33};
34use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
35use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
36use crate::values::{
37 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
38 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
39 scope_map_to_json, sort_scope_map,
40};
41
42const DEFAULT_ACCEPT: u8 = 0b0111;
47const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
48const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
49const ACCEPT_SQLITE: u8 = 1 << 2;
50const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
51
52const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
54const LOCAL_REVISION_KEY: &str = "localRevision";
55const CLIENT_ID_KEY: &str = "clientId";
56const LEASE_STATE_KEY: &str = "leaseState";
57const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
58const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63enum SubState {
64 Active,
65 Revoked,
66 Failed,
67}
68
69#[cfg(test)]
70mod observation_tests {
71 use super::*;
72 use serde_json::json;
73
74 fn client() -> SyncClient {
75 SyncClient::new(
76 "retry-test".to_owned(),
77 &json!({
78 "version": 1,
79 "tables": [{
80 "name": "tasks",
81 "primaryKey": "id",
82 "columns": [
83 { "name": "id", "type": "string", "nullable": false },
84 { "name": "project_id", "type": "string", "nullable": false }
85 ],
86 "scopes": [{ "pattern": "project:{project_id}" }]
87 }]
88 }),
89 ClientLimits::default(),
90 )
91 .expect("test client")
92 }
93
94 #[test]
95 fn background_retry_deadlines_back_off_and_reset() {
96 let mut client = client();
97 client.schedule_background_retry();
98 assert!(matches!(
99 client.drain_sync_intents().as_slice(),
100 [SyncIntent::Background { delay_ms: 250 }]
101 ));
102 client.schedule_background_retry();
103 assert!(matches!(
104 client.drain_sync_intents().as_slice(),
105 [SyncIntent::Background { delay_ms: 500 }]
106 ));
107 client.reset_background_retry();
108 client.schedule_background_retry();
109 assert!(matches!(
110 client.drain_sync_intents().as_slice(),
111 [SyncIntent::Background { delay_ms: 250 }]
112 ));
113 }
114
115 #[test]
116 fn secondary_unique_collision_preserves_existing_synced_row() {
117 let client = SyncClient::new(
118 "unique-upsert-test".to_owned(),
119 &json!({
120 "version": 1,
121 "tables": [{
122 "name": "tasks",
123 "primaryKey": "id",
124 "columns": [
125 { "name": "id", "type": "string", "nullable": false },
126 { "name": "project_id", "type": "string", "nullable": false },
127 { "name": "title", "type": "string", "nullable": false }
128 ],
129 "scopes": [{ "pattern": "project:{project_id}" }],
130 "indexes": [{
131 "name": "tasks_by_project_title",
132 "columns": ["project_id", "title"],
133 "unique": true
134 }]
135 }]
136 }),
137 ClientLimits::default(),
138 )
139 .expect("test client");
140 let table = client.schema.table("tasks").expect("tasks table");
141 let sql = client.insert_row_sql(&base_table("tasks"), table);
142
143 client
144 .conn
145 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
146 .expect("insert first row");
147 client
148 .conn
149 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
150 .expect("update same primary key");
151 client
152 .conn
153 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
154 .expect("insert second row");
155 assert!(client
156 .conn
157 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
158 .is_err());
159
160 let rows = client
161 .conn
162 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
163 .expect("prepare rows")
164 .query_map([], |row| {
165 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
166 })
167 .expect("query rows")
168 .collect::<Result<Vec<_>, _>>()
169 .expect("collect rows");
170 assert_eq!(
171 rows,
172 vec![
173 ("t1".to_owned(), "updated".to_owned()),
174 ("t2".to_owned(), "original".to_owned())
175 ]
176 );
177 }
178
179 #[test]
180 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
181 let path = std::env::temp_dir().join(format!(
182 "syncular-startup-intent-{}.db",
183 uuid::Uuid::new_v4()
184 ));
185 let schema = json!({
186 "version": 1,
187 "tables": [{
188 "name": "tasks",
189 "primaryKey": "id",
190 "columns": [
191 { "name": "id", "type": "string", "nullable": false },
192 { "name": "project_id", "type": "string", "nullable": false }
193 ],
194 "scopes": [{ "pattern": "project:{project_id}" }]
195 }]
196 });
197
198 {
199 let mut first = SyncClient::open_path_with_identity(
200 None,
201 &schema,
202 ClientLimits::default(),
203 path.to_str().expect("UTF-8 temp path"),
204 )
205 .expect("first open");
206 first
207 .set_window(
208 &WindowBase {
209 table: "tasks".to_owned(),
210 variable: "project_id".to_owned(),
211 fixed_scopes: Vec::new(),
212 params: None,
213 },
214 &["persisted".to_owned()],
215 )
216 .expect("persist window");
217 }
218
219 let mut reopened = SyncClient::open_path_with_identity(
220 None,
221 &schema,
222 ClientLimits::default(),
223 path.to_str().expect("UTF-8 temp path"),
224 )
225 .expect("reopen");
226 assert!(reopened.sync_needed());
227 assert!(matches!(
228 reopened.drain_sync_intents().as_slice(),
229 [SyncIntent::Interactive]
230 ));
231 drop(reopened);
232 std::fs::remove_file(path).expect("remove temp database");
233 }
234
235 #[test]
236 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
237 let path = std::env::temp_dir().join(format!(
238 "syncular-satisfied-schema-floor-{}.db",
239 uuid::Uuid::new_v4()
240 ));
241 let schema = json!({
242 "version": 23,
243 "tables": [{
244 "name": "tasks",
245 "primaryKey": "id",
246 "columns": [
247 { "name": "id", "type": "string", "nullable": false },
248 { "name": "project_id", "type": "string", "nullable": false }
249 ],
250 "scopes": [{ "pattern": "project:{project_id}" }]
251 }]
252 });
253
254 {
255 let mut first = SyncClient::open_path_with_identity(
256 None,
257 &schema,
258 ClientLimits::default(),
259 path.to_str().expect("UTF-8 temp path"),
260 )
261 .expect("first open");
262 first
263 .subscribe(
264 "tasks".to_owned(),
265 "tasks".to_owned(),
266 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
267 None,
268 )
269 .expect("persist subscription");
270 first.set_schema_floor(Some(SchemaFloor {
271 required_schema_version: Some(22),
272 latest_schema_version: Some(22),
273 }));
274 }
275
276 let mut reopened = SyncClient::open_path_with_identity(
277 None,
278 &schema,
279 ClientLimits::default(),
280 path.to_str().expect("UTF-8 temp path"),
281 )
282 .expect("reopen");
283 assert!(reopened.schema_floor().is_none());
284 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
285 assert!(reopened.sync_needed());
286 assert!(matches!(
287 reopened.drain_sync_intents().as_slice(),
288 [SyncIntent::Interactive]
289 ));
290 drop(reopened);
291 std::fs::remove_file(path).expect("remove temp database");
292 }
293
294 #[test]
295 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
296 let path = std::env::temp_dir().join(format!(
297 "syncular-unsatisfied-schema-floor-{}.db",
298 uuid::Uuid::new_v4()
299 ));
300 let schema = json!({
301 "version": 1,
302 "tables": [{
303 "name": "tasks",
304 "primaryKey": "id",
305 "columns": [
306 { "name": "id", "type": "string", "nullable": false },
307 { "name": "project_id", "type": "string", "nullable": false }
308 ],
309 "scopes": [{ "pattern": "project:{project_id}" }]
310 }]
311 });
312
313 {
314 let mut first = SyncClient::open_path_with_identity(
315 None,
316 &schema,
317 ClientLimits::default(),
318 path.to_str().expect("UTF-8 temp path"),
319 )
320 .expect("first open");
321 first
322 .subscribe(
323 "tasks".to_owned(),
324 "tasks".to_owned(),
325 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
326 None,
327 )
328 .expect("persist subscription");
329 first.set_schema_floor(Some(SchemaFloor {
330 required_schema_version: Some(2),
331 latest_schema_version: Some(2),
332 }));
333 }
334
335 let reopened = SyncClient::open_path_with_identity(
336 None,
337 &schema,
338 ClientLimits::default(),
339 path.to_str().expect("UTF-8 temp path"),
340 )
341 .expect("reopen");
342 assert_eq!(
343 reopened.schema_floor(),
344 Some(&SchemaFloor {
345 required_schema_version: Some(2),
346 latest_schema_version: Some(2),
347 })
348 );
349 assert!(!reopened.sync_needed());
350 drop(reopened);
351 std::fs::remove_file(path).expect("remove temp database");
352 }
353
354 #[test]
355 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
356 let path = std::env::temp_dir().join(format!(
357 "syncular-indexed-column-bump-{}.db",
358 uuid::Uuid::new_v4()
359 ));
360 let old_schema = json!({
361 "version": 1,
362 "tables": [
363 {
364 "name": "tasks",
365 "primaryKey": "id",
366 "columns": [
367 { "name": "id", "type": "string", "nullable": false },
368 { "name": "project_id", "type": "string", "nullable": false }
369 ],
370 "scopes": [{ "pattern": "project:{project_id}" }]
371 },
372 {
373 "name": "legacy",
374 "primaryKey": "id",
375 "columns": [
376 { "name": "id", "type": "string", "nullable": false },
377 { "name": "project_id", "type": "string", "nullable": false }
378 ],
379 "scopes": [{ "pattern": "project:{project_id}" }]
380 }
381 ]
382 });
383 {
384 let mut first = SyncClient::open_path_with_identity(
385 None,
386 &old_schema,
387 ClientLimits::default(),
388 path.to_str().expect("UTF-8 temp path"),
389 )
390 .expect("open old schema");
391 first
392 .subscribe(
393 "legacy-sub".to_owned(),
394 "legacy".to_owned(),
395 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
396 None,
397 )
398 .expect("persist legacy subscription");
399 }
400
401 let new_schema = json!({
402 "version": 2,
403 "tables": [{
404 "name": "tasks",
405 "primaryKey": "id",
406 "columns": [
407 { "name": "id", "type": "string", "nullable": false },
408 { "name": "project_id", "type": "string", "nullable": false },
409 { "name": "facility_membership_id", "type": "string", "nullable": true }
410 ],
411 "scopes": [{ "pattern": "project:{project_id}" }],
412 "indexes": [{
413 "name": "tasks_by_membership",
414 "columns": ["project_id", "facility_membership_id"],
415 "unique": false
416 }]
417 }]
418 });
419 let upgraded = SyncClient::open_path_with_identity(
420 None,
421 &new_schema,
422 ClientLimits::default(),
423 path.to_str().expect("UTF-8 temp path"),
424 )
425 .expect("open upgraded schema");
426 let columns = upgraded
427 .conn
428 .prepare("PRAGMA table_info(tasks)")
429 .expect("prepare columns")
430 .query_map([], |row| row.get::<_, String>(1))
431 .expect("query columns")
432 .collect::<Result<Vec<_>, _>>()
433 .expect("collect columns");
434 assert!(columns.contains(&"facility_membership_id".to_owned()));
435 let index_count: i64 = upgraded
436 .conn
437 .query_row(
438 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
439 [],
440 |row| row.get(0),
441 )
442 .expect("query index");
443 assert_eq!(index_count, 1);
444 assert!(upgraded.subscription_state("legacy-sub").is_none());
445 drop(upgraded);
446 std::fs::remove_file(path).expect("remove temp database");
447 }
448
449 #[test]
450 fn migrates_pre_envelope_outcome_journal_additively() {
451 let path = std::env::temp_dir().join(format!(
452 "syncular-outcome-migration-{}.db",
453 uuid::Uuid::new_v4()
454 ));
455 let conn = Connection::open(&path).expect("open legacy database");
456 conn.execute_batch(
457 "CREATE TABLE _syncular_commit_outcomes (
458 seq INTEGER PRIMARY KEY AUTOINCREMENT,
459 client_commit_id TEXT NOT NULL UNIQUE,
460 status TEXT NOT NULL,
461 recorded_at_ms INTEGER NOT NULL,
462 results_json TEXT NOT NULL,
463 resolution TEXT NOT NULL DEFAULT 'active',
464 resolved_at_ms INTEGER,
465 replacement_client_commit_id TEXT);",
466 )
467 .expect("create legacy outcome journal");
468 drop(conn);
469 let schema = json!({
470 "version": 1,
471 "tables": [{
472 "name": "tasks",
473 "primaryKey": "id",
474 "columns": [
475 { "name": "id", "type": "string", "nullable": false },
476 { "name": "project_id", "type": "string", "nullable": false }
477 ],
478 "scopes": [{ "pattern": "project:{project_id}" }]
479 }]
480 });
481 let client = SyncClient::open_path(
482 "migration-native".to_owned(),
483 &schema,
484 ClientLimits::default(),
485 path.to_str().expect("UTF-8 temp path"),
486 )
487 .expect("migrate database");
488 let has_operations = client
489 .conn
490 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
491 .expect("prepare table info")
492 .query_map([], |row| row.get::<_, String>(1))
493 .expect("query table info")
494 .filter_map(Result::ok)
495 .any(|column| column == "operations_json");
496 assert!(has_operations);
497 drop(client);
498 std::fs::remove_file(path).expect("remove temp database");
499 }
500
501 #[test]
502 fn durable_conflict_outcome_and_resolution_survive_reopen() {
503 let path = std::env::temp_dir().join(format!(
504 "syncular-durable-outcome-{}.db",
505 uuid::Uuid::new_v4()
506 ));
507 let schema = json!({
508 "version": 1,
509 "tables": [{
510 "name": "tasks",
511 "primaryKey": "id",
512 "columns": [
513 { "name": "id", "type": "string", "nullable": false },
514 { "name": "project_id", "type": "string", "nullable": false }
515 ],
516 "scopes": [{ "pattern": "project:{project_id}" }]
517 }]
518 });
519 let conflict = ConflictRecord {
520 client_commit_id: "losing-commit".to_owned(),
521 op_index: 0,
522 table: "tasks".to_owned(),
523 row_id: "t1".to_owned(),
524 code: "sync.version_conflict".to_owned(),
525 message: "stale base version".to_owned(),
526 server_version: 2,
527 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
528 operation: Some(CommitOperation {
529 table: "tasks".to_owned(),
530 row_id: "t1".to_owned(),
531 op: "upsert".to_owned(),
532 base_version: Some(1),
533 values: None,
534 changed_fields: None,
535 }),
536 };
537 let failed_operations = vec![
538 OutboxOp {
539 upsert: true,
540 table: "tasks".to_owned(),
541 row_id: "t1".to_owned(),
542 base_version: Some(1),
543 values: None,
544 changed_fields: None,
545 },
546 OutboxOp {
547 upsert: true,
548 table: "tasks".to_owned(),
549 row_id: "status-event-1".to_owned(),
550 base_version: Some(0),
551 values: None,
552 changed_fields: None,
553 },
554 ];
555
556 {
557 let mut first = SyncClient::open_path(
558 "durable-native".to_owned(),
559 &schema,
560 ClientLimits::default(),
561 path.to_str().expect("UTF-8 temp path"),
562 )
563 .expect("first open");
564 first
565 .begin_observation("test_outcome")
566 .expect("begin outcome");
567 first
568 .persist_commit_outcome(
569 "losing-commit",
570 CommitOutcomeStatus::Conflict,
571 &[CommitOperationOutcome::Conflict {
572 conflict: conflict.clone(),
573 }],
574 Some(&failed_operations),
575 )
576 .expect("persist outcome");
577 first.conflicts.push(conflict);
578 first
579 .finish_observation(
580 "test_outcome",
581 ChangeAccumulator {
582 conflicts: true,
583 outcomes: true,
584 ..ChangeAccumulator::default()
585 },
586 )
587 .expect("commit outcome");
588 }
589
590 {
591 let mut reopened = SyncClient::open_path(
592 "durable-native".to_owned(),
593 &schema,
594 ClientLimits::default(),
595 path.to_str().expect("UTF-8 temp path"),
596 )
597 .expect("reopen");
598 assert_eq!(reopened.conflicts().len(), 1);
599 let outcome = reopened
600 .commit_outcome("losing-commit")
601 .expect("read outcome")
602 .expect("outcome");
603 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
604 let operations = outcome.operations.expect("aggregate envelope");
605 assert_eq!(operations.len(), 2);
606 assert_eq!(operations[1].row_id, "status-event-1");
607 let resolved = reopened
608 .resolve_commit_outcome(ResolveCommitOutcomeInput {
609 client_commit_id: "losing-commit".to_owned(),
610 resolution: CommitOutcomeResolution::ResolvedKeepServer,
611 replacement_client_commit_id: None,
612 })
613 .expect("resolve");
614 assert_eq!(
615 resolved.resolution,
616 CommitOutcomeResolution::ResolvedKeepServer
617 );
618 assert!(reopened.conflicts().is_empty());
619 }
620
621 let reopened = SyncClient::open_path(
622 "durable-native".to_owned(),
623 &schema,
624 ClientLimits::default(),
625 path.to_str().expect("UTF-8 temp path"),
626 )
627 .expect("second reopen");
628 assert!(reopened.conflicts().is_empty());
629 assert_eq!(
630 reopened
631 .commit_outcome("losing-commit")
632 .expect("read outcome")
633 .expect("outcome")
634 .resolution,
635 CommitOutcomeResolution::ResolvedKeepServer
636 );
637 drop(reopened);
638 std::fs::remove_file(path).expect("remove temp database");
639 }
640
641 #[test]
642 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
643 let path =
644 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
645 let schema = json!({
646 "version": 1,
647 "tables": [{
648 "name": "tasks",
649 "primaryKey": "id",
650 "columns": [
651 { "name": "id", "type": "string", "nullable": false },
652 { "name": "project_id", "type": "string", "nullable": false }
653 ],
654 "scopes": [{ "pattern": "project:{project_id}" }]
655 }]
656 });
657 let mut client = SyncClient::open_path_with_identity(
658 Some("sidecar-client".to_owned()),
659 &schema,
660 ClientLimits::default(),
661 path.to_str().expect("UTF-8 temp path"),
662 )
663 .expect("open owner");
664 let base = WindowBase {
665 table: "tasks".to_owned(),
666 variable: "project_id".to_owned(),
667 fixed_scopes: Vec::new(),
668 params: None,
669 };
670 client
671 .set_window(&base, &["one".to_owned()])
672 .expect("set window");
673 client
674 .mutate(vec![Mutation::Upsert {
675 table: "tasks".to_owned(),
676 values: Map::from_iter([
677 ("id".to_owned(), Value::from("t1")),
678 ("project_id".to_owned(), Value::from("one")),
679 ]),
680 base_version: None,
681 }])
682 .expect("local mutate");
683
684 let coverage = [WindowCoverage {
685 base,
686 units: vec!["one".to_owned(), "missing".to_owned()],
687 }];
688 let owner = client
689 .query_snapshot(
690 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
691 &[],
692 &coverage,
693 )
694 .expect("owner snapshot");
695 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
696 let sidecar = reader
697 .query_snapshot(
698 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
699 &[],
700 &coverage,
701 )
702 .expect("sidecar snapshot");
703
704 assert_eq!(sidecar.revision, owner.revision);
705 assert_eq!(sidecar.rows, owner.rows);
706 assert_eq!(
707 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
708 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
709 );
710 assert_eq!(sidecar.revision, "2");
711 assert_eq!(sidecar.rows[0]["id"], "t1");
712 assert_eq!(sidecar.rows[0]["server_version"], -1);
713 assert!(!sidecar.coverage.complete);
714 assert_eq!(sidecar.coverage.pending.len(), 1);
715 assert_eq!(sidecar.coverage.missing.len(), 1);
716
717 drop(reader);
718 drop(client);
719 std::fs::remove_file(path).expect("remove temp database");
720 }
721
722 #[test]
723 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
724 let schema = json!({
725 "version": 1,
726 "tables": [{
727 "name": "catalogue_codes",
728 "primaryKey": "id",
729 "columns": [
730 { "name": "id", "type": "string", "nullable": false },
731 { "name": "release_id", "type": "string", "nullable": false },
732 { "name": "code", "type": "string", "nullable": false },
733 { "name": "title", "type": "string", "nullable": false }
734 ],
735 "scopes": [{ "pattern": "release:{release_id}" }],
736 "ftsIndexes": [{
737 "name": "catalogue_codes_fts",
738 "columns": ["code", "title"],
739 "tokenize": "unicode61 remove_diacritics 2"
740 }]
741 }]
742 });
743 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
744 .expect("FTS5 client");
745 let search = |client: &SyncClient, query: &str| {
746 client
747 .query(
748 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
749 &[Value::from(query)],
750 )
751 .expect("FTS query")
752 };
753
754 client
755 .mutate(vec![Mutation::Upsert {
756 table: "catalogue_codes".to_owned(),
757 values: Map::from_iter([
758 ("id".to_owned(), Value::from("c1")),
759 ("release_id".to_owned(), Value::from("r1")),
760 ("code".to_owned(), Value::from("A01")),
761 ("title".to_owned(), Value::from("Cholera")),
762 ]),
763 base_version: None,
764 }])
765 .expect("insert code");
766 assert_eq!(search(&client, "cholera").len(), 1);
767
768 client
769 .mutate(vec![Mutation::Upsert {
770 table: "catalogue_codes".to_owned(),
771 values: Map::from_iter([
772 ("id".to_owned(), Value::from("c1")),
773 ("release_id".to_owned(), Value::from("r1")),
774 ("code".to_owned(), Value::from("A01")),
775 ("title".to_owned(), Value::from("Enteric infection")),
776 ]),
777 base_version: None,
778 }])
779 .expect("update code");
780 assert!(search(&client, "cholera").is_empty());
781 assert_eq!(search(&client, "enteric").len(), 1);
782
783 client
784 .mutate(vec![Mutation::Delete {
785 table: "catalogue_codes".to_owned(),
786 row_id: "c1".to_owned(),
787 base_version: None,
788 }])
789 .expect("delete code");
790 assert!(search(&client, "enteric").is_empty());
791 }
792
793 #[test]
794 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
795 let schema = json!({
796 "version": 1,
797 "tables": [{
798 "name": "patient_notes",
799 "primaryKey": "id",
800 "columns": [
801 { "name": "id", "type": "string", "nullable": false },
802 { "name": "practice_id", "type": "string", "nullable": false },
803 { "name": "encryption_key_id", "type": "string", "nullable": false },
804 { "name": "title", "type": "string", "nullable": false }
805 ],
806 "scopes": [{ "pattern": "practice:{practice_id}" }],
807 "ftsIndexes": [{
808 "name": "patient_notes_fts",
809 "columns": ["title"],
810 "tokenize": "unicode61 remove_diacritics 2"
811 }]
812 }]
813 });
814 let mut client = SyncClient::new(
815 "local-purge-test".to_owned(),
816 &schema,
817 ClientLimits::default(),
818 )
819 .expect("local purge client");
820 client
821 .conn
822 .execute(
823 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
824 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
825 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
826 [],
827 )
828 .expect("seed base rows");
829 client.overlay_dirty.set(true);
830 client.rebuild_overlay_if_dirty();
831
832 let note = |id: &str, key_id: &str, title: &str| {
833 Map::from_iter([
834 ("id".to_owned(), Value::from(id)),
835 ("practice_id".to_owned(), Value::from("practice-1")),
836 ("encryption_key_id".to_owned(), Value::from(key_id)),
837 ("title".to_owned(), Value::from(title)),
838 ])
839 };
840 let doomed = client
841 .mutate(vec![
842 Mutation::Upsert {
843 table: "patient_notes".to_owned(),
844 values: note("target", "key-revoked", "Target changed"),
845 base_version: None,
846 },
847 Mutation::Upsert {
848 table: "patient_notes".to_owned(),
849 values: note("unrelated", "key-held", "Unrelated changed"),
850 base_version: None,
851 },
852 ])
853 .expect("doomed commit");
854 let kept = client
855 .mutate(vec![Mutation::Upsert {
856 table: "patient_notes".to_owned(),
857 values: note("kept", "key-held", "Kept optimistic"),
858 base_version: None,
859 }])
860 .expect("kept commit");
861 client.drain_change_batches();
862
863 let input = LocalDataPurgeInput {
864 purge_id: "purge-001".to_owned(),
865 targets: vec![LocalDataPurgeTarget {
866 table: "patient_notes".to_owned(),
867 selectors: BTreeMap::from([(
868 "encryption_key_id".to_owned(),
869 vec!["key-revoked".to_owned()],
870 )]),
871 }],
872 };
873 assert_eq!(
874 client.purge_local_data(&input).expect("apply purge"),
875 LocalDataPurgeResult {
876 already_applied: false,
877 purged_rows: 1,
878 dropped_commits: 1,
879 }
880 );
881 let rows = client
882 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
883 .expect("visible rows");
884 assert_eq!(rows.len(), 2);
885 assert_eq!(rows[0]["id"], "kept");
886 assert_eq!(rows[1]["id"], "unrelated");
887 assert_eq!(rows[1]["title"], "Unrelated original");
888 let fts = client
889 .query(
890 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
891 &[],
892 )
893 .expect("fts query");
894 assert!(fts.is_empty());
895 assert_eq!(client.outbox.len(), 1);
896 assert_eq!(client.outbox[0].client_commit_id, kept);
897 let outcome = client
898 .commit_outcome(&doomed)
899 .expect("read doomed outcome")
900 .expect("doomed outcome");
901 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
902 match &outcome.results[0] {
903 CommitOperationOutcome::Error { rejection } => {
904 assert_eq!(rejection.code, "client.local_data_purged");
905 assert_eq!(rejection.client_commit_id, doomed);
906 }
907 other => panic!("expected local purge rejection, got {other:?}"),
908 }
909 assert_eq!(client.drain_change_batches().len(), 1);
910 assert_eq!(
911 client.purge_local_data(&input).expect("retry purge"),
912 LocalDataPurgeResult {
913 already_applied: true,
914 purged_rows: 0,
915 dropped_commits: 0,
916 }
917 );
918 let conflicting = LocalDataPurgeInput {
919 purge_id: input.purge_id.clone(),
920 targets: vec![LocalDataPurgeTarget {
921 table: "patient_notes".to_owned(),
922 selectors: BTreeMap::from([(
923 "encryption_key_id".to_owned(),
924 vec!["key-held".to_owned()],
925 )]),
926 }],
927 };
928 assert!(client
929 .purge_local_data(&conflicting)
930 .expect_err("id collision must fail")
931 .contains("already used with a different plan"));
932 }
933}
934
935impl SubState {
936 fn name(self) -> &'static str {
937 match self {
938 SubState::Active => "active",
939 SubState::Revoked => "revoked",
940 SubState::Failed => "failed",
941 }
942 }
943
944 fn parse(value: &str) -> Self {
945 match value {
946 "revoked" => Self::Revoked,
947 "failed" => Self::Failed,
948 _ => Self::Active,
949 }
950 }
951}
952
953#[derive(Debug, Clone)]
954struct Subscription {
955 id: String,
956 table: String,
957 requested: Vec<(String, Vec<String>)>,
958 params: Option<String>,
959 cursor: i64,
960 bootstrap_state: Option<String>,
962 state: SubState,
963 reason_code: Option<String>,
964 effective: Option<Vec<(String, Vec<String>)>>,
967 synced_once: bool,
968}
969
970#[derive(Debug, Clone)]
971struct OutboxOp {
972 upsert: bool,
973 table: String,
974 row_id: String,
975 base_version: Option<i64>,
976 values: Option<Map<String, Value>>,
979 changed_fields: Option<Vec<String>>,
981}
982
983impl From<&OutboxOp> for CommitOperation {
984 fn from(operation: &OutboxOp) -> Self {
985 Self {
986 table: operation.table.clone(),
987 row_id: operation.row_id.clone(),
988 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
989 base_version: operation.base_version,
990 values: operation.values.clone(),
991 changed_fields: operation.changed_fields.clone(),
992 }
993 }
994}
995
996#[derive(Debug, Clone)]
997struct OutboxCommit {
998 client_commit_id: String,
999 ops: Vec<OutboxOp>,
1000}
1001
1002#[derive(Debug, Clone)]
1003struct CompiledLocalDataPurgeTarget {
1004 table: String,
1005 selectors: Vec<(String, Vec<String>)>,
1006}
1007
1008struct StoredCommitOutcomeRow {
1009 sequence: i64,
1010 client_commit_id: String,
1011 status: String,
1012 recorded_at_ms: i64,
1013 results_json: String,
1014 operations_json: Option<String>,
1015 resolution: String,
1016 resolved_at_ms: Option<i64>,
1017 replacement_client_commit_id: Option<String>,
1018}
1019
1020enum SectionError {
1023 FailClosed,
1024 Abort(String, String),
1025}
1026
1027struct RequestMeta {
1028 pushed_ids: Vec<String>,
1029 fresh: Vec<(String, bool)>,
1032 accept: u8,
1033 deferred_commits: usize,
1037}
1038
1039#[derive(Default)]
1040struct ChangeAccumulator {
1041 tables: BTreeMap<String, Option<BTreeSet<String>>>,
1043 windows: BTreeMap<(String, String), BTreeSet<String>>,
1044 status: bool,
1045 conflicts: bool,
1046 rejections: bool,
1047 outcomes: bool,
1048}
1049
1050impl ChangeAccumulator {
1051 fn table(&mut self, table: &str) {
1052 self.tables.insert(table.to_owned(), None);
1053 }
1054
1055 fn scope(&mut self, table: &str, key: String) {
1056 match self.tables.get_mut(table) {
1057 Some(None) => {}
1058 Some(Some(keys)) => {
1059 keys.insert(key);
1060 }
1061 None => {
1062 self.tables
1063 .insert(table.to_owned(), Some(BTreeSet::from([key])));
1064 }
1065 }
1066 }
1067
1068 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
1069 self.windows
1070 .entry((base_key.to_owned(), table.to_owned()))
1071 .or_default()
1072 .insert(unit.to_owned());
1073 }
1074
1075 fn touched(&self) -> bool {
1076 !self.tables.is_empty()
1077 || !self.windows.is_empty()
1078 || self.status
1079 || self.conflicts
1080 || self.rejections
1081 || self.outcomes
1082 }
1083}
1084
1085const PUSH_OPS_PER_REQUEST: usize = 500;
1091const MAX_LOCAL_PURGE_TARGETS: usize = 64;
1092const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
1093const MAX_LOCAL_PURGE_VALUES: usize = 128;
1094const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
1095pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
1096
1097pub struct SyncClient {
1098 conn: Connection,
1099 schema: ClientSchema,
1100 client_id: String,
1101 limits: ClientLimits,
1102 subs: Vec<Subscription>,
1103 outbox: Vec<OutboxCommit>,
1104 conflicts: Vec<ConflictRecord>,
1105 rejections: Vec<RejectionRecord>,
1106 schema_floor: Option<SchemaFloor>,
1107 lease_state: Option<LeaseState>,
1109 stopped: bool,
1111 upgrading: bool,
1113 sync_needed: bool,
1115 realtime_connected: bool,
1116 presence: HashMap<String, HashMap<String, PresencePeer>>,
1118 now_ms: Option<i64>,
1121 encryption: crate::values::EncryptionConfig,
1125 security_preflight: bool,
1128 insert_sql: RefCell<HashMap<String, String>>,
1134 overlay_dirty: Cell<bool>,
1138 change_queue: VecDeque<ClientChangeBatch>,
1140 sync_intent_queue: VecDeque<SyncIntent>,
1141 retry_delay_ms: u64,
1143}
1144
1145fn quote_ident(name: &str) -> String {
1146 format!("\"{}\"", name.replace('"', "\"\""))
1147}
1148
1149fn is_local_purge_code_like(value: &str) -> bool {
1150 let bytes = value.as_bytes();
1151 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
1152 && bytes
1153 .iter()
1154 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
1155}
1156
1157fn base_table(name: &str) -> String {
1158 quote_ident(&format!("_syncular_base_{name}"))
1159}
1160
1161type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
1166
1167fn window_base_key(base: &WindowBase) -> String {
1168 format!(
1169 "{}\0{}\0{}",
1170 base.table,
1171 base.variable,
1172 canonical_scope_json(&base.fixed_scopes)
1173 )
1174}
1175
1176fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
1178 let mut scopes = base.fixed_scopes.clone();
1179 scopes.retain(|(k, _)| k != &base.variable);
1180 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
1181 scopes
1182}
1183
1184fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
1188 let canonical = canonical_scope_json(&unit_scopes(base, unit));
1189 let digest = Sha256::digest(canonical.as_bytes());
1190 let hex = bytes_to_hex(&digest);
1191 format!("w:{}:{}", base.table, &hex[..16])
1192}
1193
1194fn visible_table(name: &str) -> String {
1195 quote_ident(name)
1196}
1197
1198const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1199
1200fn is_synced_table_name(name: &str) -> bool {
1203 if name.starts_with("sqlite_") {
1204 return false;
1205 }
1206 if name.starts_with("_syncular_base_") {
1207 return true; }
1209 !name.starts_with("_syncular_")
1211}
1212
1213fn blob_id_for(bytes: &[u8]) -> String {
1215 let digest = Sha256::digest(bytes);
1216 format!("sha256:{}", bytes_to_hex(&digest))
1217}
1218
1219enum RowParam<'a> {
1223 Cell(&'a Option<ColumnValue>),
1224 Version(i64),
1225}
1226
1227impl rusqlite::ToSql for RowParam<'_> {
1228 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1229 Ok(match self {
1230 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1231 RowParam::Cell(cell) => match cell {
1232 None => ToSqlOutput::Owned(SqlValue::Null),
1233 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1234 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1235 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1236 Some(ColumnValue::Boolean(b)) => {
1237 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1238 }
1239 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1240 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1241 }
1242 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1244 ToSqlOutput::Borrowed(ValueRef::Blob(b))
1245 }
1246 },
1247 })
1248 }
1249}
1250
1251fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1259 use ssp2::segment::ColumnType;
1260 let mismatch = || {
1261 Err(format!(
1262 "image column {:?} holds a value of the wrong type",
1263 column.name
1264 ))
1265 };
1266 match value {
1267 ValueRef::Null => {
1268 if !column.nullable {
1269 return Err(format!(
1270 "image column {:?} is NULL but not nullable",
1271 column.name
1272 ));
1273 }
1274 Ok(ToSqlOutput::Owned(SqlValue::Null))
1275 }
1276 ValueRef::Integer(i) => match column.ty {
1277 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1278 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1279 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1280 _ => mismatch(),
1281 },
1282 ValueRef::Real(_) => match column.ty {
1283 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1284 _ => mismatch(),
1285 },
1286 ValueRef::Text(t) => {
1287 std::str::from_utf8(t)
1288 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1289 match column.ty {
1290 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1291 Ok(ToSqlOutput::Borrowed(value))
1292 }
1293 _ => mismatch(),
1294 }
1295 }
1296 ValueRef::Blob(_) => match column.ty {
1297 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1299 _ => mismatch(),
1300 },
1301 }
1302}
1303
1304fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1305 use rusqlite::types::ValueRef;
1306 match value {
1307 ValueRef::Null => Value::Null,
1308 ValueRef::Integer(i) => match column.ty {
1309 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1310 ssp2::segment::ColumnType::Float => {
1311 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1312 }
1313 _ => Value::from(i),
1314 },
1315 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1316 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1317 ValueRef::Blob(b) => {
1318 let mut map = Map::new();
1319 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1320 Value::Object(map)
1321 }
1322 }
1323}
1324
1325fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1329 Ok(match value {
1330 Value::Null => SqlValue::Null,
1331 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1332 Value::Number(n) => {
1333 if let Some(i) = n.as_i64() {
1334 SqlValue::Integer(i)
1335 } else if let Some(f) = n.as_f64() {
1336 SqlValue::Real(f)
1337 } else {
1338 return Err(format!("query param number {n} is out of range"));
1339 }
1340 }
1341 Value::String(s) => SqlValue::Text(s.clone()),
1342 Value::Object(_) => {
1343 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1344 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1345 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1346 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1347 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1348 })?)
1349 } else {
1350 return Err(
1351 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1352 .to_owned(),
1353 );
1354 }
1355 }
1356 Value::Array(_) => return Err("query array params are not supported".to_owned()),
1357 })
1358}
1359
1360fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1365 use rusqlite::types::ValueRef;
1366 match value {
1367 ValueRef::Null => Value::Null,
1368 ValueRef::Integer(i) => {
1369 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1373 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1374 Value::from(i)
1375 } else {
1376 let mut map = Map::new();
1377 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1378 Value::Object(map)
1379 }
1380 }
1381 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1382 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1383 ValueRef::Blob(b) => {
1384 let mut map = Map::new();
1385 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1386 Value::Object(map)
1387 }
1388 }
1389}
1390
1391fn query_connection(
1392 conn: &Connection,
1393 sql: &str,
1394 params: &[QueryValue],
1395) -> Result<Vec<QueryRow>, String> {
1396 crate::query_guard::assert_read_only_query(sql)?;
1397 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1398 let bound: Vec<SqlValue> = params
1399 .iter()
1400 .map(json_param_to_sql)
1401 .collect::<Result<_, _>>()?;
1402 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
1403 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1404 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1405 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1406 let mut sql_rows = stmt
1407 .query(bound_refs.as_slice())
1408 .map_err(|e| e.to_string())?;
1409 let mut out = Vec::new();
1410 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1411 let mut record = Map::new();
1412 for (i, name) in column_names.iter().enumerate() {
1413 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1414 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1415 }
1416 out.push(record);
1417 }
1418 Ok(out)
1419}
1420
1421fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1422 let mut stmt = conn
1423 .prepare(
1424 "SELECT windows.unit, subscriptions.state_json
1425 FROM _syncular_windows AS windows
1426 JOIN _syncular_subscriptions AS subscriptions
1427 ON subscriptions.id = windows.sub_id
1428 WHERE windows.base = ?1
1429 ORDER BY windows.unit ASC",
1430 )
1431 .map_err(|error| error.to_string())?;
1432 let rows = stmt
1433 .query_map(rusqlite::params![window_base_key(base)], |row| {
1434 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1435 })
1436 .map_err(|error| error.to_string())?;
1437 let mut units = Vec::new();
1438 let mut pending = Vec::new();
1439 for row in rows {
1440 let (unit, raw) = row.map_err(|error| error.to_string())?;
1441 let state: Value = serde_json::from_str(&raw)
1442 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1443 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1444 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1445 || state
1446 .get("bootstrapState")
1447 .is_some_and(|value| !value.is_null());
1448 if is_pending {
1449 pending.push(unit.clone());
1450 }
1451 units.push(unit);
1452 }
1453 Ok(WindowState { units, pending })
1454}
1455
1456fn snapshot_connection(
1457 conn: &Connection,
1458 sql: &str,
1459 params: &[Value],
1460 coverage: &[WindowCoverage],
1461) -> Result<QuerySnapshot, String> {
1462 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1463 .map_err(|error| error.to_string())?;
1464 let result = (|| {
1465 let revision = conn
1466 .query_row(
1467 "SELECT value FROM _syncular_meta WHERE key = ?1",
1468 rusqlite::params![LOCAL_REVISION_KEY],
1469 |row| row.get::<_, String>(0),
1470 )
1471 .ok()
1472 .and_then(|value| value.parse::<u64>().ok())
1473 .unwrap_or(0);
1474 let rows = query_connection(conn, sql, params)?;
1475 let mut pending = Vec::new();
1476 let mut missing = Vec::new();
1477 for requested in coverage {
1478 let base_key = window_base_key(&requested.base);
1479 let state = persisted_window_state(conn, &requested.base)?;
1480 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1481 let reference = WindowUnitRef {
1482 base_key: base_key.clone(),
1483 unit: unit.clone(),
1484 };
1485 if !state.units.iter().any(|held| held == &unit) {
1486 missing.push(reference);
1487 } else if state.pending.iter().any(|held| held == &unit) {
1488 pending.push(reference);
1489 }
1490 }
1491 }
1492 Ok(QuerySnapshot {
1493 revision: revision.to_string(),
1494 rows,
1495 coverage: CoverageSnapshot {
1496 complete: pending.is_empty() && missing.is_empty(),
1497 pending,
1498 missing,
1499 },
1500 })
1501 })();
1502 match result {
1503 Ok(snapshot) => {
1504 conn.execute_batch("RELEASE syncular_snapshot_read")
1505 .map_err(|error| error.to_string())?;
1506 Ok(snapshot)
1507 }
1508 Err(error) => {
1509 let _ = conn.execute_batch(
1510 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1511 );
1512 Err(error)
1513 }
1514 }
1515}
1516
1517pub struct FileQuerySnapshotReader {
1522 path: String,
1523 conn: Option<Connection>,
1524}
1525
1526impl FileQuerySnapshotReader {
1527 #[must_use]
1528 pub fn new(path: impl Into<String>) -> Self {
1529 Self {
1530 path: path.into(),
1531 conn: None,
1532 }
1533 }
1534
1535 fn connection(&mut self) -> Result<&Connection, String> {
1536 if self.conn.is_none() {
1537 let conn = Connection::open_with_flags(
1538 &self.path,
1539 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1540 )
1541 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1542 conn.busy_timeout(std::time::Duration::from_millis(250))
1543 .map_err(|error| error.to_string())?;
1544 self.conn = Some(conn);
1545 }
1546 self.conn
1547 .as_ref()
1548 .ok_or_else(|| "read sidecar connection missing".to_owned())
1549 }
1550
1551 pub fn query_snapshot(
1552 &mut self,
1553 sql: &str,
1554 params: &[Value],
1555 coverage: &[WindowCoverage],
1556 ) -> Result<QuerySnapshot, String> {
1557 snapshot_connection(self.connection()?, sql, params, coverage)
1558 }
1559}
1560
1561impl SyncClient {
1562 pub fn new_with_identity(
1563 client_id: Option<String>,
1564 schema_json: &Value,
1565 limits: ClientLimits,
1566 ) -> Result<Self, String> {
1567 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1568 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1569 Self::with_connection(resolved, schema_json, limits, conn)
1570 }
1571
1572 pub fn new(
1573 client_id: String,
1574 schema_json: &Value,
1575 limits: ClientLimits,
1576 ) -> Result<Self, String> {
1577 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1578 Self::with_connection(client_id, schema_json, limits, conn)
1579 }
1580
1581 pub fn open_path(
1587 client_id: String,
1588 schema_json: &Value,
1589 limits: ClientLimits,
1590 path: &str,
1591 ) -> Result<Self, String> {
1592 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1593 Self::with_connection(client_id, schema_json, limits, conn)
1594 }
1595
1596 pub fn open_path_with_identity(
1597 client_id: Option<String>,
1598 schema_json: &Value,
1599 limits: ClientLimits,
1600 path: &str,
1601 ) -> Result<Self, String> {
1602 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1603 conn.busy_timeout(std::time::Duration::from_millis(250))
1609 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1610 conn.pragma_update(None, "journal_mode", "WAL")
1611 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
1612 let persisted = conn
1613 .query_row(
1614 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
1615 [],
1616 |row| row.get::<_, String>(0),
1617 )
1618 .ok();
1619 let resolved = persisted
1620 .clone()
1621 .or(client_id.clone())
1622 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1623 if let (Some(existing), Some(requested)) = (persisted, client_id) {
1624 if existing != requested {
1625 return Err(format!(
1626 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
1627 ));
1628 }
1629 }
1630 Self::with_connection(resolved, schema_json, limits, conn)
1631 }
1632
1633 #[must_use]
1634 pub fn client_id(&self) -> &str {
1635 &self.client_id
1636 }
1637
1638 pub fn with_connection(
1645 client_id: String,
1646 schema_json: &Value,
1647 limits: ClientLimits,
1648 conn: Connection,
1649 ) -> Result<Self, String> {
1650 if limits.outcome_retention_max_entries == Some(0) {
1651 return Err(
1652 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
1653 );
1654 }
1655 let schema = parse_schema_json(schema_json)?;
1656 let mut client = SyncClient {
1657 conn,
1658 schema,
1659 client_id,
1660 limits,
1661 subs: Vec::new(),
1662 outbox: Vec::new(),
1663 conflicts: Vec::new(),
1664 rejections: Vec::new(),
1665 schema_floor: None,
1666 lease_state: None,
1667 stopped: false,
1668 upgrading: false,
1669 sync_needed: false,
1670 realtime_connected: false,
1671 presence: HashMap::new(),
1672 now_ms: None,
1673 encryption: crate::values::EncryptionConfig::default(),
1674 security_preflight: false,
1675 insert_sql: RefCell::new(HashMap::new()),
1676 overlay_dirty: Cell::new(false),
1677 change_queue: VecDeque::new(),
1678 sync_intent_queue: VecDeque::new(),
1679 retry_delay_ms: 250,
1680 };
1681 client
1685 .conn
1686 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
1687 client.create_bookkeeping_tables()?;
1691 match client.get_meta(CLIENT_ID_KEY) {
1692 Some(existing) if existing != client.client_id => {
1693 return Err(format!(
1694 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
1695 client.client_id
1696 ));
1697 }
1698 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
1699 _ => {}
1700 }
1701 client.restore_persisted_state()?;
1702 let marker = client
1703 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1704 .and_then(|value| value.parse::<i32>().ok());
1705 match marker {
1706 None => {
1707 client.create_synced_tables()?;
1708 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
1709 }
1710 Some(version) if version == client.schema.version => {
1711 client.create_synced_tables()?;
1712 }
1713 Some(_) => client.run_schema_reset()?,
1714 }
1715 client.clear_satisfied_persisted_schema_floor();
1716 client.prune_unknown_subscriptions()?;
1717 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
1718 client.overlay_dirty.set(true);
1721 client.rebuild_overlay();
1722 }
1723 client.enqueue_startup_sync_if_needed();
1729 Ok(client)
1730 }
1731
1732 pub fn set_now_ms(&mut self, now_ms: i64) {
1735 self.now_ms = Some(now_ms);
1736 }
1737
1738 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
1742 self.encryption = encryption;
1743 }
1744
1745 #[must_use]
1746 pub fn security_lifecycle(&self) -> &'static str {
1747 if self.security_preflight {
1748 "preflight"
1749 } else {
1750 "active"
1751 }
1752 }
1753
1754 #[must_use]
1755 pub fn security_preflight(&self) -> bool {
1756 self.security_preflight
1757 }
1758
1759 pub fn begin_security_preflight(&mut self) {
1761 self.security_preflight = true;
1762 self.encryption = crate::values::EncryptionConfig::default();
1763 self.sync_intent_queue.clear();
1764 }
1765
1766 pub fn activate_security(
1768 &mut self,
1769 encryption: crate::values::EncryptionConfig,
1770 ) -> Result<(), String> {
1771 if !self.security_preflight {
1772 return Err(
1773 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
1774 );
1775 }
1776 self.encryption = encryption;
1777 self.security_preflight = false;
1778 self.enqueue_startup_sync_if_needed();
1779 Ok(())
1780 }
1781
1782 fn clock_now_ms(&self) -> i64 {
1783 self.now_ms.unwrap_or_else(|| {
1784 std::time::SystemTime::now()
1785 .duration_since(std::time::UNIX_EPOCH)
1786 .map(|d| d.as_millis() as i64)
1787 .unwrap_or(0)
1788 })
1789 }
1790
1791 fn create_bookkeeping_tables(&self) -> Result<(), String> {
1792 self.conn
1794 .execute_batch(
1795 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
1796 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1797 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
1798 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
1799 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1800 client_commit_id TEXT NOT NULL UNIQUE,
1801 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
1802 recorded_at_ms INTEGER NOT NULL,
1803 results_json TEXT NOT NULL,
1804 operations_json TEXT,
1805 resolution TEXT NOT NULL DEFAULT 'active'
1806 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
1807 resolved_at_ms INTEGER,
1808 replacement_client_commit_id TEXT);
1809 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
1810 ON _syncular_commit_outcomes(resolution, seq);
1811 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
1812 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
1813 CREATE TABLE IF NOT EXISTS _syncular_meta (
1814 key TEXT PRIMARY KEY, value TEXT NOT NULL);
1815 CREATE TABLE IF NOT EXISTS _syncular_windows (
1816 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
1817 PRIMARY KEY (base, unit));
1818 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
1819 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
1820 effective_scopes TEXT NOT NULL);",
1821 )
1822 .map_err(|e| e.to_string())?;
1823 let _ = self
1826 .conn
1827 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
1828 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
1829 self.set_meta(LOCAL_REVISION_KEY, "0");
1830 }
1831 if self.schema_has_blobs() {
1836 self.conn
1837 .execute_batch(
1838 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
1839 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
1840 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
1841 created_at_ms INTEGER NOT NULL,
1842 last_used_ms INTEGER NOT NULL DEFAULT 0);
1843 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
1844 media_type TEXT, created_at_ms INTEGER NOT NULL);",
1845 )
1846 .map_err(|e| e.to_string())?;
1847 let _ = self.conn.execute_batch(
1851 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
1852 );
1853 }
1854 Ok(())
1855 }
1856
1857 fn schema_has_blobs(&self) -> bool {
1859 self.schema
1860 .tables
1861 .iter()
1862 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
1863 }
1864
1865 fn get_meta(&self, key: &str) -> Option<String> {
1868 self.conn
1869 .query_row(
1870 "SELECT value FROM _syncular_meta WHERE key = ?1",
1871 rusqlite::params![key],
1872 |row| row.get::<_, String>(0),
1873 )
1874 .ok()
1875 }
1876
1877 fn set_meta(&self, key: &str, value: &str) {
1878 let _ = self.conn.execute(
1879 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
1880 rusqlite::params![key, value],
1881 );
1882 }
1883
1884 fn delete_meta(&self, key: &str) {
1885 let _ = self.conn.execute(
1886 "DELETE FROM _syncular_meta WHERE key = ?1",
1887 rusqlite::params![key],
1888 );
1889 }
1890
1891 fn restore_persisted_state(&mut self) -> Result<(), String> {
1892 self.subs = {
1893 let mut stmt = self
1894 .conn
1895 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
1896 .map_err(|error| error.to_string())?;
1897 let rows = stmt
1898 .query_map([], |row| {
1899 Ok((
1900 row.get::<_, String>(0)?,
1901 row.get::<_, String>(1)?,
1902 row.get::<_, String>(2)?,
1903 ))
1904 })
1905 .map_err(|error| error.to_string())?;
1906 let mut subscriptions = Vec::new();
1907 for row in rows {
1908 let (id, table, raw) = row.map_err(|error| error.to_string())?;
1909 let state: Value = serde_json::from_str(&raw)
1910 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
1911 let requested = json_to_scope_map(
1912 state.get("requested").unwrap_or(&Value::Object(Map::new())),
1913 )?;
1914 let effective = state
1915 .get("effectiveScopes")
1916 .filter(|value| !value.is_null())
1917 .map(json_to_scope_map)
1918 .transpose()?;
1919 subscriptions.push(Subscription {
1920 id,
1921 table,
1922 requested,
1923 params: state
1924 .get("params")
1925 .and_then(Value::as_str)
1926 .map(str::to_owned),
1927 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
1928 bootstrap_state: state
1929 .get("bootstrapState")
1930 .and_then(Value::as_str)
1931 .map(str::to_owned),
1932 state: SubState::parse(
1933 state
1934 .get("status")
1935 .and_then(Value::as_str)
1936 .unwrap_or("active"),
1937 ),
1938 reason_code: state
1939 .get("reasonCode")
1940 .and_then(Value::as_str)
1941 .map(str::to_owned),
1942 effective,
1943 synced_once: state
1944 .get("syncedOnce")
1945 .and_then(Value::as_bool)
1946 .unwrap_or(false),
1947 });
1948 }
1949 subscriptions
1950 };
1951
1952 self.outbox = {
1953 let mut stmt = self
1954 .conn
1955 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
1956 .map_err(|error| error.to_string())?;
1957 let rows = stmt
1958 .query_map([], |row| {
1959 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1960 })
1961 .map_err(|error| error.to_string())?;
1962 let mut commits = Vec::new();
1963 for row in rows {
1964 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
1965 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
1966 format!("invalid persisted outbox {client_commit_id:?}: {error}")
1967 })?;
1968 let mut ops = Vec::with_capacity(entries.len());
1969 for entry in entries {
1970 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
1971 ops.push(OutboxOp {
1972 upsert: op == "upsert",
1973 table: entry
1974 .get("table")
1975 .and_then(Value::as_str)
1976 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
1977 .to_owned(),
1978 row_id: entry
1979 .get("rowId")
1980 .and_then(Value::as_str)
1981 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
1982 .to_owned(),
1983 base_version: entry.get("baseVersion").and_then(Value::as_i64),
1984 values: entry.get("values").and_then(Value::as_object).cloned(),
1985 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
1986 |values| {
1987 values
1988 .iter()
1989 .filter_map(Value::as_str)
1990 .map(str::to_owned)
1991 .collect()
1992 },
1993 ),
1994 });
1995 }
1996 commits.push(OutboxCommit {
1997 client_commit_id,
1998 ops,
1999 });
2000 }
2001 commits
2002 };
2003
2004 self.prune_commit_outcomes()?;
2005 let active = self.commit_outcomes(CommitOutcomeQuery {
2006 active_only: true,
2007 ..CommitOutcomeQuery::default()
2008 })?;
2009 self.conflicts = active
2010 .iter()
2011 .flat_map(|outcome| outcome.results.iter())
2012 .filter_map(|result| match result {
2013 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2014 _ => None,
2015 })
2016 .collect();
2017 self.rejections = active
2018 .iter()
2019 .flat_map(|outcome| outcome.results.iter())
2020 .filter_map(|result| match result {
2021 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2022 _ => None,
2023 })
2024 .collect();
2025
2026 self.lease_state = self
2027 .get_meta(LEASE_STATE_KEY)
2028 .map(|raw| serde_json::from_str(&raw))
2029 .transpose()
2030 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2031 self.schema_floor = self
2032 .get_meta(SCHEMA_FLOOR_KEY)
2033 .map(|raw| serde_json::from_str(&raw))
2034 .transpose()
2035 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2036 self.stopped = self.schema_floor.is_some();
2037 Ok(())
2038 }
2039
2040 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
2043 let valid_tables: BTreeSet<String> = self
2044 .schema
2045 .tables
2046 .iter()
2047 .map(|table| table.name.clone())
2048 .collect();
2049 let stale_ids: Vec<String> = self
2050 .subs
2051 .iter()
2052 .filter(|sub| !valid_tables.contains(&sub.table))
2053 .map(|sub| sub.id.clone())
2054 .collect();
2055 for id in &stale_ids {
2056 self.conn
2057 .execute(
2058 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2059 rusqlite::params![id],
2060 )
2061 .map_err(|error| error.to_string())?;
2062 self.conn
2063 .execute(
2064 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2065 rusqlite::params![id],
2066 )
2067 .map_err(|error| error.to_string())?;
2068 self.conn
2069 .execute(
2070 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2071 rusqlite::params![id],
2072 )
2073 .map_err(|error| error.to_string())?;
2074 }
2075 self.subs.retain(|sub| valid_tables.contains(&sub.table));
2076 Ok(())
2077 }
2078
2079 #[must_use]
2080 pub fn local_revision(&self) -> u64 {
2081 self.get_meta(LOCAL_REVISION_KEY)
2082 .and_then(|value| value.parse().ok())
2083 .unwrap_or(0)
2084 }
2085
2086 #[must_use]
2087 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2088 SyncStatusSnapshot {
2089 current_schema_version: self.schema.version,
2090 outbox: self.outbox.len(),
2091 upgrading: self.upgrading,
2092 lease_state: self.lease_state.clone(),
2093 schema_floor: self.schema_floor.clone(),
2094 sync_needed: self.sync_needed,
2095 }
2096 }
2097
2098 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
2099 self.change_queue.drain(..).collect()
2100 }
2101
2102 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
2103 self.sync_intent_queue.drain(..).collect()
2104 }
2105
2106 fn schedule_background_retry(&mut self) {
2107 self.sync_intent_queue.push_back(SyncIntent::Background {
2108 delay_ms: self.retry_delay_ms,
2109 });
2110 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
2111 }
2112
2113 fn reset_background_retry(&mut self) {
2114 self.retry_delay_ms = 250;
2115 }
2116
2117 fn retryable_transport_code(code: &str) -> bool {
2118 code == "transport.failed" || code == "sync.transport_failed"
2119 }
2120
2121 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
2122 if self.sync_needed != value {
2123 if self.begin_observation("syncular_status").is_ok() {
2124 self.sync_needed = value;
2125 let batch = ChangeAccumulator {
2126 status: true,
2127 ..ChangeAccumulator::default()
2128 };
2129 if self.finish_observation("syncular_status", batch).is_err() {
2130 self.rollback_observation("syncular_status");
2131 }
2132 } else {
2133 self.sync_needed = value;
2134 }
2135 }
2136 if value && interactive {
2137 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2138 }
2139 }
2140
2141 fn begin_observation(&self, name: &str) -> Result<(), String> {
2142 self.conn
2143 .execute_batch(&format!("SAVEPOINT {name}"))
2144 .map_err(|error| error.to_string())
2145 }
2146
2147 fn rollback_observation(&self, name: &str) {
2148 let _ = self
2149 .conn
2150 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
2151 }
2152
2153 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
2154 if !batch.touched() {
2155 self.conn
2156 .execute_batch(&format!("RELEASE {name}"))
2157 .map_err(|error| error.to_string())?;
2158 return Ok(());
2159 }
2160 let revision = self
2161 .local_revision()
2162 .checked_add(1)
2163 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
2164 self.conn
2165 .execute(
2166 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
2167 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
2168 )
2169 .map_err(|error| error.to_string())?;
2170 let status = batch.status.then(|| self.status_snapshot());
2171 let event = ClientChangeBatch {
2172 revision: revision.to_string(),
2173 tables: batch
2174 .tables
2175 .into_iter()
2176 .map(|(table, scope_keys)| TableChange {
2177 table,
2178 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
2179 })
2180 .collect(),
2181 windows: batch
2182 .windows
2183 .into_iter()
2184 .map(|((base_key, table), units)| WindowChange {
2185 base_key,
2186 table,
2187 units: units.into_iter().collect(),
2188 })
2189 .collect(),
2190 status,
2191 conflicts_changed: batch.conflicts,
2192 rejections_changed: batch.rejections,
2193 outcomes_changed: batch.outcomes,
2194 };
2195 self.conn
2196 .execute_batch(&format!("RELEASE {name}"))
2197 .map_err(|error| error.to_string())?;
2198 self.change_queue.push_back(event);
2199 Ok(())
2200 }
2201
2202 fn record_scope_map(
2203 &self,
2204 batch: &mut ChangeAccumulator,
2205 table_name: &str,
2206 scopes: &[(String, Vec<String>)],
2207 ) {
2208 let Some(table) = self.schema.table(table_name) else {
2209 return;
2210 };
2211 for (variable, values) in scopes {
2212 let Some(scope) = table
2213 .scope_variables
2214 .iter()
2215 .find(|scope| &scope.variable == variable)
2216 else {
2217 continue;
2218 };
2219 for value in values {
2220 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2221 }
2222 }
2223 }
2224
2225 fn record_row_scopes(
2227 &self,
2228 batch: &mut ChangeAccumulator,
2229 table_name: &str,
2230 row_id: &str,
2231 base: bool,
2232 ) -> bool {
2233 let Some(table) = self.schema.table(table_name) else {
2234 return false;
2235 };
2236 if table.scope_variables.is_empty() {
2237 return false;
2238 }
2239 let columns = table
2240 .scope_variables
2241 .iter()
2242 .map(|scope| quote_ident(&scope.column))
2243 .collect::<Vec<_>>()
2244 .join(", ");
2245 let full_table = if base {
2246 base_table(table_name)
2247 } else {
2248 visible_table(table_name)
2249 };
2250 let sql = format!(
2251 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
2252 quote_ident(&table.primary_key)
2253 );
2254 let Ok(mut stmt) = self.conn.prepare(&sql) else {
2255 return false;
2256 };
2257 let values = stmt.query_row(rusqlite::params![row_id], |row| {
2258 let mut values = Vec::with_capacity(table.scope_variables.len());
2259 for index in 0..table.scope_variables.len() {
2260 values.push(row.get::<_, Option<String>>(index)?);
2261 }
2262 Ok(values)
2263 });
2264 let Ok(values) = values else {
2265 return false;
2266 };
2267 let mut recorded = false;
2268 for (scope, value) in table.scope_variables.iter().zip(values) {
2269 if let Some(value) = value {
2270 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2271 recorded = true;
2272 }
2273 }
2274 recorded
2275 }
2276
2277 fn record_commit_changes(
2278 &self,
2279 batch: &mut ChangeAccumulator,
2280 tables: &[String],
2281 changes: &[ssp2::model::Change],
2282 ) {
2283 for change in changes {
2284 let Some(table_name) = tables.get(change.table_index as usize) else {
2285 continue;
2286 };
2287 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
2288 if let Some(table) = self.schema.table(table_name) {
2289 for (variable, value) in &change.scopes {
2290 if let Some(scope) = table
2291 .scope_variables
2292 .iter()
2293 .find(|scope| &scope.variable == variable)
2294 {
2295 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2296 precise = true;
2297 }
2298 }
2299 }
2300 if !precise {
2301 batch.table(table_name);
2302 }
2303 }
2304 }
2305
2306 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
2307 if effective.is_empty() {
2308 return false;
2309 }
2310 let Some(table) = self.schema.table(table_name) else {
2311 return false;
2312 };
2313 let mut clauses = Vec::new();
2314 let mut params = Vec::new();
2315 for (variable, values) in effective {
2316 let Some(column) = table.scope_column(variable) else {
2317 return false;
2318 };
2319 if values.is_empty() {
2320 return false;
2321 }
2322 let placeholders = values
2323 .iter()
2324 .map(|value| {
2325 params.push(SqlValue::Text(value.clone()));
2326 "?"
2327 })
2328 .collect::<Vec<_>>()
2329 .join(", ");
2330 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
2331 }
2332 let sql = format!(
2333 "SELECT 1 FROM {} WHERE {} LIMIT 1",
2334 base_table(table_name),
2335 clauses.join(" AND ")
2336 );
2337 self.conn
2338 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
2339 .is_ok()
2340 }
2341
2342 pub fn upgrading(&self) -> bool {
2344 self.upgrading
2345 }
2346
2347 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
2353 let new_schema = parse_schema_json(schema_json)?;
2354 let marker: Option<i32> = self
2355 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
2356 .and_then(|v| v.parse().ok());
2357 self.schema = new_schema;
2358 if marker != Some(self.schema.version) {
2359 self.run_schema_reset()?;
2360 }
2361 self.prune_unknown_subscriptions()?;
2362 self.enqueue_startup_sync_if_needed();
2366 Ok(())
2367 }
2368
2369 fn enqueue_startup_sync_if_needed(&mut self) {
2370 let startup_work = !self.stopped
2371 && (!self.outbox.is_empty()
2372 || self.subs.iter().any(|sub| sub.state == SubState::Active));
2373 if startup_work {
2374 self.sync_needed = true;
2375 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2376 }
2377 }
2378
2379 fn clear_satisfied_persisted_schema_floor(&mut self) {
2386 let satisfied = self
2387 .schema_floor
2388 .as_ref()
2389 .and_then(|floor| floor.required_schema_version)
2390 .is_some_and(|required| self.schema.version >= required);
2391 if !satisfied {
2392 return;
2393 }
2394 self.schema_floor = None;
2395 self.stopped = false;
2396 self.delete_meta(SCHEMA_FLOOR_KEY);
2397 }
2398
2399 fn run_schema_reset(&mut self) -> Result<(), String> {
2405 self.begin_observation("syncular_schema_reset")?;
2406 let mut batch = ChangeAccumulator::default();
2407 let result = self.run_schema_reset_observed(&mut batch);
2408 if let Err(error) = result {
2409 self.rollback_observation("syncular_schema_reset");
2410 return Err(error);
2411 }
2412 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
2413 self.rollback_observation("syncular_schema_reset");
2414 return Err(error);
2415 }
2416 Ok(())
2417 }
2418
2419 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
2420 self.upgrading = true;
2421 batch.status = true;
2422 for table in &self.schema.tables {
2423 batch.table(&table.name);
2424 }
2425 for (base_key, unit, table) in self.load_registered_window_units() {
2426 batch.window(&base_key, &table, &unit);
2427 }
2428 self.insert_sql.borrow_mut().clear();
2430 self.overlay_dirty.set(true);
2431 let virtual_tables: Vec<String> = {
2439 let mut stmt = self
2440 .conn
2441 .prepare(
2442 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
2443 )
2444 .map_err(|e| e.to_string())?;
2445 let rows = stmt
2446 .query_map([], |row| row.get::<_, String>(0))
2447 .map_err(|e| e.to_string())?;
2448 rows.filter_map(Result::ok)
2449 .filter(|name| is_synced_table_name(name))
2450 .collect()
2451 };
2452 for name in virtual_tables {
2453 self.conn
2454 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2455 .map_err(|e| e.to_string())?;
2456 }
2457 let existing: Vec<String> = {
2458 let mut stmt = self
2459 .conn
2460 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
2461 .map_err(|e| e.to_string())?;
2462 let rows = stmt
2463 .query_map([], |row| row.get::<_, String>(0))
2464 .map_err(|e| e.to_string())?;
2465 rows.filter_map(Result::ok)
2466 .filter(|name| is_synced_table_name(name))
2467 .collect()
2468 };
2469 for name in &existing {
2470 if let Some(table) = name.strip_prefix("_syncular_base_") {
2471 batch.table(table);
2472 } else if !name.starts_with("_syncular_") {
2473 batch.table(name);
2474 }
2475 }
2476 for name in existing {
2477 self.conn
2478 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2479 .map_err(|e| e.to_string())?;
2480 }
2481 self.create_synced_tables()?;
2483 for sub in &mut self.subs {
2485 sub.cursor = -1;
2486 sub.bootstrap_state = None;
2487 sub.effective = None;
2488 sub.state = SubState::Active;
2489 sub.reason_code = None;
2490 sub.synced_once = false;
2491 }
2492 let subs = self.subs.clone();
2493 for sub in &subs {
2494 self.persist_sub(sub);
2495 }
2496 self.stopped = false;
2498 self.schema_floor = None;
2499 self.delete_meta(SCHEMA_FLOOR_KEY);
2500 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
2502 if self.drop_incompatible_outbox()? {
2506 batch.rejections = true;
2507 batch.status = true;
2508 batch.outcomes = true;
2509 }
2510 self.rebuild_overlay();
2512 Ok(())
2513 }
2514
2515 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
2519 let schema = &self.schema;
2520 let incompatible = self
2521 .outbox
2522 .iter()
2523 .filter(|commit| {
2524 commit.ops.iter().any(|op| {
2525 if !op.upsert {
2526 return false;
2527 }
2528 match schema.table(&op.table) {
2529 None => true,
2530 Some(table) => op.values.as_ref().is_some_and(|values| {
2531 values
2532 .keys()
2533 .any(|key| !table.columns.iter().any(|c| &c.name == key))
2534 }),
2535 }
2536 })
2537 })
2538 .cloned()
2539 .collect::<Vec<_>>();
2540 if incompatible.is_empty() {
2541 return Ok(false);
2542 }
2543 let mut rejections = Vec::new();
2544 for commit in &incompatible {
2545 let results = commit
2546 .ops
2547 .iter()
2548 .enumerate()
2549 .map(|(op_index, operation)| {
2550 let rejection = RejectionRecord {
2551 client_commit_id: commit.client_commit_id.clone(),
2552 op_index: op_index as i32,
2553 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
2554 message: "the persisted commit cannot encode under the current schema"
2555 .to_owned(),
2556 retryable: false,
2557 details: None,
2558 operation: Some(CommitOperation::from(operation)),
2559 };
2560 rejections.push(rejection.clone());
2561 CommitOperationOutcome::Error { rejection }
2562 })
2563 .collect::<Vec<_>>();
2564 self.persist_commit_outcome(
2565 &commit.client_commit_id,
2566 CommitOutcomeStatus::Rejected,
2567 &results,
2568 Some(&commit.ops),
2569 )?;
2570 self.delete_outbox_persisted(&commit.client_commit_id)?;
2571 }
2572 self.prune_commit_outcomes()?;
2573 let incompatible_ids = incompatible
2574 .iter()
2575 .map(|commit| commit.client_commit_id.as_str())
2576 .collect::<BTreeSet<_>>();
2577 self.outbox
2578 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
2579 self.rejections.extend(rejections);
2580 Ok(true)
2581 }
2582
2583 fn create_synced_tables(&self) -> Result<(), String> {
2586 for table in &self.schema.tables {
2587 for (full, index_prefix) in [
2591 (base_table(&table.name), "_syncular_base_"),
2592 (visible_table(&table.name), ""),
2593 ] {
2594 let mut cols: Vec<String> =
2595 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
2596 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
2597 let sql = format!(
2598 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
2599 cols.join(", "),
2600 quote_ident(&table.primary_key)
2601 );
2602 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
2603 for index in &table.indexes {
2607 let unique = if index.unique { "UNIQUE " } else { "" };
2608 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
2609 let cols_sql = index
2610 .columns
2611 .iter()
2612 .map(|c| quote_ident(c))
2613 .collect::<Vec<_>>()
2614 .join(", ");
2615 let index_sql = format!(
2616 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
2617 );
2618 self.conn
2619 .execute(&index_sql, [])
2620 .map_err(|e| e.to_string())?;
2621 }
2622 }
2623 }
2624 for table in &self.schema.tables {
2627 for index in &table.fts_indexes {
2628 self.create_fts_projection(table, index)?;
2629 }
2630 }
2631 Ok(())
2632 }
2633
2634 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
2635 let count: i64 = self
2636 .conn
2637 .query_row(
2638 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
2639 rusqlite::params![index.name],
2640 |row| row.get(0),
2641 )
2642 .map_err(|e| e.to_string())?;
2643 Ok(count > 0)
2644 }
2645
2646 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
2647 for suffix in ["ai", "ad", "au"] {
2648 self.conn
2649 .execute(
2650 &format!(
2651 "DROP TRIGGER IF EXISTS {}",
2652 quote_ident(&format!("{}_{suffix}", index.name))
2653 ),
2654 [],
2655 )
2656 .map_err(|e| e.to_string())?;
2657 }
2658 Ok(())
2659 }
2660
2661 fn create_fts_triggers(
2662 &self,
2663 table: &TableSchema,
2664 index: &FtsIndexSchema,
2665 ) -> Result<(), String> {
2666 let fts = quote_ident(&index.name);
2667 let source = visible_table(&table.name);
2668 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2669 let pk = quote_ident(&table.primary_key);
2670 let projection_columns = std::iter::once(source_id.clone())
2671 .chain(index.columns.iter().map(|column| quote_ident(column)))
2672 .collect::<Vec<_>>()
2673 .join(", ");
2674 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
2675 .chain(
2676 index
2677 .columns
2678 .iter()
2679 .map(|column| format!("new.{}", quote_ident(column))),
2680 )
2681 .collect::<Vec<_>>()
2682 .join(", ");
2683 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
2684 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
2685 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
2686 let sql = format!(
2687 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
2688 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
2689 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
2690 ai = quote_ident(&format!("{}_ai", index.name)),
2691 ad = quote_ident(&format!("{}_ad", index.name)),
2692 au = quote_ident(&format!("{}_au", index.name)),
2693 );
2694 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2695 }
2696
2697 fn rebuild_fts_projection(
2698 &self,
2699 table: &TableSchema,
2700 index: &FtsIndexSchema,
2701 ) -> Result<(), String> {
2702 let fts = quote_ident(&index.name);
2703 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2704 let pk = quote_ident(&table.primary_key);
2705 let indexed_columns = index
2706 .columns
2707 .iter()
2708 .map(|column| quote_ident(column))
2709 .collect::<Vec<_>>();
2710 let projection_columns = std::iter::once(source_id)
2711 .chain(indexed_columns.iter().cloned())
2712 .collect::<Vec<_>>()
2713 .join(", ");
2714 let sql = format!(
2715 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
2716 columns = indexed_columns.join(", "),
2717 source = visible_table(&table.name),
2718 );
2719 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2720 }
2721
2722 fn create_fts_projection(
2723 &self,
2724 table: &TableSchema,
2725 index: &FtsIndexSchema,
2726 ) -> Result<(), String> {
2727 let existed = self.fts_projection_exists(index)?;
2728 let tokenizer = index.tokenize.replace('\'', "''");
2729 let columns = index
2730 .columns
2731 .iter()
2732 .map(|column| quote_ident(column))
2733 .collect::<Vec<_>>()
2734 .join(", ");
2735 let sql = format!(
2736 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
2737 fts = quote_ident(&index.name),
2738 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
2739 );
2740 self.conn.execute(&sql, []).map_err(|error| {
2741 format!(
2742 "cannot create local FTS5 projection {:?}: {error}",
2743 index.name
2744 )
2745 })?;
2746 self.create_fts_triggers(table, index)?;
2747 if !existed {
2748 self.rebuild_fts_projection(table, index)?;
2749 }
2750 Ok(())
2751 }
2752
2753 fn persist_sub(&self, sub: &Subscription) {
2756 let state = serde_json::json!({
2757 "requested": scope_map_to_json(&sub.requested),
2758 "params": sub.params,
2759 "cursor": sub.cursor,
2760 "bootstrapState": sub.bootstrap_state,
2761 "status": sub.state.name(),
2762 "reasonCode": sub.reason_code,
2763 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2764 "syncedOnce": sub.synced_once,
2765 });
2766 let _ = self.conn.execute(
2767 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
2768 rusqlite::params![sub.id, sub.table, state.to_string()],
2769 );
2770 }
2771
2772 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
2773 let ops: Vec<Value> = commit
2774 .ops
2775 .iter()
2776 .map(|op| {
2777 serde_json::json!({
2778 "op": if op.upsert { "upsert" } else { "delete" },
2779 "table": op.table,
2780 "rowId": op.row_id,
2781 "baseVersion": op.base_version,
2782 "values": op.values.clone().map(Value::Object),
2783 "changedFields": op.changed_fields,
2784 })
2785 })
2786 .collect();
2787 let _ = self.conn.execute(
2788 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
2789 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
2790 );
2791 }
2792
2793 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
2794 self.conn
2795 .execute(
2796 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
2797 rusqlite::params![client_commit_id],
2798 )
2799 .map(|_| ())
2800 .map_err(|error| error.to_string())
2801 }
2802
2803 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
2804 match status {
2805 CommitOutcomeStatus::Applied => "applied",
2806 CommitOutcomeStatus::Cached => "cached",
2807 CommitOutcomeStatus::Conflict => "conflict",
2808 CommitOutcomeStatus::Rejected => "rejected",
2809 }
2810 }
2811
2812 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
2813 match resolution {
2814 CommitOutcomeResolution::Active => "active",
2815 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
2816 CommitOutcomeResolution::Superseded => "superseded",
2817 CommitOutcomeResolution::Dismissed => "dismissed",
2818 }
2819 }
2820
2821 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
2822 match value {
2823 "applied" => Ok(CommitOutcomeStatus::Applied),
2824 "cached" => Ok(CommitOutcomeStatus::Cached),
2825 "conflict" => Ok(CommitOutcomeStatus::Conflict),
2826 "rejected" => Ok(CommitOutcomeStatus::Rejected),
2827 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
2828 }
2829 }
2830
2831 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
2832 match value {
2833 "active" => Ok(CommitOutcomeResolution::Active),
2834 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
2835 "superseded" => Ok(CommitOutcomeResolution::Superseded),
2836 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
2837 _ => Err(format!(
2838 "invalid persisted commit outcome resolution {value:?}"
2839 )),
2840 }
2841 }
2842
2843 fn persist_commit_outcome(
2844 &self,
2845 client_commit_id: &str,
2846 status: CommitOutcomeStatus,
2847 results: &[CommitOperationOutcome],
2848 operations: Option<&[OutboxOp]>,
2849 ) -> Result<(), String> {
2850 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
2851 let operations_json = operations
2852 .map(|items| {
2853 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
2854 })
2855 .transpose()
2856 .map_err(|error| error.to_string())?;
2857 self.conn
2858 .execute(
2859 "INSERT INTO _syncular_commit_outcomes (
2860 client_commit_id, status, recorded_at_ms, results_json,
2861 operations_json, resolution
2862 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
2863 rusqlite::params![
2864 client_commit_id,
2865 Self::outcome_status_name(status),
2866 self.clock_now_ms(),
2867 results_json,
2868 operations_json
2869 ],
2870 )
2871 .map(|_| ())
2872 .map_err(|error| error.to_string())
2873 }
2874
2875 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
2876 let StoredCommitOutcomeRow {
2877 sequence,
2878 client_commit_id,
2879 status,
2880 recorded_at_ms,
2881 results_json,
2882 operations_json,
2883 resolution,
2884 resolved_at_ms,
2885 replacement_client_commit_id,
2886 } = row;
2887 Ok(CommitOutcome {
2888 sequence,
2889 client_commit_id,
2890 status: Self::parse_outcome_status(&status)?,
2891 recorded_at_ms,
2892 results: serde_json::from_str(&results_json)
2893 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
2894 operations: operations_json
2895 .map(|value| {
2896 serde_json::from_str(&value).map_err(|error| {
2897 format!("invalid persisted commit outcome operations: {error}")
2898 })
2899 })
2900 .transpose()?,
2901 resolution: Self::parse_outcome_resolution(&resolution)?,
2902 resolved_at_ms,
2903 replacement_client_commit_id,
2904 })
2905 }
2906
2907 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
2908 let row = self
2909 .conn
2910 .query_row(
2911 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2912 resolution, resolved_at_ms, replacement_client_commit_id
2913 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
2914 rusqlite::params![client_commit_id],
2915 |row| {
2916 Ok(StoredCommitOutcomeRow {
2917 sequence: row.get(0)?,
2918 client_commit_id: row.get(1)?,
2919 status: row.get(2)?,
2920 recorded_at_ms: row.get(3)?,
2921 results_json: row.get(4)?,
2922 operations_json: row.get(5)?,
2923 resolution: row.get(6)?,
2924 resolved_at_ms: row.get(7)?,
2925 replacement_client_commit_id: row.get(8)?,
2926 })
2927 },
2928 )
2929 .optional()
2930 .map_err(|error| error.to_string())?;
2931 row.map(Self::outcome_from_row).transpose()
2932 }
2933
2934 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
2935 if query.limit == Some(0) {
2936 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
2937 }
2938 let mut sql = String::from(
2939 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2940 resolution, resolved_at_ms, replacement_client_commit_id
2941 FROM _syncular_commit_outcomes",
2942 );
2943 if query.active_only {
2944 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
2945 }
2946 sql.push_str(" ORDER BY seq DESC");
2947 if let Some(limit) = query.limit {
2948 sql.push_str(&format!(" LIMIT {limit}"));
2949 }
2950 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
2951 let rows = stmt
2952 .query_map([], |row| {
2953 Ok(StoredCommitOutcomeRow {
2954 sequence: row.get(0)?,
2955 client_commit_id: row.get(1)?,
2956 status: row.get(2)?,
2957 recorded_at_ms: row.get(3)?,
2958 results_json: row.get(4)?,
2959 operations_json: row.get(5)?,
2960 resolution: row.get(6)?,
2961 resolved_at_ms: row.get(7)?,
2962 replacement_client_commit_id: row.get(8)?,
2963 })
2964 })
2965 .map_err(|error| error.to_string())?;
2966 let mut outcomes = Vec::new();
2967 for row in rows {
2968 outcomes.push(Self::outcome_from_row(
2969 row.map_err(|error| error.to_string())?,
2970 )?);
2971 }
2972 Ok(outcomes)
2973 }
2974
2975 fn prune_commit_outcomes(&self) -> Result<(), String> {
2976 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
2977 let count = self
2978 .conn
2979 .query_row(
2980 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
2981 [],
2982 |row| row.get::<_, i64>(0),
2983 )
2984 .map_err(|error| error.to_string())? as usize;
2985 let excess = count.saturating_sub(max_entries);
2986 if excess == 0 {
2987 return Ok(());
2988 }
2989 let mut stmt = self
2990 .conn
2991 .prepare(
2992 "SELECT seq FROM _syncular_commit_outcomes
2993 WHERE status IN ('applied', 'cached') OR resolution != 'active'
2994 ORDER BY seq ASC LIMIT ?1",
2995 )
2996 .map_err(|error| error.to_string())?;
2997 let rows = stmt
2998 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
2999 .map_err(|error| error.to_string())?;
3000 let sequences = rows
3001 .collect::<Result<Vec<_>, _>>()
3002 .map_err(|error| error.to_string())?;
3003 drop(stmt);
3004 for sequence in sequences {
3005 self.conn
3006 .execute(
3007 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
3008 rusqlite::params![sequence],
3009 )
3010 .map_err(|error| error.to_string())?;
3011 }
3012 Ok(())
3013 }
3014
3015 pub fn subscribe(
3018 &mut self,
3019 id: String,
3020 table: String,
3021 scopes: Vec<(String, Vec<String>)>,
3022 params: Option<String>,
3023 ) -> Result<(), String> {
3024 if self.schema.table(&table).is_none() {
3025 return Err(format!("unknown table {table:?}"));
3026 }
3027 let sub = Subscription {
3028 id: id.clone(),
3029 table,
3030 requested: scopes,
3031 params,
3032 cursor: -1,
3033 bootstrap_state: None,
3034 state: SubState::Active,
3035 reason_code: None,
3036 effective: None,
3037 synced_once: false,
3038 };
3039 self.persist_sub(&sub);
3040 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
3041 *existing = sub;
3042 } else {
3043 self.subs.push(sub);
3044 }
3045 Ok(())
3046 }
3047
3048 pub fn unsubscribe(&mut self, id: &str) {
3049 self.subs.retain(|s| s.id != id);
3050 let _ = self.conn.execute(
3051 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
3052 rusqlite::params![id],
3053 );
3054 }
3055
3056 pub fn set_window(
3064 &mut self,
3065 base: &WindowBase,
3066 units: &[String],
3067 ) -> Result<CommandEffects, String> {
3068 let table = self
3069 .schema
3070 .table(&base.table)
3071 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
3072 if table.scope_column(&base.variable).is_none() {
3073 return Err(format!(
3074 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
3075 base.table, base.variable
3076 ));
3077 }
3078 let base_key = window_base_key(base);
3079 let wanted: std::collections::HashSet<&String> = units.iter().collect();
3080 let live = self.load_window_units(&base_key);
3081 self.begin_observation("syncular_window")?;
3082 let mut batch = ChangeAccumulator::default();
3083 let mut changed = false;
3084
3085 for unit in units {
3087 if live.iter().any(|(u, _)| u == unit) {
3088 continue;
3089 }
3090 let sub_id = derive_sub_id(base, unit);
3091 self.delete_pending_evict(&sub_id);
3092 self.insert_window_unit(&base_key, unit, &sub_id);
3093 self.subscribe(
3094 sub_id,
3095 base.table.clone(),
3096 unit_scopes(base, unit),
3097 base.params.clone(),
3098 )?;
3099 batch.window(&base_key, &base.table, unit);
3100 changed = true;
3101 }
3102
3103 for (unit, sub_id) in live {
3105 if wanted.contains(&unit) {
3106 continue;
3107 }
3108 let effective = self
3109 .subs
3110 .iter()
3111 .find(|sub| sub.id == sub_id)
3112 .and_then(|sub| sub.effective.clone())
3113 .unwrap_or_else(|| unit_scopes(base, &unit));
3114 self.record_scope_map(&mut batch, &base.table, &effective);
3115 batch.window(&base_key, &base.table, &unit);
3116 self.evict_unit(&base_key, base, &unit, &sub_id);
3117 changed = true;
3118 }
3119 if let Err(error) = self.finish_observation("syncular_window", batch) {
3120 self.rollback_observation("syncular_window");
3121 return Err(error);
3122 }
3123 Ok(if changed {
3124 CommandEffects::interactive()
3125 } else {
3126 CommandEffects::none()
3127 })
3128 }
3129
3130 pub fn window_state(&self, base: &WindowBase) -> WindowState {
3135 let mut units = Vec::new();
3136 let mut pending = Vec::new();
3137 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
3138 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
3139 Some(sub) => {
3140 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
3141 }
3142 None => true,
3143 };
3144 if is_pending {
3145 pending.push(unit.clone());
3146 }
3147 units.push(unit);
3148 }
3149 WindowState { units, pending }
3150 }
3151
3152 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
3153 let mut stmt = match self
3154 .conn
3155 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
3156 {
3157 Ok(stmt) => stmt,
3158 Err(_) => return Vec::new(),
3159 };
3160 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
3161 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
3162 });
3163 match rows {
3164 Ok(rows) => rows.filter_map(Result::ok).collect(),
3165 Err(_) => Vec::new(),
3166 }
3167 }
3168
3169 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
3170 let mut stmt = match self.conn.prepare(
3171 "SELECT windows.base, windows.unit, subscriptions.tbl
3172 FROM _syncular_windows AS windows
3173 JOIN _syncular_subscriptions AS subscriptions
3174 ON subscriptions.id = windows.sub_id
3175 ORDER BY windows.base, windows.unit",
3176 ) {
3177 Ok(stmt) => stmt,
3178 Err(_) => return Vec::new(),
3179 };
3180 let rows = stmt.query_map([], |row| {
3181 Ok((
3182 row.get::<_, String>(0)?,
3183 row.get::<_, String>(1)?,
3184 row.get::<_, String>(2)?,
3185 ))
3186 });
3187 match rows {
3188 Ok(rows) => rows.filter_map(Result::ok).collect(),
3189 Err(_) => Vec::new(),
3190 }
3191 }
3192
3193 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
3194 self.conn
3195 .query_row(
3196 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
3197 rusqlite::params![sub_id],
3198 |row| Ok((row.get(0)?, row.get(1)?)),
3199 )
3200 .ok()
3201 }
3202
3203 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
3204 let _ = self.conn.execute(
3205 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
3206 rusqlite::params![base_key, unit, sub_id],
3207 );
3208 }
3209
3210 fn delete_window_unit(&self, base_key: &str, unit: &str) {
3211 let _ = self.conn.execute(
3212 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
3213 rusqlite::params![base_key, unit],
3214 );
3215 }
3216
3217 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
3224 let effective = self
3225 .subs
3226 .iter()
3227 .find(|s| s.id == sub_id)
3228 .and_then(|s| s.effective.clone())
3229 .unwrap_or_else(|| unit_scopes(base, unit));
3230 let pinned = self.pinned_row_ids(&base.table);
3231 let deferred = self
3232 .evict_scope_rows(&base.table, &effective, &pinned)
3233 .unwrap_or(false);
3234 self.delete_window_unit(base_key, unit);
3235 self.unsubscribe(sub_id);
3236 if deferred {
3237 self.save_pending_evict(sub_id, &base.table, &effective);
3238 } else {
3239 self.delete_pending_evict(sub_id);
3240 }
3241 self.rebuild_overlay();
3242 }
3243
3244 fn evict_scope_rows(
3248 &mut self,
3249 table_name: &str,
3250 effective: &[(String, Vec<String>)],
3251 pinned: &std::collections::HashSet<String>,
3252 ) -> Result<bool, ()> {
3253 if effective.is_empty() {
3254 return Ok(false);
3255 }
3256 let table = self.schema.table(table_name).ok_or(())?.clone();
3257 let mut clauses = Vec::new();
3258 let mut params: Vec<SqlValue> = Vec::new();
3259 for (variable, values) in effective {
3260 let column = table.scope_column(variable).ok_or(())?;
3261 let placeholders: Vec<String> = values
3262 .iter()
3263 .map(|v| {
3264 params.push(SqlValue::Text(v.clone()));
3265 "?".to_owned()
3266 })
3267 .collect();
3268 clauses.push(format!(
3269 "{} IN ({})",
3270 quote_ident(column),
3271 placeholders.join(", ")
3272 ));
3273 }
3274 let mut sql = format!(
3275 "DELETE FROM {} WHERE {}",
3276 base_table(table_name),
3277 clauses.join(" AND ")
3278 );
3279 if !pinned.is_empty() {
3280 let pk = quote_ident(&table.primary_key);
3281 let holes: Vec<String> = pinned
3282 .iter()
3283 .map(|id| {
3284 params.push(SqlValue::Text(id.clone()));
3285 "?".to_owned()
3286 })
3287 .collect();
3288 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
3289 }
3290 self.overlay_dirty.set(true);
3291 self.conn
3292 .execute(&sql, rusqlite::params_from_iter(params))
3293 .map_err(|_| ())?;
3294 if pinned.is_empty() {
3295 return Ok(false);
3296 }
3297 let mut where_params: Vec<SqlValue> = Vec::new();
3300 let mut where_clauses = Vec::new();
3301 for (variable, values) in effective {
3302 let column = table.scope_column(variable).ok_or(())?;
3303 let placeholders: Vec<String> = values
3304 .iter()
3305 .map(|v| {
3306 where_params.push(SqlValue::Text(v.clone()));
3307 "?".to_owned()
3308 })
3309 .collect();
3310 where_clauses.push(format!(
3311 "{} IN ({})",
3312 quote_ident(column),
3313 placeholders.join(", ")
3314 ));
3315 }
3316 let pk = quote_ident(&table.primary_key);
3317 let select = format!(
3318 "SELECT {} FROM {} WHERE {}",
3319 pk,
3320 base_table(table_name),
3321 where_clauses.join(" AND ")
3322 );
3323 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
3324 let survivors: Vec<String> = stmt
3325 .query_map(rusqlite::params_from_iter(where_params), |row| {
3326 row.get::<_, String>(0)
3327 })
3328 .map_err(|_| ())?
3329 .filter_map(Result::ok)
3330 .collect();
3331 Ok(survivors.iter().any(|id| pinned.contains(id)))
3332 }
3333
3334 fn drain_pending_evictions(&mut self) {
3338 let pending = self.load_pending_evictions();
3339 if pending.is_empty() {
3340 return;
3341 }
3342 for (sub_id, table_name, effective) in pending {
3343 if self.schema.table(&table_name).is_none() {
3344 self.delete_pending_evict(&sub_id);
3345 continue;
3346 }
3347 let pinned = self.pinned_row_ids(&table_name);
3348 let deferred = self
3349 .evict_scope_rows(&table_name, &effective, &pinned)
3350 .unwrap_or(false);
3351 if !deferred {
3352 self.delete_pending_evict(&sub_id);
3353 }
3354 }
3355 self.rebuild_overlay_if_dirty();
3356 }
3357
3358 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
3361 let mut pinned = std::collections::HashSet::new();
3362 for commit in &self.outbox {
3363 for op in &commit.ops {
3364 if op.table == table {
3365 pinned.insert(op.row_id.clone());
3366 }
3367 }
3368 }
3369 pinned
3370 }
3371
3372 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
3373 let _ = self.conn.execute(
3374 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
3375 VALUES (?1, ?2, ?3)",
3376 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
3377 );
3378 }
3379
3380 fn delete_pending_evict(&self, sub_id: &str) {
3381 let _ = self.conn.execute(
3382 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
3383 rusqlite::params![sub_id],
3384 );
3385 }
3386
3387 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
3388 let mut stmt = match self
3389 .conn
3390 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
3391 {
3392 Ok(stmt) => stmt,
3393 Err(_) => return Vec::new(),
3394 };
3395 let rows = stmt.query_map([], |row| {
3396 Ok((
3397 row.get::<_, String>(0)?,
3398 row.get::<_, String>(1)?,
3399 row.get::<_, String>(2)?,
3400 ))
3401 });
3402 let mut out = Vec::new();
3403 if let Ok(rows) = rows {
3404 for entry in rows.filter_map(Result::ok) {
3405 let (sub_id, table, json) = entry;
3406 if let Ok(value) = serde_json::from_str::<Value>(&json) {
3407 if let Ok(effective) = json_to_scope_map(&value) {
3408 out.push((sub_id, table, effective));
3409 }
3410 }
3411 }
3412 }
3413 out
3414 }
3415
3416 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
3418 if mutations.is_empty() {
3419 return Err("a commit must contain at least one operation (§6.1)".to_owned());
3420 }
3421 let mut ops = Vec::with_capacity(mutations.len());
3422 for mutation in mutations {
3423 match mutation {
3424 Mutation::Upsert {
3425 table,
3426 values,
3427 base_version,
3428 } => {
3429 let schema_table = self
3430 .schema
3431 .table(&table)
3432 .ok_or_else(|| format!("unknown table {table:?}"))?;
3433 let values = normalize_values_casing(schema_table, values)?;
3437 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
3438 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
3441 ops.push(OutboxOp {
3442 upsert: true,
3443 table,
3444 row_id,
3445 base_version,
3446 values: Some(values),
3447 changed_fields: None,
3448 });
3449 }
3450 Mutation::Delete {
3451 table,
3452 row_id,
3453 base_version,
3454 } => {
3455 if self.schema.table(&table).is_none() {
3456 return Err(format!("unknown table {table:?}"));
3457 }
3458 ops.push(OutboxOp {
3459 upsert: false,
3460 table,
3461 row_id,
3462 base_version,
3463 values: None,
3464 changed_fields: None,
3465 });
3466 }
3467 }
3468 }
3469 self.record_outbox_commit(ops)
3470 }
3471
3472 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
3473 let commit = OutboxCommit {
3474 client_commit_id: uuid::Uuid::new_v4().to_string(),
3475 ops,
3476 };
3477 self.begin_observation("syncular_mutation")?;
3478 let mut batch = ChangeAccumulator::default();
3479 for op in &commit.ops {
3480 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
3481 if let Some(values) = &op.values {
3482 if let Some(table) = self.schema.table(&op.table) {
3483 for scope in &table.scope_variables {
3484 if let Some(Value::String(value)) = values.get(&scope.column) {
3485 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
3486 precise = true;
3487 }
3488 }
3489 }
3490 }
3491 if !precise {
3492 batch.table(&op.table);
3493 }
3494 }
3495 self.persist_outbox_insert(&commit);
3496 let id = commit.client_commit_id.clone();
3497 self.outbox.push(commit);
3498 self.overlay_dirty.set(true);
3499 self.rebuild_overlay();
3500 batch.status = true;
3501 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
3502 self.rollback_observation("syncular_mutation");
3503 return Err(error);
3504 }
3505 Ok(id)
3506 }
3507
3508 pub fn patch(
3513 &mut self,
3514 table: &str,
3515 row_id: &str,
3516 partial: Map<String, Value>,
3517 base_version: Option<i64>,
3518 ) -> Result<String, String> {
3519 let schema_table = self
3520 .schema
3521 .table(table)
3522 .ok_or_else(|| format!("unknown table {table:?}"))?;
3523 let partial = normalize_values_casing(schema_table, partial)?;
3524 let mut values = self
3525 .read_rows(table)?
3526 .into_iter()
3527 .find(|row| row.row_id == row_id)
3528 .map(|row| row.values)
3529 .ok_or_else(|| {
3530 format!(
3531 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
3532 )
3533 })?;
3534 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
3535 changed_fields.sort();
3536 values.extend(partial);
3537 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
3538 if row_id_from_values != row_id {
3539 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
3540 }
3541 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
3542 self.record_outbox_commit(vec![OutboxOp {
3543 upsert: true,
3544 table: table.to_owned(),
3545 row_id: row_id.to_owned(),
3546 base_version,
3547 values: Some(values),
3548 changed_fields: Some(changed_fields),
3549 }])
3550 }
3551
3552 pub fn pending_commit_ids(&self) -> Vec<String> {
3553 self.outbox
3554 .iter()
3555 .map(|c| c.client_commit_id.clone())
3556 .collect()
3557 }
3558
3559 pub fn conflicts(&self) -> &[ConflictRecord] {
3560 &self.conflicts
3561 }
3562
3563 pub fn rejections(&self) -> &[RejectionRecord] {
3564 &self.rejections
3565 }
3566
3567 pub fn resolve_commit_outcome(
3568 &mut self,
3569 input: ResolveCommitOutcomeInput,
3570 ) -> Result<CommitOutcome, String> {
3571 let current = self
3572 .commit_outcome(&input.client_commit_id)?
3573 .ok_or_else(|| {
3574 format!(
3575 "sync.outcome_not_found: no durable outcome exists for {:?}",
3576 input.client_commit_id
3577 )
3578 })?;
3579 if current.resolution != CommitOutcomeResolution::Active {
3580 return Ok(current);
3581 }
3582 if input.resolution == CommitOutcomeResolution::Active {
3583 return Err("sync.invalid_request: resolution must leave active state".to_owned());
3584 }
3585 match input.resolution {
3586 CommitOutcomeResolution::Superseded => {
3587 let replacement = input
3588 .replacement_client_commit_id
3589 .as_deref()
3590 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
3591 .ok_or_else(|| {
3592 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
3593 .to_owned()
3594 })?;
3595 let _ = replacement;
3596 }
3597 _ if input.replacement_client_commit_id.is_some() => {
3598 return Err(
3599 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
3600 .to_owned(),
3601 );
3602 }
3603 _ => {}
3604 }
3605 let allowed = match current.status {
3606 CommitOutcomeStatus::Conflict => matches!(
3607 input.resolution,
3608 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
3609 ),
3610 CommitOutcomeStatus::Rejected => {
3611 input.resolution == CommitOutcomeResolution::Superseded
3612 }
3613 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
3614 input.resolution == CommitOutcomeResolution::Dismissed
3615 }
3616 };
3617 if !allowed {
3618 return Err(format!(
3619 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
3620 input.resolution, current.status
3621 ));
3622 }
3623
3624 self.begin_observation("syncular_outcome_resolution")?;
3625 let result = (|| {
3626 self.conn
3627 .execute(
3628 "UPDATE _syncular_commit_outcomes
3629 SET resolution = ?1, resolved_at_ms = ?2,
3630 replacement_client_commit_id = ?3
3631 WHERE client_commit_id = ?4 AND resolution = 'active'",
3632 rusqlite::params![
3633 Self::outcome_resolution_name(input.resolution),
3634 self.clock_now_ms(),
3635 input.replacement_client_commit_id,
3636 input.client_commit_id
3637 ],
3638 )
3639 .map_err(|error| error.to_string())?;
3640 let resolved = self
3641 .commit_outcome(¤t.client_commit_id)?
3642 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
3643 self.prune_commit_outcomes()?;
3644 Ok(resolved)
3645 })();
3646 let resolved = match result {
3647 Ok(outcome) => outcome,
3648 Err(error) => {
3649 self.rollback_observation("syncular_outcome_resolution");
3650 return Err(error);
3651 }
3652 };
3653 self.conflicts
3654 .retain(|record| record.client_commit_id != current.client_commit_id);
3655 self.rejections
3656 .retain(|record| record.client_commit_id != current.client_commit_id);
3657 let batch = ChangeAccumulator {
3658 conflicts: current.status == CommitOutcomeStatus::Conflict,
3659 rejections: current.status == CommitOutcomeStatus::Rejected,
3660 outcomes: true,
3661 ..ChangeAccumulator::default()
3662 };
3663 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
3664 self.rollback_observation("syncular_outcome_resolution");
3665 return Err(error);
3666 }
3667 Ok(resolved)
3668 }
3669
3670 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
3671 self.schema_floor.as_ref()
3672 }
3673
3674 pub fn lease_state(&self) -> Option<&LeaseState> {
3676 self.lease_state.as_ref()
3677 }
3678
3679 fn record_lease_error(&mut self, code: &str) {
3682 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
3683 return;
3684 }
3685 let mut next = self.lease_state.clone().unwrap_or_default();
3686 next.error_code = Some(code.to_owned());
3687 self.set_lease_state(Some(next));
3688 }
3689
3690 fn set_lease_state(&mut self, next: Option<LeaseState>) {
3691 if self.lease_state == next {
3692 return;
3693 }
3694 if self.begin_observation("syncular_lease").is_err() {
3695 return;
3696 }
3697 self.lease_state = next;
3698 if let Some(lease) = &self.lease_state {
3699 if let Ok(json) = serde_json::to_string(lease) {
3700 self.set_meta(LEASE_STATE_KEY, &json);
3701 }
3702 } else {
3703 self.delete_meta(LEASE_STATE_KEY);
3704 }
3705 let batch = ChangeAccumulator {
3706 status: true,
3707 ..ChangeAccumulator::default()
3708 };
3709 if self.finish_observation("syncular_lease", batch).is_err() {
3710 self.rollback_observation("syncular_lease");
3711 }
3712 }
3713
3714 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
3715 if self.schema_floor == next {
3716 return;
3717 }
3718 if self.begin_observation("syncular_schema_floor").is_err() {
3719 return;
3720 }
3721 self.schema_floor = next;
3722 self.stopped = self.schema_floor.is_some();
3723 if let Some(floor) = &self.schema_floor {
3724 if let Ok(json) = serde_json::to_string(floor) {
3725 self.set_meta(SCHEMA_FLOOR_KEY, &json);
3726 }
3727 } else {
3728 self.delete_meta(SCHEMA_FLOOR_KEY);
3729 }
3730 let batch = ChangeAccumulator {
3731 status: true,
3732 ..ChangeAccumulator::default()
3733 };
3734 if self
3735 .finish_observation("syncular_schema_floor", batch)
3736 .is_err()
3737 {
3738 self.rollback_observation("syncular_schema_floor");
3739 }
3740 }
3741
3742 fn set_upgrading(&mut self, value: bool) {
3743 if self.upgrading == value {
3744 return;
3745 }
3746 if self.begin_observation("syncular_upgrading").is_err() {
3747 return;
3748 }
3749 self.upgrading = value;
3750 let batch = ChangeAccumulator {
3751 status: true,
3752 ..ChangeAccumulator::default()
3753 };
3754 if self
3755 .finish_observation("syncular_upgrading", batch)
3756 .is_err()
3757 {
3758 self.rollback_observation("syncular_upgrading");
3759 }
3760 }
3761
3762 pub fn sync_needed(&self) -> bool {
3763 self.sync_needed
3764 }
3765
3766 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
3767 let sub = self.subs.iter().find(|s| s.id == id)?;
3768 Some(SubscriptionStateView {
3769 id: sub.id.clone(),
3770 table: sub.table.clone(),
3771 status: sub.state.name().to_owned(),
3772 cursor: sub.cursor,
3773 has_resume_token: sub.bootstrap_state.is_some(),
3774 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3775 reason_code: sub.reason_code.clone(),
3776 })
3777 }
3778
3779 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
3780 let schema_table = self
3781 .schema
3782 .table(table)
3783 .ok_or_else(|| format!("unknown table {table:?}"))?;
3784 let sql = format!(
3785 "SELECT * FROM {} ORDER BY {} ASC",
3786 visible_table(table),
3787 quote_ident(&schema_table.primary_key)
3788 );
3789 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
3790 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
3791 let mut out = Vec::new();
3792 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
3793 let mut values = Map::new();
3794 for (i, column) in schema_table.columns.iter().enumerate() {
3795 let value = row.get_ref(i).map_err(|e| e.to_string())?;
3796 values.insert(column.name.clone(), sql_ref_to_json(column, value));
3797 }
3798 let version: i64 = row
3799 .get(schema_table.columns.len())
3800 .map_err(|e| e.to_string())?;
3801 let row_id = match values.get(&schema_table.primary_key) {
3802 Some(Value::String(s)) => s.clone(),
3803 Some(Value::Number(n)) => n.to_string(),
3804 Some(Value::Bool(b)) => b.to_string(),
3805 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
3806 };
3807 out.push(RowState {
3808 row_id,
3809 version,
3810 values,
3811 });
3812 }
3813 Ok(out)
3814 }
3815
3816 #[cfg(feature = "crdt-yjs")]
3832 fn crdt_column_bytes(
3833 &self,
3834 table: &str,
3835 row_id: &str,
3836 column: &str,
3837 ) -> Result<Option<Vec<u8>>, String> {
3838 let schema_table = self
3839 .schema
3840 .table(table)
3841 .ok_or_else(|| format!("unknown table {table:?}"))?;
3842 let col = schema_table
3843 .columns
3844 .iter()
3845 .find(|c| c.name == column)
3846 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
3847 if col.ty != ColumnType::Crdt {
3848 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
3849 }
3850 let sql = format!(
3851 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
3852 quote_ident(column),
3853 visible_table(table),
3854 quote_ident(&schema_table.primary_key)
3855 );
3856 let bytes: Option<Vec<u8>> = self
3857 .conn
3858 .query_row(&sql, rusqlite::params![row_id], |row| {
3859 row.get::<_, Option<Vec<u8>>>(0)
3860 })
3861 .map_err(|e| match e {
3862 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
3863 other => other.to_string(),
3864 })?;
3865 Ok(bytes)
3866 }
3867
3868 #[cfg(feature = "crdt-yjs")]
3872 pub fn crdt_text(
3873 &self,
3874 table: &str,
3875 row_id: &str,
3876 column: &str,
3877 name: &str,
3878 ) -> Result<String, String> {
3879 let bytes = self
3880 .crdt_column_bytes(table, row_id, column)?
3881 .unwrap_or_default();
3882 crate::crdt::text(&bytes, name)
3883 }
3884
3885 #[cfg(feature = "crdt-yjs")]
3889 pub fn crdt_insert_text(
3890 &mut self,
3891 table: &str,
3892 row_id: &str,
3893 column: &str,
3894 name: &str,
3895 index: u32,
3896 value: &str,
3897 ) -> Result<String, String> {
3898 let current = self
3899 .crdt_column_bytes(table, row_id, column)?
3900 .unwrap_or_default();
3901 let update = crate::crdt::insert_text(¤t, name, index, value)?;
3902 self.crdt_push_update(table, row_id, column, &update)
3903 }
3904
3905 #[cfg(feature = "crdt-yjs")]
3908 pub fn crdt_delete_text(
3909 &mut self,
3910 table: &str,
3911 row_id: &str,
3912 column: &str,
3913 name: &str,
3914 index: u32,
3915 len: u32,
3916 ) -> Result<String, String> {
3917 let current = self
3918 .crdt_column_bytes(table, row_id, column)?
3919 .unwrap_or_default();
3920 let update = crate::crdt::delete_text(¤t, name, index, len)?;
3921 self.crdt_push_update(table, row_id, column, &update)
3922 }
3923
3924 #[cfg(feature = "crdt-yjs")]
3929 pub fn crdt_apply_update(
3930 &mut self,
3931 table: &str,
3932 row_id: &str,
3933 column: &str,
3934 update: &[u8],
3935 ) -> Result<String, String> {
3936 let current = self
3937 .crdt_column_bytes(table, row_id, column)?
3938 .unwrap_or_default();
3939 let next = crate::crdt::apply_update(¤t, update)?;
3940 self.crdt_push_update(table, row_id, column, &next)
3941 }
3942
3943 #[cfg(feature = "crdt-yjs")]
3949 fn crdt_push_update(
3950 &mut self,
3951 table: &str,
3952 row_id: &str,
3953 column: &str,
3954 crdt_bytes: &[u8],
3955 ) -> Result<String, String> {
3956 let schema_table = self
3957 .schema
3958 .table(table)
3959 .ok_or_else(|| format!("unknown table {table:?}"))?
3960 .clone();
3961 let mut values: Map<String, Value> = self
3964 .read_rows(table)?
3965 .into_iter()
3966 .find(|r| r.row_id == row_id)
3967 .map(|r| r.values)
3968 .unwrap_or_else(|| {
3969 let mut map = Map::new();
3970 map.insert(
3971 schema_table.primary_key.clone(),
3972 Value::from(row_id.to_owned()),
3973 );
3974 map
3975 });
3976 let mut bytes_obj = Map::new();
3978 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
3979 values.insert(column.to_owned(), Value::Object(bytes_obj));
3980 self.mutate(vec![Mutation::Upsert {
3981 table: table.to_owned(),
3982 values,
3983 base_version: None,
3984 }])
3985 }
3986
3987 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
4002 query_connection(&self.conn, sql, params)
4003 }
4004
4005 pub fn query_snapshot(
4007 &mut self,
4008 sql: &str,
4009 params: &[QueryValue],
4010 coverage: &[WindowCoverage],
4011 ) -> Result<QuerySnapshot, String> {
4012 snapshot_connection(&self.conn, sql, params, coverage)
4013 }
4014
4015 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
4018 let mut frames = vec![Frame::ReqHeader {
4019 client_id: self.client_id.clone(),
4020 schema_version: self.schema.version,
4021 }];
4022 let mut pushed_ids = Vec::new();
4023 let mut ops_in_request = 0usize;
4024 let mut deferred_commits = 0usize;
4025 for (index, commit) in self.outbox.iter().enumerate() {
4026 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
4031 deferred_commits = self.outbox.len() - index;
4032 break;
4033 }
4034 ops_in_request += commit.ops.len();
4035 let operations = commit
4036 .ops
4037 .iter()
4038 .map(|op| {
4039 let payload = op.values.as_ref().and_then(|values| {
4040 let table = self.schema.table(&op.table)?;
4041 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
4045 });
4046 ssp2::model::Operation {
4047 table: op.table.clone(),
4048 row_id: op.row_id.clone(),
4049 op: if op.upsert { Op::Upsert } else { Op::Delete },
4050 base_version: op.base_version,
4051 payload,
4052 }
4053 })
4054 .collect();
4055 frames.push(Frame::PushCommit {
4056 client_commit_id: commit.client_commit_id.clone(),
4057 operations,
4058 });
4059 pushed_ids.push(commit.client_commit_id.clone());
4060 }
4061 let accept = self.limits.accept.unwrap_or(if url_capable {
4064 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
4065 } else {
4066 DEFAULT_ACCEPT
4067 });
4068 frames.push(Frame::PullHeader {
4069 limit_commits: self.limits.limit_commits.unwrap_or(0),
4070 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
4071 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
4072 accept,
4073 });
4074 let mut fresh = Vec::new();
4075 for sub in &self.subs {
4076 if sub.state != SubState::Active {
4077 continue;
4078 }
4079 let mut scopes = sub.requested.clone();
4080 sort_scope_map(&mut scopes);
4081 frames.push(Frame::Subscription {
4082 id: sub.id.clone(),
4083 table: sub.table.clone(),
4084 scopes,
4085 params: sub.params.clone().map(RawJson),
4086 cursor: sub.cursor,
4087 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
4088 });
4089 fresh.push((
4090 sub.id.clone(),
4091 sub.cursor < 0 && sub.bootstrap_state.is_none(),
4092 ));
4093 }
4094 let message = Message {
4095 msg_kind: MsgKind::Request,
4096 frames,
4097 };
4098 (
4099 message,
4100 RequestMeta {
4101 pushed_ids,
4102 fresh,
4103 accept,
4104 deferred_commits,
4105 },
4106 )
4107 }
4108
4109 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
4112 if self.stopped {
4113 return SyncOutcome::Ok(SyncReport {
4116 schema_floor: self.schema_floor.clone(),
4117 ..SyncReport::default()
4118 });
4119 }
4120 self.set_sync_needed(false, false);
4123 if self.schema_has_blobs() {
4126 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
4127 if Self::retryable_transport_code(&code) {
4128 self.schedule_background_retry();
4129 }
4130 return SyncOutcome::Failed {
4131 error_code: code,
4132 message,
4133 };
4134 }
4135 }
4136 let (message, meta) = self.build_request(transport.supports_url_fetch());
4137 let request_bytes = encode_message(&message);
4138 let round = if self.realtime_connected {
4142 transport.realtime_sync(&request_bytes)
4143 } else {
4144 transport.sync(&request_bytes)
4145 };
4146 let response_bytes = match round {
4147 Ok(bytes) => bytes,
4148 Err(TransportError { code, message }) => {
4149 self.record_lease_error(&code);
4152 if Self::retryable_transport_code(&code) {
4153 self.schedule_background_retry();
4154 }
4155 return SyncOutcome::Failed {
4156 error_code: code,
4157 message,
4158 };
4159 }
4160 };
4161 let response = match decode_message(&response_bytes) {
4162 Ok(message) => message,
4163 Err(error) => {
4164 return SyncOutcome::Failed {
4167 error_code: error.code.as_str().to_owned(),
4168 message: error.detail,
4169 };
4170 }
4171 };
4172 if response.msg_kind != MsgKind::Response {
4173 return SyncOutcome::Failed {
4174 error_code: "sync.invalid_request".to_owned(),
4175 message: "expected a response message".to_owned(),
4176 };
4177 }
4178 let outcome = self.process_response(transport, response, &meta);
4179 match &outcome {
4180 SyncOutcome::Ok(_) => self.reset_background_retry(),
4181 SyncOutcome::Failed { error_code, .. }
4182 if Self::retryable_transport_code(error_code) =>
4183 {
4184 self.schedule_background_retry();
4185 }
4186 SyncOutcome::Failed { .. } => {}
4187 }
4188 if meta.deferred_commits > 0 {
4189 self.set_sync_needed(true, true);
4192 }
4193 outcome
4194 }
4195
4196 pub fn sync_until_idle(
4197 &mut self,
4198 transport: &mut dyn Transport,
4199 max_rounds: Option<u32>,
4200 ) -> SyncOutcome {
4201 let rounds = max_rounds.unwrap_or(12).max(1);
4202 let mut aggregate = SyncReport::default();
4203 for _ in 0..rounds {
4204 match self.sync(transport) {
4205 SyncOutcome::Failed {
4206 error_code,
4207 message,
4208 } => {
4209 return SyncOutcome::Failed {
4210 error_code,
4211 message,
4212 };
4213 }
4214 SyncOutcome::Ok(report) => {
4215 aggregate.pushed += report.pushed;
4216 aggregate.applied.extend(report.applied.iter().cloned());
4217 aggregate.rejected.extend(report.rejected.iter().cloned());
4218 aggregate.retryable.extend(report.retryable.iter().cloned());
4219 aggregate.conflicts += report.conflicts;
4220 aggregate.commits_applied += report.commits_applied;
4221 aggregate.segment_rows_applied += report.segment_rows_applied;
4222 aggregate.bootstrapping = report.bootstrapping.clone();
4223 aggregate.resets.extend(report.resets.iter().cloned());
4224 aggregate.revoked.extend(report.revoked.iter().cloned());
4225 aggregate.failed.extend(report.failed.iter().cloned());
4226 if report.schema_floor.is_some() {
4227 aggregate.schema_floor = report.schema_floor.clone();
4228 }
4229 let more = !report.bootstrapping.is_empty()
4235 || report.commits_applied > 0
4236 || report.segment_rows_applied > 0
4237 || !report.resets.is_empty()
4238 || self.sync_needed;
4239 if !more {
4240 break;
4241 }
4242 }
4243 }
4244 }
4245 SyncOutcome::Ok(aggregate)
4246 }
4247
4248 fn process_response(
4249 &mut self,
4250 transport: &mut dyn Transport,
4251 response: Message,
4252 meta: &RequestMeta,
4253 ) -> SyncOutcome {
4254 let mut report = SyncReport {
4255 pushed: meta.pushed_ids.len() as u32,
4256 ..SyncReport::default()
4257 };
4258 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
4259 HashMap::new();
4260 for frame in &response.frames {
4261 if let Frame::PushResultDetails {
4262 client_commit_id,
4263 entries,
4264 } = frame
4265 {
4266 let details = rejection_details_by_commit
4267 .entry(client_commit_id.clone())
4268 .or_default();
4269 for entry in entries {
4270 let parsed = match RejectionDetails::parse(&entry.details.0) {
4271 Ok(value) => value,
4272 Err(message) => {
4273 return SyncOutcome::Failed {
4274 error_code: "sync.invalid_request".to_owned(),
4275 message,
4276 };
4277 }
4278 };
4279 details.insert(entry.op_index, parsed);
4280 }
4281 }
4282 }
4283 let mut frames = response.frames.into_iter();
4284 match frames.next() {
4285 Some(Frame::RespHeader {
4286 required_schema_version,
4287 latest_schema_version,
4288 }) => {
4289 if let Some(required) = required_schema_version {
4290 let floor = SchemaFloor {
4293 required_schema_version: Some(required),
4294 latest_schema_version,
4295 };
4296 self.set_schema_floor(Some(floor.clone()));
4297 report.schema_floor = Some(floor);
4298 return SyncOutcome::Ok(report);
4299 }
4300 }
4301 _ => {
4302 return SyncOutcome::Failed {
4303 error_code: "sync.invalid_request".to_owned(),
4304 message: "response does not start with RESP_HEADER".to_owned(),
4305 };
4306 }
4307 }
4308
4309 let mut failure: Option<(String, String)> = None;
4310 while let Some(frame) = frames.next() {
4311 match frame {
4312 Frame::PushResult {
4313 client_commit_id,
4314 status,
4315 commit_seq: _,
4316 results,
4317 } => {
4318 self.handle_push_result(
4319 &client_commit_id,
4320 status,
4321 &results,
4322 rejection_details_by_commit.get(&client_commit_id),
4323 &mut report,
4324 );
4325 }
4326 Frame::PushResultDetails { .. } => {}
4327 Frame::SubStart {
4328 id,
4329 status,
4330 reason_code,
4331 effective_scopes,
4332 bootstrap: _,
4333 } => {
4334 let mut body = Vec::new();
4335 let mut sub_end: Option<(i64, Option<String>)> = None;
4336 for inner in frames.by_ref() {
4337 match inner {
4338 Frame::SubEnd {
4339 next_cursor,
4340 bootstrap_state,
4341 } => {
4342 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
4343 break;
4344 }
4345 Frame::Unknown { .. } => {}
4346 other => body.push(other),
4347 }
4348 }
4349 let Some((next_cursor, bootstrap_state)) = sub_end else {
4350 failure = Some((
4351 "sync.invalid_request".to_owned(),
4352 "subscription section without SUB_END".to_owned(),
4353 ));
4354 break;
4355 };
4356 if let Err(SectionError::Abort(code, message)) = self.process_section(
4357 transport,
4358 &id,
4359 status,
4360 &reason_code,
4361 effective_scopes,
4362 body,
4363 next_cursor,
4364 bootstrap_state,
4365 meta,
4366 &mut report,
4367 ) {
4368 failure = Some((code, message));
4369 break;
4370 }
4371 }
4372 Frame::Lease {
4373 lease_id,
4374 expires_at_ms,
4375 } => {
4376 self.set_lease_state(Some(LeaseState {
4379 lease_id: Some(lease_id),
4380 expires_at_ms: Some(expires_at_ms),
4381 error_code: None,
4382 }));
4383 }
4384 Frame::Error { code, message, .. } => {
4385 failure = Some((code, message));
4388 break;
4389 }
4390 Frame::Unknown { .. } => {}
4391 _ => {
4392 failure = Some((
4393 "sync.invalid_request".to_owned(),
4394 "unexpected frame in response".to_owned(),
4395 ));
4396 break;
4397 }
4398 }
4399 }
4400
4401 if self.overlay_dirty.get() {
4404 self.rebuild_overlay();
4405 }
4406 self.reconcile_blob_refcounts(false);
4413
4414 if let Some((error_code, message)) = failure {
4415 return SyncOutcome::Failed {
4416 error_code,
4417 message,
4418 };
4419 }
4420 self.drain_pending_evictions();
4423 self.ack_after_pull(transport);
4424 if self.upgrading && report.bootstrapping.is_empty() {
4427 self.set_upgrading(false);
4428 }
4429 SyncOutcome::Ok(report)
4430 }
4431
4432 fn handle_push_result(
4435 &mut self,
4436 client_commit_id: &str,
4437 status: PushStatus,
4438 results: &[OpResult],
4439 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
4440 report: &mut SyncReport,
4441 ) {
4442 let Some(index) = self
4443 .outbox
4444 .iter()
4445 .position(|c| c.client_commit_id == client_commit_id)
4446 else {
4447 return;
4448 };
4449 if self.begin_observation("syncular_push_result").is_err() {
4450 return;
4451 }
4452 let mut batch = ChangeAccumulator::default();
4453 let operations = self.outbox[index].ops.clone();
4454 match status {
4455 PushStatus::Applied | PushStatus::Cached => {
4456 let journal_results = results
4459 .iter()
4460 .map(|result| {
4461 let op_index = match result {
4462 OpResult::Applied { op_index }
4463 | OpResult::Conflict { op_index, .. }
4464 | OpResult::Error { op_index, .. } => *op_index,
4465 };
4466 CommitOperationOutcome::Applied { op_index }
4467 })
4468 .collect::<Vec<_>>();
4469 let outcome_status = if status == PushStatus::Applied {
4470 CommitOutcomeStatus::Applied
4471 } else {
4472 CommitOutcomeStatus::Cached
4473 };
4474 if self
4475 .persist_commit_outcome(
4476 client_commit_id,
4477 outcome_status,
4478 &journal_results,
4479 None,
4480 )
4481 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4482 .and_then(|()| self.prune_commit_outcomes())
4483 .is_err()
4484 {
4485 self.rollback_observation("syncular_push_result");
4486 return;
4487 }
4488 report.applied.push(client_commit_id.to_owned());
4489 self.outbox.remove(index);
4490 self.overlay_dirty.set(true);
4491 batch.status = true;
4492 batch.outcomes = true;
4493 }
4494 PushStatus::Rejected => {
4495 if results.iter().any(|result| {
4496 matches!(
4497 result,
4498 OpResult::Error {
4499 code,
4500 retryable: true,
4501 ..
4502 } if code == "sync.idempotency_cache_miss"
4503 )
4504 }) {
4505 report.retryable.push(client_commit_id.to_owned());
4508 if self
4509 .finish_observation("syncular_push_result", batch)
4510 .is_err()
4511 {
4512 self.rollback_observation("syncular_push_result");
4513 }
4514 return;
4515 }
4516
4517 let mut journal_results = Vec::with_capacity(results.len());
4518 let mut conflicts = Vec::new();
4519 let mut rejections = Vec::new();
4520 for result in results {
4521 match result {
4522 OpResult::Applied { op_index } => {
4523 journal_results.push(CommitOperationOutcome::Applied {
4524 op_index: *op_index,
4525 });
4526 }
4527 OpResult::Conflict {
4528 op_index,
4529 code,
4530 message,
4531 server_version,
4532 server_row,
4533 } => {
4534 let operation = operations
4535 .get(*op_index as usize)
4536 .map(CommitOperation::from);
4537 let (table, row_id) = operation
4538 .as_ref()
4539 .map(|op| (op.table.clone(), op.row_id.clone()))
4540 .unwrap_or_default();
4541 let server_row_json = self
4542 .schema
4543 .table(&table)
4544 .and_then(|t| {
4545 decode_row_bytes(t, server_row, &self.encryption)
4546 .ok()
4547 .map(|row| (t, row))
4548 })
4549 .map(|(t, row)| {
4550 let mut map = Map::new();
4551 for (i, column) in t.columns.iter().enumerate() {
4552 map.insert(
4553 column.name.clone(),
4554 column_value_to_json(row.get(i).unwrap_or(&None)),
4555 );
4556 }
4557 map
4558 })
4559 .unwrap_or_default();
4560 let conflict = ConflictRecord {
4561 client_commit_id: client_commit_id.to_owned(),
4562 op_index: *op_index,
4563 table,
4564 row_id,
4565 code: code.clone(),
4566 message: message.clone(),
4567 server_version: *server_version,
4568 server_row: server_row_json,
4569 operation,
4570 };
4571 journal_results.push(CommitOperationOutcome::Conflict {
4572 conflict: conflict.clone(),
4573 });
4574 conflicts.push(conflict);
4575 }
4576 OpResult::Error {
4577 op_index,
4578 code,
4579 message,
4580 retryable,
4581 } => {
4582 let rejection = RejectionRecord {
4583 client_commit_id: client_commit_id.to_owned(),
4584 op_index: *op_index,
4585 code: code.clone(),
4586 message: message.clone(),
4587 retryable: *retryable,
4588 details: rejection_details
4589 .and_then(|details| details.get(op_index))
4590 .cloned(),
4591 operation: operations
4592 .get(*op_index as usize)
4593 .map(CommitOperation::from),
4594 };
4595 journal_results.push(CommitOperationOutcome::Error {
4596 rejection: rejection.clone(),
4597 });
4598 rejections.push(rejection);
4599 }
4600 }
4601 }
4602 let outcome_status = if conflicts.is_empty() {
4603 CommitOutcomeStatus::Rejected
4604 } else {
4605 CommitOutcomeStatus::Conflict
4606 };
4607 if self
4608 .persist_commit_outcome(
4609 client_commit_id,
4610 outcome_status,
4611 &journal_results,
4612 Some(&operations),
4613 )
4614 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4615 .and_then(|()| self.prune_commit_outcomes())
4616 .is_err()
4617 {
4618 self.rollback_observation("syncular_push_result");
4619 return;
4620 }
4621 report.conflicts += conflicts.len() as u32;
4622 report.rejected.push(client_commit_id.to_owned());
4623 batch.conflicts = !conflicts.is_empty();
4624 batch.rejections = !rejections.is_empty();
4625 batch.status = true;
4626 batch.outcomes = true;
4627 self.conflicts.extend(conflicts);
4628 self.rejections.extend(rejections);
4629 self.outbox.remove(index);
4630 self.overlay_dirty.set(true);
4631 }
4632 }
4633 if batch.status {
4634 for operation in &operations {
4635 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
4636 batch.table(&operation.table);
4637 }
4638 }
4639 self.rebuild_overlay_if_dirty();
4640 }
4641 if self
4642 .finish_observation("syncular_push_result", batch)
4643 .is_err()
4644 {
4645 self.rollback_observation("syncular_push_result");
4646 }
4647 }
4648
4649 #[allow(clippy::too_many_arguments)]
4652 fn process_section(
4653 &mut self,
4654 transport: &mut dyn Transport,
4655 id: &str,
4656 status: SubStatus,
4657 reason_code: &str,
4658 effective_scopes: Vec<(String, Vec<String>)>,
4659 body: Vec<Frame>,
4660 next_cursor: i64,
4661 bootstrap_state: Option<String>,
4662 meta: &RequestMeta,
4663 report: &mut SyncReport,
4664 ) -> Result<(), SectionError> {
4665 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4666 return Ok(()); };
4668 match status {
4669 SubStatus::Revoked => {
4670 self.begin_observation("syncular_revocation")
4671 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4672 let mut batch = ChangeAccumulator::default();
4673 let registered = self.window_unit_by_sub_id(id);
4674 let (table, effective) = {
4676 let sub = &self.subs[sub_index];
4677 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4678 };
4679 let purged = self.purge_scope_rows(&table, &effective);
4680 match purged {
4681 Ok(()) => {
4682 self.record_scope_map(&mut batch, &table, &effective);
4683 let sub = &mut self.subs[sub_index];
4684 sub.state = SubState::Revoked;
4685 sub.reason_code = Some(if reason_code.is_empty() {
4686 "sync.scope_revoked".to_owned()
4687 } else {
4688 reason_code.to_owned()
4689 });
4690 report.revoked.push(id.to_owned());
4691 let doomed_effective = effective;
4692 let sub_table = table;
4693 self.persist_sub(&self.subs[sub_index].clone());
4694 let dropped = self
4695 .drop_doomed_outbox(&sub_table, &doomed_effective)
4696 .map_err(|message| {
4697 SectionError::Abort("storage.failed".to_owned(), message)
4698 })?;
4699 if dropped {
4700 batch.status = true;
4701 batch.rejections = true;
4702 batch.outcomes = true;
4703 }
4704 self.reconcile_blob_refcounts(true);
4707 }
4708 Err(()) => {
4709 let sub = &mut self.subs[sub_index];
4712 sub.state = SubState::Failed;
4713 sub.reason_code = Some("sync.scope_revoked".to_owned());
4714 report.failed.push(id.to_owned());
4715 self.persist_sub(&self.subs[sub_index].clone());
4716 }
4717 }
4718 if let Some((base_key, unit)) = registered {
4719 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4720 }
4721 self.rebuild_overlay_if_dirty();
4722 self.finish_observation("syncular_revocation", batch)
4723 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4724 Ok(())
4725 }
4726 SubStatus::Reset => {
4727 self.begin_observation("syncular_reset")
4728 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4729 let mut batch = ChangeAccumulator::default();
4730 let registered = self.window_unit_by_sub_id(id);
4731 let sub = &mut self.subs[sub_index];
4734 sub.cursor = -1;
4735 sub.bootstrap_state = None;
4736 report.resets.push(id.to_owned());
4737 self.persist_sub(&self.subs[sub_index].clone());
4738 if let Some((base_key, unit)) = registered {
4739 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4740 }
4741 self.finish_observation("syncular_reset", batch)
4742 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4743 Ok(())
4744 }
4745 SubStatus::Active => {
4746 let fresh = meta
4747 .fresh
4748 .iter()
4749 .find(|(fid, _)| fid == id)
4750 .map(|(_, f)| *f)
4751 .unwrap_or(false);
4752 let was_pending = self.subs[sub_index].cursor < 0
4753 || self.subs[sub_index].bootstrap_state.is_some();
4754 let registered = self.window_unit_by_sub_id(id);
4755 self.subs[sub_index].effective = Some(effective_scopes);
4757 self.begin_observation("syncular_section")
4758 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4759 let mut batch = ChangeAccumulator::default();
4760 let outcome = self.apply_section_body(
4761 transport, sub_index, body, fresh, meta, report, &mut batch,
4762 );
4763 match outcome {
4764 Ok(()) => {
4765 let sub = &mut self.subs[sub_index];
4766 sub.cursor = next_cursor;
4769 sub.bootstrap_state = bootstrap_state;
4770 sub.synced_once = true;
4771 if sub.bootstrap_state.is_some() {
4772 report.bootstrapping.push(id.to_owned());
4773 }
4774 let completed =
4775 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
4776 self.persist_sub(&self.subs[sub_index].clone());
4777 if completed {
4778 if let Some((base_key, unit)) = registered.clone() {
4779 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4780 }
4781 }
4782 self.rebuild_overlay_if_dirty();
4783 self.finish_observation("syncular_section", batch)
4784 .map_err(|message| {
4785 SectionError::Abort("storage.failed".to_owned(), message)
4786 })?;
4787 Ok(())
4788 }
4789 Err(SectionError::FailClosed) => {
4790 self.rollback_observation("syncular_section");
4793 self.begin_observation("syncular_section_failure")
4794 .map_err(|message| {
4795 SectionError::Abort("storage.failed".to_owned(), message)
4796 })?;
4797 let mut failure_batch = ChangeAccumulator::default();
4798 let sub = &mut self.subs[sub_index];
4799 sub.state = SubState::Failed;
4800 sub.reason_code = Some("sync.scope_revoked".to_owned());
4801 report.failed.push(id.to_owned());
4802 self.persist_sub(&self.subs[sub_index].clone());
4803 if let Some((base_key, unit)) = registered {
4804 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
4805 }
4806 self.finish_observation("syncular_section_failure", failure_batch)
4807 .map_err(|message| {
4808 SectionError::Abort("storage.failed".to_owned(), message)
4809 })?;
4810 Ok(())
4811 }
4812 Err(SectionError::Abort(code, message)) => {
4813 self.rollback_observation("syncular_section");
4816 Err(SectionError::Abort(code, message))
4817 }
4818 }
4819 }
4820 }
4821 }
4822
4823 #[allow(clippy::too_many_arguments)]
4827 fn apply_section_body(
4828 &mut self,
4829 transport: &mut dyn Transport,
4830 sub_index: usize,
4831 body: Vec<Frame>,
4832 fresh: bool,
4833 meta: &RequestMeta,
4834 report: &mut SyncReport,
4835 batch: &mut ChangeAccumulator,
4836 ) -> Result<(), SectionError> {
4837 let mut saw_segment = false;
4838 for frame in body {
4839 match frame {
4840 Frame::Commit {
4841 tables, changes, ..
4842 } => {
4843 self.record_commit_changes(batch, &tables, &changes);
4844 self.apply_commit_changes(&tables, &changes)
4845 .map_err(|(c, m)| SectionError::Abort(c, m))?;
4846 report.commits_applied += 1;
4847 }
4848 Frame::SegmentInline { payload } => {
4849 let segment = decode_rows_segment(&payload)
4850 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
4851 let first = !saw_segment;
4852 saw_segment = true;
4853 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4854 let cleared =
4855 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4856 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4857 if applied > 0 || cleared {
4858 batch.table(&segment.table);
4859 }
4860 report.segment_rows_applied += applied;
4861 }
4862 Frame::SegmentRef {
4863 segment_id,
4864 media_type,
4865 table,
4866 row_count,
4867 as_of_commit_seq,
4868 scope_digest,
4869 row_cursor,
4870 next_row_cursor,
4871 url,
4872 url_expires_at_ms,
4873 ..
4874 } => {
4875 let advertised = match media_type {
4878 MediaType::Rows => {
4879 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
4880 || meta.accept & ACCEPT_INLINE_ROWS != 0
4881 }
4882 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
4883 };
4884 if !advertised {
4885 return Err(SectionError::Abort(
4886 "sync.invalid_request".to_owned(),
4887 format!(
4888 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
4889 media_type.name()
4890 ),
4891 ));
4892 }
4893 let bytes = if let Some(url) = url {
4894 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
4899 return Err(SectionError::Abort(
4900 "sync.invalid_request".to_owned(),
4901 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
4902 .to_owned(),
4903 ));
4904 }
4905 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
4907 return Err(SectionError::Abort(
4908 "sync.segment_expired".to_owned(),
4909 format!(
4910 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
4911 ),
4912 ));
4913 }
4914 transport
4915 .fetch_url(&url)
4916 .map_err(|e| SectionError::Abort(e.code, e.message))?
4917 } else {
4918 let requested_scopes_json =
4919 canonical_scope_json(&self.subs[sub_index].requested);
4920 transport
4921 .download_segment(&SegmentRequest {
4922 segment_id: segment_id.clone(),
4923 table,
4924 requested_scopes_json,
4925 })
4926 .map_err(|e| SectionError::Abort(e.code, e.message))?
4927 };
4928 let digest = Sha256::digest(&bytes);
4930 let expected = segment_id
4931 .strip_prefix("sha256:")
4932 .unwrap_or(segment_id.as_str());
4933 if bytes_to_hex(&digest) != expected {
4934 return Err(SectionError::Abort(
4935 "sync.invalid_request".to_owned(),
4936 "segment bytes do not match the content address (§5.1)".to_owned(),
4937 ));
4938 }
4939 if media_type == MediaType::Sqlite {
4940 if row_cursor.is_some() || next_row_cursor.is_some() {
4943 return Err(SectionError::Abort(
4944 "sync.invalid_request".to_owned(),
4945 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
4946 .to_owned(),
4947 ));
4948 }
4949 let first = !saw_segment;
4950 saw_segment = true;
4951 let sub_table = self.subs[sub_index].table.clone();
4952 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4953 let cleared =
4954 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
4955 let applied = self.apply_sqlite_segment(
4956 sub_index,
4957 &bytes,
4958 fresh && first,
4959 row_count,
4960 as_of_commit_seq,
4961 &scope_digest,
4962 )?;
4963 if applied > 0 || cleared {
4964 batch.table(&sub_table);
4965 }
4966 report.segment_rows_applied += applied;
4967 } else {
4968 let segment = decode_rows_segment(&bytes).map_err(|e| {
4969 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
4970 })?;
4971 let first = row_cursor.is_none();
4972 saw_segment = true;
4973 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4974 let cleared =
4975 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4976 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4977 if applied > 0 || cleared {
4978 batch.table(&segment.table);
4979 }
4980 report.segment_rows_applied += applied;
4981 }
4982 }
4983 Frame::Unknown { .. } => {}
4984 _ => {
4985 return Err(SectionError::Abort(
4986 "sync.invalid_request".to_owned(),
4987 "unexpected frame inside a subscription section".to_owned(),
4988 ));
4989 }
4990 }
4991 }
4992 Ok(())
4993 }
4994
4995 fn apply_commit_changes(
4996 &mut self,
4997 tables: &[String],
4998 changes: &[ssp2::model::Change],
4999 ) -> Result<(), (String, String)> {
5000 for change in changes {
5001 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
5002 (
5003 "sync.invalid_request".to_owned(),
5004 "change tableIndex out of range".to_owned(),
5005 )
5006 })?;
5007 let table = self.schema.table(table_name).ok_or_else(|| {
5008 (
5009 "sync.schema_mismatch".to_owned(),
5010 format!("change targets unknown table {table_name:?}"),
5011 )
5012 })?;
5013 match change.op {
5014 Op::Upsert => {
5015 let payload = change.row.as_ref().ok_or_else(|| {
5016 (
5017 "sync.invalid_request".to_owned(),
5018 "upsert change without row payload".to_owned(),
5019 )
5020 })?;
5021 let row = decode_row_bytes(table, payload, &self.encryption)
5023 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
5024 let version = change.row_version.unwrap_or(0);
5025 let table_name = table.name.clone();
5026 self.write_base_row(&table_name, &row, version)
5027 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
5028 }
5029 Op::Delete => {
5030 self.delete_base_row(table_name, &change.row_id)
5031 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
5032 }
5033 }
5034 }
5035 Ok(())
5036 }
5037
5038 fn apply_segment(
5043 &mut self,
5044 sub_index: usize,
5045 segment: &RowsSegment,
5046 first_fresh_page: bool,
5047 ) -> Result<u32, SectionError> {
5048 let (sub_table, effective) = {
5049 let sub = &self.subs[sub_index];
5050 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5051 };
5052 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5053 SectionError::Abort(
5054 "sync.schema_mismatch".to_owned(),
5055 format!("subscription table {sub_table:?} is not in the client schema"),
5056 )
5057 })?;
5058 let matches = segment.table == table.name
5063 && segment.schema_version == self.schema.version
5064 && segment.columns.len() == table.wire_columns.len()
5065 && segment
5066 .columns
5067 .iter()
5068 .zip(table.wire_columns.iter())
5069 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
5070 if !matches {
5071 return Err(SectionError::Abort(
5072 "sync.schema_mismatch".to_owned(),
5073 "segment column table does not match the generated schema (§5.2)".to_owned(),
5074 ));
5075 }
5076 if first_fresh_page {
5077 self.purge_scope_rows(&table.name, &effective)
5081 .map_err(|()| SectionError::FailClosed)?;
5082 }
5083 let mut applied = 0u32;
5084 for block in &segment.blocks {
5085 for row in block {
5086 let decrypted;
5091 let values = if table.has_encrypted_columns() {
5092 let mut values = row.values.clone();
5093 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
5094 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
5095 decrypted = values;
5096 &decrypted
5097 } else {
5098 &row.values
5099 };
5100 self.write_base_row(&table.name, values, row.server_version)
5103 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
5104 applied += 1;
5105 }
5106 }
5107 Ok(applied)
5108 }
5109
5110 fn apply_sqlite_segment(
5118 &mut self,
5119 sub_index: usize,
5120 bytes: &[u8],
5121 first_fresh_page: bool,
5122 row_count: i64,
5123 as_of_commit_seq: i64,
5124 scope_digest: &str,
5125 ) -> Result<u32, SectionError> {
5126 let invalid = |detail: &str| {
5127 SectionError::Abort(
5128 "sync.invalid_request".to_owned(),
5129 format!("sqlite segment rejected: {detail} (§5.3)"),
5130 )
5131 };
5132 let (sub_table, effective) = {
5133 let sub = &self.subs[sub_index];
5134 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5135 };
5136 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5137 SectionError::Abort(
5138 "sync.schema_mismatch".to_owned(),
5139 format!("subscription table {sub_table:?} is not in the client schema"),
5140 )
5141 })?;
5142
5143 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
5144 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
5145 let img = match rusqlite::Connection::open_with_flags(
5146 &path,
5147 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
5148 ) {
5149 Ok(conn) => conn,
5150 Err(_) => {
5151 let _ = std::fs::remove_file(&path);
5152 return Err(invalid("bytes do not open as a SQLite database"));
5153 }
5154 };
5155 let outcome = self.apply_sqlite_image(
5156 &img,
5157 &table,
5158 first_fresh_page,
5159 &effective,
5160 row_count,
5161 as_of_commit_seq,
5162 scope_digest,
5163 );
5164 drop(img);
5165 let _ = std::fs::remove_file(&path);
5166 outcome
5167 }
5168
5169 #[allow(clippy::too_many_arguments)]
5170 fn apply_sqlite_image(
5171 &mut self,
5172 img: &rusqlite::Connection,
5173 table: &crate::schema::TableSchema,
5174 first_fresh_page: bool,
5175 effective: &[(String, Vec<String>)],
5176 row_count: i64,
5177 as_of_commit_seq: i64,
5178 scope_digest: &str,
5179 ) -> Result<u32, SectionError> {
5180 let invalid = |detail: String| {
5181 SectionError::Abort(
5182 "sync.invalid_request".to_owned(),
5183 format!("sqlite segment rejected: {detail} (§5.3)"),
5184 )
5185 };
5186
5187 type MetaRow = (i64, String, i64, i64, String, i64, i64);
5190 let meta: MetaRow = img
5191 .query_row(
5192 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
5193 \"scopeDigest\", \"rowCount\",
5194 (SELECT count(*) FROM _syncular_segment)
5195 FROM _syncular_segment",
5196 [],
5197 |row| {
5198 Ok((
5199 row.get(0)?,
5200 row.get(1)?,
5201 row.get(2)?,
5202 row.get(3)?,
5203 row.get(4)?,
5204 row.get(5)?,
5205 row.get(6)?,
5206 ))
5207 },
5208 )
5209 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
5210 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
5211 if meta_count != 1 {
5212 return Err(invalid(format!(
5213 "_syncular_segment must contain exactly one row, found {meta_count}"
5214 )));
5215 }
5216 if format != 1 {
5217 return Err(invalid(format!("format {format}")));
5218 }
5219 if meta_table != table.name {
5220 return Err(invalid(format!("image table {meta_table:?}")));
5221 }
5222 if schema_version != i64::from(self.schema.version) {
5223 return Err(invalid(format!("schemaVersion {schema_version}")));
5224 }
5225 if pin != as_of_commit_seq {
5226 return Err(invalid(format!("asOfCommitSeq {pin}")));
5227 }
5228 if digest != scope_digest {
5229 return Err(invalid("scopeDigest mismatch".to_owned()));
5230 }
5231 if meta_rows != row_count {
5232 return Err(invalid(format!("rowCount {meta_rows}")));
5233 }
5234
5235 let mut names: Vec<String> = Vec::new();
5237 {
5238 let mut stmt = img
5239 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
5240 .map_err(|_| invalid("image data table missing".to_owned()))?;
5241 let mut rows = stmt
5242 .query([])
5243 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5244 while let Some(row) = rows
5245 .next()
5246 .map_err(|_| invalid("image data table unreadable".to_owned()))?
5247 {
5248 names.push(
5249 row.get::<_, String>(1)
5250 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
5251 );
5252 }
5253 }
5254 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
5255 expected.push("_syncular_version");
5256 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
5257 return Err(SectionError::Abort(
5258 "sync.schema_mismatch".to_owned(),
5259 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
5260 ));
5261 }
5262
5263 if first_fresh_page {
5266 self.purge_scope_rows(&table.name, effective)
5267 .map_err(|()| SectionError::FailClosed)?;
5268 }
5269 self.overlay_dirty.set(true);
5275 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
5283 table.indexes.iter().filter(|i| !i.unique).collect()
5284 } else {
5285 Vec::new()
5286 };
5287 for index in &bulk_indexes {
5288 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
5289 self.conn
5290 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
5291 .map_err(|e| invalid(e.to_string()))?;
5292 }
5293 let insert = self.insert_row_sql(&base_table(&table.name), table);
5294 let applied = {
5295 let mut ins = self
5296 .conn
5297 .prepare_cached(&insert)
5298 .map_err(|e| invalid(e.to_string()))?;
5299 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
5300 let mut stmt = img
5301 .prepare(&format!(
5302 "SELECT {} FROM {}",
5303 column_list.join(", "),
5304 quote_ident(&table.name)
5305 ))
5306 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5307 let mut rows = stmt
5308 .query([])
5309 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5310 let version_index = table.columns.len();
5311 let mut applied = 0u32;
5312 while let Some(row) = rows
5313 .next()
5314 .map_err(|_| invalid("image row unreadable".to_owned()))?
5315 {
5316 for (i, column) in table.columns.iter().enumerate() {
5317 let cell = row
5318 .get_ref(i)
5319 .map_err(|_| invalid("image row unreadable".to_owned()))?;
5320 let param = image_cell_param(column, cell).map_err(&invalid)?;
5321 ins.raw_bind_parameter(i + 1, param)
5322 .map_err(|e| invalid(e.to_string()))?;
5323 }
5324 let version: i64 = row
5325 .get(version_index)
5326 .map_err(|_| invalid("image row unreadable".to_owned()))?;
5327 if version < 1 {
5328 return Err(invalid(format!(
5329 "row _syncular_version must be >= 1, got {version}"
5330 )));
5331 }
5332 ins.raw_bind_parameter(version_index + 1, version)
5333 .map_err(|e| invalid(e.to_string()))?;
5334 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
5335 applied += 1;
5336 }
5337 applied
5338 };
5339 for index in &bulk_indexes {
5340 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
5341 let cols_sql = index
5342 .columns
5343 .iter()
5344 .map(|c| quote_ident(c))
5345 .collect::<Vec<_>>()
5346 .join(", ");
5347 self.conn
5348 .execute(
5349 &format!(
5350 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
5351 base_table(&table.name)
5352 ),
5353 [],
5354 )
5355 .map_err(|e| invalid(e.to_string()))?;
5356 }
5357 if i64::from(applied) != row_count {
5358 return Err(invalid(format!(
5359 "image holds {applied} rows, descriptor says {row_count}"
5360 )));
5361 }
5362 Ok(applied)
5363 }
5364
5365 fn compile_local_data_purge(
5368 &self,
5369 input: &LocalDataPurgeInput,
5370 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
5371 let invalid = |message: String| format!("sync.invalid_request: {message}");
5372 if input.purge_id.is_empty()
5373 || input.purge_id.len() > 128
5374 || !is_local_purge_code_like(&input.purge_id)
5375 {
5376 return Err(invalid(
5377 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
5378 ));
5379 }
5380 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
5381 return Err(invalid(format!(
5382 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
5383 )));
5384 }
5385
5386 let mut deduplicated: BTreeMap<
5387 String,
5388 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
5389 > = BTreeMap::new();
5390 for target in &input.targets {
5391 let table = self.schema.table(&target.table).ok_or_else(|| {
5392 invalid(format!(
5393 "local purge names unknown table {:?}",
5394 target.table
5395 ))
5396 })?;
5397 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
5398 return Err(invalid(format!(
5399 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
5400 target.table
5401 )));
5402 }
5403 let mut selectors = Vec::with_capacity(target.selectors.len());
5404 let mut canonical_selectors = BTreeMap::new();
5405 for (column_name, raw_values) in &target.selectors {
5406 let Some((column_index, column)) = table
5407 .columns
5408 .iter()
5409 .enumerate()
5410 .find(|(_, column)| column.name == *column_name)
5411 else {
5412 return Err(invalid(format!(
5413 "local purge target {:?} names unknown column {:?}",
5414 target.table, column_name
5415 )));
5416 };
5417 let encrypted = table
5418 .encrypted_columns
5419 .iter()
5420 .any(|candidate| candidate.index == column_index);
5421 if column.ty != ColumnType::String || encrypted {
5422 return Err(invalid(format!(
5423 "local purge selector {:?}.{:?} must be a plaintext string column",
5424 target.table, column_name
5425 )));
5426 }
5427 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
5428 return Err(invalid(format!(
5429 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
5430 target.table, column_name
5431 )));
5432 }
5433 let mut values = raw_values.clone();
5434 values.sort();
5435 values.dedup();
5436 if values.iter().any(|value| {
5437 value.is_empty()
5438 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
5439 || !is_local_purge_code_like(value)
5440 }) {
5441 return Err(invalid(format!(
5442 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
5443 )));
5444 }
5445 selectors.push((column_name.clone(), values.clone()));
5446 canonical_selectors.insert(column_name.clone(), values);
5447 }
5448 let canonical = LocalDataPurgeTarget {
5449 table: target.table.clone(),
5450 selectors: canonical_selectors,
5451 };
5452 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
5453 deduplicated.insert(
5454 key,
5455 (
5456 CompiledLocalDataPurgeTarget {
5457 table: target.table.clone(),
5458 selectors,
5459 },
5460 canonical,
5461 ),
5462 );
5463 }
5464 let targets = deduplicated
5465 .values()
5466 .map(|(compiled, _)| compiled.clone())
5467 .collect::<Vec<_>>();
5468 let canonical = deduplicated
5469 .values()
5470 .map(|(_, target)| target.clone())
5471 .collect::<Vec<_>>();
5472 let canonical_plan =
5473 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
5474 Ok((targets, canonical_plan))
5475 }
5476
5477 fn local_purge_base_row_ids(
5478 &self,
5479 targets: &[CompiledLocalDataPurgeTarget],
5480 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
5481 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
5482 for target in targets {
5483 let table = self
5484 .schema
5485 .table(&target.table)
5486 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
5487 let mut clauses = Vec::with_capacity(target.selectors.len());
5488 let mut params = Vec::new();
5489 for (column, values) in &target.selectors {
5490 clauses.push(format!(
5491 "{} IN ({})",
5492 quote_ident(column),
5493 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
5494 ));
5495 params.extend(values.iter().cloned().map(SqlValue::Text));
5496 }
5497 let sql = format!(
5498 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
5499 quote_ident(&table.primary_key),
5500 base_table(&target.table),
5501 clauses.join(" AND ")
5502 );
5503 let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
5504 let rows = statement
5505 .query_map(rusqlite::params_from_iter(params), |row| {
5506 row.get::<_, String>(0)
5507 })
5508 .map_err(|error| error.to_string())?;
5509 let ids = by_table.entry(target.table.clone()).or_default();
5510 for row in rows {
5511 ids.insert(row.map_err(|error| error.to_string())?);
5512 }
5513 }
5514 Ok(by_table)
5515 }
5516
5517 fn local_purge_values_match(
5518 target: &CompiledLocalDataPurgeTarget,
5519 values: &Map<String, Value>,
5520 ) -> bool {
5521 target.selectors.iter().all(|(column, allowed)| {
5522 values
5523 .get(column)
5524 .and_then(Value::as_str)
5525 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
5526 })
5527 }
5528
5529 pub fn purge_local_data(
5533 &mut self,
5534 input: &LocalDataPurgeInput,
5535 ) -> Result<LocalDataPurgeResult, String> {
5536 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
5537 let meta_key = format!("localPurge:{}", input.purge_id);
5538 let applied_plan = self
5539 .conn
5540 .query_row(
5541 "SELECT value FROM _syncular_meta WHERE key = ?1",
5542 rusqlite::params![meta_key],
5543 |row| row.get::<_, String>(0),
5544 )
5545 .optional()
5546 .map_err(|error| error.to_string())?;
5547 if let Some(applied_plan) = applied_plan {
5548 if applied_plan != canonical_plan {
5549 return Err(format!(
5550 "sync.invalid_request: local purge id {:?} was already used with a different plan",
5551 input.purge_id
5552 ));
5553 }
5554 return Ok(LocalDataPurgeResult {
5555 already_applied: true,
5556 purged_rows: 0,
5557 dropped_commits: 0,
5558 });
5559 }
5560
5561 let prior_outbox = self.outbox.clone();
5562 let prior_rejection_count = self.rejections.len();
5563 let prior_overlay_dirty = self.overlay_dirty.get();
5564 self.begin_observation("syncular_local_purge")?;
5565 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
5566 let row_ids = self.local_purge_base_row_ids(&targets)?;
5567 let doomed = self
5568 .outbox
5569 .iter()
5570 .filter(|commit| {
5571 commit.ops.iter().any(|operation| {
5572 let matching_targets = targets
5573 .iter()
5574 .filter(|target| target.table == operation.table)
5575 .collect::<Vec<_>>();
5576 if matching_targets.is_empty() {
5577 return false;
5578 }
5579 if row_ids
5580 .get(&operation.table)
5581 .is_some_and(|ids| ids.contains(&operation.row_id))
5582 {
5583 return true;
5584 }
5585 operation.values.as_ref().is_some_and(|values| {
5586 matching_targets
5587 .iter()
5588 .any(|target| Self::local_purge_values_match(target, values))
5589 })
5590 })
5591 })
5592 .cloned()
5593 .collect::<Vec<_>>();
5594 let doomed_ids = doomed
5595 .iter()
5596 .map(|commit| commit.client_commit_id.clone())
5597 .collect::<BTreeSet<_>>();
5598 let mut batch = ChangeAccumulator::default();
5599 let mut rejections = Vec::new();
5600 for commit in &doomed {
5601 for operation in &commit.ops {
5602 batch.table(&operation.table);
5603 }
5604 let results = commit
5605 .ops
5606 .iter()
5607 .enumerate()
5608 .map(|(op_index, operation)| {
5609 let rejection = RejectionRecord {
5610 client_commit_id: commit.client_commit_id.clone(),
5611 op_index: op_index as i32,
5612 code: "client.local_data_purged".to_owned(),
5613 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
5614 retryable: false,
5615 details: None,
5616 operation: Some(CommitOperation::from(operation)),
5617 };
5618 rejections.push(rejection.clone());
5619 CommitOperationOutcome::Error { rejection }
5620 })
5621 .collect::<Vec<_>>();
5622 self.persist_commit_outcome(
5623 &commit.client_commit_id,
5624 CommitOutcomeStatus::Rejected,
5625 &results,
5626 Some(&commit.ops),
5627 )?;
5628 self.delete_outbox_persisted(&commit.client_commit_id)?;
5629 }
5630 if !doomed.is_empty() {
5631 self.outbox
5632 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
5633 self.rejections.extend(rejections);
5634 self.prune_commit_outcomes()?;
5635 self.overlay_dirty.set(true);
5636 batch.status = true;
5637 batch.rejections = true;
5638 batch.outcomes = true;
5639 }
5640
5641 let mut purged_rows = 0usize;
5642 for (table, ids) in &row_ids {
5643 if ids.is_empty() {
5644 continue;
5645 }
5646 batch.table(table);
5647 for row_id in ids {
5648 self.delete_base_row(table, row_id)?;
5649 purged_rows += 1;
5650 }
5651 }
5652 self.rebuild_overlay_if_dirty();
5653 self.reconcile_blob_refcounts(true);
5654 self.conn
5655 .execute(
5656 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
5657 rusqlite::params![meta_key, canonical_plan],
5658 )
5659 .map_err(|error| error.to_string())?;
5660 Ok((
5661 batch,
5662 LocalDataPurgeResult {
5663 already_applied: false,
5664 purged_rows,
5665 dropped_commits: doomed.len(),
5666 },
5667 ))
5668 })();
5669 let (batch, result) = match applied {
5670 Ok(value) => value,
5671 Err(error) => {
5672 self.rollback_observation("syncular_local_purge");
5673 self.outbox = prior_outbox;
5674 self.rejections.truncate(prior_rejection_count);
5675 self.overlay_dirty.set(prior_overlay_dirty);
5676 return Err(error);
5677 }
5678 };
5679 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
5680 self.rollback_observation("syncular_local_purge");
5681 self.outbox = prior_outbox;
5682 self.rejections.truncate(prior_rejection_count);
5683 self.overlay_dirty.set(prior_overlay_dirty);
5684 return Err(error);
5685 }
5686 Ok(result)
5687 }
5688
5689 fn purge_scope_rows(
5694 &mut self,
5695 table_name: &str,
5696 effective: &[(String, Vec<String>)],
5697 ) -> Result<(), ()> {
5698 if effective.is_empty() {
5699 return Ok(());
5700 }
5701 let table = self.schema.table(table_name).ok_or(())?.clone();
5702 let mut clauses = Vec::new();
5703 let mut params: Vec<SqlValue> = Vec::new();
5704 for (variable, values) in effective {
5705 let column = table.scope_column(variable).ok_or(())?;
5706 let placeholders: Vec<String> = values
5707 .iter()
5708 .map(|v| {
5709 params.push(SqlValue::Text(v.clone()));
5710 "?".to_owned()
5711 })
5712 .collect();
5713 clauses.push(format!(
5714 "{} IN ({})",
5715 quote_ident(column),
5716 placeholders.join(", ")
5717 ));
5718 }
5719 let sql = format!(
5720 "DELETE FROM {} WHERE {}",
5721 base_table(table_name),
5722 clauses.join(" AND ")
5723 );
5724 self.overlay_dirty.set(true);
5725 self.conn
5726 .execute(&sql, rusqlite::params_from_iter(params))
5727 .map_err(|_| ())?;
5728 Ok(())
5729 }
5730
5731 fn drop_doomed_outbox(
5734 &mut self,
5735 table_name: &str,
5736 effective: &[(String, Vec<String>)],
5737 ) -> Result<bool, String> {
5738 if effective.is_empty() {
5739 return Ok(false);
5740 }
5741 let Some(table) = self.schema.table(table_name).cloned() else {
5742 return Ok(false);
5743 };
5744 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
5745 for (variable, values) in effective {
5746 match table.scope_column(variable) {
5747 Some(column) => mappings.push((column, values)),
5748 None => return Ok(false), }
5750 }
5751 let doomed: Vec<OutboxCommit> = self
5752 .outbox
5753 .iter()
5754 .filter(|commit| {
5755 commit.ops.iter().any(|op| {
5756 op.upsert
5757 && op.table == table_name
5758 && op.values.as_ref().is_some_and(|values| {
5759 mappings
5760 .iter()
5761 .all(|(column, allowed)| match values.get(*column) {
5762 Some(Value::String(s)) => allowed.contains(s),
5763 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
5764 _ => false,
5765 })
5766 })
5767 })
5768 })
5769 .cloned()
5770 .collect();
5771 if doomed.is_empty() {
5772 return Ok(false);
5773 }
5774 let mut rejections = Vec::new();
5775 for commit in &doomed {
5776 let results = commit
5777 .ops
5778 .iter()
5779 .enumerate()
5780 .map(|(op_index, operation)| {
5781 let rejection = RejectionRecord {
5782 client_commit_id: commit.client_commit_id.clone(),
5783 op_index: op_index as i32,
5784 code: "sync.scope_revoked".to_owned(),
5785 message: "the commit was dropped because its effective scope was revoked"
5786 .to_owned(),
5787 retryable: false,
5788 details: None,
5789 operation: Some(CommitOperation::from(operation)),
5790 };
5791 rejections.push(rejection.clone());
5792 CommitOperationOutcome::Error { rejection }
5793 })
5794 .collect::<Vec<_>>();
5795 self.persist_commit_outcome(
5796 &commit.client_commit_id,
5797 CommitOutcomeStatus::Rejected,
5798 &results,
5799 Some(&commit.ops),
5800 )?;
5801 self.delete_outbox_persisted(&commit.client_commit_id)?;
5802 }
5803 self.prune_commit_outcomes()?;
5804 let doomed_ids = doomed
5805 .iter()
5806 .map(|commit| commit.client_commit_id.as_str())
5807 .collect::<BTreeSet<_>>();
5808 self.outbox
5809 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
5810 self.rejections.extend(rejections);
5811 self.overlay_dirty.set(true);
5812 Ok(true)
5813 }
5814
5815 pub fn upload_blob(
5822 &mut self,
5823 bytes: &[u8],
5824 media_type: Option<String>,
5825 name: Option<String>,
5826 ) -> Result<Value, String> {
5827 let blob_id = blob_id_for(bytes);
5828 let now = self.clock_now_ms();
5829 self.conn
5830 .execute(
5831 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
5832 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
5833 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
5834 )
5835 .map_err(|e| e.to_string())?;
5836 self.conn
5837 .execute(
5838 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
5839 rusqlite::params![blob_id, media_type, now],
5840 )
5841 .map_err(|e| e.to_string())?;
5842 self.enforce_blob_cache_cap();
5845 let mut obj = Map::new();
5846 obj.insert("blobId".to_owned(), Value::from(blob_id));
5847 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
5848 if let Some(mt) = media_type {
5849 obj.insert("mediaType".to_owned(), Value::from(mt));
5850 }
5851 if let Some(n) = name {
5852 obj.insert("name".to_owned(), Value::from(n));
5853 }
5854 Ok(Value::Object(obj))
5855 }
5856
5857 pub fn fetch_blob(
5861 &mut self,
5862 transport: &mut dyn Transport,
5863 blob_id_or_ref: &str,
5864 ) -> Result<Value, (String, String)> {
5865 let simple = |m: String| ("client.failed".to_owned(), m);
5866 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
5867 blob_id_or_ref.to_owned()
5868 } else {
5869 let value: Value = serde_json::from_str(blob_id_or_ref)
5870 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
5871 value
5872 .get("blobId")
5873 .and_then(Value::as_str)
5874 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
5875 .to_owned()
5876 };
5877 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
5878 return Ok(cached);
5879 }
5880 let bytes = match transport
5886 .blob_download(&blob_id)
5887 .map_err(|e| (e.code, e.message))?
5888 {
5889 BlobDownload::Bytes(bytes) => bytes,
5890 BlobDownload::Url {
5891 url,
5892 url_expires_at_ms,
5893 } => {
5894 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5896 return Err((
5897 "sync.segment_expired".to_owned(),
5898 format!(
5899 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
5900 ),
5901 ));
5902 }
5903 transport
5904 .fetch_blob_url(&url)
5905 .map_err(|e| (e.code, e.message))?
5906 }
5907 };
5908 if blob_id_for(&bytes) != blob_id {
5910 return Err(simple(format!(
5911 "blob content address mismatch for {blob_id}"
5912 )));
5913 }
5914 let now = self.clock_now_ms();
5915 self.conn
5916 .execute(
5917 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
5918 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
5919 )
5920 .map_err(|e| simple(e.to_string()))?;
5921 self.enforce_blob_cache_cap();
5922 self.get_cached_blob(&blob_id)
5923 .map_err(simple)?
5924 .ok_or_else(|| simple("blob cache write failed".to_owned()))
5925 }
5926
5927 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
5928 let _ = self.conn.execute(
5931 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
5932 rusqlite::params![self.clock_now_ms(), blob_id],
5933 );
5934 let mut stmt = self
5935 .conn
5936 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
5937 .map_err(|e| e.to_string())?;
5938 let mut rows = stmt
5939 .query(rusqlite::params![blob_id])
5940 .map_err(|e| e.to_string())?;
5941 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
5942 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
5943 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
5944 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
5945 let mut obj = Map::new();
5946 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
5947 obj.insert("byteLength".to_owned(), Value::from(byte_length));
5948 let mut bytes_obj = Map::new();
5949 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
5950 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
5951 if let Some(mt) = media_type {
5952 obj.insert("mediaType".to_owned(), Value::from(mt));
5953 }
5954 return Ok(Some(Value::Object(obj)));
5955 }
5956 Ok(None)
5957 }
5958
5959 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
5961 let pending: Vec<(String, Option<String>)> = {
5962 let mut stmt = self
5963 .conn
5964 .prepare(
5965 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
5966 )
5967 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5968 let rows = stmt
5969 .query_map([], |row| {
5970 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
5971 })
5972 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5973 rows.filter_map(Result::ok).collect()
5974 };
5975 for (blob_id, media_type) in pending {
5976 let bytes: Option<Vec<u8>> = self
5977 .conn
5978 .query_row(
5979 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
5980 rusqlite::params![blob_id],
5981 |row| row.get(0),
5982 )
5983 .ok();
5984 if let Some(bytes) = bytes {
5985 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
5986 }
5987 let _ = self.conn.execute(
5988 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
5989 rusqlite::params![blob_id],
5990 );
5991 }
5992 Ok(())
5993 }
5994
5995 fn upload_one(
6002 &self,
6003 transport: &mut dyn Transport,
6004 blob_id: &str,
6005 bytes: &[u8],
6006 media_type: Option<&str>,
6007 ) -> Result<(), TransportError> {
6008 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
6009 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
6011 url,
6012 url_expires_at_ms,
6013 } => {
6014 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
6015 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
6016 return Ok(());
6017 }
6018 }
6020 BlobUploadGrant::None => {
6021 }
6023 }
6024 transport.blob_upload(blob_id, bytes, media_type)
6025 }
6026
6027 fn enforce_blob_cache_cap(&self) {
6036 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
6037 return;
6038 };
6039 let mut total: i64 = self
6040 .conn
6041 .query_row(
6042 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
6043 [],
6044 |row| row.get(0),
6045 )
6046 .unwrap_or(0);
6047 if total <= max_bytes {
6048 return;
6049 }
6050 let candidates: Vec<(String, i64)> = {
6051 let Ok(mut stmt) = self.conn.prepare(
6052 "SELECT blob_id, byte_length FROM _syncular_blobs
6053 WHERE refcount = 0
6054 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
6055 ORDER BY last_used_ms ASC, created_at_ms ASC",
6056 ) else {
6057 return;
6058 };
6059 let Ok(rows) = stmt.query_map([], |row| {
6060 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
6061 }) else {
6062 return;
6063 };
6064 rows.filter_map(Result::ok).collect()
6065 };
6066 for (blob_id, byte_length) in candidates {
6067 if total <= max_bytes {
6068 break;
6069 }
6070 let _ = self.conn.execute(
6071 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
6072 rusqlite::params![blob_id],
6073 );
6074 total -= byte_length;
6075 }
6076 }
6077
6078 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
6082 if !self.schema_has_blobs() {
6083 return;
6084 }
6085 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
6086 for table in self.schema.tables.clone() {
6087 let blob_cols: Vec<String> = table
6088 .columns
6089 .iter()
6090 .filter(|c| c.ty == ColumnType::BlobRef)
6091 .map(|c| c.name.clone())
6092 .collect();
6093 for column in blob_cols {
6094 let sql = format!(
6095 "SELECT {} FROM {} WHERE {} IS NOT NULL",
6096 quote_ident(&column),
6097 base_table(&table.name),
6098 quote_ident(&column)
6099 );
6100 let Ok(mut stmt) = self.conn.prepare(&sql) else {
6101 continue;
6102 };
6103 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
6104 continue;
6105 };
6106 for raw in rows.flatten().flatten() {
6107 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
6108 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
6109 *counts.entry(id.to_owned()).or_insert(0) += 1;
6110 }
6111 }
6112 }
6113 }
6114 }
6115 let _ = self
6116 .conn
6117 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
6118 for (blob_id, count) in &counts {
6119 let _ = self.conn.execute(
6120 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
6121 rusqlite::params![count, blob_id],
6122 );
6123 }
6124 if delete_orphans {
6125 let _ = self.conn.execute(
6126 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
6127 [],
6128 );
6129 }
6130 }
6131
6132 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
6137 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
6138 return sql.clone();
6139 }
6140 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
6141 columns.push(quote_ident("_syncular_version"));
6142 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
6143 let primary_key = quote_ident(&table.primary_key);
6144 let updates = columns
6145 .iter()
6146 .filter(|column| **column != primary_key)
6147 .map(|column| format!("{column}=excluded.{column}"))
6148 .collect::<Vec<_>>()
6149 .join(", ");
6150 let sql = format!(
6151 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
6152 columns.join(", "),
6153 placeholders.join(", ")
6154 );
6155 self.insert_sql
6156 .borrow_mut()
6157 .insert(full_table.to_owned(), sql.clone());
6158 sql
6159 }
6160
6161 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
6162 self.overlay_dirty.set(true);
6163 self.write_row(&base_table(table_name), table_name, row, version)
6164 }
6165
6166 fn write_row(
6167 &self,
6168 full_table: &str,
6169 table_name: &str,
6170 row: &Row,
6171 version: i64,
6172 ) -> Result<(), String> {
6173 let table = self
6174 .schema
6175 .table(table_name)
6176 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
6177 let sql = self.insert_row_sql(full_table, table);
6178 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
6179 let params = row
6180 .iter()
6181 .map(RowParam::Cell)
6182 .chain(std::iter::once(RowParam::Version(version)));
6183 stmt.execute(rusqlite::params_from_iter(params))
6184 .map_err(|e| e.to_string())?;
6185 Ok(())
6186 }
6187
6188 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
6189 let table = self
6190 .schema
6191 .table(table_name)
6192 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
6193 self.overlay_dirty.set(true);
6194 let sql = format!(
6195 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
6196 base_table(table_name),
6197 quote_ident(&table.primary_key)
6198 );
6199 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
6200 stmt.execute(rusqlite::params![row_id])
6201 .map_err(|e| e.to_string())?;
6202 Ok(())
6203 }
6204
6205 fn rebuild_overlay_if_dirty(&mut self) {
6208 if self.overlay_dirty.get() {
6209 self.rebuild_overlay();
6210 }
6211 }
6212
6213 fn rebuild_overlay(&mut self) {
6217 self.exec("SAVEPOINT syncular_overlay");
6218 for table in self.schema.tables.clone() {
6219 for index in &table.fts_indexes {
6220 let _ = self.drop_fts_triggers(index);
6221 }
6222 let visible = visible_table(&table.name);
6223 let base = base_table(&table.name);
6224 self.exec(&format!("DELETE FROM {visible}"));
6225 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
6226 }
6227 for commit in self.outbox.clone() {
6228 for op in &commit.ops {
6229 let Some(table) = self.schema.table(&op.table).cloned() else {
6230 continue;
6231 };
6232 if op.upsert {
6233 let Some(values) = op.values.as_ref() else {
6234 continue;
6235 };
6236 let mut row: Row = Vec::with_capacity(table.columns.len());
6237 let mut ok = true;
6238 for column in &table.columns {
6239 match json_to_column_value(column, values.get(&column.name)) {
6240 Ok(v) => row.push(v),
6241 Err(_) => {
6242 ok = false;
6243 break;
6244 }
6245 }
6246 }
6247 if ok {
6248 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
6249 }
6250 } else {
6251 let sql = format!(
6252 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
6253 visible_table(&table.name),
6254 quote_ident(&table.primary_key)
6255 );
6256 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
6257 }
6258 }
6259 }
6260 for table in self.schema.tables.clone() {
6261 for index in &table.fts_indexes {
6262 let _ = self.rebuild_fts_projection(&table, index);
6263 let _ = self.create_fts_triggers(&table, index);
6264 }
6265 }
6266 self.exec("RELEASE syncular_overlay");
6267 self.overlay_dirty.set(false);
6268 }
6269
6270 fn exec(&self, sql: &str) {
6271 let _ = self.conn.execute_batch(sql);
6272 }
6273
6274 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
6277 transport
6278 .realtime_connect_for_client(&self.client_id)
6279 .map_err(|e| format!("{}: {}", e.code, e.message))?;
6280 self.realtime_connected = true;
6281 Ok(())
6282 }
6283
6284 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
6285 let _ = transport.realtime_close();
6286 self.realtime_connected = false;
6287 self.presence.clear(); }
6289
6290 pub fn set_presence(
6296 &mut self,
6297 transport: &mut dyn Transport,
6298 scope_key: &str,
6299 doc: Option<&Value>,
6300 ) -> Result<(), String> {
6301 if !self.realtime_connected {
6302 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
6303 }
6304 let text = encode_presence_publish(scope_key, doc);
6305 transport
6306 .realtime_send(&text)
6307 .map_err(|e| format!("{}: {}", e.code, e.message))
6308 }
6309
6310 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
6312 self.presence
6313 .get(scope_key)
6314 .map(|peers| peers.values().cloned().collect())
6315 .unwrap_or_default()
6316 }
6317
6318 fn apply_presence(
6320 &mut self,
6321 scope_key: String,
6322 kind: Option<PresenceKind>,
6323 actor_id: Option<String>,
6324 client_id: Option<String>,
6325 doc: Option<Value>,
6326 error: Option<String>,
6327 ) {
6328 if error.is_some() {
6331 return;
6332 }
6333 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
6334 return;
6335 };
6336 let peer_key = format!("{actor_id} {client_id}");
6337 match kind {
6338 PresenceKind::Leave => {
6339 if let Some(peers) = self.presence.get_mut(&scope_key) {
6340 peers.remove(&peer_key);
6341 if peers.is_empty() {
6342 self.presence.remove(&scope_key);
6343 }
6344 }
6345 }
6346 _ => {
6347 let doc = match doc {
6348 Some(Value::Object(_)) => doc.unwrap(),
6349 _ => return,
6350 };
6351 self.presence.entry(scope_key).or_default().insert(
6352 peer_key,
6353 PresencePeer {
6354 actor_id,
6355 client_id,
6356 doc,
6357 },
6358 );
6359 }
6360 }
6361 }
6362
6363 pub fn on_realtime_text(&mut self, text: &str) {
6365 match parse_control(text) {
6366 Ok(ControlMessage::Hello { requires_sync, .. }) => {
6367 if requires_sync {
6368 self.set_sync_needed(true, true);
6370 }
6371 }
6372 Ok(ControlMessage::Presence {
6373 scope_key,
6374 kind,
6375 actor_id,
6376 client_id,
6377 doc,
6378 error,
6379 ..
6380 }) => {
6381 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
6382 }
6383 Ok(ControlMessage::Wake { .. }) => {
6384 self.set_sync_needed(true, true);
6386 }
6387 _ => {}
6388 }
6389 }
6390
6391 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
6394 if self.stopped {
6395 return;
6396 }
6397 let message = match decode_message(bytes) {
6398 Ok(m) if m.msg_kind == MsgKind::Response => m,
6399 _ => {
6400 self.set_sync_needed(true, true);
6401 return;
6402 }
6403 };
6404 let mut frames = message.frames.into_iter();
6405 let mut applied_cursor: Option<i64> = None;
6406 let mut any_covered = false;
6407 let mut dropped = false;
6408 while let Some(frame) = frames.next() {
6409 let Frame::SubStart {
6410 id,
6411 status,
6412 effective_scopes,
6413 ..
6414 } = frame
6415 else {
6416 continue;
6417 };
6418 let mut body = Vec::new();
6419 let mut next_cursor: Option<i64> = None;
6420 for inner in frames.by_ref() {
6421 match inner {
6422 Frame::SubEnd {
6423 next_cursor: nc, ..
6424 } => {
6425 next_cursor = Some(nc);
6426 break;
6427 }
6428 Frame::Unknown { .. } => {}
6429 other => body.push(other),
6430 }
6431 }
6432 let Some(next_cursor) = next_cursor else {
6433 dropped = true;
6434 break;
6435 };
6436 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
6437 dropped = true;
6438 continue;
6439 };
6440 let sub = &self.subs[sub_index];
6441 if status != SubStatus::Active
6444 || sub.state != SubState::Active
6445 || sub.bootstrap_state.is_some()
6446 || !sub.synced_once
6447 {
6448 dropped = true;
6449 continue;
6450 }
6451 if next_cursor <= sub.cursor {
6452 any_covered = true;
6454 continue;
6455 }
6456 let previous_effective = self.subs[sub_index].effective.clone();
6457 let previous_cursor = self.subs[sub_index].cursor;
6458 if self.begin_observation("syncular_delta").is_err() {
6459 dropped = true;
6460 continue;
6461 }
6462 self.subs[sub_index].effective = Some(effective_scopes);
6463 let mut batch = ChangeAccumulator::default();
6464 let mut failed = false;
6465 for inner in body {
6466 if let Frame::Commit {
6467 tables, changes, ..
6468 } = inner
6469 {
6470 self.record_commit_changes(&mut batch, &tables, &changes);
6471 if self.apply_commit_changes(&tables, &changes).is_err() {
6472 failed = true;
6473 break;
6474 }
6475 }
6476 }
6477 if failed {
6478 self.rollback_observation("syncular_delta");
6479 self.subs[sub_index].effective = previous_effective;
6480 self.subs[sub_index].cursor = previous_cursor;
6481 self.overlay_dirty.set(true);
6482 self.rebuild_overlay();
6483 dropped = true;
6484 continue;
6485 }
6486 let sub = &mut self.subs[sub_index];
6487 sub.cursor = next_cursor;
6488 self.persist_sub(&self.subs[sub_index].clone());
6489 self.rebuild_overlay_if_dirty();
6490 if self.finish_observation("syncular_delta", batch).is_err() {
6491 self.rollback_observation("syncular_delta");
6492 self.subs[sub_index].effective = previous_effective;
6493 self.subs[sub_index].cursor = previous_cursor;
6494 self.overlay_dirty.set(true);
6495 self.rebuild_overlay();
6496 dropped = true;
6497 continue;
6498 }
6499 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
6500 }
6501 if let Some(cursor) = applied_cursor {
6502 self.reconcile_blob_refcounts(false);
6503 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
6505 let _ = transport.realtime_send(&ack);
6506 } else if !any_covered || dropped {
6507 self.set_sync_needed(true, true);
6509 }
6510 }
6511
6512 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
6516 if !self.realtime_connected {
6517 return;
6518 }
6519 let floor = self
6520 .subs
6521 .iter()
6522 .filter(|s| {
6523 s.state == SubState::Active
6524 && s.bootstrap_state.is_none()
6525 && s.synced_once
6526 && s.cursor >= 0
6527 })
6528 .map(|s| s.cursor)
6529 .min();
6530 if let Some(cursor) = floor {
6531 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
6532 let _ = transport.realtime_send(&ack);
6533 }
6534 }
6535}