1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, CommitOperation, CommitOperationOutcome,
27 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
28 ConflictRecord, CoverageSnapshot, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult,
29 LocalDataPurgeTarget, Mutation, PresencePeer, QueryRow, QuerySnapshot, QueryValue,
30 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
31 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
32 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
33};
34use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
35use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
36use crate::values::{
37 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
38 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
39 scope_map_to_json, sort_scope_map,
40};
41
42const DEFAULT_ACCEPT: u8 = 0b0111;
47const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
48const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
49const ACCEPT_SQLITE: u8 = 1 << 2;
50const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
51
52const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
54const LOCAL_REVISION_KEY: &str = "localRevision";
55const CLIENT_ID_KEY: &str = "clientId";
56const LEASE_STATE_KEY: &str = "leaseState";
57const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
58const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63enum SubState {
64 Active,
65 Revoked,
66 Failed,
67}
68
69#[cfg(test)]
70mod observation_tests {
71 use super::*;
72 use serde_json::json;
73
74 fn client() -> SyncClient {
75 SyncClient::new(
76 "retry-test".to_owned(),
77 &json!({
78 "version": 1,
79 "tables": [{
80 "name": "tasks",
81 "primaryKey": "id",
82 "columns": [
83 { "name": "id", "type": "string", "nullable": false },
84 { "name": "project_id", "type": "string", "nullable": false }
85 ],
86 "scopes": [{ "pattern": "project:{project_id}" }]
87 }]
88 }),
89 ClientLimits::default(),
90 )
91 .expect("test client")
92 }
93
94 #[test]
95 fn background_retry_deadlines_back_off_and_reset() {
96 let mut client = client();
97 client.schedule_background_retry();
98 assert!(matches!(
99 client.drain_sync_intents().as_slice(),
100 [SyncIntent::Background { delay_ms: 250 }]
101 ));
102 client.schedule_background_retry();
103 assert!(matches!(
104 client.drain_sync_intents().as_slice(),
105 [SyncIntent::Background { delay_ms: 500 }]
106 ));
107 client.reset_background_retry();
108 client.schedule_background_retry();
109 assert!(matches!(
110 client.drain_sync_intents().as_slice(),
111 [SyncIntent::Background { delay_ms: 250 }]
112 ));
113 }
114
115 #[test]
116 fn secondary_unique_collision_preserves_existing_synced_row() {
117 let client = SyncClient::new(
118 "unique-upsert-test".to_owned(),
119 &json!({
120 "version": 1,
121 "tables": [{
122 "name": "tasks",
123 "primaryKey": "id",
124 "columns": [
125 { "name": "id", "type": "string", "nullable": false },
126 { "name": "project_id", "type": "string", "nullable": false },
127 { "name": "title", "type": "string", "nullable": false }
128 ],
129 "scopes": [{ "pattern": "project:{project_id}" }],
130 "indexes": [{
131 "name": "tasks_by_project_title",
132 "columns": ["project_id", "title"],
133 "unique": true
134 }]
135 }]
136 }),
137 ClientLimits::default(),
138 )
139 .expect("test client");
140 let table = client.schema.table("tasks").expect("tasks table");
141 let sql = client.insert_row_sql(&base_table("tasks"), table);
142
143 client
144 .conn
145 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
146 .expect("insert first row");
147 client
148 .conn
149 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
150 .expect("update same primary key");
151 client
152 .conn
153 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
154 .expect("insert second row");
155 assert!(client
156 .conn
157 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
158 .is_err());
159
160 let rows = client
161 .conn
162 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
163 .expect("prepare rows")
164 .query_map([], |row| {
165 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
166 })
167 .expect("query rows")
168 .collect::<Result<Vec<_>, _>>()
169 .expect("collect rows");
170 assert_eq!(
171 rows,
172 vec![
173 ("t1".to_owned(), "updated".to_owned()),
174 ("t2".to_owned(), "original".to_owned())
175 ]
176 );
177 }
178
179 #[test]
180 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
181 let path = std::env::temp_dir().join(format!(
182 "syncular-startup-intent-{}.db",
183 uuid::Uuid::new_v4()
184 ));
185 let schema = json!({
186 "version": 1,
187 "tables": [{
188 "name": "tasks",
189 "primaryKey": "id",
190 "columns": [
191 { "name": "id", "type": "string", "nullable": false },
192 { "name": "project_id", "type": "string", "nullable": false }
193 ],
194 "scopes": [{ "pattern": "project:{project_id}" }]
195 }]
196 });
197
198 {
199 let mut first = SyncClient::open_path_with_identity(
200 None,
201 &schema,
202 ClientLimits::default(),
203 path.to_str().expect("UTF-8 temp path"),
204 )
205 .expect("first open");
206 first
207 .set_window(
208 &WindowBase {
209 table: "tasks".to_owned(),
210 variable: "project_id".to_owned(),
211 fixed_scopes: Vec::new(),
212 params: None,
213 },
214 &["persisted".to_owned()],
215 )
216 .expect("persist window");
217 }
218
219 let mut reopened = SyncClient::open_path_with_identity(
220 None,
221 &schema,
222 ClientLimits::default(),
223 path.to_str().expect("UTF-8 temp path"),
224 )
225 .expect("reopen");
226 assert!(reopened.sync_needed());
227 assert!(matches!(
228 reopened.drain_sync_intents().as_slice(),
229 [SyncIntent::Interactive]
230 ));
231 drop(reopened);
232 std::fs::remove_file(path).expect("remove temp database");
233 }
234
235 #[test]
236 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
237 let path = std::env::temp_dir().join(format!(
238 "syncular-satisfied-schema-floor-{}.db",
239 uuid::Uuid::new_v4()
240 ));
241 let schema = json!({
242 "version": 23,
243 "tables": [{
244 "name": "tasks",
245 "primaryKey": "id",
246 "columns": [
247 { "name": "id", "type": "string", "nullable": false },
248 { "name": "project_id", "type": "string", "nullable": false }
249 ],
250 "scopes": [{ "pattern": "project:{project_id}" }]
251 }]
252 });
253
254 {
255 let mut first = SyncClient::open_path_with_identity(
256 None,
257 &schema,
258 ClientLimits::default(),
259 path.to_str().expect("UTF-8 temp path"),
260 )
261 .expect("first open");
262 first
263 .subscribe(
264 "tasks".to_owned(),
265 "tasks".to_owned(),
266 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
267 None,
268 )
269 .expect("persist subscription");
270 first.set_schema_floor(Some(SchemaFloor {
271 required_schema_version: Some(22),
272 latest_schema_version: Some(22),
273 }));
274 }
275
276 let mut reopened = SyncClient::open_path_with_identity(
277 None,
278 &schema,
279 ClientLimits::default(),
280 path.to_str().expect("UTF-8 temp path"),
281 )
282 .expect("reopen");
283 assert!(reopened.schema_floor().is_none());
284 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
285 assert!(reopened.sync_needed());
286 assert!(matches!(
287 reopened.drain_sync_intents().as_slice(),
288 [SyncIntent::Interactive]
289 ));
290 drop(reopened);
291 std::fs::remove_file(path).expect("remove temp database");
292 }
293
294 #[test]
295 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
296 let path = std::env::temp_dir().join(format!(
297 "syncular-unsatisfied-schema-floor-{}.db",
298 uuid::Uuid::new_v4()
299 ));
300 let schema = json!({
301 "version": 1,
302 "tables": [{
303 "name": "tasks",
304 "primaryKey": "id",
305 "columns": [
306 { "name": "id", "type": "string", "nullable": false },
307 { "name": "project_id", "type": "string", "nullable": false }
308 ],
309 "scopes": [{ "pattern": "project:{project_id}" }]
310 }]
311 });
312
313 {
314 let mut first = SyncClient::open_path_with_identity(
315 None,
316 &schema,
317 ClientLimits::default(),
318 path.to_str().expect("UTF-8 temp path"),
319 )
320 .expect("first open");
321 first
322 .subscribe(
323 "tasks".to_owned(),
324 "tasks".to_owned(),
325 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
326 None,
327 )
328 .expect("persist subscription");
329 first.set_schema_floor(Some(SchemaFloor {
330 required_schema_version: Some(2),
331 latest_schema_version: Some(2),
332 }));
333 }
334
335 let reopened = SyncClient::open_path_with_identity(
336 None,
337 &schema,
338 ClientLimits::default(),
339 path.to_str().expect("UTF-8 temp path"),
340 )
341 .expect("reopen");
342 assert_eq!(
343 reopened.schema_floor(),
344 Some(&SchemaFloor {
345 required_schema_version: Some(2),
346 latest_schema_version: Some(2),
347 })
348 );
349 assert!(!reopened.sync_needed());
350 drop(reopened);
351 std::fs::remove_file(path).expect("remove temp database");
352 }
353
354 #[test]
355 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
356 let path = std::env::temp_dir().join(format!(
357 "syncular-indexed-column-bump-{}.db",
358 uuid::Uuid::new_v4()
359 ));
360 let old_schema = json!({
361 "version": 1,
362 "tables": [
363 {
364 "name": "tasks",
365 "primaryKey": "id",
366 "columns": [
367 { "name": "id", "type": "string", "nullable": false },
368 { "name": "project_id", "type": "string", "nullable": false }
369 ],
370 "scopes": [{ "pattern": "project:{project_id}" }]
371 },
372 {
373 "name": "legacy",
374 "primaryKey": "id",
375 "columns": [
376 { "name": "id", "type": "string", "nullable": false },
377 { "name": "project_id", "type": "string", "nullable": false }
378 ],
379 "scopes": [{ "pattern": "project:{project_id}" }]
380 }
381 ]
382 });
383 {
384 let mut first = SyncClient::open_path_with_identity(
385 None,
386 &old_schema,
387 ClientLimits::default(),
388 path.to_str().expect("UTF-8 temp path"),
389 )
390 .expect("open old schema");
391 first
392 .subscribe(
393 "legacy-sub".to_owned(),
394 "legacy".to_owned(),
395 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
396 None,
397 )
398 .expect("persist legacy subscription");
399 }
400
401 let new_schema = json!({
402 "version": 2,
403 "tables": [{
404 "name": "tasks",
405 "primaryKey": "id",
406 "columns": [
407 { "name": "id", "type": "string", "nullable": false },
408 { "name": "project_id", "type": "string", "nullable": false },
409 { "name": "facility_membership_id", "type": "string", "nullable": true }
410 ],
411 "scopes": [{ "pattern": "project:{project_id}" }],
412 "indexes": [{
413 "name": "tasks_by_membership",
414 "columns": ["project_id", "facility_membership_id"],
415 "unique": false
416 }]
417 }]
418 });
419 let upgraded = SyncClient::open_path_with_identity(
420 None,
421 &new_schema,
422 ClientLimits::default(),
423 path.to_str().expect("UTF-8 temp path"),
424 )
425 .expect("open upgraded schema");
426 let columns = upgraded
427 .conn
428 .prepare("PRAGMA table_info(tasks)")
429 .expect("prepare columns")
430 .query_map([], |row| row.get::<_, String>(1))
431 .expect("query columns")
432 .collect::<Result<Vec<_>, _>>()
433 .expect("collect columns");
434 assert!(columns.contains(&"facility_membership_id".to_owned()));
435 let index_count: i64 = upgraded
436 .conn
437 .query_row(
438 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
439 [],
440 |row| row.get(0),
441 )
442 .expect("query index");
443 assert_eq!(index_count, 1);
444 assert!(upgraded.subscription_state("legacy-sub").is_none());
445 drop(upgraded);
446 std::fs::remove_file(path).expect("remove temp database");
447 }
448
449 #[test]
450 fn migrates_pre_envelope_outcome_journal_additively() {
451 let path = std::env::temp_dir().join(format!(
452 "syncular-outcome-migration-{}.db",
453 uuid::Uuid::new_v4()
454 ));
455 let conn = Connection::open(&path).expect("open legacy database");
456 conn.execute_batch(
457 "CREATE TABLE _syncular_commit_outcomes (
458 seq INTEGER PRIMARY KEY AUTOINCREMENT,
459 client_commit_id TEXT NOT NULL UNIQUE,
460 status TEXT NOT NULL,
461 recorded_at_ms INTEGER NOT NULL,
462 results_json TEXT NOT NULL,
463 resolution TEXT NOT NULL DEFAULT 'active',
464 resolved_at_ms INTEGER,
465 replacement_client_commit_id TEXT);",
466 )
467 .expect("create legacy outcome journal");
468 drop(conn);
469 let schema = json!({
470 "version": 1,
471 "tables": [{
472 "name": "tasks",
473 "primaryKey": "id",
474 "columns": [
475 { "name": "id", "type": "string", "nullable": false },
476 { "name": "project_id", "type": "string", "nullable": false }
477 ],
478 "scopes": [{ "pattern": "project:{project_id}" }]
479 }]
480 });
481 let client = SyncClient::open_path(
482 "migration-native".to_owned(),
483 &schema,
484 ClientLimits::default(),
485 path.to_str().expect("UTF-8 temp path"),
486 )
487 .expect("migrate database");
488 let has_operations = client
489 .conn
490 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
491 .expect("prepare table info")
492 .query_map([], |row| row.get::<_, String>(1))
493 .expect("query table info")
494 .filter_map(Result::ok)
495 .any(|column| column == "operations_json");
496 assert!(has_operations);
497 drop(client);
498 std::fs::remove_file(path).expect("remove temp database");
499 }
500
501 #[test]
502 fn durable_conflict_outcome_and_resolution_survive_reopen() {
503 let path = std::env::temp_dir().join(format!(
504 "syncular-durable-outcome-{}.db",
505 uuid::Uuid::new_v4()
506 ));
507 let schema = json!({
508 "version": 1,
509 "tables": [{
510 "name": "tasks",
511 "primaryKey": "id",
512 "columns": [
513 { "name": "id", "type": "string", "nullable": false },
514 { "name": "project_id", "type": "string", "nullable": false }
515 ],
516 "scopes": [{ "pattern": "project:{project_id}" }]
517 }]
518 });
519 let conflict = ConflictRecord {
520 client_commit_id: "losing-commit".to_owned(),
521 op_index: 0,
522 table: "tasks".to_owned(),
523 row_id: "t1".to_owned(),
524 code: "sync.version_conflict".to_owned(),
525 message: "stale base version".to_owned(),
526 server_version: 2,
527 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
528 operation: Some(CommitOperation {
529 table: "tasks".to_owned(),
530 row_id: "t1".to_owned(),
531 op: "upsert".to_owned(),
532 base_version: Some(1),
533 values: None,
534 changed_fields: None,
535 }),
536 };
537 let failed_operations = vec![
538 OutboxOp {
539 upsert: true,
540 table: "tasks".to_owned(),
541 row_id: "t1".to_owned(),
542 base_version: Some(1),
543 values: None,
544 changed_fields: None,
545 },
546 OutboxOp {
547 upsert: true,
548 table: "tasks".to_owned(),
549 row_id: "status-event-1".to_owned(),
550 base_version: Some(0),
551 values: None,
552 changed_fields: None,
553 },
554 ];
555
556 {
557 let mut first = SyncClient::open_path(
558 "durable-native".to_owned(),
559 &schema,
560 ClientLimits::default(),
561 path.to_str().expect("UTF-8 temp path"),
562 )
563 .expect("first open");
564 first
565 .begin_observation("test_outcome")
566 .expect("begin outcome");
567 first
568 .persist_commit_outcome(
569 "losing-commit",
570 CommitOutcomeStatus::Conflict,
571 &[CommitOperationOutcome::Conflict {
572 conflict: conflict.clone(),
573 }],
574 Some(&failed_operations),
575 )
576 .expect("persist outcome");
577 first.conflicts.push(conflict);
578 first
579 .finish_observation(
580 "test_outcome",
581 ChangeAccumulator {
582 conflicts: true,
583 outcomes: true,
584 ..ChangeAccumulator::default()
585 },
586 )
587 .expect("commit outcome");
588 }
589
590 {
591 let mut reopened = SyncClient::open_path(
592 "durable-native".to_owned(),
593 &schema,
594 ClientLimits::default(),
595 path.to_str().expect("UTF-8 temp path"),
596 )
597 .expect("reopen");
598 assert_eq!(reopened.conflicts().len(), 1);
599 let outcome = reopened
600 .commit_outcome("losing-commit")
601 .expect("read outcome")
602 .expect("outcome");
603 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
604 let operations = outcome.operations.expect("aggregate envelope");
605 assert_eq!(operations.len(), 2);
606 assert_eq!(operations[1].row_id, "status-event-1");
607 let resolved = reopened
608 .resolve_commit_outcome(ResolveCommitOutcomeInput {
609 client_commit_id: "losing-commit".to_owned(),
610 resolution: CommitOutcomeResolution::ResolvedKeepServer,
611 replacement_client_commit_id: None,
612 })
613 .expect("resolve");
614 assert_eq!(
615 resolved.resolution,
616 CommitOutcomeResolution::ResolvedKeepServer
617 );
618 assert!(reopened.conflicts().is_empty());
619 }
620
621 let reopened = SyncClient::open_path(
622 "durable-native".to_owned(),
623 &schema,
624 ClientLimits::default(),
625 path.to_str().expect("UTF-8 temp path"),
626 )
627 .expect("second reopen");
628 assert!(reopened.conflicts().is_empty());
629 assert_eq!(
630 reopened
631 .commit_outcome("losing-commit")
632 .expect("read outcome")
633 .expect("outcome")
634 .resolution,
635 CommitOutcomeResolution::ResolvedKeepServer
636 );
637 drop(reopened);
638 std::fs::remove_file(path).expect("remove temp database");
639 }
640
641 #[test]
642 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
643 let path =
644 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
645 let schema = json!({
646 "version": 1,
647 "tables": [{
648 "name": "tasks",
649 "primaryKey": "id",
650 "columns": [
651 { "name": "id", "type": "string", "nullable": false },
652 { "name": "project_id", "type": "string", "nullable": false }
653 ],
654 "scopes": [{ "pattern": "project:{project_id}" }]
655 }]
656 });
657 let mut client = SyncClient::open_path_with_identity(
658 Some("sidecar-client".to_owned()),
659 &schema,
660 ClientLimits::default(),
661 path.to_str().expect("UTF-8 temp path"),
662 )
663 .expect("open owner");
664 let base = WindowBase {
665 table: "tasks".to_owned(),
666 variable: "project_id".to_owned(),
667 fixed_scopes: Vec::new(),
668 params: None,
669 };
670 client
671 .set_window(&base, &["one".to_owned()])
672 .expect("set window");
673 client
674 .mutate(vec![Mutation::Upsert {
675 table: "tasks".to_owned(),
676 values: Map::from_iter([
677 ("id".to_owned(), Value::from("t1")),
678 ("project_id".to_owned(), Value::from("one")),
679 ]),
680 base_version: None,
681 }])
682 .expect("local mutate");
683
684 let coverage = [WindowCoverage {
685 base,
686 units: vec!["one".to_owned(), "missing".to_owned()],
687 }];
688 let owner = client
689 .query_snapshot(
690 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
691 &[],
692 &coverage,
693 )
694 .expect("owner snapshot");
695 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
696 let sidecar = reader
697 .query_snapshot(
698 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
699 &[],
700 &coverage,
701 )
702 .expect("sidecar snapshot");
703
704 assert_eq!(sidecar.revision, owner.revision);
705 assert_eq!(sidecar.rows, owner.rows);
706 assert_eq!(
707 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
708 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
709 );
710 assert_eq!(sidecar.revision, "2");
711 assert_eq!(sidecar.rows[0]["id"], "t1");
712 assert_eq!(sidecar.rows[0]["server_version"], -1);
713 assert!(!sidecar.coverage.complete);
714 assert_eq!(sidecar.coverage.pending.len(), 1);
715 assert_eq!(sidecar.coverage.missing.len(), 1);
716
717 drop(reader);
718 drop(client);
719 std::fs::remove_file(path).expect("remove temp database");
720 }
721
722 #[test]
723 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
724 let schema = json!({
725 "version": 1,
726 "tables": [{
727 "name": "catalogue_codes",
728 "primaryKey": "id",
729 "columns": [
730 { "name": "id", "type": "string", "nullable": false },
731 { "name": "release_id", "type": "string", "nullable": false },
732 { "name": "code", "type": "string", "nullable": false },
733 { "name": "title", "type": "string", "nullable": false }
734 ],
735 "scopes": [{ "pattern": "release:{release_id}" }],
736 "ftsIndexes": [{
737 "name": "catalogue_codes_fts",
738 "columns": ["code", "title"],
739 "tokenize": "unicode61 remove_diacritics 2"
740 }]
741 }]
742 });
743 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
744 .expect("FTS5 client");
745 let search = |client: &SyncClient, query: &str| {
746 client
747 .query(
748 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
749 &[Value::from(query)],
750 )
751 .expect("FTS query")
752 };
753
754 client
755 .mutate(vec![Mutation::Upsert {
756 table: "catalogue_codes".to_owned(),
757 values: Map::from_iter([
758 ("id".to_owned(), Value::from("c1")),
759 ("release_id".to_owned(), Value::from("r1")),
760 ("code".to_owned(), Value::from("A01")),
761 ("title".to_owned(), Value::from("Cholera")),
762 ]),
763 base_version: None,
764 }])
765 .expect("insert code");
766 assert_eq!(search(&client, "cholera").len(), 1);
767
768 client
769 .mutate(vec![Mutation::Upsert {
770 table: "catalogue_codes".to_owned(),
771 values: Map::from_iter([
772 ("id".to_owned(), Value::from("c1")),
773 ("release_id".to_owned(), Value::from("r1")),
774 ("code".to_owned(), Value::from("A01")),
775 ("title".to_owned(), Value::from("Enteric infection")),
776 ]),
777 base_version: None,
778 }])
779 .expect("update code");
780 assert!(search(&client, "cholera").is_empty());
781 assert_eq!(search(&client, "enteric").len(), 1);
782
783 client
784 .mutate(vec![Mutation::Delete {
785 table: "catalogue_codes".to_owned(),
786 row_id: "c1".to_owned(),
787 base_version: None,
788 }])
789 .expect("delete code");
790 assert!(search(&client, "enteric").is_empty());
791 }
792
793 #[test]
794 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
795 let schema = json!({
796 "version": 1,
797 "tables": [{
798 "name": "patient_notes",
799 "primaryKey": "id",
800 "columns": [
801 { "name": "id", "type": "string", "nullable": false },
802 { "name": "practice_id", "type": "string", "nullable": false },
803 { "name": "encryption_key_id", "type": "string", "nullable": false },
804 { "name": "title", "type": "string", "nullable": false }
805 ],
806 "scopes": [{ "pattern": "practice:{practice_id}" }],
807 "ftsIndexes": [{
808 "name": "patient_notes_fts",
809 "columns": ["title"],
810 "tokenize": "unicode61 remove_diacritics 2"
811 }]
812 }]
813 });
814 let mut client = SyncClient::new(
815 "local-purge-test".to_owned(),
816 &schema,
817 ClientLimits::default(),
818 )
819 .expect("local purge client");
820 client
821 .conn
822 .execute(
823 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
824 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
825 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
826 [],
827 )
828 .expect("seed base rows");
829 client.overlay_dirty.set(true);
830 client.rebuild_overlay_if_dirty();
831
832 let note = |id: &str, key_id: &str, title: &str| {
833 Map::from_iter([
834 ("id".to_owned(), Value::from(id)),
835 ("practice_id".to_owned(), Value::from("practice-1")),
836 ("encryption_key_id".to_owned(), Value::from(key_id)),
837 ("title".to_owned(), Value::from(title)),
838 ])
839 };
840 let doomed = client
841 .mutate(vec![
842 Mutation::Upsert {
843 table: "patient_notes".to_owned(),
844 values: note("target", "key-revoked", "Target changed"),
845 base_version: None,
846 },
847 Mutation::Upsert {
848 table: "patient_notes".to_owned(),
849 values: note("unrelated", "key-held", "Unrelated changed"),
850 base_version: None,
851 },
852 ])
853 .expect("doomed commit");
854 let kept = client
855 .mutate(vec![Mutation::Upsert {
856 table: "patient_notes".to_owned(),
857 values: note("kept", "key-held", "Kept optimistic"),
858 base_version: None,
859 }])
860 .expect("kept commit");
861 client.drain_change_batches();
862
863 let input = LocalDataPurgeInput {
864 purge_id: "purge-001".to_owned(),
865 targets: vec![LocalDataPurgeTarget {
866 table: "patient_notes".to_owned(),
867 selectors: BTreeMap::from([(
868 "encryption_key_id".to_owned(),
869 vec!["key-revoked".to_owned()],
870 )]),
871 }],
872 };
873 assert_eq!(
874 client.purge_local_data(&input).expect("apply purge"),
875 LocalDataPurgeResult {
876 already_applied: false,
877 purged_rows: 1,
878 dropped_commits: 1,
879 }
880 );
881 let rows = client
882 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
883 .expect("visible rows");
884 assert_eq!(rows.len(), 2);
885 assert_eq!(rows[0]["id"], "kept");
886 assert_eq!(rows[1]["id"], "unrelated");
887 assert_eq!(rows[1]["title"], "Unrelated original");
888 let fts = client
889 .query(
890 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
891 &[],
892 )
893 .expect("fts query");
894 assert!(fts.is_empty());
895 assert_eq!(client.outbox.len(), 1);
896 assert_eq!(client.outbox[0].client_commit_id, kept);
897 let outcome = client
898 .commit_outcome(&doomed)
899 .expect("read doomed outcome")
900 .expect("doomed outcome");
901 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
902 match &outcome.results[0] {
903 CommitOperationOutcome::Error { rejection } => {
904 assert_eq!(rejection.code, "client.local_data_purged");
905 assert_eq!(rejection.client_commit_id, doomed);
906 }
907 other => panic!("expected local purge rejection, got {other:?}"),
908 }
909 assert_eq!(client.drain_change_batches().len(), 1);
910 assert_eq!(
911 client.purge_local_data(&input).expect("retry purge"),
912 LocalDataPurgeResult {
913 already_applied: true,
914 purged_rows: 0,
915 dropped_commits: 0,
916 }
917 );
918 let conflicting = LocalDataPurgeInput {
919 purge_id: input.purge_id.clone(),
920 targets: vec![LocalDataPurgeTarget {
921 table: "patient_notes".to_owned(),
922 selectors: BTreeMap::from([(
923 "encryption_key_id".to_owned(),
924 vec!["key-held".to_owned()],
925 )]),
926 }],
927 };
928 assert!(client
929 .purge_local_data(&conflicting)
930 .expect_err("id collision must fail")
931 .contains("already used with a different plan"));
932 }
933}
934
935impl SubState {
936 fn name(self) -> &'static str {
937 match self {
938 SubState::Active => "active",
939 SubState::Revoked => "revoked",
940 SubState::Failed => "failed",
941 }
942 }
943
944 fn parse(value: &str) -> Self {
945 match value {
946 "revoked" => Self::Revoked,
947 "failed" => Self::Failed,
948 _ => Self::Active,
949 }
950 }
951}
952
953#[derive(Debug, Clone)]
954struct Subscription {
955 id: String,
956 table: String,
957 requested: Vec<(String, Vec<String>)>,
958 params: Option<String>,
959 cursor: i64,
960 bootstrap_state: Option<String>,
962 state: SubState,
963 reason_code: Option<String>,
964 effective: Option<Vec<(String, Vec<String>)>>,
967 synced_once: bool,
968}
969
970#[derive(Debug, Clone)]
971struct OutboxOp {
972 upsert: bool,
973 table: String,
974 row_id: String,
975 base_version: Option<i64>,
976 values: Option<Map<String, Value>>,
979 changed_fields: Option<Vec<String>>,
981}
982
983impl From<&OutboxOp> for CommitOperation {
984 fn from(operation: &OutboxOp) -> Self {
985 Self {
986 table: operation.table.clone(),
987 row_id: operation.row_id.clone(),
988 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
989 base_version: operation.base_version,
990 values: operation.values.clone(),
991 changed_fields: operation.changed_fields.clone(),
992 }
993 }
994}
995
996#[derive(Debug, Clone)]
997struct OutboxCommit {
998 client_commit_id: String,
999 ops: Vec<OutboxOp>,
1000}
1001
1002#[derive(Debug, Clone)]
1003struct CompiledLocalDataPurgeTarget {
1004 table: String,
1005 selectors: Vec<(String, Vec<String>)>,
1006}
1007
1008struct StoredCommitOutcomeRow {
1009 sequence: i64,
1010 client_commit_id: String,
1011 status: String,
1012 recorded_at_ms: i64,
1013 results_json: String,
1014 operations_json: Option<String>,
1015 resolution: String,
1016 resolved_at_ms: Option<i64>,
1017 replacement_client_commit_id: Option<String>,
1018}
1019
1020enum SectionError {
1023 FailClosed,
1024 Abort(String, String),
1025}
1026
1027struct RequestMeta {
1028 pushed_ids: Vec<String>,
1029 fresh: Vec<(String, bool)>,
1032 accept: u8,
1033 deferred_commits: usize,
1037}
1038
1039#[derive(Default)]
1040struct ChangeAccumulator {
1041 tables: BTreeMap<String, Option<BTreeSet<String>>>,
1043 windows: BTreeMap<(String, String), BTreeSet<String>>,
1044 status: bool,
1045 conflicts: bool,
1046 rejections: bool,
1047 outcomes: bool,
1048}
1049
1050impl ChangeAccumulator {
1051 fn table(&mut self, table: &str) {
1052 self.tables.insert(table.to_owned(), None);
1053 }
1054
1055 fn scope(&mut self, table: &str, key: String) {
1056 match self.tables.get_mut(table) {
1057 Some(None) => {}
1058 Some(Some(keys)) => {
1059 keys.insert(key);
1060 }
1061 None => {
1062 self.tables
1063 .insert(table.to_owned(), Some(BTreeSet::from([key])));
1064 }
1065 }
1066 }
1067
1068 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
1069 self.windows
1070 .entry((base_key.to_owned(), table.to_owned()))
1071 .or_default()
1072 .insert(unit.to_owned());
1073 }
1074
1075 fn touched(&self) -> bool {
1076 !self.tables.is_empty()
1077 || !self.windows.is_empty()
1078 || self.status
1079 || self.conflicts
1080 || self.rejections
1081 || self.outcomes
1082 }
1083}
1084
1085const PUSH_OPS_PER_REQUEST: usize = 500;
1091const MAX_LOCAL_PURGE_TARGETS: usize = 64;
1092const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
1093const MAX_LOCAL_PURGE_VALUES: usize = 128;
1094const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
1095
1096pub struct SyncClient {
1097 conn: Connection,
1098 schema: ClientSchema,
1099 client_id: String,
1100 limits: ClientLimits,
1101 subs: Vec<Subscription>,
1102 outbox: Vec<OutboxCommit>,
1103 conflicts: Vec<ConflictRecord>,
1104 rejections: Vec<RejectionRecord>,
1105 schema_floor: Option<SchemaFloor>,
1106 lease_state: Option<LeaseState>,
1108 stopped: bool,
1110 upgrading: bool,
1112 sync_needed: bool,
1114 realtime_connected: bool,
1115 presence: HashMap<String, HashMap<String, PresencePeer>>,
1117 now_ms: Option<i64>,
1120 encryption: crate::values::EncryptionConfig,
1124 insert_sql: RefCell<HashMap<String, String>>,
1130 overlay_dirty: Cell<bool>,
1134 change_queue: VecDeque<ClientChangeBatch>,
1136 sync_intent_queue: VecDeque<SyncIntent>,
1137 retry_delay_ms: u64,
1139}
1140
1141fn quote_ident(name: &str) -> String {
1142 format!("\"{}\"", name.replace('"', "\"\""))
1143}
1144
1145fn is_local_purge_code_like(value: &str) -> bool {
1146 let bytes = value.as_bytes();
1147 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
1148 && bytes
1149 .iter()
1150 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
1151}
1152
1153fn base_table(name: &str) -> String {
1154 quote_ident(&format!("_syncular_base_{name}"))
1155}
1156
1157type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
1162
1163fn window_base_key(base: &WindowBase) -> String {
1164 format!(
1165 "{}\0{}\0{}",
1166 base.table,
1167 base.variable,
1168 canonical_scope_json(&base.fixed_scopes)
1169 )
1170}
1171
1172fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
1174 let mut scopes = base.fixed_scopes.clone();
1175 scopes.retain(|(k, _)| k != &base.variable);
1176 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
1177 scopes
1178}
1179
1180fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
1184 let canonical = canonical_scope_json(&unit_scopes(base, unit));
1185 let digest = Sha256::digest(canonical.as_bytes());
1186 let hex = bytes_to_hex(&digest);
1187 format!("w:{}:{}", base.table, &hex[..16])
1188}
1189
1190fn visible_table(name: &str) -> String {
1191 quote_ident(name)
1192}
1193
1194const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1195
1196fn is_synced_table_name(name: &str) -> bool {
1199 if name.starts_with("sqlite_") {
1200 return false;
1201 }
1202 if name.starts_with("_syncular_base_") {
1203 return true; }
1205 !name.starts_with("_syncular_")
1207}
1208
1209fn blob_id_for(bytes: &[u8]) -> String {
1211 let digest = Sha256::digest(bytes);
1212 format!("sha256:{}", bytes_to_hex(&digest))
1213}
1214
1215enum RowParam<'a> {
1219 Cell(&'a Option<ColumnValue>),
1220 Version(i64),
1221}
1222
1223impl rusqlite::ToSql for RowParam<'_> {
1224 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1225 Ok(match self {
1226 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1227 RowParam::Cell(cell) => match cell {
1228 None => ToSqlOutput::Owned(SqlValue::Null),
1229 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1230 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1231 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1232 Some(ColumnValue::Boolean(b)) => {
1233 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1234 }
1235 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1236 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1237 }
1238 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1240 ToSqlOutput::Borrowed(ValueRef::Blob(b))
1241 }
1242 },
1243 })
1244 }
1245}
1246
1247fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1255 use ssp2::segment::ColumnType;
1256 let mismatch = || {
1257 Err(format!(
1258 "image column {:?} holds a value of the wrong type",
1259 column.name
1260 ))
1261 };
1262 match value {
1263 ValueRef::Null => {
1264 if !column.nullable {
1265 return Err(format!(
1266 "image column {:?} is NULL but not nullable",
1267 column.name
1268 ));
1269 }
1270 Ok(ToSqlOutput::Owned(SqlValue::Null))
1271 }
1272 ValueRef::Integer(i) => match column.ty {
1273 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1274 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1275 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1276 _ => mismatch(),
1277 },
1278 ValueRef::Real(_) => match column.ty {
1279 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1280 _ => mismatch(),
1281 },
1282 ValueRef::Text(t) => {
1283 std::str::from_utf8(t)
1284 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1285 match column.ty {
1286 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1287 Ok(ToSqlOutput::Borrowed(value))
1288 }
1289 _ => mismatch(),
1290 }
1291 }
1292 ValueRef::Blob(_) => match column.ty {
1293 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1295 _ => mismatch(),
1296 },
1297 }
1298}
1299
1300fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1301 use rusqlite::types::ValueRef;
1302 match value {
1303 ValueRef::Null => Value::Null,
1304 ValueRef::Integer(i) => match column.ty {
1305 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1306 ssp2::segment::ColumnType::Float => {
1307 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1308 }
1309 _ => Value::from(i),
1310 },
1311 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1312 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1313 ValueRef::Blob(b) => {
1314 let mut map = Map::new();
1315 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1316 Value::Object(map)
1317 }
1318 }
1319}
1320
1321fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1325 Ok(match value {
1326 Value::Null => SqlValue::Null,
1327 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1328 Value::Number(n) => {
1329 if let Some(i) = n.as_i64() {
1330 SqlValue::Integer(i)
1331 } else if let Some(f) = n.as_f64() {
1332 SqlValue::Real(f)
1333 } else {
1334 return Err(format!("query param number {n} is out of range"));
1335 }
1336 }
1337 Value::String(s) => SqlValue::Text(s.clone()),
1338 Value::Object(_) => {
1339 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1340 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1341 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1342 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1343 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1344 })?)
1345 } else {
1346 return Err(
1347 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1348 .to_owned(),
1349 );
1350 }
1351 }
1352 Value::Array(_) => return Err("query array params are not supported".to_owned()),
1353 })
1354}
1355
1356fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1361 use rusqlite::types::ValueRef;
1362 match value {
1363 ValueRef::Null => Value::Null,
1364 ValueRef::Integer(i) => {
1365 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1369 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1370 Value::from(i)
1371 } else {
1372 let mut map = Map::new();
1373 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1374 Value::Object(map)
1375 }
1376 }
1377 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1378 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1379 ValueRef::Blob(b) => {
1380 let mut map = Map::new();
1381 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1382 Value::Object(map)
1383 }
1384 }
1385}
1386
1387fn query_connection(
1388 conn: &Connection,
1389 sql: &str,
1390 params: &[QueryValue],
1391) -> Result<Vec<QueryRow>, String> {
1392 crate::query_guard::assert_read_only_query(sql)?;
1393 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1394 let bound: Vec<SqlValue> = params
1395 .iter()
1396 .map(json_param_to_sql)
1397 .collect::<Result<_, _>>()?;
1398 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
1399 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1400 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1401 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1402 let mut sql_rows = stmt
1403 .query(bound_refs.as_slice())
1404 .map_err(|e| e.to_string())?;
1405 let mut out = Vec::new();
1406 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1407 let mut record = Map::new();
1408 for (i, name) in column_names.iter().enumerate() {
1409 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1410 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1411 }
1412 out.push(record);
1413 }
1414 Ok(out)
1415}
1416
1417fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1418 let mut stmt = conn
1419 .prepare(
1420 "SELECT windows.unit, subscriptions.state_json
1421 FROM _syncular_windows AS windows
1422 JOIN _syncular_subscriptions AS subscriptions
1423 ON subscriptions.id = windows.sub_id
1424 WHERE windows.base = ?1
1425 ORDER BY windows.unit ASC",
1426 )
1427 .map_err(|error| error.to_string())?;
1428 let rows = stmt
1429 .query_map(rusqlite::params![window_base_key(base)], |row| {
1430 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1431 })
1432 .map_err(|error| error.to_string())?;
1433 let mut units = Vec::new();
1434 let mut pending = Vec::new();
1435 for row in rows {
1436 let (unit, raw) = row.map_err(|error| error.to_string())?;
1437 let state: Value = serde_json::from_str(&raw)
1438 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1439 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1440 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1441 || state
1442 .get("bootstrapState")
1443 .is_some_and(|value| !value.is_null());
1444 if is_pending {
1445 pending.push(unit.clone());
1446 }
1447 units.push(unit);
1448 }
1449 Ok(WindowState { units, pending })
1450}
1451
1452fn snapshot_connection(
1453 conn: &Connection,
1454 sql: &str,
1455 params: &[Value],
1456 coverage: &[WindowCoverage],
1457) -> Result<QuerySnapshot, String> {
1458 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1459 .map_err(|error| error.to_string())?;
1460 let result = (|| {
1461 let revision = conn
1462 .query_row(
1463 "SELECT value FROM _syncular_meta WHERE key = ?1",
1464 rusqlite::params![LOCAL_REVISION_KEY],
1465 |row| row.get::<_, String>(0),
1466 )
1467 .ok()
1468 .and_then(|value| value.parse::<u64>().ok())
1469 .unwrap_or(0);
1470 let rows = query_connection(conn, sql, params)?;
1471 let mut pending = Vec::new();
1472 let mut missing = Vec::new();
1473 for requested in coverage {
1474 let base_key = window_base_key(&requested.base);
1475 let state = persisted_window_state(conn, &requested.base)?;
1476 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1477 let reference = WindowUnitRef {
1478 base_key: base_key.clone(),
1479 unit: unit.clone(),
1480 };
1481 if !state.units.iter().any(|held| held == &unit) {
1482 missing.push(reference);
1483 } else if state.pending.iter().any(|held| held == &unit) {
1484 pending.push(reference);
1485 }
1486 }
1487 }
1488 Ok(QuerySnapshot {
1489 revision: revision.to_string(),
1490 rows,
1491 coverage: CoverageSnapshot {
1492 complete: pending.is_empty() && missing.is_empty(),
1493 pending,
1494 missing,
1495 },
1496 })
1497 })();
1498 match result {
1499 Ok(snapshot) => {
1500 conn.execute_batch("RELEASE syncular_snapshot_read")
1501 .map_err(|error| error.to_string())?;
1502 Ok(snapshot)
1503 }
1504 Err(error) => {
1505 let _ = conn.execute_batch(
1506 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1507 );
1508 Err(error)
1509 }
1510 }
1511}
1512
1513pub struct FileQuerySnapshotReader {
1518 path: String,
1519 conn: Option<Connection>,
1520}
1521
1522impl FileQuerySnapshotReader {
1523 #[must_use]
1524 pub fn new(path: impl Into<String>) -> Self {
1525 Self {
1526 path: path.into(),
1527 conn: None,
1528 }
1529 }
1530
1531 fn connection(&mut self) -> Result<&Connection, String> {
1532 if self.conn.is_none() {
1533 let conn = Connection::open_with_flags(
1534 &self.path,
1535 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1536 )
1537 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1538 conn.busy_timeout(std::time::Duration::from_millis(250))
1539 .map_err(|error| error.to_string())?;
1540 self.conn = Some(conn);
1541 }
1542 self.conn
1543 .as_ref()
1544 .ok_or_else(|| "read sidecar connection missing".to_owned())
1545 }
1546
1547 pub fn query_snapshot(
1548 &mut self,
1549 sql: &str,
1550 params: &[Value],
1551 coverage: &[WindowCoverage],
1552 ) -> Result<QuerySnapshot, String> {
1553 snapshot_connection(self.connection()?, sql, params, coverage)
1554 }
1555}
1556
1557impl SyncClient {
1558 pub fn new_with_identity(
1559 client_id: Option<String>,
1560 schema_json: &Value,
1561 limits: ClientLimits,
1562 ) -> Result<Self, String> {
1563 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1564 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1565 Self::with_connection(resolved, schema_json, limits, conn)
1566 }
1567
1568 pub fn new(
1569 client_id: String,
1570 schema_json: &Value,
1571 limits: ClientLimits,
1572 ) -> Result<Self, String> {
1573 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1574 Self::with_connection(client_id, schema_json, limits, conn)
1575 }
1576
1577 pub fn open_path(
1583 client_id: String,
1584 schema_json: &Value,
1585 limits: ClientLimits,
1586 path: &str,
1587 ) -> Result<Self, String> {
1588 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1589 Self::with_connection(client_id, schema_json, limits, conn)
1590 }
1591
1592 pub fn open_path_with_identity(
1593 client_id: Option<String>,
1594 schema_json: &Value,
1595 limits: ClientLimits,
1596 path: &str,
1597 ) -> Result<Self, String> {
1598 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1599 conn.busy_timeout(std::time::Duration::from_millis(250))
1605 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1606 conn.pragma_update(None, "journal_mode", "WAL")
1607 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
1608 let persisted = conn
1609 .query_row(
1610 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
1611 [],
1612 |row| row.get::<_, String>(0),
1613 )
1614 .ok();
1615 let resolved = persisted
1616 .clone()
1617 .or(client_id.clone())
1618 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1619 if let (Some(existing), Some(requested)) = (persisted, client_id) {
1620 if existing != requested {
1621 return Err(format!(
1622 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
1623 ));
1624 }
1625 }
1626 Self::with_connection(resolved, schema_json, limits, conn)
1627 }
1628
1629 #[must_use]
1630 pub fn client_id(&self) -> &str {
1631 &self.client_id
1632 }
1633
1634 pub fn with_connection(
1641 client_id: String,
1642 schema_json: &Value,
1643 limits: ClientLimits,
1644 conn: Connection,
1645 ) -> Result<Self, String> {
1646 if limits.outcome_retention_max_entries == Some(0) {
1647 return Err(
1648 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
1649 );
1650 }
1651 let schema = parse_schema_json(schema_json)?;
1652 let mut client = SyncClient {
1653 conn,
1654 schema,
1655 client_id,
1656 limits,
1657 subs: Vec::new(),
1658 outbox: Vec::new(),
1659 conflicts: Vec::new(),
1660 rejections: Vec::new(),
1661 schema_floor: None,
1662 lease_state: None,
1663 stopped: false,
1664 upgrading: false,
1665 sync_needed: false,
1666 realtime_connected: false,
1667 presence: HashMap::new(),
1668 now_ms: None,
1669 encryption: crate::values::EncryptionConfig::default(),
1670 insert_sql: RefCell::new(HashMap::new()),
1671 overlay_dirty: Cell::new(false),
1672 change_queue: VecDeque::new(),
1673 sync_intent_queue: VecDeque::new(),
1674 retry_delay_ms: 250,
1675 };
1676 client
1680 .conn
1681 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
1682 client.create_bookkeeping_tables()?;
1686 match client.get_meta(CLIENT_ID_KEY) {
1687 Some(existing) if existing != client.client_id => {
1688 return Err(format!(
1689 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
1690 client.client_id
1691 ));
1692 }
1693 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
1694 _ => {}
1695 }
1696 client.restore_persisted_state()?;
1697 let marker = client
1698 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1699 .and_then(|value| value.parse::<i32>().ok());
1700 match marker {
1701 None => {
1702 client.create_synced_tables()?;
1703 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
1704 }
1705 Some(version) if version == client.schema.version => {
1706 client.create_synced_tables()?;
1707 }
1708 Some(_) => client.run_schema_reset()?,
1709 }
1710 client.clear_satisfied_persisted_schema_floor();
1711 client.prune_unknown_subscriptions()?;
1712 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
1713 client.overlay_dirty.set(true);
1716 client.rebuild_overlay();
1717 }
1718 client.enqueue_startup_sync_if_needed();
1724 Ok(client)
1725 }
1726
1727 pub fn set_now_ms(&mut self, now_ms: i64) {
1730 self.now_ms = Some(now_ms);
1731 }
1732
1733 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
1737 self.encryption = encryption;
1738 }
1739
1740 fn clock_now_ms(&self) -> i64 {
1741 self.now_ms.unwrap_or_else(|| {
1742 std::time::SystemTime::now()
1743 .duration_since(std::time::UNIX_EPOCH)
1744 .map(|d| d.as_millis() as i64)
1745 .unwrap_or(0)
1746 })
1747 }
1748
1749 fn create_bookkeeping_tables(&self) -> Result<(), String> {
1750 self.conn
1752 .execute_batch(
1753 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
1754 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1755 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
1756 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
1757 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1758 client_commit_id TEXT NOT NULL UNIQUE,
1759 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
1760 recorded_at_ms INTEGER NOT NULL,
1761 results_json TEXT NOT NULL,
1762 operations_json TEXT,
1763 resolution TEXT NOT NULL DEFAULT 'active'
1764 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
1765 resolved_at_ms INTEGER,
1766 replacement_client_commit_id TEXT);
1767 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
1768 ON _syncular_commit_outcomes(resolution, seq);
1769 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
1770 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
1771 CREATE TABLE IF NOT EXISTS _syncular_meta (
1772 key TEXT PRIMARY KEY, value TEXT NOT NULL);
1773 CREATE TABLE IF NOT EXISTS _syncular_windows (
1774 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
1775 PRIMARY KEY (base, unit));
1776 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
1777 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
1778 effective_scopes TEXT NOT NULL);",
1779 )
1780 .map_err(|e| e.to_string())?;
1781 let _ = self
1784 .conn
1785 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
1786 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
1787 self.set_meta(LOCAL_REVISION_KEY, "0");
1788 }
1789 if self.schema_has_blobs() {
1794 self.conn
1795 .execute_batch(
1796 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
1797 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
1798 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
1799 created_at_ms INTEGER NOT NULL,
1800 last_used_ms INTEGER NOT NULL DEFAULT 0);
1801 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
1802 media_type TEXT, created_at_ms INTEGER NOT NULL);",
1803 )
1804 .map_err(|e| e.to_string())?;
1805 let _ = self.conn.execute_batch(
1809 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
1810 );
1811 }
1812 Ok(())
1813 }
1814
1815 fn schema_has_blobs(&self) -> bool {
1817 self.schema
1818 .tables
1819 .iter()
1820 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
1821 }
1822
1823 fn get_meta(&self, key: &str) -> Option<String> {
1826 self.conn
1827 .query_row(
1828 "SELECT value FROM _syncular_meta WHERE key = ?1",
1829 rusqlite::params![key],
1830 |row| row.get::<_, String>(0),
1831 )
1832 .ok()
1833 }
1834
1835 fn set_meta(&self, key: &str, value: &str) {
1836 let _ = self.conn.execute(
1837 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
1838 rusqlite::params![key, value],
1839 );
1840 }
1841
1842 fn delete_meta(&self, key: &str) {
1843 let _ = self.conn.execute(
1844 "DELETE FROM _syncular_meta WHERE key = ?1",
1845 rusqlite::params![key],
1846 );
1847 }
1848
1849 fn restore_persisted_state(&mut self) -> Result<(), String> {
1850 self.subs = {
1851 let mut stmt = self
1852 .conn
1853 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
1854 .map_err(|error| error.to_string())?;
1855 let rows = stmt
1856 .query_map([], |row| {
1857 Ok((
1858 row.get::<_, String>(0)?,
1859 row.get::<_, String>(1)?,
1860 row.get::<_, String>(2)?,
1861 ))
1862 })
1863 .map_err(|error| error.to_string())?;
1864 let mut subscriptions = Vec::new();
1865 for row in rows {
1866 let (id, table, raw) = row.map_err(|error| error.to_string())?;
1867 let state: Value = serde_json::from_str(&raw)
1868 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
1869 let requested = json_to_scope_map(
1870 state.get("requested").unwrap_or(&Value::Object(Map::new())),
1871 )?;
1872 let effective = state
1873 .get("effectiveScopes")
1874 .filter(|value| !value.is_null())
1875 .map(json_to_scope_map)
1876 .transpose()?;
1877 subscriptions.push(Subscription {
1878 id,
1879 table,
1880 requested,
1881 params: state
1882 .get("params")
1883 .and_then(Value::as_str)
1884 .map(str::to_owned),
1885 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
1886 bootstrap_state: state
1887 .get("bootstrapState")
1888 .and_then(Value::as_str)
1889 .map(str::to_owned),
1890 state: SubState::parse(
1891 state
1892 .get("status")
1893 .and_then(Value::as_str)
1894 .unwrap_or("active"),
1895 ),
1896 reason_code: state
1897 .get("reasonCode")
1898 .and_then(Value::as_str)
1899 .map(str::to_owned),
1900 effective,
1901 synced_once: state
1902 .get("syncedOnce")
1903 .and_then(Value::as_bool)
1904 .unwrap_or(false),
1905 });
1906 }
1907 subscriptions
1908 };
1909
1910 self.outbox = {
1911 let mut stmt = self
1912 .conn
1913 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
1914 .map_err(|error| error.to_string())?;
1915 let rows = stmt
1916 .query_map([], |row| {
1917 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1918 })
1919 .map_err(|error| error.to_string())?;
1920 let mut commits = Vec::new();
1921 for row in rows {
1922 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
1923 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
1924 format!("invalid persisted outbox {client_commit_id:?}: {error}")
1925 })?;
1926 let mut ops = Vec::with_capacity(entries.len());
1927 for entry in entries {
1928 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
1929 ops.push(OutboxOp {
1930 upsert: op == "upsert",
1931 table: entry
1932 .get("table")
1933 .and_then(Value::as_str)
1934 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
1935 .to_owned(),
1936 row_id: entry
1937 .get("rowId")
1938 .and_then(Value::as_str)
1939 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
1940 .to_owned(),
1941 base_version: entry.get("baseVersion").and_then(Value::as_i64),
1942 values: entry.get("values").and_then(Value::as_object).cloned(),
1943 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
1944 |values| {
1945 values
1946 .iter()
1947 .filter_map(Value::as_str)
1948 .map(str::to_owned)
1949 .collect()
1950 },
1951 ),
1952 });
1953 }
1954 commits.push(OutboxCommit {
1955 client_commit_id,
1956 ops,
1957 });
1958 }
1959 commits
1960 };
1961
1962 self.prune_commit_outcomes()?;
1963 let active = self.commit_outcomes(CommitOutcomeQuery {
1964 active_only: true,
1965 ..CommitOutcomeQuery::default()
1966 })?;
1967 self.conflicts = active
1968 .iter()
1969 .flat_map(|outcome| outcome.results.iter())
1970 .filter_map(|result| match result {
1971 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
1972 _ => None,
1973 })
1974 .collect();
1975 self.rejections = active
1976 .iter()
1977 .flat_map(|outcome| outcome.results.iter())
1978 .filter_map(|result| match result {
1979 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
1980 _ => None,
1981 })
1982 .collect();
1983
1984 self.lease_state = self
1985 .get_meta(LEASE_STATE_KEY)
1986 .map(|raw| serde_json::from_str(&raw))
1987 .transpose()
1988 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
1989 self.schema_floor = self
1990 .get_meta(SCHEMA_FLOOR_KEY)
1991 .map(|raw| serde_json::from_str(&raw))
1992 .transpose()
1993 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
1994 self.stopped = self.schema_floor.is_some();
1995 Ok(())
1996 }
1997
1998 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
2001 let valid_tables: BTreeSet<String> = self
2002 .schema
2003 .tables
2004 .iter()
2005 .map(|table| table.name.clone())
2006 .collect();
2007 let stale_ids: Vec<String> = self
2008 .subs
2009 .iter()
2010 .filter(|sub| !valid_tables.contains(&sub.table))
2011 .map(|sub| sub.id.clone())
2012 .collect();
2013 for id in &stale_ids {
2014 self.conn
2015 .execute(
2016 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2017 rusqlite::params![id],
2018 )
2019 .map_err(|error| error.to_string())?;
2020 self.conn
2021 .execute(
2022 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2023 rusqlite::params![id],
2024 )
2025 .map_err(|error| error.to_string())?;
2026 self.conn
2027 .execute(
2028 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2029 rusqlite::params![id],
2030 )
2031 .map_err(|error| error.to_string())?;
2032 }
2033 self.subs.retain(|sub| valid_tables.contains(&sub.table));
2034 Ok(())
2035 }
2036
2037 #[must_use]
2038 pub fn local_revision(&self) -> u64 {
2039 self.get_meta(LOCAL_REVISION_KEY)
2040 .and_then(|value| value.parse().ok())
2041 .unwrap_or(0)
2042 }
2043
2044 #[must_use]
2045 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2046 SyncStatusSnapshot {
2047 current_schema_version: self.schema.version,
2048 outbox: self.outbox.len(),
2049 upgrading: self.upgrading,
2050 lease_state: self.lease_state.clone(),
2051 schema_floor: self.schema_floor.clone(),
2052 sync_needed: self.sync_needed,
2053 }
2054 }
2055
2056 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
2057 self.change_queue.drain(..).collect()
2058 }
2059
2060 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
2061 self.sync_intent_queue.drain(..).collect()
2062 }
2063
2064 fn schedule_background_retry(&mut self) {
2065 self.sync_intent_queue.push_back(SyncIntent::Background {
2066 delay_ms: self.retry_delay_ms,
2067 });
2068 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
2069 }
2070
2071 fn reset_background_retry(&mut self) {
2072 self.retry_delay_ms = 250;
2073 }
2074
2075 fn retryable_transport_code(code: &str) -> bool {
2076 code == "transport.failed" || code == "sync.transport_failed"
2077 }
2078
2079 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
2080 if self.sync_needed != value {
2081 if self.begin_observation("syncular_status").is_ok() {
2082 self.sync_needed = value;
2083 let batch = ChangeAccumulator {
2084 status: true,
2085 ..ChangeAccumulator::default()
2086 };
2087 if self.finish_observation("syncular_status", batch).is_err() {
2088 self.rollback_observation("syncular_status");
2089 }
2090 } else {
2091 self.sync_needed = value;
2092 }
2093 }
2094 if value && interactive {
2095 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2096 }
2097 }
2098
2099 fn begin_observation(&self, name: &str) -> Result<(), String> {
2100 self.conn
2101 .execute_batch(&format!("SAVEPOINT {name}"))
2102 .map_err(|error| error.to_string())
2103 }
2104
2105 fn rollback_observation(&self, name: &str) {
2106 let _ = self
2107 .conn
2108 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
2109 }
2110
2111 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
2112 if !batch.touched() {
2113 self.conn
2114 .execute_batch(&format!("RELEASE {name}"))
2115 .map_err(|error| error.to_string())?;
2116 return Ok(());
2117 }
2118 let revision = self
2119 .local_revision()
2120 .checked_add(1)
2121 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
2122 self.conn
2123 .execute(
2124 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
2125 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
2126 )
2127 .map_err(|error| error.to_string())?;
2128 let status = batch.status.then(|| self.status_snapshot());
2129 let event = ClientChangeBatch {
2130 revision: revision.to_string(),
2131 tables: batch
2132 .tables
2133 .into_iter()
2134 .map(|(table, scope_keys)| TableChange {
2135 table,
2136 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
2137 })
2138 .collect(),
2139 windows: batch
2140 .windows
2141 .into_iter()
2142 .map(|((base_key, table), units)| WindowChange {
2143 base_key,
2144 table,
2145 units: units.into_iter().collect(),
2146 })
2147 .collect(),
2148 status,
2149 conflicts_changed: batch.conflicts,
2150 rejections_changed: batch.rejections,
2151 outcomes_changed: batch.outcomes,
2152 };
2153 self.conn
2154 .execute_batch(&format!("RELEASE {name}"))
2155 .map_err(|error| error.to_string())?;
2156 self.change_queue.push_back(event);
2157 Ok(())
2158 }
2159
2160 fn record_scope_map(
2161 &self,
2162 batch: &mut ChangeAccumulator,
2163 table_name: &str,
2164 scopes: &[(String, Vec<String>)],
2165 ) {
2166 let Some(table) = self.schema.table(table_name) else {
2167 return;
2168 };
2169 for (variable, values) in scopes {
2170 let Some(scope) = table
2171 .scope_variables
2172 .iter()
2173 .find(|scope| &scope.variable == variable)
2174 else {
2175 continue;
2176 };
2177 for value in values {
2178 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2179 }
2180 }
2181 }
2182
2183 fn record_row_scopes(
2185 &self,
2186 batch: &mut ChangeAccumulator,
2187 table_name: &str,
2188 row_id: &str,
2189 base: bool,
2190 ) -> bool {
2191 let Some(table) = self.schema.table(table_name) else {
2192 return false;
2193 };
2194 if table.scope_variables.is_empty() {
2195 return false;
2196 }
2197 let columns = table
2198 .scope_variables
2199 .iter()
2200 .map(|scope| quote_ident(&scope.column))
2201 .collect::<Vec<_>>()
2202 .join(", ");
2203 let full_table = if base {
2204 base_table(table_name)
2205 } else {
2206 visible_table(table_name)
2207 };
2208 let sql = format!(
2209 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
2210 quote_ident(&table.primary_key)
2211 );
2212 let Ok(mut stmt) = self.conn.prepare(&sql) else {
2213 return false;
2214 };
2215 let values = stmt.query_row(rusqlite::params![row_id], |row| {
2216 let mut values = Vec::with_capacity(table.scope_variables.len());
2217 for index in 0..table.scope_variables.len() {
2218 values.push(row.get::<_, Option<String>>(index)?);
2219 }
2220 Ok(values)
2221 });
2222 let Ok(values) = values else {
2223 return false;
2224 };
2225 let mut recorded = false;
2226 for (scope, value) in table.scope_variables.iter().zip(values) {
2227 if let Some(value) = value {
2228 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2229 recorded = true;
2230 }
2231 }
2232 recorded
2233 }
2234
2235 fn record_commit_changes(
2236 &self,
2237 batch: &mut ChangeAccumulator,
2238 tables: &[String],
2239 changes: &[ssp2::model::Change],
2240 ) {
2241 for change in changes {
2242 let Some(table_name) = tables.get(change.table_index as usize) else {
2243 continue;
2244 };
2245 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
2246 if let Some(table) = self.schema.table(table_name) {
2247 for (variable, value) in &change.scopes {
2248 if let Some(scope) = table
2249 .scope_variables
2250 .iter()
2251 .find(|scope| &scope.variable == variable)
2252 {
2253 batch.scope(table_name, format!("{}:{value}", scope.prefix));
2254 precise = true;
2255 }
2256 }
2257 }
2258 if !precise {
2259 batch.table(table_name);
2260 }
2261 }
2262 }
2263
2264 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
2265 if effective.is_empty() {
2266 return false;
2267 }
2268 let Some(table) = self.schema.table(table_name) else {
2269 return false;
2270 };
2271 let mut clauses = Vec::new();
2272 let mut params = Vec::new();
2273 for (variable, values) in effective {
2274 let Some(column) = table.scope_column(variable) else {
2275 return false;
2276 };
2277 if values.is_empty() {
2278 return false;
2279 }
2280 let placeholders = values
2281 .iter()
2282 .map(|value| {
2283 params.push(SqlValue::Text(value.clone()));
2284 "?"
2285 })
2286 .collect::<Vec<_>>()
2287 .join(", ");
2288 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
2289 }
2290 let sql = format!(
2291 "SELECT 1 FROM {} WHERE {} LIMIT 1",
2292 base_table(table_name),
2293 clauses.join(" AND ")
2294 );
2295 self.conn
2296 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
2297 .is_ok()
2298 }
2299
2300 pub fn upgrading(&self) -> bool {
2302 self.upgrading
2303 }
2304
2305 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
2311 let new_schema = parse_schema_json(schema_json)?;
2312 let marker: Option<i32> = self
2313 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
2314 .and_then(|v| v.parse().ok());
2315 self.schema = new_schema;
2316 if marker != Some(self.schema.version) {
2317 self.run_schema_reset()?;
2318 }
2319 self.prune_unknown_subscriptions()?;
2320 self.enqueue_startup_sync_if_needed();
2324 Ok(())
2325 }
2326
2327 fn enqueue_startup_sync_if_needed(&mut self) {
2328 let startup_work = !self.stopped
2329 && (!self.outbox.is_empty()
2330 || self.subs.iter().any(|sub| sub.state == SubState::Active));
2331 if startup_work {
2332 self.sync_needed = true;
2333 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2334 }
2335 }
2336
2337 fn clear_satisfied_persisted_schema_floor(&mut self) {
2344 let satisfied = self
2345 .schema_floor
2346 .as_ref()
2347 .and_then(|floor| floor.required_schema_version)
2348 .is_some_and(|required| self.schema.version >= required);
2349 if !satisfied {
2350 return;
2351 }
2352 self.schema_floor = None;
2353 self.stopped = false;
2354 self.delete_meta(SCHEMA_FLOOR_KEY);
2355 }
2356
2357 fn run_schema_reset(&mut self) -> Result<(), String> {
2363 self.begin_observation("syncular_schema_reset")?;
2364 let mut batch = ChangeAccumulator::default();
2365 let result = self.run_schema_reset_observed(&mut batch);
2366 if let Err(error) = result {
2367 self.rollback_observation("syncular_schema_reset");
2368 return Err(error);
2369 }
2370 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
2371 self.rollback_observation("syncular_schema_reset");
2372 return Err(error);
2373 }
2374 Ok(())
2375 }
2376
2377 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
2378 self.upgrading = true;
2379 batch.status = true;
2380 for table in &self.schema.tables {
2381 batch.table(&table.name);
2382 }
2383 for (base_key, unit, table) in self.load_registered_window_units() {
2384 batch.window(&base_key, &table, &unit);
2385 }
2386 self.insert_sql.borrow_mut().clear();
2388 self.overlay_dirty.set(true);
2389 let virtual_tables: Vec<String> = {
2397 let mut stmt = self
2398 .conn
2399 .prepare(
2400 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
2401 )
2402 .map_err(|e| e.to_string())?;
2403 let rows = stmt
2404 .query_map([], |row| row.get::<_, String>(0))
2405 .map_err(|e| e.to_string())?;
2406 rows.filter_map(Result::ok)
2407 .filter(|name| is_synced_table_name(name))
2408 .collect()
2409 };
2410 for name in virtual_tables {
2411 self.conn
2412 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2413 .map_err(|e| e.to_string())?;
2414 }
2415 let existing: Vec<String> = {
2416 let mut stmt = self
2417 .conn
2418 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
2419 .map_err(|e| e.to_string())?;
2420 let rows = stmt
2421 .query_map([], |row| row.get::<_, String>(0))
2422 .map_err(|e| e.to_string())?;
2423 rows.filter_map(Result::ok)
2424 .filter(|name| is_synced_table_name(name))
2425 .collect()
2426 };
2427 for name in &existing {
2428 if let Some(table) = name.strip_prefix("_syncular_base_") {
2429 batch.table(table);
2430 } else if !name.starts_with("_syncular_") {
2431 batch.table(name);
2432 }
2433 }
2434 for name in existing {
2435 self.conn
2436 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2437 .map_err(|e| e.to_string())?;
2438 }
2439 self.create_synced_tables()?;
2441 for sub in &mut self.subs {
2443 sub.cursor = -1;
2444 sub.bootstrap_state = None;
2445 sub.effective = None;
2446 sub.state = SubState::Active;
2447 sub.reason_code = None;
2448 sub.synced_once = false;
2449 }
2450 let subs = self.subs.clone();
2451 for sub in &subs {
2452 self.persist_sub(sub);
2453 }
2454 self.stopped = false;
2456 self.schema_floor = None;
2457 self.delete_meta(SCHEMA_FLOOR_KEY);
2458 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
2460 if self.drop_incompatible_outbox()? {
2464 batch.rejections = true;
2465 batch.status = true;
2466 batch.outcomes = true;
2467 }
2468 self.rebuild_overlay();
2470 Ok(())
2471 }
2472
2473 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
2477 let schema = &self.schema;
2478 let incompatible = self
2479 .outbox
2480 .iter()
2481 .filter(|commit| {
2482 commit.ops.iter().any(|op| {
2483 if !op.upsert {
2484 return false;
2485 }
2486 match schema.table(&op.table) {
2487 None => true,
2488 Some(table) => op.values.as_ref().is_some_and(|values| {
2489 values
2490 .keys()
2491 .any(|key| !table.columns.iter().any(|c| &c.name == key))
2492 }),
2493 }
2494 })
2495 })
2496 .cloned()
2497 .collect::<Vec<_>>();
2498 if incompatible.is_empty() {
2499 return Ok(false);
2500 }
2501 let mut rejections = Vec::new();
2502 for commit in &incompatible {
2503 let results = commit
2504 .ops
2505 .iter()
2506 .enumerate()
2507 .map(|(op_index, operation)| {
2508 let rejection = RejectionRecord {
2509 client_commit_id: commit.client_commit_id.clone(),
2510 op_index: op_index as i32,
2511 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
2512 message: "the persisted commit cannot encode under the current schema"
2513 .to_owned(),
2514 retryable: false,
2515 details: None,
2516 operation: Some(CommitOperation::from(operation)),
2517 };
2518 rejections.push(rejection.clone());
2519 CommitOperationOutcome::Error { rejection }
2520 })
2521 .collect::<Vec<_>>();
2522 self.persist_commit_outcome(
2523 &commit.client_commit_id,
2524 CommitOutcomeStatus::Rejected,
2525 &results,
2526 Some(&commit.ops),
2527 )?;
2528 self.delete_outbox_persisted(&commit.client_commit_id)?;
2529 }
2530 self.prune_commit_outcomes()?;
2531 let incompatible_ids = incompatible
2532 .iter()
2533 .map(|commit| commit.client_commit_id.as_str())
2534 .collect::<BTreeSet<_>>();
2535 self.outbox
2536 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
2537 self.rejections.extend(rejections);
2538 Ok(true)
2539 }
2540
2541 fn create_synced_tables(&self) -> Result<(), String> {
2544 for table in &self.schema.tables {
2545 for (full, index_prefix) in [
2549 (base_table(&table.name), "_syncular_base_"),
2550 (visible_table(&table.name), ""),
2551 ] {
2552 let mut cols: Vec<String> =
2553 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
2554 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
2555 let sql = format!(
2556 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
2557 cols.join(", "),
2558 quote_ident(&table.primary_key)
2559 );
2560 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
2561 for index in &table.indexes {
2565 let unique = if index.unique { "UNIQUE " } else { "" };
2566 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
2567 let cols_sql = index
2568 .columns
2569 .iter()
2570 .map(|c| quote_ident(c))
2571 .collect::<Vec<_>>()
2572 .join(", ");
2573 let index_sql = format!(
2574 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
2575 );
2576 self.conn
2577 .execute(&index_sql, [])
2578 .map_err(|e| e.to_string())?;
2579 }
2580 }
2581 }
2582 for table in &self.schema.tables {
2585 for index in &table.fts_indexes {
2586 self.create_fts_projection(table, index)?;
2587 }
2588 }
2589 Ok(())
2590 }
2591
2592 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
2593 let count: i64 = self
2594 .conn
2595 .query_row(
2596 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
2597 rusqlite::params![index.name],
2598 |row| row.get(0),
2599 )
2600 .map_err(|e| e.to_string())?;
2601 Ok(count > 0)
2602 }
2603
2604 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
2605 for suffix in ["ai", "ad", "au"] {
2606 self.conn
2607 .execute(
2608 &format!(
2609 "DROP TRIGGER IF EXISTS {}",
2610 quote_ident(&format!("{}_{suffix}", index.name))
2611 ),
2612 [],
2613 )
2614 .map_err(|e| e.to_string())?;
2615 }
2616 Ok(())
2617 }
2618
2619 fn create_fts_triggers(
2620 &self,
2621 table: &TableSchema,
2622 index: &FtsIndexSchema,
2623 ) -> Result<(), String> {
2624 let fts = quote_ident(&index.name);
2625 let source = visible_table(&table.name);
2626 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2627 let pk = quote_ident(&table.primary_key);
2628 let projection_columns = std::iter::once(source_id.clone())
2629 .chain(index.columns.iter().map(|column| quote_ident(column)))
2630 .collect::<Vec<_>>()
2631 .join(", ");
2632 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
2633 .chain(
2634 index
2635 .columns
2636 .iter()
2637 .map(|column| format!("new.{}", quote_ident(column))),
2638 )
2639 .collect::<Vec<_>>()
2640 .join(", ");
2641 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
2642 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
2643 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
2644 let sql = format!(
2645 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
2646 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
2647 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
2648 ai = quote_ident(&format!("{}_ai", index.name)),
2649 ad = quote_ident(&format!("{}_ad", index.name)),
2650 au = quote_ident(&format!("{}_au", index.name)),
2651 );
2652 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2653 }
2654
2655 fn rebuild_fts_projection(
2656 &self,
2657 table: &TableSchema,
2658 index: &FtsIndexSchema,
2659 ) -> Result<(), String> {
2660 let fts = quote_ident(&index.name);
2661 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2662 let pk = quote_ident(&table.primary_key);
2663 let indexed_columns = index
2664 .columns
2665 .iter()
2666 .map(|column| quote_ident(column))
2667 .collect::<Vec<_>>();
2668 let projection_columns = std::iter::once(source_id)
2669 .chain(indexed_columns.iter().cloned())
2670 .collect::<Vec<_>>()
2671 .join(", ");
2672 let sql = format!(
2673 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
2674 columns = indexed_columns.join(", "),
2675 source = visible_table(&table.name),
2676 );
2677 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2678 }
2679
2680 fn create_fts_projection(
2681 &self,
2682 table: &TableSchema,
2683 index: &FtsIndexSchema,
2684 ) -> Result<(), String> {
2685 let existed = self.fts_projection_exists(index)?;
2686 let tokenizer = index.tokenize.replace('\'', "''");
2687 let columns = index
2688 .columns
2689 .iter()
2690 .map(|column| quote_ident(column))
2691 .collect::<Vec<_>>()
2692 .join(", ");
2693 let sql = format!(
2694 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
2695 fts = quote_ident(&index.name),
2696 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
2697 );
2698 self.conn.execute(&sql, []).map_err(|error| {
2699 format!(
2700 "cannot create local FTS5 projection {:?}: {error}",
2701 index.name
2702 )
2703 })?;
2704 self.create_fts_triggers(table, index)?;
2705 if !existed {
2706 self.rebuild_fts_projection(table, index)?;
2707 }
2708 Ok(())
2709 }
2710
2711 fn persist_sub(&self, sub: &Subscription) {
2714 let state = serde_json::json!({
2715 "requested": scope_map_to_json(&sub.requested),
2716 "params": sub.params,
2717 "cursor": sub.cursor,
2718 "bootstrapState": sub.bootstrap_state,
2719 "status": sub.state.name(),
2720 "reasonCode": sub.reason_code,
2721 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2722 "syncedOnce": sub.synced_once,
2723 });
2724 let _ = self.conn.execute(
2725 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
2726 rusqlite::params![sub.id, sub.table, state.to_string()],
2727 );
2728 }
2729
2730 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
2731 let ops: Vec<Value> = commit
2732 .ops
2733 .iter()
2734 .map(|op| {
2735 serde_json::json!({
2736 "op": if op.upsert { "upsert" } else { "delete" },
2737 "table": op.table,
2738 "rowId": op.row_id,
2739 "baseVersion": op.base_version,
2740 "values": op.values.clone().map(Value::Object),
2741 "changedFields": op.changed_fields,
2742 })
2743 })
2744 .collect();
2745 let _ = self.conn.execute(
2746 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
2747 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
2748 );
2749 }
2750
2751 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
2752 self.conn
2753 .execute(
2754 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
2755 rusqlite::params![client_commit_id],
2756 )
2757 .map(|_| ())
2758 .map_err(|error| error.to_string())
2759 }
2760
2761 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
2762 match status {
2763 CommitOutcomeStatus::Applied => "applied",
2764 CommitOutcomeStatus::Cached => "cached",
2765 CommitOutcomeStatus::Conflict => "conflict",
2766 CommitOutcomeStatus::Rejected => "rejected",
2767 }
2768 }
2769
2770 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
2771 match resolution {
2772 CommitOutcomeResolution::Active => "active",
2773 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
2774 CommitOutcomeResolution::Superseded => "superseded",
2775 CommitOutcomeResolution::Dismissed => "dismissed",
2776 }
2777 }
2778
2779 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
2780 match value {
2781 "applied" => Ok(CommitOutcomeStatus::Applied),
2782 "cached" => Ok(CommitOutcomeStatus::Cached),
2783 "conflict" => Ok(CommitOutcomeStatus::Conflict),
2784 "rejected" => Ok(CommitOutcomeStatus::Rejected),
2785 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
2786 }
2787 }
2788
2789 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
2790 match value {
2791 "active" => Ok(CommitOutcomeResolution::Active),
2792 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
2793 "superseded" => Ok(CommitOutcomeResolution::Superseded),
2794 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
2795 _ => Err(format!(
2796 "invalid persisted commit outcome resolution {value:?}"
2797 )),
2798 }
2799 }
2800
2801 fn persist_commit_outcome(
2802 &self,
2803 client_commit_id: &str,
2804 status: CommitOutcomeStatus,
2805 results: &[CommitOperationOutcome],
2806 operations: Option<&[OutboxOp]>,
2807 ) -> Result<(), String> {
2808 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
2809 let operations_json = operations
2810 .map(|items| {
2811 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
2812 })
2813 .transpose()
2814 .map_err(|error| error.to_string())?;
2815 self.conn
2816 .execute(
2817 "INSERT INTO _syncular_commit_outcomes (
2818 client_commit_id, status, recorded_at_ms, results_json,
2819 operations_json, resolution
2820 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
2821 rusqlite::params![
2822 client_commit_id,
2823 Self::outcome_status_name(status),
2824 self.clock_now_ms(),
2825 results_json,
2826 operations_json
2827 ],
2828 )
2829 .map(|_| ())
2830 .map_err(|error| error.to_string())
2831 }
2832
2833 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
2834 let StoredCommitOutcomeRow {
2835 sequence,
2836 client_commit_id,
2837 status,
2838 recorded_at_ms,
2839 results_json,
2840 operations_json,
2841 resolution,
2842 resolved_at_ms,
2843 replacement_client_commit_id,
2844 } = row;
2845 Ok(CommitOutcome {
2846 sequence,
2847 client_commit_id,
2848 status: Self::parse_outcome_status(&status)?,
2849 recorded_at_ms,
2850 results: serde_json::from_str(&results_json)
2851 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
2852 operations: operations_json
2853 .map(|value| {
2854 serde_json::from_str(&value).map_err(|error| {
2855 format!("invalid persisted commit outcome operations: {error}")
2856 })
2857 })
2858 .transpose()?,
2859 resolution: Self::parse_outcome_resolution(&resolution)?,
2860 resolved_at_ms,
2861 replacement_client_commit_id,
2862 })
2863 }
2864
2865 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
2866 let row = self
2867 .conn
2868 .query_row(
2869 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2870 resolution, resolved_at_ms, replacement_client_commit_id
2871 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
2872 rusqlite::params![client_commit_id],
2873 |row| {
2874 Ok(StoredCommitOutcomeRow {
2875 sequence: row.get(0)?,
2876 client_commit_id: row.get(1)?,
2877 status: row.get(2)?,
2878 recorded_at_ms: row.get(3)?,
2879 results_json: row.get(4)?,
2880 operations_json: row.get(5)?,
2881 resolution: row.get(6)?,
2882 resolved_at_ms: row.get(7)?,
2883 replacement_client_commit_id: row.get(8)?,
2884 })
2885 },
2886 )
2887 .optional()
2888 .map_err(|error| error.to_string())?;
2889 row.map(Self::outcome_from_row).transpose()
2890 }
2891
2892 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
2893 if query.limit == Some(0) {
2894 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
2895 }
2896 let mut sql = String::from(
2897 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2898 resolution, resolved_at_ms, replacement_client_commit_id
2899 FROM _syncular_commit_outcomes",
2900 );
2901 if query.active_only {
2902 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
2903 }
2904 sql.push_str(" ORDER BY seq DESC");
2905 if let Some(limit) = query.limit {
2906 sql.push_str(&format!(" LIMIT {limit}"));
2907 }
2908 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
2909 let rows = stmt
2910 .query_map([], |row| {
2911 Ok(StoredCommitOutcomeRow {
2912 sequence: row.get(0)?,
2913 client_commit_id: row.get(1)?,
2914 status: row.get(2)?,
2915 recorded_at_ms: row.get(3)?,
2916 results_json: row.get(4)?,
2917 operations_json: row.get(5)?,
2918 resolution: row.get(6)?,
2919 resolved_at_ms: row.get(7)?,
2920 replacement_client_commit_id: row.get(8)?,
2921 })
2922 })
2923 .map_err(|error| error.to_string())?;
2924 let mut outcomes = Vec::new();
2925 for row in rows {
2926 outcomes.push(Self::outcome_from_row(
2927 row.map_err(|error| error.to_string())?,
2928 )?);
2929 }
2930 Ok(outcomes)
2931 }
2932
2933 fn prune_commit_outcomes(&self) -> Result<(), String> {
2934 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
2935 let count = self
2936 .conn
2937 .query_row(
2938 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
2939 [],
2940 |row| row.get::<_, i64>(0),
2941 )
2942 .map_err(|error| error.to_string())? as usize;
2943 let excess = count.saturating_sub(max_entries);
2944 if excess == 0 {
2945 return Ok(());
2946 }
2947 let mut stmt = self
2948 .conn
2949 .prepare(
2950 "SELECT seq FROM _syncular_commit_outcomes
2951 WHERE status IN ('applied', 'cached') OR resolution != 'active'
2952 ORDER BY seq ASC LIMIT ?1",
2953 )
2954 .map_err(|error| error.to_string())?;
2955 let rows = stmt
2956 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
2957 .map_err(|error| error.to_string())?;
2958 let sequences = rows
2959 .collect::<Result<Vec<_>, _>>()
2960 .map_err(|error| error.to_string())?;
2961 drop(stmt);
2962 for sequence in sequences {
2963 self.conn
2964 .execute(
2965 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
2966 rusqlite::params![sequence],
2967 )
2968 .map_err(|error| error.to_string())?;
2969 }
2970 Ok(())
2971 }
2972
2973 pub fn subscribe(
2976 &mut self,
2977 id: String,
2978 table: String,
2979 scopes: Vec<(String, Vec<String>)>,
2980 params: Option<String>,
2981 ) -> Result<(), String> {
2982 if self.schema.table(&table).is_none() {
2983 return Err(format!("unknown table {table:?}"));
2984 }
2985 let sub = Subscription {
2986 id: id.clone(),
2987 table,
2988 requested: scopes,
2989 params,
2990 cursor: -1,
2991 bootstrap_state: None,
2992 state: SubState::Active,
2993 reason_code: None,
2994 effective: None,
2995 synced_once: false,
2996 };
2997 self.persist_sub(&sub);
2998 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
2999 *existing = sub;
3000 } else {
3001 self.subs.push(sub);
3002 }
3003 Ok(())
3004 }
3005
3006 pub fn unsubscribe(&mut self, id: &str) {
3007 self.subs.retain(|s| s.id != id);
3008 let _ = self.conn.execute(
3009 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
3010 rusqlite::params![id],
3011 );
3012 }
3013
3014 pub fn set_window(
3022 &mut self,
3023 base: &WindowBase,
3024 units: &[String],
3025 ) -> Result<CommandEffects, String> {
3026 let table = self
3027 .schema
3028 .table(&base.table)
3029 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
3030 if table.scope_column(&base.variable).is_none() {
3031 return Err(format!(
3032 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
3033 base.table, base.variable
3034 ));
3035 }
3036 let base_key = window_base_key(base);
3037 let wanted: std::collections::HashSet<&String> = units.iter().collect();
3038 let live = self.load_window_units(&base_key);
3039 self.begin_observation("syncular_window")?;
3040 let mut batch = ChangeAccumulator::default();
3041 let mut changed = false;
3042
3043 for unit in units {
3045 if live.iter().any(|(u, _)| u == unit) {
3046 continue;
3047 }
3048 let sub_id = derive_sub_id(base, unit);
3049 self.delete_pending_evict(&sub_id);
3050 self.insert_window_unit(&base_key, unit, &sub_id);
3051 self.subscribe(
3052 sub_id,
3053 base.table.clone(),
3054 unit_scopes(base, unit),
3055 base.params.clone(),
3056 )?;
3057 batch.window(&base_key, &base.table, unit);
3058 changed = true;
3059 }
3060
3061 for (unit, sub_id) in live {
3063 if wanted.contains(&unit) {
3064 continue;
3065 }
3066 let effective = self
3067 .subs
3068 .iter()
3069 .find(|sub| sub.id == sub_id)
3070 .and_then(|sub| sub.effective.clone())
3071 .unwrap_or_else(|| unit_scopes(base, &unit));
3072 self.record_scope_map(&mut batch, &base.table, &effective);
3073 batch.window(&base_key, &base.table, &unit);
3074 self.evict_unit(&base_key, base, &unit, &sub_id);
3075 changed = true;
3076 }
3077 if let Err(error) = self.finish_observation("syncular_window", batch) {
3078 self.rollback_observation("syncular_window");
3079 return Err(error);
3080 }
3081 Ok(if changed {
3082 CommandEffects::interactive()
3083 } else {
3084 CommandEffects::none()
3085 })
3086 }
3087
3088 pub fn window_state(&self, base: &WindowBase) -> WindowState {
3093 let mut units = Vec::new();
3094 let mut pending = Vec::new();
3095 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
3096 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
3097 Some(sub) => {
3098 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
3099 }
3100 None => true,
3101 };
3102 if is_pending {
3103 pending.push(unit.clone());
3104 }
3105 units.push(unit);
3106 }
3107 WindowState { units, pending }
3108 }
3109
3110 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
3111 let mut stmt = match self
3112 .conn
3113 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
3114 {
3115 Ok(stmt) => stmt,
3116 Err(_) => return Vec::new(),
3117 };
3118 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
3119 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
3120 });
3121 match rows {
3122 Ok(rows) => rows.filter_map(Result::ok).collect(),
3123 Err(_) => Vec::new(),
3124 }
3125 }
3126
3127 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
3128 let mut stmt = match self.conn.prepare(
3129 "SELECT windows.base, windows.unit, subscriptions.tbl
3130 FROM _syncular_windows AS windows
3131 JOIN _syncular_subscriptions AS subscriptions
3132 ON subscriptions.id = windows.sub_id
3133 ORDER BY windows.base, windows.unit",
3134 ) {
3135 Ok(stmt) => stmt,
3136 Err(_) => return Vec::new(),
3137 };
3138 let rows = stmt.query_map([], |row| {
3139 Ok((
3140 row.get::<_, String>(0)?,
3141 row.get::<_, String>(1)?,
3142 row.get::<_, String>(2)?,
3143 ))
3144 });
3145 match rows {
3146 Ok(rows) => rows.filter_map(Result::ok).collect(),
3147 Err(_) => Vec::new(),
3148 }
3149 }
3150
3151 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
3152 self.conn
3153 .query_row(
3154 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
3155 rusqlite::params![sub_id],
3156 |row| Ok((row.get(0)?, row.get(1)?)),
3157 )
3158 .ok()
3159 }
3160
3161 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
3162 let _ = self.conn.execute(
3163 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
3164 rusqlite::params![base_key, unit, sub_id],
3165 );
3166 }
3167
3168 fn delete_window_unit(&self, base_key: &str, unit: &str) {
3169 let _ = self.conn.execute(
3170 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
3171 rusqlite::params![base_key, unit],
3172 );
3173 }
3174
3175 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
3182 let effective = self
3183 .subs
3184 .iter()
3185 .find(|s| s.id == sub_id)
3186 .and_then(|s| s.effective.clone())
3187 .unwrap_or_else(|| unit_scopes(base, unit));
3188 let pinned = self.pinned_row_ids(&base.table);
3189 let deferred = self
3190 .evict_scope_rows(&base.table, &effective, &pinned)
3191 .unwrap_or(false);
3192 self.delete_window_unit(base_key, unit);
3193 self.unsubscribe(sub_id);
3194 if deferred {
3195 self.save_pending_evict(sub_id, &base.table, &effective);
3196 } else {
3197 self.delete_pending_evict(sub_id);
3198 }
3199 self.rebuild_overlay();
3200 }
3201
3202 fn evict_scope_rows(
3206 &mut self,
3207 table_name: &str,
3208 effective: &[(String, Vec<String>)],
3209 pinned: &std::collections::HashSet<String>,
3210 ) -> Result<bool, ()> {
3211 if effective.is_empty() {
3212 return Ok(false);
3213 }
3214 let table = self.schema.table(table_name).ok_or(())?.clone();
3215 let mut clauses = Vec::new();
3216 let mut params: Vec<SqlValue> = Vec::new();
3217 for (variable, values) in effective {
3218 let column = table.scope_column(variable).ok_or(())?;
3219 let placeholders: Vec<String> = values
3220 .iter()
3221 .map(|v| {
3222 params.push(SqlValue::Text(v.clone()));
3223 "?".to_owned()
3224 })
3225 .collect();
3226 clauses.push(format!(
3227 "{} IN ({})",
3228 quote_ident(column),
3229 placeholders.join(", ")
3230 ));
3231 }
3232 let mut sql = format!(
3233 "DELETE FROM {} WHERE {}",
3234 base_table(table_name),
3235 clauses.join(" AND ")
3236 );
3237 if !pinned.is_empty() {
3238 let pk = quote_ident(&table.primary_key);
3239 let holes: Vec<String> = pinned
3240 .iter()
3241 .map(|id| {
3242 params.push(SqlValue::Text(id.clone()));
3243 "?".to_owned()
3244 })
3245 .collect();
3246 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
3247 }
3248 self.overlay_dirty.set(true);
3249 self.conn
3250 .execute(&sql, rusqlite::params_from_iter(params))
3251 .map_err(|_| ())?;
3252 if pinned.is_empty() {
3253 return Ok(false);
3254 }
3255 let mut where_params: Vec<SqlValue> = Vec::new();
3258 let mut where_clauses = Vec::new();
3259 for (variable, values) in effective {
3260 let column = table.scope_column(variable).ok_or(())?;
3261 let placeholders: Vec<String> = values
3262 .iter()
3263 .map(|v| {
3264 where_params.push(SqlValue::Text(v.clone()));
3265 "?".to_owned()
3266 })
3267 .collect();
3268 where_clauses.push(format!(
3269 "{} IN ({})",
3270 quote_ident(column),
3271 placeholders.join(", ")
3272 ));
3273 }
3274 let pk = quote_ident(&table.primary_key);
3275 let select = format!(
3276 "SELECT {} FROM {} WHERE {}",
3277 pk,
3278 base_table(table_name),
3279 where_clauses.join(" AND ")
3280 );
3281 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
3282 let survivors: Vec<String> = stmt
3283 .query_map(rusqlite::params_from_iter(where_params), |row| {
3284 row.get::<_, String>(0)
3285 })
3286 .map_err(|_| ())?
3287 .filter_map(Result::ok)
3288 .collect();
3289 Ok(survivors.iter().any(|id| pinned.contains(id)))
3290 }
3291
3292 fn drain_pending_evictions(&mut self) {
3296 let pending = self.load_pending_evictions();
3297 if pending.is_empty() {
3298 return;
3299 }
3300 for (sub_id, table_name, effective) in pending {
3301 if self.schema.table(&table_name).is_none() {
3302 self.delete_pending_evict(&sub_id);
3303 continue;
3304 }
3305 let pinned = self.pinned_row_ids(&table_name);
3306 let deferred = self
3307 .evict_scope_rows(&table_name, &effective, &pinned)
3308 .unwrap_or(false);
3309 if !deferred {
3310 self.delete_pending_evict(&sub_id);
3311 }
3312 }
3313 self.rebuild_overlay_if_dirty();
3314 }
3315
3316 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
3319 let mut pinned = std::collections::HashSet::new();
3320 for commit in &self.outbox {
3321 for op in &commit.ops {
3322 if op.table == table {
3323 pinned.insert(op.row_id.clone());
3324 }
3325 }
3326 }
3327 pinned
3328 }
3329
3330 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
3331 let _ = self.conn.execute(
3332 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
3333 VALUES (?1, ?2, ?3)",
3334 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
3335 );
3336 }
3337
3338 fn delete_pending_evict(&self, sub_id: &str) {
3339 let _ = self.conn.execute(
3340 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
3341 rusqlite::params![sub_id],
3342 );
3343 }
3344
3345 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
3346 let mut stmt = match self
3347 .conn
3348 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
3349 {
3350 Ok(stmt) => stmt,
3351 Err(_) => return Vec::new(),
3352 };
3353 let rows = stmt.query_map([], |row| {
3354 Ok((
3355 row.get::<_, String>(0)?,
3356 row.get::<_, String>(1)?,
3357 row.get::<_, String>(2)?,
3358 ))
3359 });
3360 let mut out = Vec::new();
3361 if let Ok(rows) = rows {
3362 for entry in rows.filter_map(Result::ok) {
3363 let (sub_id, table, json) = entry;
3364 if let Ok(value) = serde_json::from_str::<Value>(&json) {
3365 if let Ok(effective) = json_to_scope_map(&value) {
3366 out.push((sub_id, table, effective));
3367 }
3368 }
3369 }
3370 }
3371 out
3372 }
3373
3374 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
3376 if mutations.is_empty() {
3377 return Err("a commit must contain at least one operation (§6.1)".to_owned());
3378 }
3379 let mut ops = Vec::with_capacity(mutations.len());
3380 for mutation in mutations {
3381 match mutation {
3382 Mutation::Upsert {
3383 table,
3384 values,
3385 base_version,
3386 } => {
3387 let schema_table = self
3388 .schema
3389 .table(&table)
3390 .ok_or_else(|| format!("unknown table {table:?}"))?;
3391 let values = normalize_values_casing(schema_table, values)?;
3395 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
3396 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
3399 ops.push(OutboxOp {
3400 upsert: true,
3401 table,
3402 row_id,
3403 base_version,
3404 values: Some(values),
3405 changed_fields: None,
3406 });
3407 }
3408 Mutation::Delete {
3409 table,
3410 row_id,
3411 base_version,
3412 } => {
3413 if self.schema.table(&table).is_none() {
3414 return Err(format!("unknown table {table:?}"));
3415 }
3416 ops.push(OutboxOp {
3417 upsert: false,
3418 table,
3419 row_id,
3420 base_version,
3421 values: None,
3422 changed_fields: None,
3423 });
3424 }
3425 }
3426 }
3427 self.record_outbox_commit(ops)
3428 }
3429
3430 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
3431 let commit = OutboxCommit {
3432 client_commit_id: uuid::Uuid::new_v4().to_string(),
3433 ops,
3434 };
3435 self.begin_observation("syncular_mutation")?;
3436 let mut batch = ChangeAccumulator::default();
3437 for op in &commit.ops {
3438 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
3439 if let Some(values) = &op.values {
3440 if let Some(table) = self.schema.table(&op.table) {
3441 for scope in &table.scope_variables {
3442 if let Some(Value::String(value)) = values.get(&scope.column) {
3443 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
3444 precise = true;
3445 }
3446 }
3447 }
3448 }
3449 if !precise {
3450 batch.table(&op.table);
3451 }
3452 }
3453 self.persist_outbox_insert(&commit);
3454 let id = commit.client_commit_id.clone();
3455 self.outbox.push(commit);
3456 self.overlay_dirty.set(true);
3457 self.rebuild_overlay();
3458 batch.status = true;
3459 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
3460 self.rollback_observation("syncular_mutation");
3461 return Err(error);
3462 }
3463 Ok(id)
3464 }
3465
3466 pub fn patch(
3471 &mut self,
3472 table: &str,
3473 row_id: &str,
3474 partial: Map<String, Value>,
3475 base_version: Option<i64>,
3476 ) -> Result<String, String> {
3477 let schema_table = self
3478 .schema
3479 .table(table)
3480 .ok_or_else(|| format!("unknown table {table:?}"))?;
3481 let partial = normalize_values_casing(schema_table, partial)?;
3482 let mut values = self
3483 .read_rows(table)?
3484 .into_iter()
3485 .find(|row| row.row_id == row_id)
3486 .map(|row| row.values)
3487 .ok_or_else(|| {
3488 format!(
3489 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
3490 )
3491 })?;
3492 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
3493 changed_fields.sort();
3494 values.extend(partial);
3495 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
3496 if row_id_from_values != row_id {
3497 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
3498 }
3499 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
3500 self.record_outbox_commit(vec![OutboxOp {
3501 upsert: true,
3502 table: table.to_owned(),
3503 row_id: row_id.to_owned(),
3504 base_version,
3505 values: Some(values),
3506 changed_fields: Some(changed_fields),
3507 }])
3508 }
3509
3510 pub fn pending_commit_ids(&self) -> Vec<String> {
3511 self.outbox
3512 .iter()
3513 .map(|c| c.client_commit_id.clone())
3514 .collect()
3515 }
3516
3517 pub fn conflicts(&self) -> &[ConflictRecord] {
3518 &self.conflicts
3519 }
3520
3521 pub fn rejections(&self) -> &[RejectionRecord] {
3522 &self.rejections
3523 }
3524
3525 pub fn resolve_commit_outcome(
3526 &mut self,
3527 input: ResolveCommitOutcomeInput,
3528 ) -> Result<CommitOutcome, String> {
3529 let current = self
3530 .commit_outcome(&input.client_commit_id)?
3531 .ok_or_else(|| {
3532 format!(
3533 "sync.outcome_not_found: no durable outcome exists for {:?}",
3534 input.client_commit_id
3535 )
3536 })?;
3537 if current.resolution != CommitOutcomeResolution::Active {
3538 return Ok(current);
3539 }
3540 if input.resolution == CommitOutcomeResolution::Active {
3541 return Err("sync.invalid_request: resolution must leave active state".to_owned());
3542 }
3543 match input.resolution {
3544 CommitOutcomeResolution::Superseded => {
3545 let replacement = input
3546 .replacement_client_commit_id
3547 .as_deref()
3548 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
3549 .ok_or_else(|| {
3550 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
3551 .to_owned()
3552 })?;
3553 let _ = replacement;
3554 }
3555 _ if input.replacement_client_commit_id.is_some() => {
3556 return Err(
3557 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
3558 .to_owned(),
3559 );
3560 }
3561 _ => {}
3562 }
3563 let allowed = match current.status {
3564 CommitOutcomeStatus::Conflict => matches!(
3565 input.resolution,
3566 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
3567 ),
3568 CommitOutcomeStatus::Rejected => {
3569 input.resolution == CommitOutcomeResolution::Superseded
3570 }
3571 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
3572 input.resolution == CommitOutcomeResolution::Dismissed
3573 }
3574 };
3575 if !allowed {
3576 return Err(format!(
3577 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
3578 input.resolution, current.status
3579 ));
3580 }
3581
3582 self.begin_observation("syncular_outcome_resolution")?;
3583 let result = (|| {
3584 self.conn
3585 .execute(
3586 "UPDATE _syncular_commit_outcomes
3587 SET resolution = ?1, resolved_at_ms = ?2,
3588 replacement_client_commit_id = ?3
3589 WHERE client_commit_id = ?4 AND resolution = 'active'",
3590 rusqlite::params![
3591 Self::outcome_resolution_name(input.resolution),
3592 self.clock_now_ms(),
3593 input.replacement_client_commit_id,
3594 input.client_commit_id
3595 ],
3596 )
3597 .map_err(|error| error.to_string())?;
3598 let resolved = self
3599 .commit_outcome(¤t.client_commit_id)?
3600 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
3601 self.prune_commit_outcomes()?;
3602 Ok(resolved)
3603 })();
3604 let resolved = match result {
3605 Ok(outcome) => outcome,
3606 Err(error) => {
3607 self.rollback_observation("syncular_outcome_resolution");
3608 return Err(error);
3609 }
3610 };
3611 self.conflicts
3612 .retain(|record| record.client_commit_id != current.client_commit_id);
3613 self.rejections
3614 .retain(|record| record.client_commit_id != current.client_commit_id);
3615 let batch = ChangeAccumulator {
3616 conflicts: current.status == CommitOutcomeStatus::Conflict,
3617 rejections: current.status == CommitOutcomeStatus::Rejected,
3618 outcomes: true,
3619 ..ChangeAccumulator::default()
3620 };
3621 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
3622 self.rollback_observation("syncular_outcome_resolution");
3623 return Err(error);
3624 }
3625 Ok(resolved)
3626 }
3627
3628 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
3629 self.schema_floor.as_ref()
3630 }
3631
3632 pub fn lease_state(&self) -> Option<&LeaseState> {
3634 self.lease_state.as_ref()
3635 }
3636
3637 fn record_lease_error(&mut self, code: &str) {
3640 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
3641 return;
3642 }
3643 let mut next = self.lease_state.clone().unwrap_or_default();
3644 next.error_code = Some(code.to_owned());
3645 self.set_lease_state(Some(next));
3646 }
3647
3648 fn set_lease_state(&mut self, next: Option<LeaseState>) {
3649 if self.lease_state == next {
3650 return;
3651 }
3652 if self.begin_observation("syncular_lease").is_err() {
3653 return;
3654 }
3655 self.lease_state = next;
3656 if let Some(lease) = &self.lease_state {
3657 if let Ok(json) = serde_json::to_string(lease) {
3658 self.set_meta(LEASE_STATE_KEY, &json);
3659 }
3660 } else {
3661 self.delete_meta(LEASE_STATE_KEY);
3662 }
3663 let batch = ChangeAccumulator {
3664 status: true,
3665 ..ChangeAccumulator::default()
3666 };
3667 if self.finish_observation("syncular_lease", batch).is_err() {
3668 self.rollback_observation("syncular_lease");
3669 }
3670 }
3671
3672 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
3673 if self.schema_floor == next {
3674 return;
3675 }
3676 if self.begin_observation("syncular_schema_floor").is_err() {
3677 return;
3678 }
3679 self.schema_floor = next;
3680 self.stopped = self.schema_floor.is_some();
3681 if let Some(floor) = &self.schema_floor {
3682 if let Ok(json) = serde_json::to_string(floor) {
3683 self.set_meta(SCHEMA_FLOOR_KEY, &json);
3684 }
3685 } else {
3686 self.delete_meta(SCHEMA_FLOOR_KEY);
3687 }
3688 let batch = ChangeAccumulator {
3689 status: true,
3690 ..ChangeAccumulator::default()
3691 };
3692 if self
3693 .finish_observation("syncular_schema_floor", batch)
3694 .is_err()
3695 {
3696 self.rollback_observation("syncular_schema_floor");
3697 }
3698 }
3699
3700 fn set_upgrading(&mut self, value: bool) {
3701 if self.upgrading == value {
3702 return;
3703 }
3704 if self.begin_observation("syncular_upgrading").is_err() {
3705 return;
3706 }
3707 self.upgrading = value;
3708 let batch = ChangeAccumulator {
3709 status: true,
3710 ..ChangeAccumulator::default()
3711 };
3712 if self
3713 .finish_observation("syncular_upgrading", batch)
3714 .is_err()
3715 {
3716 self.rollback_observation("syncular_upgrading");
3717 }
3718 }
3719
3720 pub fn sync_needed(&self) -> bool {
3721 self.sync_needed
3722 }
3723
3724 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
3725 let sub = self.subs.iter().find(|s| s.id == id)?;
3726 Some(SubscriptionStateView {
3727 id: sub.id.clone(),
3728 table: sub.table.clone(),
3729 status: sub.state.name().to_owned(),
3730 cursor: sub.cursor,
3731 has_resume_token: sub.bootstrap_state.is_some(),
3732 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3733 reason_code: sub.reason_code.clone(),
3734 })
3735 }
3736
3737 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
3738 let schema_table = self
3739 .schema
3740 .table(table)
3741 .ok_or_else(|| format!("unknown table {table:?}"))?;
3742 let sql = format!(
3743 "SELECT * FROM {} ORDER BY {} ASC",
3744 visible_table(table),
3745 quote_ident(&schema_table.primary_key)
3746 );
3747 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
3748 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
3749 let mut out = Vec::new();
3750 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
3751 let mut values = Map::new();
3752 for (i, column) in schema_table.columns.iter().enumerate() {
3753 let value = row.get_ref(i).map_err(|e| e.to_string())?;
3754 values.insert(column.name.clone(), sql_ref_to_json(column, value));
3755 }
3756 let version: i64 = row
3757 .get(schema_table.columns.len())
3758 .map_err(|e| e.to_string())?;
3759 let row_id = match values.get(&schema_table.primary_key) {
3760 Some(Value::String(s)) => s.clone(),
3761 Some(Value::Number(n)) => n.to_string(),
3762 Some(Value::Bool(b)) => b.to_string(),
3763 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
3764 };
3765 out.push(RowState {
3766 row_id,
3767 version,
3768 values,
3769 });
3770 }
3771 Ok(out)
3772 }
3773
3774 #[cfg(feature = "crdt-yjs")]
3790 fn crdt_column_bytes(
3791 &self,
3792 table: &str,
3793 row_id: &str,
3794 column: &str,
3795 ) -> Result<Option<Vec<u8>>, String> {
3796 let schema_table = self
3797 .schema
3798 .table(table)
3799 .ok_or_else(|| format!("unknown table {table:?}"))?;
3800 let col = schema_table
3801 .columns
3802 .iter()
3803 .find(|c| c.name == column)
3804 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
3805 if col.ty != ColumnType::Crdt {
3806 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
3807 }
3808 let sql = format!(
3809 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
3810 quote_ident(column),
3811 visible_table(table),
3812 quote_ident(&schema_table.primary_key)
3813 );
3814 let bytes: Option<Vec<u8>> = self
3815 .conn
3816 .query_row(&sql, rusqlite::params![row_id], |row| {
3817 row.get::<_, Option<Vec<u8>>>(0)
3818 })
3819 .map_err(|e| match e {
3820 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
3821 other => other.to_string(),
3822 })?;
3823 Ok(bytes)
3824 }
3825
3826 #[cfg(feature = "crdt-yjs")]
3830 pub fn crdt_text(
3831 &self,
3832 table: &str,
3833 row_id: &str,
3834 column: &str,
3835 name: &str,
3836 ) -> Result<String, String> {
3837 let bytes = self
3838 .crdt_column_bytes(table, row_id, column)?
3839 .unwrap_or_default();
3840 crate::crdt::text(&bytes, name)
3841 }
3842
3843 #[cfg(feature = "crdt-yjs")]
3847 pub fn crdt_insert_text(
3848 &mut self,
3849 table: &str,
3850 row_id: &str,
3851 column: &str,
3852 name: &str,
3853 index: u32,
3854 value: &str,
3855 ) -> Result<String, String> {
3856 let current = self
3857 .crdt_column_bytes(table, row_id, column)?
3858 .unwrap_or_default();
3859 let update = crate::crdt::insert_text(¤t, name, index, value)?;
3860 self.crdt_push_update(table, row_id, column, &update)
3861 }
3862
3863 #[cfg(feature = "crdt-yjs")]
3866 pub fn crdt_delete_text(
3867 &mut self,
3868 table: &str,
3869 row_id: &str,
3870 column: &str,
3871 name: &str,
3872 index: u32,
3873 len: u32,
3874 ) -> Result<String, String> {
3875 let current = self
3876 .crdt_column_bytes(table, row_id, column)?
3877 .unwrap_or_default();
3878 let update = crate::crdt::delete_text(¤t, name, index, len)?;
3879 self.crdt_push_update(table, row_id, column, &update)
3880 }
3881
3882 #[cfg(feature = "crdt-yjs")]
3887 pub fn crdt_apply_update(
3888 &mut self,
3889 table: &str,
3890 row_id: &str,
3891 column: &str,
3892 update: &[u8],
3893 ) -> Result<String, String> {
3894 let current = self
3895 .crdt_column_bytes(table, row_id, column)?
3896 .unwrap_or_default();
3897 let next = crate::crdt::apply_update(¤t, update)?;
3898 self.crdt_push_update(table, row_id, column, &next)
3899 }
3900
3901 #[cfg(feature = "crdt-yjs")]
3907 fn crdt_push_update(
3908 &mut self,
3909 table: &str,
3910 row_id: &str,
3911 column: &str,
3912 crdt_bytes: &[u8],
3913 ) -> Result<String, String> {
3914 let schema_table = self
3915 .schema
3916 .table(table)
3917 .ok_or_else(|| format!("unknown table {table:?}"))?
3918 .clone();
3919 let mut values: Map<String, Value> = self
3922 .read_rows(table)?
3923 .into_iter()
3924 .find(|r| r.row_id == row_id)
3925 .map(|r| r.values)
3926 .unwrap_or_else(|| {
3927 let mut map = Map::new();
3928 map.insert(
3929 schema_table.primary_key.clone(),
3930 Value::from(row_id.to_owned()),
3931 );
3932 map
3933 });
3934 let mut bytes_obj = Map::new();
3936 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
3937 values.insert(column.to_owned(), Value::Object(bytes_obj));
3938 self.mutate(vec![Mutation::Upsert {
3939 table: table.to_owned(),
3940 values,
3941 base_version: None,
3942 }])
3943 }
3944
3945 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
3960 query_connection(&self.conn, sql, params)
3961 }
3962
3963 pub fn query_snapshot(
3965 &mut self,
3966 sql: &str,
3967 params: &[QueryValue],
3968 coverage: &[WindowCoverage],
3969 ) -> Result<QuerySnapshot, String> {
3970 snapshot_connection(&self.conn, sql, params, coverage)
3971 }
3972
3973 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
3976 let mut frames = vec![Frame::ReqHeader {
3977 client_id: self.client_id.clone(),
3978 schema_version: self.schema.version,
3979 }];
3980 let mut pushed_ids = Vec::new();
3981 let mut ops_in_request = 0usize;
3982 let mut deferred_commits = 0usize;
3983 for (index, commit) in self.outbox.iter().enumerate() {
3984 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
3989 deferred_commits = self.outbox.len() - index;
3990 break;
3991 }
3992 ops_in_request += commit.ops.len();
3993 let operations = commit
3994 .ops
3995 .iter()
3996 .map(|op| {
3997 let payload = op.values.as_ref().and_then(|values| {
3998 let table = self.schema.table(&op.table)?;
3999 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
4003 });
4004 ssp2::model::Operation {
4005 table: op.table.clone(),
4006 row_id: op.row_id.clone(),
4007 op: if op.upsert { Op::Upsert } else { Op::Delete },
4008 base_version: op.base_version,
4009 payload,
4010 }
4011 })
4012 .collect();
4013 frames.push(Frame::PushCommit {
4014 client_commit_id: commit.client_commit_id.clone(),
4015 operations,
4016 });
4017 pushed_ids.push(commit.client_commit_id.clone());
4018 }
4019 let accept = self.limits.accept.unwrap_or(if url_capable {
4022 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
4023 } else {
4024 DEFAULT_ACCEPT
4025 });
4026 frames.push(Frame::PullHeader {
4027 limit_commits: self.limits.limit_commits.unwrap_or(0),
4028 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
4029 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
4030 accept,
4031 });
4032 let mut fresh = Vec::new();
4033 for sub in &self.subs {
4034 if sub.state != SubState::Active {
4035 continue;
4036 }
4037 let mut scopes = sub.requested.clone();
4038 sort_scope_map(&mut scopes);
4039 frames.push(Frame::Subscription {
4040 id: sub.id.clone(),
4041 table: sub.table.clone(),
4042 scopes,
4043 params: sub.params.clone().map(RawJson),
4044 cursor: sub.cursor,
4045 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
4046 });
4047 fresh.push((
4048 sub.id.clone(),
4049 sub.cursor < 0 && sub.bootstrap_state.is_none(),
4050 ));
4051 }
4052 let message = Message {
4053 msg_kind: MsgKind::Request,
4054 frames,
4055 };
4056 (
4057 message,
4058 RequestMeta {
4059 pushed_ids,
4060 fresh,
4061 accept,
4062 deferred_commits,
4063 },
4064 )
4065 }
4066
4067 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
4070 if self.stopped {
4071 return SyncOutcome::Ok(SyncReport {
4074 schema_floor: self.schema_floor.clone(),
4075 ..SyncReport::default()
4076 });
4077 }
4078 self.set_sync_needed(false, false);
4081 if self.schema_has_blobs() {
4084 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
4085 if Self::retryable_transport_code(&code) {
4086 self.schedule_background_retry();
4087 }
4088 return SyncOutcome::Failed {
4089 error_code: code,
4090 message,
4091 };
4092 }
4093 }
4094 let (message, meta) = self.build_request(transport.supports_url_fetch());
4095 let request_bytes = encode_message(&message);
4096 let round = if self.realtime_connected {
4100 transport.realtime_sync(&request_bytes)
4101 } else {
4102 transport.sync(&request_bytes)
4103 };
4104 let response_bytes = match round {
4105 Ok(bytes) => bytes,
4106 Err(TransportError { code, message }) => {
4107 self.record_lease_error(&code);
4110 if Self::retryable_transport_code(&code) {
4111 self.schedule_background_retry();
4112 }
4113 return SyncOutcome::Failed {
4114 error_code: code,
4115 message,
4116 };
4117 }
4118 };
4119 let response = match decode_message(&response_bytes) {
4120 Ok(message) => message,
4121 Err(error) => {
4122 return SyncOutcome::Failed {
4125 error_code: error.code.as_str().to_owned(),
4126 message: error.detail,
4127 };
4128 }
4129 };
4130 if response.msg_kind != MsgKind::Response {
4131 return SyncOutcome::Failed {
4132 error_code: "sync.invalid_request".to_owned(),
4133 message: "expected a response message".to_owned(),
4134 };
4135 }
4136 let outcome = self.process_response(transport, response, &meta);
4137 match &outcome {
4138 SyncOutcome::Ok(_) => self.reset_background_retry(),
4139 SyncOutcome::Failed { error_code, .. }
4140 if Self::retryable_transport_code(error_code) =>
4141 {
4142 self.schedule_background_retry();
4143 }
4144 SyncOutcome::Failed { .. } => {}
4145 }
4146 if meta.deferred_commits > 0 {
4147 self.set_sync_needed(true, true);
4150 }
4151 outcome
4152 }
4153
4154 pub fn sync_until_idle(
4155 &mut self,
4156 transport: &mut dyn Transport,
4157 max_rounds: Option<u32>,
4158 ) -> SyncOutcome {
4159 let rounds = max_rounds.unwrap_or(12).max(1);
4160 let mut aggregate = SyncReport::default();
4161 for _ in 0..rounds {
4162 match self.sync(transport) {
4163 SyncOutcome::Failed {
4164 error_code,
4165 message,
4166 } => {
4167 return SyncOutcome::Failed {
4168 error_code,
4169 message,
4170 };
4171 }
4172 SyncOutcome::Ok(report) => {
4173 aggregate.pushed += report.pushed;
4174 aggregate.applied.extend(report.applied.iter().cloned());
4175 aggregate.rejected.extend(report.rejected.iter().cloned());
4176 aggregate.retryable.extend(report.retryable.iter().cloned());
4177 aggregate.conflicts += report.conflicts;
4178 aggregate.commits_applied += report.commits_applied;
4179 aggregate.segment_rows_applied += report.segment_rows_applied;
4180 aggregate.bootstrapping = report.bootstrapping.clone();
4181 aggregate.resets.extend(report.resets.iter().cloned());
4182 aggregate.revoked.extend(report.revoked.iter().cloned());
4183 aggregate.failed.extend(report.failed.iter().cloned());
4184 if report.schema_floor.is_some() {
4185 aggregate.schema_floor = report.schema_floor.clone();
4186 }
4187 let more = !report.bootstrapping.is_empty()
4193 || report.commits_applied > 0
4194 || report.segment_rows_applied > 0
4195 || !report.resets.is_empty()
4196 || self.sync_needed;
4197 if !more {
4198 break;
4199 }
4200 }
4201 }
4202 }
4203 SyncOutcome::Ok(aggregate)
4204 }
4205
4206 fn process_response(
4207 &mut self,
4208 transport: &mut dyn Transport,
4209 response: Message,
4210 meta: &RequestMeta,
4211 ) -> SyncOutcome {
4212 let mut report = SyncReport {
4213 pushed: meta.pushed_ids.len() as u32,
4214 ..SyncReport::default()
4215 };
4216 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
4217 HashMap::new();
4218 for frame in &response.frames {
4219 if let Frame::PushResultDetails {
4220 client_commit_id,
4221 entries,
4222 } = frame
4223 {
4224 let details = rejection_details_by_commit
4225 .entry(client_commit_id.clone())
4226 .or_default();
4227 for entry in entries {
4228 let parsed = match RejectionDetails::parse(&entry.details.0) {
4229 Ok(value) => value,
4230 Err(message) => {
4231 return SyncOutcome::Failed {
4232 error_code: "sync.invalid_request".to_owned(),
4233 message,
4234 };
4235 }
4236 };
4237 details.insert(entry.op_index, parsed);
4238 }
4239 }
4240 }
4241 let mut frames = response.frames.into_iter();
4242 match frames.next() {
4243 Some(Frame::RespHeader {
4244 required_schema_version,
4245 latest_schema_version,
4246 }) => {
4247 if let Some(required) = required_schema_version {
4248 let floor = SchemaFloor {
4251 required_schema_version: Some(required),
4252 latest_schema_version,
4253 };
4254 self.set_schema_floor(Some(floor.clone()));
4255 report.schema_floor = Some(floor);
4256 return SyncOutcome::Ok(report);
4257 }
4258 }
4259 _ => {
4260 return SyncOutcome::Failed {
4261 error_code: "sync.invalid_request".to_owned(),
4262 message: "response does not start with RESP_HEADER".to_owned(),
4263 };
4264 }
4265 }
4266
4267 let mut failure: Option<(String, String)> = None;
4268 while let Some(frame) = frames.next() {
4269 match frame {
4270 Frame::PushResult {
4271 client_commit_id,
4272 status,
4273 commit_seq: _,
4274 results,
4275 } => {
4276 self.handle_push_result(
4277 &client_commit_id,
4278 status,
4279 &results,
4280 rejection_details_by_commit.get(&client_commit_id),
4281 &mut report,
4282 );
4283 }
4284 Frame::PushResultDetails { .. } => {}
4285 Frame::SubStart {
4286 id,
4287 status,
4288 reason_code,
4289 effective_scopes,
4290 bootstrap: _,
4291 } => {
4292 let mut body = Vec::new();
4293 let mut sub_end: Option<(i64, Option<String>)> = None;
4294 for inner in frames.by_ref() {
4295 match inner {
4296 Frame::SubEnd {
4297 next_cursor,
4298 bootstrap_state,
4299 } => {
4300 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
4301 break;
4302 }
4303 Frame::Unknown { .. } => {}
4304 other => body.push(other),
4305 }
4306 }
4307 let Some((next_cursor, bootstrap_state)) = sub_end else {
4308 failure = Some((
4309 "sync.invalid_request".to_owned(),
4310 "subscription section without SUB_END".to_owned(),
4311 ));
4312 break;
4313 };
4314 if let Err(SectionError::Abort(code, message)) = self.process_section(
4315 transport,
4316 &id,
4317 status,
4318 &reason_code,
4319 effective_scopes,
4320 body,
4321 next_cursor,
4322 bootstrap_state,
4323 meta,
4324 &mut report,
4325 ) {
4326 failure = Some((code, message));
4327 break;
4328 }
4329 }
4330 Frame::Lease {
4331 lease_id,
4332 expires_at_ms,
4333 } => {
4334 self.set_lease_state(Some(LeaseState {
4337 lease_id: Some(lease_id),
4338 expires_at_ms: Some(expires_at_ms),
4339 error_code: None,
4340 }));
4341 }
4342 Frame::Error { code, message, .. } => {
4343 failure = Some((code, message));
4346 break;
4347 }
4348 Frame::Unknown { .. } => {}
4349 _ => {
4350 failure = Some((
4351 "sync.invalid_request".to_owned(),
4352 "unexpected frame in response".to_owned(),
4353 ));
4354 break;
4355 }
4356 }
4357 }
4358
4359 if self.overlay_dirty.get() {
4362 self.rebuild_overlay();
4363 }
4364 self.reconcile_blob_refcounts(false);
4371
4372 if let Some((error_code, message)) = failure {
4373 return SyncOutcome::Failed {
4374 error_code,
4375 message,
4376 };
4377 }
4378 self.drain_pending_evictions();
4381 self.ack_after_pull(transport);
4382 if self.upgrading && report.bootstrapping.is_empty() {
4385 self.set_upgrading(false);
4386 }
4387 SyncOutcome::Ok(report)
4388 }
4389
4390 fn handle_push_result(
4393 &mut self,
4394 client_commit_id: &str,
4395 status: PushStatus,
4396 results: &[OpResult],
4397 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
4398 report: &mut SyncReport,
4399 ) {
4400 let Some(index) = self
4401 .outbox
4402 .iter()
4403 .position(|c| c.client_commit_id == client_commit_id)
4404 else {
4405 return;
4406 };
4407 if self.begin_observation("syncular_push_result").is_err() {
4408 return;
4409 }
4410 let mut batch = ChangeAccumulator::default();
4411 let operations = self.outbox[index].ops.clone();
4412 match status {
4413 PushStatus::Applied | PushStatus::Cached => {
4414 let journal_results = results
4417 .iter()
4418 .map(|result| {
4419 let op_index = match result {
4420 OpResult::Applied { op_index }
4421 | OpResult::Conflict { op_index, .. }
4422 | OpResult::Error { op_index, .. } => *op_index,
4423 };
4424 CommitOperationOutcome::Applied { op_index }
4425 })
4426 .collect::<Vec<_>>();
4427 let outcome_status = if status == PushStatus::Applied {
4428 CommitOutcomeStatus::Applied
4429 } else {
4430 CommitOutcomeStatus::Cached
4431 };
4432 if self
4433 .persist_commit_outcome(
4434 client_commit_id,
4435 outcome_status,
4436 &journal_results,
4437 None,
4438 )
4439 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4440 .and_then(|()| self.prune_commit_outcomes())
4441 .is_err()
4442 {
4443 self.rollback_observation("syncular_push_result");
4444 return;
4445 }
4446 report.applied.push(client_commit_id.to_owned());
4447 self.outbox.remove(index);
4448 self.overlay_dirty.set(true);
4449 batch.status = true;
4450 batch.outcomes = true;
4451 }
4452 PushStatus::Rejected => {
4453 if results.iter().any(|result| {
4454 matches!(
4455 result,
4456 OpResult::Error {
4457 code,
4458 retryable: true,
4459 ..
4460 } if code == "sync.idempotency_cache_miss"
4461 )
4462 }) {
4463 report.retryable.push(client_commit_id.to_owned());
4466 if self
4467 .finish_observation("syncular_push_result", batch)
4468 .is_err()
4469 {
4470 self.rollback_observation("syncular_push_result");
4471 }
4472 return;
4473 }
4474
4475 let mut journal_results = Vec::with_capacity(results.len());
4476 let mut conflicts = Vec::new();
4477 let mut rejections = Vec::new();
4478 for result in results {
4479 match result {
4480 OpResult::Applied { op_index } => {
4481 journal_results.push(CommitOperationOutcome::Applied {
4482 op_index: *op_index,
4483 });
4484 }
4485 OpResult::Conflict {
4486 op_index,
4487 code,
4488 message,
4489 server_version,
4490 server_row,
4491 } => {
4492 let operation = operations
4493 .get(*op_index as usize)
4494 .map(CommitOperation::from);
4495 let (table, row_id) = operation
4496 .as_ref()
4497 .map(|op| (op.table.clone(), op.row_id.clone()))
4498 .unwrap_or_default();
4499 let server_row_json = self
4500 .schema
4501 .table(&table)
4502 .and_then(|t| {
4503 decode_row_bytes(t, server_row, &self.encryption)
4504 .ok()
4505 .map(|row| (t, row))
4506 })
4507 .map(|(t, row)| {
4508 let mut map = Map::new();
4509 for (i, column) in t.columns.iter().enumerate() {
4510 map.insert(
4511 column.name.clone(),
4512 column_value_to_json(row.get(i).unwrap_or(&None)),
4513 );
4514 }
4515 map
4516 })
4517 .unwrap_or_default();
4518 let conflict = ConflictRecord {
4519 client_commit_id: client_commit_id.to_owned(),
4520 op_index: *op_index,
4521 table,
4522 row_id,
4523 code: code.clone(),
4524 message: message.clone(),
4525 server_version: *server_version,
4526 server_row: server_row_json,
4527 operation,
4528 };
4529 journal_results.push(CommitOperationOutcome::Conflict {
4530 conflict: conflict.clone(),
4531 });
4532 conflicts.push(conflict);
4533 }
4534 OpResult::Error {
4535 op_index,
4536 code,
4537 message,
4538 retryable,
4539 } => {
4540 let rejection = RejectionRecord {
4541 client_commit_id: client_commit_id.to_owned(),
4542 op_index: *op_index,
4543 code: code.clone(),
4544 message: message.clone(),
4545 retryable: *retryable,
4546 details: rejection_details
4547 .and_then(|details| details.get(op_index))
4548 .cloned(),
4549 operation: operations
4550 .get(*op_index as usize)
4551 .map(CommitOperation::from),
4552 };
4553 journal_results.push(CommitOperationOutcome::Error {
4554 rejection: rejection.clone(),
4555 });
4556 rejections.push(rejection);
4557 }
4558 }
4559 }
4560 let outcome_status = if conflicts.is_empty() {
4561 CommitOutcomeStatus::Rejected
4562 } else {
4563 CommitOutcomeStatus::Conflict
4564 };
4565 if self
4566 .persist_commit_outcome(
4567 client_commit_id,
4568 outcome_status,
4569 &journal_results,
4570 Some(&operations),
4571 )
4572 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4573 .and_then(|()| self.prune_commit_outcomes())
4574 .is_err()
4575 {
4576 self.rollback_observation("syncular_push_result");
4577 return;
4578 }
4579 report.conflicts += conflicts.len() as u32;
4580 report.rejected.push(client_commit_id.to_owned());
4581 batch.conflicts = !conflicts.is_empty();
4582 batch.rejections = !rejections.is_empty();
4583 batch.status = true;
4584 batch.outcomes = true;
4585 self.conflicts.extend(conflicts);
4586 self.rejections.extend(rejections);
4587 self.outbox.remove(index);
4588 self.overlay_dirty.set(true);
4589 }
4590 }
4591 if batch.status {
4592 for operation in &operations {
4593 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
4594 batch.table(&operation.table);
4595 }
4596 }
4597 self.rebuild_overlay_if_dirty();
4598 }
4599 if self
4600 .finish_observation("syncular_push_result", batch)
4601 .is_err()
4602 {
4603 self.rollback_observation("syncular_push_result");
4604 }
4605 }
4606
4607 #[allow(clippy::too_many_arguments)]
4610 fn process_section(
4611 &mut self,
4612 transport: &mut dyn Transport,
4613 id: &str,
4614 status: SubStatus,
4615 reason_code: &str,
4616 effective_scopes: Vec<(String, Vec<String>)>,
4617 body: Vec<Frame>,
4618 next_cursor: i64,
4619 bootstrap_state: Option<String>,
4620 meta: &RequestMeta,
4621 report: &mut SyncReport,
4622 ) -> Result<(), SectionError> {
4623 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4624 return Ok(()); };
4626 match status {
4627 SubStatus::Revoked => {
4628 self.begin_observation("syncular_revocation")
4629 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4630 let mut batch = ChangeAccumulator::default();
4631 let registered = self.window_unit_by_sub_id(id);
4632 let (table, effective) = {
4634 let sub = &self.subs[sub_index];
4635 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4636 };
4637 let purged = self.purge_scope_rows(&table, &effective);
4638 match purged {
4639 Ok(()) => {
4640 self.record_scope_map(&mut batch, &table, &effective);
4641 let sub = &mut self.subs[sub_index];
4642 sub.state = SubState::Revoked;
4643 sub.reason_code = Some(if reason_code.is_empty() {
4644 "sync.scope_revoked".to_owned()
4645 } else {
4646 reason_code.to_owned()
4647 });
4648 report.revoked.push(id.to_owned());
4649 let doomed_effective = effective;
4650 let sub_table = table;
4651 self.persist_sub(&self.subs[sub_index].clone());
4652 let dropped = self
4653 .drop_doomed_outbox(&sub_table, &doomed_effective)
4654 .map_err(|message| {
4655 SectionError::Abort("storage.failed".to_owned(), message)
4656 })?;
4657 if dropped {
4658 batch.status = true;
4659 batch.rejections = true;
4660 batch.outcomes = true;
4661 }
4662 self.reconcile_blob_refcounts(true);
4665 }
4666 Err(()) => {
4667 let sub = &mut self.subs[sub_index];
4670 sub.state = SubState::Failed;
4671 sub.reason_code = Some("sync.scope_revoked".to_owned());
4672 report.failed.push(id.to_owned());
4673 self.persist_sub(&self.subs[sub_index].clone());
4674 }
4675 }
4676 if let Some((base_key, unit)) = registered {
4677 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4678 }
4679 self.rebuild_overlay_if_dirty();
4680 self.finish_observation("syncular_revocation", batch)
4681 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4682 Ok(())
4683 }
4684 SubStatus::Reset => {
4685 self.begin_observation("syncular_reset")
4686 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4687 let mut batch = ChangeAccumulator::default();
4688 let registered = self.window_unit_by_sub_id(id);
4689 let sub = &mut self.subs[sub_index];
4692 sub.cursor = -1;
4693 sub.bootstrap_state = None;
4694 report.resets.push(id.to_owned());
4695 self.persist_sub(&self.subs[sub_index].clone());
4696 if let Some((base_key, unit)) = registered {
4697 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4698 }
4699 self.finish_observation("syncular_reset", batch)
4700 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4701 Ok(())
4702 }
4703 SubStatus::Active => {
4704 let fresh = meta
4705 .fresh
4706 .iter()
4707 .find(|(fid, _)| fid == id)
4708 .map(|(_, f)| *f)
4709 .unwrap_or(false);
4710 let was_pending = self.subs[sub_index].cursor < 0
4711 || self.subs[sub_index].bootstrap_state.is_some();
4712 let registered = self.window_unit_by_sub_id(id);
4713 self.subs[sub_index].effective = Some(effective_scopes);
4715 self.begin_observation("syncular_section")
4716 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4717 let mut batch = ChangeAccumulator::default();
4718 let outcome = self.apply_section_body(
4719 transport, sub_index, body, fresh, meta, report, &mut batch,
4720 );
4721 match outcome {
4722 Ok(()) => {
4723 let sub = &mut self.subs[sub_index];
4724 sub.cursor = next_cursor;
4727 sub.bootstrap_state = bootstrap_state;
4728 sub.synced_once = true;
4729 if sub.bootstrap_state.is_some() {
4730 report.bootstrapping.push(id.to_owned());
4731 }
4732 let completed =
4733 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
4734 self.persist_sub(&self.subs[sub_index].clone());
4735 if completed {
4736 if let Some((base_key, unit)) = registered.clone() {
4737 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4738 }
4739 }
4740 self.rebuild_overlay_if_dirty();
4741 self.finish_observation("syncular_section", batch)
4742 .map_err(|message| {
4743 SectionError::Abort("storage.failed".to_owned(), message)
4744 })?;
4745 Ok(())
4746 }
4747 Err(SectionError::FailClosed) => {
4748 self.rollback_observation("syncular_section");
4751 self.begin_observation("syncular_section_failure")
4752 .map_err(|message| {
4753 SectionError::Abort("storage.failed".to_owned(), message)
4754 })?;
4755 let mut failure_batch = ChangeAccumulator::default();
4756 let sub = &mut self.subs[sub_index];
4757 sub.state = SubState::Failed;
4758 sub.reason_code = Some("sync.scope_revoked".to_owned());
4759 report.failed.push(id.to_owned());
4760 self.persist_sub(&self.subs[sub_index].clone());
4761 if let Some((base_key, unit)) = registered {
4762 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
4763 }
4764 self.finish_observation("syncular_section_failure", failure_batch)
4765 .map_err(|message| {
4766 SectionError::Abort("storage.failed".to_owned(), message)
4767 })?;
4768 Ok(())
4769 }
4770 Err(SectionError::Abort(code, message)) => {
4771 self.rollback_observation("syncular_section");
4774 Err(SectionError::Abort(code, message))
4775 }
4776 }
4777 }
4778 }
4779 }
4780
4781 #[allow(clippy::too_many_arguments)]
4785 fn apply_section_body(
4786 &mut self,
4787 transport: &mut dyn Transport,
4788 sub_index: usize,
4789 body: Vec<Frame>,
4790 fresh: bool,
4791 meta: &RequestMeta,
4792 report: &mut SyncReport,
4793 batch: &mut ChangeAccumulator,
4794 ) -> Result<(), SectionError> {
4795 let mut saw_segment = false;
4796 for frame in body {
4797 match frame {
4798 Frame::Commit {
4799 tables, changes, ..
4800 } => {
4801 self.record_commit_changes(batch, &tables, &changes);
4802 self.apply_commit_changes(&tables, &changes)
4803 .map_err(|(c, m)| SectionError::Abort(c, m))?;
4804 report.commits_applied += 1;
4805 }
4806 Frame::SegmentInline { payload } => {
4807 let segment = decode_rows_segment(&payload)
4808 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
4809 let first = !saw_segment;
4810 saw_segment = true;
4811 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4812 let cleared =
4813 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4814 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4815 if applied > 0 || cleared {
4816 batch.table(&segment.table);
4817 }
4818 report.segment_rows_applied += applied;
4819 }
4820 Frame::SegmentRef {
4821 segment_id,
4822 media_type,
4823 table,
4824 row_count,
4825 as_of_commit_seq,
4826 scope_digest,
4827 row_cursor,
4828 next_row_cursor,
4829 url,
4830 url_expires_at_ms,
4831 ..
4832 } => {
4833 let advertised = match media_type {
4836 MediaType::Rows => {
4837 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
4838 || meta.accept & ACCEPT_INLINE_ROWS != 0
4839 }
4840 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
4841 };
4842 if !advertised {
4843 return Err(SectionError::Abort(
4844 "sync.invalid_request".to_owned(),
4845 format!(
4846 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
4847 media_type.name()
4848 ),
4849 ));
4850 }
4851 let bytes = if let Some(url) = url {
4852 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
4857 return Err(SectionError::Abort(
4858 "sync.invalid_request".to_owned(),
4859 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
4860 .to_owned(),
4861 ));
4862 }
4863 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
4865 return Err(SectionError::Abort(
4866 "sync.segment_expired".to_owned(),
4867 format!(
4868 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
4869 ),
4870 ));
4871 }
4872 transport
4873 .fetch_url(&url)
4874 .map_err(|e| SectionError::Abort(e.code, e.message))?
4875 } else {
4876 let requested_scopes_json =
4877 canonical_scope_json(&self.subs[sub_index].requested);
4878 transport
4879 .download_segment(&SegmentRequest {
4880 segment_id: segment_id.clone(),
4881 table,
4882 requested_scopes_json,
4883 })
4884 .map_err(|e| SectionError::Abort(e.code, e.message))?
4885 };
4886 let digest = Sha256::digest(&bytes);
4888 let expected = segment_id
4889 .strip_prefix("sha256:")
4890 .unwrap_or(segment_id.as_str());
4891 if bytes_to_hex(&digest) != expected {
4892 return Err(SectionError::Abort(
4893 "sync.invalid_request".to_owned(),
4894 "segment bytes do not match the content address (§5.1)".to_owned(),
4895 ));
4896 }
4897 if media_type == MediaType::Sqlite {
4898 if row_cursor.is_some() || next_row_cursor.is_some() {
4901 return Err(SectionError::Abort(
4902 "sync.invalid_request".to_owned(),
4903 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
4904 .to_owned(),
4905 ));
4906 }
4907 let first = !saw_segment;
4908 saw_segment = true;
4909 let sub_table = self.subs[sub_index].table.clone();
4910 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4911 let cleared =
4912 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
4913 let applied = self.apply_sqlite_segment(
4914 sub_index,
4915 &bytes,
4916 fresh && first,
4917 row_count,
4918 as_of_commit_seq,
4919 &scope_digest,
4920 )?;
4921 if applied > 0 || cleared {
4922 batch.table(&sub_table);
4923 }
4924 report.segment_rows_applied += applied;
4925 } else {
4926 let segment = decode_rows_segment(&bytes).map_err(|e| {
4927 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
4928 })?;
4929 let first = row_cursor.is_none();
4930 saw_segment = true;
4931 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4932 let cleared =
4933 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4934 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4935 if applied > 0 || cleared {
4936 batch.table(&segment.table);
4937 }
4938 report.segment_rows_applied += applied;
4939 }
4940 }
4941 Frame::Unknown { .. } => {}
4942 _ => {
4943 return Err(SectionError::Abort(
4944 "sync.invalid_request".to_owned(),
4945 "unexpected frame inside a subscription section".to_owned(),
4946 ));
4947 }
4948 }
4949 }
4950 Ok(())
4951 }
4952
4953 fn apply_commit_changes(
4954 &mut self,
4955 tables: &[String],
4956 changes: &[ssp2::model::Change],
4957 ) -> Result<(), (String, String)> {
4958 for change in changes {
4959 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
4960 (
4961 "sync.invalid_request".to_owned(),
4962 "change tableIndex out of range".to_owned(),
4963 )
4964 })?;
4965 let table = self.schema.table(table_name).ok_or_else(|| {
4966 (
4967 "sync.schema_mismatch".to_owned(),
4968 format!("change targets unknown table {table_name:?}"),
4969 )
4970 })?;
4971 match change.op {
4972 Op::Upsert => {
4973 let payload = change.row.as_ref().ok_or_else(|| {
4974 (
4975 "sync.invalid_request".to_owned(),
4976 "upsert change without row payload".to_owned(),
4977 )
4978 })?;
4979 let row = decode_row_bytes(table, payload, &self.encryption)
4981 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4982 let version = change.row_version.unwrap_or(0);
4983 let table_name = table.name.clone();
4984 self.write_base_row(&table_name, &row, version)
4985 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4986 }
4987 Op::Delete => {
4988 self.delete_base_row(table_name, &change.row_id)
4989 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4990 }
4991 }
4992 }
4993 Ok(())
4994 }
4995
4996 fn apply_segment(
5001 &mut self,
5002 sub_index: usize,
5003 segment: &RowsSegment,
5004 first_fresh_page: bool,
5005 ) -> Result<u32, SectionError> {
5006 let (sub_table, effective) = {
5007 let sub = &self.subs[sub_index];
5008 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5009 };
5010 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5011 SectionError::Abort(
5012 "sync.schema_mismatch".to_owned(),
5013 format!("subscription table {sub_table:?} is not in the client schema"),
5014 )
5015 })?;
5016 let matches = segment.table == table.name
5021 && segment.schema_version == self.schema.version
5022 && segment.columns.len() == table.wire_columns.len()
5023 && segment
5024 .columns
5025 .iter()
5026 .zip(table.wire_columns.iter())
5027 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
5028 if !matches {
5029 return Err(SectionError::Abort(
5030 "sync.schema_mismatch".to_owned(),
5031 "segment column table does not match the generated schema (§5.2)".to_owned(),
5032 ));
5033 }
5034 if first_fresh_page {
5035 self.purge_scope_rows(&table.name, &effective)
5039 .map_err(|()| SectionError::FailClosed)?;
5040 }
5041 let mut applied = 0u32;
5042 for block in &segment.blocks {
5043 for row in block {
5044 let decrypted;
5049 let values = if table.has_encrypted_columns() {
5050 let mut values = row.values.clone();
5051 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
5052 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
5053 decrypted = values;
5054 &decrypted
5055 } else {
5056 &row.values
5057 };
5058 self.write_base_row(&table.name, values, row.server_version)
5061 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
5062 applied += 1;
5063 }
5064 }
5065 Ok(applied)
5066 }
5067
5068 fn apply_sqlite_segment(
5076 &mut self,
5077 sub_index: usize,
5078 bytes: &[u8],
5079 first_fresh_page: bool,
5080 row_count: i64,
5081 as_of_commit_seq: i64,
5082 scope_digest: &str,
5083 ) -> Result<u32, SectionError> {
5084 let invalid = |detail: &str| {
5085 SectionError::Abort(
5086 "sync.invalid_request".to_owned(),
5087 format!("sqlite segment rejected: {detail} (§5.3)"),
5088 )
5089 };
5090 let (sub_table, effective) = {
5091 let sub = &self.subs[sub_index];
5092 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5093 };
5094 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
5095 SectionError::Abort(
5096 "sync.schema_mismatch".to_owned(),
5097 format!("subscription table {sub_table:?} is not in the client schema"),
5098 )
5099 })?;
5100
5101 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
5102 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
5103 let img = match rusqlite::Connection::open_with_flags(
5104 &path,
5105 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
5106 ) {
5107 Ok(conn) => conn,
5108 Err(_) => {
5109 let _ = std::fs::remove_file(&path);
5110 return Err(invalid("bytes do not open as a SQLite database"));
5111 }
5112 };
5113 let outcome = self.apply_sqlite_image(
5114 &img,
5115 &table,
5116 first_fresh_page,
5117 &effective,
5118 row_count,
5119 as_of_commit_seq,
5120 scope_digest,
5121 );
5122 drop(img);
5123 let _ = std::fs::remove_file(&path);
5124 outcome
5125 }
5126
5127 #[allow(clippy::too_many_arguments)]
5128 fn apply_sqlite_image(
5129 &mut self,
5130 img: &rusqlite::Connection,
5131 table: &crate::schema::TableSchema,
5132 first_fresh_page: bool,
5133 effective: &[(String, Vec<String>)],
5134 row_count: i64,
5135 as_of_commit_seq: i64,
5136 scope_digest: &str,
5137 ) -> Result<u32, SectionError> {
5138 let invalid = |detail: String| {
5139 SectionError::Abort(
5140 "sync.invalid_request".to_owned(),
5141 format!("sqlite segment rejected: {detail} (§5.3)"),
5142 )
5143 };
5144
5145 type MetaRow = (i64, String, i64, i64, String, i64, i64);
5148 let meta: MetaRow = img
5149 .query_row(
5150 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
5151 \"scopeDigest\", \"rowCount\",
5152 (SELECT count(*) FROM _syncular_segment)
5153 FROM _syncular_segment",
5154 [],
5155 |row| {
5156 Ok((
5157 row.get(0)?,
5158 row.get(1)?,
5159 row.get(2)?,
5160 row.get(3)?,
5161 row.get(4)?,
5162 row.get(5)?,
5163 row.get(6)?,
5164 ))
5165 },
5166 )
5167 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
5168 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
5169 if meta_count != 1 {
5170 return Err(invalid(format!(
5171 "_syncular_segment must contain exactly one row, found {meta_count}"
5172 )));
5173 }
5174 if format != 1 {
5175 return Err(invalid(format!("format {format}")));
5176 }
5177 if meta_table != table.name {
5178 return Err(invalid(format!("image table {meta_table:?}")));
5179 }
5180 if schema_version != i64::from(self.schema.version) {
5181 return Err(invalid(format!("schemaVersion {schema_version}")));
5182 }
5183 if pin != as_of_commit_seq {
5184 return Err(invalid(format!("asOfCommitSeq {pin}")));
5185 }
5186 if digest != scope_digest {
5187 return Err(invalid("scopeDigest mismatch".to_owned()));
5188 }
5189 if meta_rows != row_count {
5190 return Err(invalid(format!("rowCount {meta_rows}")));
5191 }
5192
5193 let mut names: Vec<String> = Vec::new();
5195 {
5196 let mut stmt = img
5197 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
5198 .map_err(|_| invalid("image data table missing".to_owned()))?;
5199 let mut rows = stmt
5200 .query([])
5201 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5202 while let Some(row) = rows
5203 .next()
5204 .map_err(|_| invalid("image data table unreadable".to_owned()))?
5205 {
5206 names.push(
5207 row.get::<_, String>(1)
5208 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
5209 );
5210 }
5211 }
5212 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
5213 expected.push("_syncular_version");
5214 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
5215 return Err(SectionError::Abort(
5216 "sync.schema_mismatch".to_owned(),
5217 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
5218 ));
5219 }
5220
5221 if first_fresh_page {
5224 self.purge_scope_rows(&table.name, effective)
5225 .map_err(|()| SectionError::FailClosed)?;
5226 }
5227 self.overlay_dirty.set(true);
5233 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
5241 table.indexes.iter().filter(|i| !i.unique).collect()
5242 } else {
5243 Vec::new()
5244 };
5245 for index in &bulk_indexes {
5246 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
5247 self.conn
5248 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
5249 .map_err(|e| invalid(e.to_string()))?;
5250 }
5251 let insert = self.insert_row_sql(&base_table(&table.name), table);
5252 let applied = {
5253 let mut ins = self
5254 .conn
5255 .prepare_cached(&insert)
5256 .map_err(|e| invalid(e.to_string()))?;
5257 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
5258 let mut stmt = img
5259 .prepare(&format!(
5260 "SELECT {} FROM {}",
5261 column_list.join(", "),
5262 quote_ident(&table.name)
5263 ))
5264 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5265 let mut rows = stmt
5266 .query([])
5267 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
5268 let version_index = table.columns.len();
5269 let mut applied = 0u32;
5270 while let Some(row) = rows
5271 .next()
5272 .map_err(|_| invalid("image row unreadable".to_owned()))?
5273 {
5274 for (i, column) in table.columns.iter().enumerate() {
5275 let cell = row
5276 .get_ref(i)
5277 .map_err(|_| invalid("image row unreadable".to_owned()))?;
5278 let param = image_cell_param(column, cell).map_err(&invalid)?;
5279 ins.raw_bind_parameter(i + 1, param)
5280 .map_err(|e| invalid(e.to_string()))?;
5281 }
5282 let version: i64 = row
5283 .get(version_index)
5284 .map_err(|_| invalid("image row unreadable".to_owned()))?;
5285 if version < 1 {
5286 return Err(invalid(format!(
5287 "row _syncular_version must be >= 1, got {version}"
5288 )));
5289 }
5290 ins.raw_bind_parameter(version_index + 1, version)
5291 .map_err(|e| invalid(e.to_string()))?;
5292 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
5293 applied += 1;
5294 }
5295 applied
5296 };
5297 for index in &bulk_indexes {
5298 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
5299 let cols_sql = index
5300 .columns
5301 .iter()
5302 .map(|c| quote_ident(c))
5303 .collect::<Vec<_>>()
5304 .join(", ");
5305 self.conn
5306 .execute(
5307 &format!(
5308 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
5309 base_table(&table.name)
5310 ),
5311 [],
5312 )
5313 .map_err(|e| invalid(e.to_string()))?;
5314 }
5315 if i64::from(applied) != row_count {
5316 return Err(invalid(format!(
5317 "image holds {applied} rows, descriptor says {row_count}"
5318 )));
5319 }
5320 Ok(applied)
5321 }
5322
5323 fn compile_local_data_purge(
5326 &self,
5327 input: &LocalDataPurgeInput,
5328 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
5329 let invalid = |message: String| format!("sync.invalid_request: {message}");
5330 if input.purge_id.is_empty()
5331 || input.purge_id.len() > 128
5332 || !is_local_purge_code_like(&input.purge_id)
5333 {
5334 return Err(invalid(
5335 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
5336 ));
5337 }
5338 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
5339 return Err(invalid(format!(
5340 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
5341 )));
5342 }
5343
5344 let mut deduplicated: BTreeMap<
5345 String,
5346 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
5347 > = BTreeMap::new();
5348 for target in &input.targets {
5349 let table = self.schema.table(&target.table).ok_or_else(|| {
5350 invalid(format!(
5351 "local purge names unknown table {:?}",
5352 target.table
5353 ))
5354 })?;
5355 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
5356 return Err(invalid(format!(
5357 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
5358 target.table
5359 )));
5360 }
5361 let mut selectors = Vec::with_capacity(target.selectors.len());
5362 let mut canonical_selectors = BTreeMap::new();
5363 for (column_name, raw_values) in &target.selectors {
5364 let Some((column_index, column)) = table
5365 .columns
5366 .iter()
5367 .enumerate()
5368 .find(|(_, column)| column.name == *column_name)
5369 else {
5370 return Err(invalid(format!(
5371 "local purge target {:?} names unknown column {:?}",
5372 target.table, column_name
5373 )));
5374 };
5375 let encrypted = table
5376 .encrypted_columns
5377 .iter()
5378 .any(|candidate| candidate.index == column_index);
5379 if column.ty != ColumnType::String || encrypted {
5380 return Err(invalid(format!(
5381 "local purge selector {:?}.{:?} must be a plaintext string column",
5382 target.table, column_name
5383 )));
5384 }
5385 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
5386 return Err(invalid(format!(
5387 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
5388 target.table, column_name
5389 )));
5390 }
5391 let mut values = raw_values.clone();
5392 values.sort();
5393 values.dedup();
5394 if values.iter().any(|value| {
5395 value.is_empty()
5396 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
5397 || !is_local_purge_code_like(value)
5398 }) {
5399 return Err(invalid(format!(
5400 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
5401 )));
5402 }
5403 selectors.push((column_name.clone(), values.clone()));
5404 canonical_selectors.insert(column_name.clone(), values);
5405 }
5406 let canonical = LocalDataPurgeTarget {
5407 table: target.table.clone(),
5408 selectors: canonical_selectors,
5409 };
5410 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
5411 deduplicated.insert(
5412 key,
5413 (
5414 CompiledLocalDataPurgeTarget {
5415 table: target.table.clone(),
5416 selectors,
5417 },
5418 canonical,
5419 ),
5420 );
5421 }
5422 let targets = deduplicated
5423 .values()
5424 .map(|(compiled, _)| compiled.clone())
5425 .collect::<Vec<_>>();
5426 let canonical = deduplicated
5427 .values()
5428 .map(|(_, target)| target.clone())
5429 .collect::<Vec<_>>();
5430 let canonical_plan =
5431 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
5432 Ok((targets, canonical_plan))
5433 }
5434
5435 fn local_purge_base_row_ids(
5436 &self,
5437 targets: &[CompiledLocalDataPurgeTarget],
5438 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
5439 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
5440 for target in targets {
5441 let table = self
5442 .schema
5443 .table(&target.table)
5444 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
5445 let mut clauses = Vec::with_capacity(target.selectors.len());
5446 let mut params = Vec::new();
5447 for (column, values) in &target.selectors {
5448 clauses.push(format!(
5449 "{} IN ({})",
5450 quote_ident(column),
5451 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
5452 ));
5453 params.extend(values.iter().cloned().map(SqlValue::Text));
5454 }
5455 let sql = format!(
5456 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
5457 quote_ident(&table.primary_key),
5458 base_table(&target.table),
5459 clauses.join(" AND ")
5460 );
5461 let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
5462 let rows = statement
5463 .query_map(rusqlite::params_from_iter(params), |row| {
5464 row.get::<_, String>(0)
5465 })
5466 .map_err(|error| error.to_string())?;
5467 let ids = by_table.entry(target.table.clone()).or_default();
5468 for row in rows {
5469 ids.insert(row.map_err(|error| error.to_string())?);
5470 }
5471 }
5472 Ok(by_table)
5473 }
5474
5475 fn local_purge_values_match(
5476 target: &CompiledLocalDataPurgeTarget,
5477 values: &Map<String, Value>,
5478 ) -> bool {
5479 target.selectors.iter().all(|(column, allowed)| {
5480 values
5481 .get(column)
5482 .and_then(Value::as_str)
5483 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
5484 })
5485 }
5486
5487 pub fn purge_local_data(
5491 &mut self,
5492 input: &LocalDataPurgeInput,
5493 ) -> Result<LocalDataPurgeResult, String> {
5494 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
5495 let meta_key = format!("localPurge:{}", input.purge_id);
5496 let applied_plan = self
5497 .conn
5498 .query_row(
5499 "SELECT value FROM _syncular_meta WHERE key = ?1",
5500 rusqlite::params![meta_key],
5501 |row| row.get::<_, String>(0),
5502 )
5503 .optional()
5504 .map_err(|error| error.to_string())?;
5505 if let Some(applied_plan) = applied_plan {
5506 if applied_plan != canonical_plan {
5507 return Err(format!(
5508 "sync.invalid_request: local purge id {:?} was already used with a different plan",
5509 input.purge_id
5510 ));
5511 }
5512 return Ok(LocalDataPurgeResult {
5513 already_applied: true,
5514 purged_rows: 0,
5515 dropped_commits: 0,
5516 });
5517 }
5518
5519 let prior_outbox = self.outbox.clone();
5520 let prior_rejection_count = self.rejections.len();
5521 let prior_overlay_dirty = self.overlay_dirty.get();
5522 self.begin_observation("syncular_local_purge")?;
5523 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
5524 let row_ids = self.local_purge_base_row_ids(&targets)?;
5525 let doomed = self
5526 .outbox
5527 .iter()
5528 .filter(|commit| {
5529 commit.ops.iter().any(|operation| {
5530 let matching_targets = targets
5531 .iter()
5532 .filter(|target| target.table == operation.table)
5533 .collect::<Vec<_>>();
5534 if matching_targets.is_empty() {
5535 return false;
5536 }
5537 if row_ids
5538 .get(&operation.table)
5539 .is_some_and(|ids| ids.contains(&operation.row_id))
5540 {
5541 return true;
5542 }
5543 operation.values.as_ref().is_some_and(|values| {
5544 matching_targets
5545 .iter()
5546 .any(|target| Self::local_purge_values_match(target, values))
5547 })
5548 })
5549 })
5550 .cloned()
5551 .collect::<Vec<_>>();
5552 let doomed_ids = doomed
5553 .iter()
5554 .map(|commit| commit.client_commit_id.clone())
5555 .collect::<BTreeSet<_>>();
5556 let mut batch = ChangeAccumulator::default();
5557 let mut rejections = Vec::new();
5558 for commit in &doomed {
5559 for operation in &commit.ops {
5560 batch.table(&operation.table);
5561 }
5562 let results = commit
5563 .ops
5564 .iter()
5565 .enumerate()
5566 .map(|(op_index, operation)| {
5567 let rejection = RejectionRecord {
5568 client_commit_id: commit.client_commit_id.clone(),
5569 op_index: op_index as i32,
5570 code: "client.local_data_purged".to_owned(),
5571 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
5572 retryable: false,
5573 details: None,
5574 operation: Some(CommitOperation::from(operation)),
5575 };
5576 rejections.push(rejection.clone());
5577 CommitOperationOutcome::Error { rejection }
5578 })
5579 .collect::<Vec<_>>();
5580 self.persist_commit_outcome(
5581 &commit.client_commit_id,
5582 CommitOutcomeStatus::Rejected,
5583 &results,
5584 Some(&commit.ops),
5585 )?;
5586 self.delete_outbox_persisted(&commit.client_commit_id)?;
5587 }
5588 if !doomed.is_empty() {
5589 self.outbox
5590 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
5591 self.rejections.extend(rejections);
5592 self.prune_commit_outcomes()?;
5593 self.overlay_dirty.set(true);
5594 batch.status = true;
5595 batch.rejections = true;
5596 batch.outcomes = true;
5597 }
5598
5599 let mut purged_rows = 0usize;
5600 for (table, ids) in &row_ids {
5601 if ids.is_empty() {
5602 continue;
5603 }
5604 batch.table(table);
5605 for row_id in ids {
5606 self.delete_base_row(table, row_id)?;
5607 purged_rows += 1;
5608 }
5609 }
5610 self.rebuild_overlay_if_dirty();
5611 self.reconcile_blob_refcounts(true);
5612 self.conn
5613 .execute(
5614 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
5615 rusqlite::params![meta_key, canonical_plan],
5616 )
5617 .map_err(|error| error.to_string())?;
5618 Ok((
5619 batch,
5620 LocalDataPurgeResult {
5621 already_applied: false,
5622 purged_rows,
5623 dropped_commits: doomed.len(),
5624 },
5625 ))
5626 })();
5627 let (batch, result) = match applied {
5628 Ok(value) => value,
5629 Err(error) => {
5630 self.rollback_observation("syncular_local_purge");
5631 self.outbox = prior_outbox;
5632 self.rejections.truncate(prior_rejection_count);
5633 self.overlay_dirty.set(prior_overlay_dirty);
5634 return Err(error);
5635 }
5636 };
5637 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
5638 self.rollback_observation("syncular_local_purge");
5639 self.outbox = prior_outbox;
5640 self.rejections.truncate(prior_rejection_count);
5641 self.overlay_dirty.set(prior_overlay_dirty);
5642 return Err(error);
5643 }
5644 Ok(result)
5645 }
5646
5647 fn purge_scope_rows(
5652 &mut self,
5653 table_name: &str,
5654 effective: &[(String, Vec<String>)],
5655 ) -> Result<(), ()> {
5656 if effective.is_empty() {
5657 return Ok(());
5658 }
5659 let table = self.schema.table(table_name).ok_or(())?.clone();
5660 let mut clauses = Vec::new();
5661 let mut params: Vec<SqlValue> = Vec::new();
5662 for (variable, values) in effective {
5663 let column = table.scope_column(variable).ok_or(())?;
5664 let placeholders: Vec<String> = values
5665 .iter()
5666 .map(|v| {
5667 params.push(SqlValue::Text(v.clone()));
5668 "?".to_owned()
5669 })
5670 .collect();
5671 clauses.push(format!(
5672 "{} IN ({})",
5673 quote_ident(column),
5674 placeholders.join(", ")
5675 ));
5676 }
5677 let sql = format!(
5678 "DELETE FROM {} WHERE {}",
5679 base_table(table_name),
5680 clauses.join(" AND ")
5681 );
5682 self.overlay_dirty.set(true);
5683 self.conn
5684 .execute(&sql, rusqlite::params_from_iter(params))
5685 .map_err(|_| ())?;
5686 Ok(())
5687 }
5688
5689 fn drop_doomed_outbox(
5692 &mut self,
5693 table_name: &str,
5694 effective: &[(String, Vec<String>)],
5695 ) -> Result<bool, String> {
5696 if effective.is_empty() {
5697 return Ok(false);
5698 }
5699 let Some(table) = self.schema.table(table_name).cloned() else {
5700 return Ok(false);
5701 };
5702 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
5703 for (variable, values) in effective {
5704 match table.scope_column(variable) {
5705 Some(column) => mappings.push((column, values)),
5706 None => return Ok(false), }
5708 }
5709 let doomed: Vec<OutboxCommit> = self
5710 .outbox
5711 .iter()
5712 .filter(|commit| {
5713 commit.ops.iter().any(|op| {
5714 op.upsert
5715 && op.table == table_name
5716 && op.values.as_ref().is_some_and(|values| {
5717 mappings
5718 .iter()
5719 .all(|(column, allowed)| match values.get(*column) {
5720 Some(Value::String(s)) => allowed.contains(s),
5721 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
5722 _ => false,
5723 })
5724 })
5725 })
5726 })
5727 .cloned()
5728 .collect();
5729 if doomed.is_empty() {
5730 return Ok(false);
5731 }
5732 let mut rejections = Vec::new();
5733 for commit in &doomed {
5734 let results = commit
5735 .ops
5736 .iter()
5737 .enumerate()
5738 .map(|(op_index, operation)| {
5739 let rejection = RejectionRecord {
5740 client_commit_id: commit.client_commit_id.clone(),
5741 op_index: op_index as i32,
5742 code: "sync.scope_revoked".to_owned(),
5743 message: "the commit was dropped because its effective scope was revoked"
5744 .to_owned(),
5745 retryable: false,
5746 details: None,
5747 operation: Some(CommitOperation::from(operation)),
5748 };
5749 rejections.push(rejection.clone());
5750 CommitOperationOutcome::Error { rejection }
5751 })
5752 .collect::<Vec<_>>();
5753 self.persist_commit_outcome(
5754 &commit.client_commit_id,
5755 CommitOutcomeStatus::Rejected,
5756 &results,
5757 Some(&commit.ops),
5758 )?;
5759 self.delete_outbox_persisted(&commit.client_commit_id)?;
5760 }
5761 self.prune_commit_outcomes()?;
5762 let doomed_ids = doomed
5763 .iter()
5764 .map(|commit| commit.client_commit_id.as_str())
5765 .collect::<BTreeSet<_>>();
5766 self.outbox
5767 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
5768 self.rejections.extend(rejections);
5769 self.overlay_dirty.set(true);
5770 Ok(true)
5771 }
5772
5773 pub fn upload_blob(
5780 &mut self,
5781 bytes: &[u8],
5782 media_type: Option<String>,
5783 name: Option<String>,
5784 ) -> Result<Value, String> {
5785 let blob_id = blob_id_for(bytes);
5786 let now = self.clock_now_ms();
5787 self.conn
5788 .execute(
5789 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
5790 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
5791 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
5792 )
5793 .map_err(|e| e.to_string())?;
5794 self.conn
5795 .execute(
5796 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
5797 rusqlite::params![blob_id, media_type, now],
5798 )
5799 .map_err(|e| e.to_string())?;
5800 self.enforce_blob_cache_cap();
5803 let mut obj = Map::new();
5804 obj.insert("blobId".to_owned(), Value::from(blob_id));
5805 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
5806 if let Some(mt) = media_type {
5807 obj.insert("mediaType".to_owned(), Value::from(mt));
5808 }
5809 if let Some(n) = name {
5810 obj.insert("name".to_owned(), Value::from(n));
5811 }
5812 Ok(Value::Object(obj))
5813 }
5814
5815 pub fn fetch_blob(
5819 &mut self,
5820 transport: &mut dyn Transport,
5821 blob_id_or_ref: &str,
5822 ) -> Result<Value, (String, String)> {
5823 let simple = |m: String| ("client.failed".to_owned(), m);
5824 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
5825 blob_id_or_ref.to_owned()
5826 } else {
5827 let value: Value = serde_json::from_str(blob_id_or_ref)
5828 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
5829 value
5830 .get("blobId")
5831 .and_then(Value::as_str)
5832 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
5833 .to_owned()
5834 };
5835 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
5836 return Ok(cached);
5837 }
5838 let bytes = match transport
5844 .blob_download(&blob_id)
5845 .map_err(|e| (e.code, e.message))?
5846 {
5847 BlobDownload::Bytes(bytes) => bytes,
5848 BlobDownload::Url {
5849 url,
5850 url_expires_at_ms,
5851 } => {
5852 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5854 return Err((
5855 "sync.segment_expired".to_owned(),
5856 format!(
5857 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
5858 ),
5859 ));
5860 }
5861 transport
5862 .fetch_blob_url(&url)
5863 .map_err(|e| (e.code, e.message))?
5864 }
5865 };
5866 if blob_id_for(&bytes) != blob_id {
5868 return Err(simple(format!(
5869 "blob content address mismatch for {blob_id}"
5870 )));
5871 }
5872 let now = self.clock_now_ms();
5873 self.conn
5874 .execute(
5875 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
5876 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
5877 )
5878 .map_err(|e| simple(e.to_string()))?;
5879 self.enforce_blob_cache_cap();
5880 self.get_cached_blob(&blob_id)
5881 .map_err(simple)?
5882 .ok_or_else(|| simple("blob cache write failed".to_owned()))
5883 }
5884
5885 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
5886 let _ = self.conn.execute(
5889 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
5890 rusqlite::params![self.clock_now_ms(), blob_id],
5891 );
5892 let mut stmt = self
5893 .conn
5894 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
5895 .map_err(|e| e.to_string())?;
5896 let mut rows = stmt
5897 .query(rusqlite::params![blob_id])
5898 .map_err(|e| e.to_string())?;
5899 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
5900 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
5901 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
5902 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
5903 let mut obj = Map::new();
5904 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
5905 obj.insert("byteLength".to_owned(), Value::from(byte_length));
5906 let mut bytes_obj = Map::new();
5907 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
5908 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
5909 if let Some(mt) = media_type {
5910 obj.insert("mediaType".to_owned(), Value::from(mt));
5911 }
5912 return Ok(Some(Value::Object(obj)));
5913 }
5914 Ok(None)
5915 }
5916
5917 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
5919 let pending: Vec<(String, Option<String>)> = {
5920 let mut stmt = self
5921 .conn
5922 .prepare(
5923 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
5924 )
5925 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5926 let rows = stmt
5927 .query_map([], |row| {
5928 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
5929 })
5930 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5931 rows.filter_map(Result::ok).collect()
5932 };
5933 for (blob_id, media_type) in pending {
5934 let bytes: Option<Vec<u8>> = self
5935 .conn
5936 .query_row(
5937 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
5938 rusqlite::params![blob_id],
5939 |row| row.get(0),
5940 )
5941 .ok();
5942 if let Some(bytes) = bytes {
5943 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
5944 }
5945 let _ = self.conn.execute(
5946 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
5947 rusqlite::params![blob_id],
5948 );
5949 }
5950 Ok(())
5951 }
5952
5953 fn upload_one(
5960 &self,
5961 transport: &mut dyn Transport,
5962 blob_id: &str,
5963 bytes: &[u8],
5964 media_type: Option<&str>,
5965 ) -> Result<(), TransportError> {
5966 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
5967 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
5969 url,
5970 url_expires_at_ms,
5971 } => {
5972 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
5973 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
5974 return Ok(());
5975 }
5976 }
5978 BlobUploadGrant::None => {
5979 }
5981 }
5982 transport.blob_upload(blob_id, bytes, media_type)
5983 }
5984
5985 fn enforce_blob_cache_cap(&self) {
5994 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
5995 return;
5996 };
5997 let mut total: i64 = self
5998 .conn
5999 .query_row(
6000 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
6001 [],
6002 |row| row.get(0),
6003 )
6004 .unwrap_or(0);
6005 if total <= max_bytes {
6006 return;
6007 }
6008 let candidates: Vec<(String, i64)> = {
6009 let Ok(mut stmt) = self.conn.prepare(
6010 "SELECT blob_id, byte_length FROM _syncular_blobs
6011 WHERE refcount = 0
6012 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
6013 ORDER BY last_used_ms ASC, created_at_ms ASC",
6014 ) else {
6015 return;
6016 };
6017 let Ok(rows) = stmt.query_map([], |row| {
6018 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
6019 }) else {
6020 return;
6021 };
6022 rows.filter_map(Result::ok).collect()
6023 };
6024 for (blob_id, byte_length) in candidates {
6025 if total <= max_bytes {
6026 break;
6027 }
6028 let _ = self.conn.execute(
6029 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
6030 rusqlite::params![blob_id],
6031 );
6032 total -= byte_length;
6033 }
6034 }
6035
6036 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
6040 if !self.schema_has_blobs() {
6041 return;
6042 }
6043 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
6044 for table in self.schema.tables.clone() {
6045 let blob_cols: Vec<String> = table
6046 .columns
6047 .iter()
6048 .filter(|c| c.ty == ColumnType::BlobRef)
6049 .map(|c| c.name.clone())
6050 .collect();
6051 for column in blob_cols {
6052 let sql = format!(
6053 "SELECT {} FROM {} WHERE {} IS NOT NULL",
6054 quote_ident(&column),
6055 base_table(&table.name),
6056 quote_ident(&column)
6057 );
6058 let Ok(mut stmt) = self.conn.prepare(&sql) else {
6059 continue;
6060 };
6061 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
6062 continue;
6063 };
6064 for raw in rows.flatten().flatten() {
6065 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
6066 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
6067 *counts.entry(id.to_owned()).or_insert(0) += 1;
6068 }
6069 }
6070 }
6071 }
6072 }
6073 let _ = self
6074 .conn
6075 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
6076 for (blob_id, count) in &counts {
6077 let _ = self.conn.execute(
6078 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
6079 rusqlite::params![count, blob_id],
6080 );
6081 }
6082 if delete_orphans {
6083 let _ = self.conn.execute(
6084 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
6085 [],
6086 );
6087 }
6088 }
6089
6090 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
6095 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
6096 return sql.clone();
6097 }
6098 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
6099 columns.push(quote_ident("_syncular_version"));
6100 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
6101 let primary_key = quote_ident(&table.primary_key);
6102 let updates = columns
6103 .iter()
6104 .filter(|column| **column != primary_key)
6105 .map(|column| format!("{column}=excluded.{column}"))
6106 .collect::<Vec<_>>()
6107 .join(", ");
6108 let sql = format!(
6109 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
6110 columns.join(", "),
6111 placeholders.join(", ")
6112 );
6113 self.insert_sql
6114 .borrow_mut()
6115 .insert(full_table.to_owned(), sql.clone());
6116 sql
6117 }
6118
6119 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
6120 self.overlay_dirty.set(true);
6121 self.write_row(&base_table(table_name), table_name, row, version)
6122 }
6123
6124 fn write_row(
6125 &self,
6126 full_table: &str,
6127 table_name: &str,
6128 row: &Row,
6129 version: i64,
6130 ) -> Result<(), String> {
6131 let table = self
6132 .schema
6133 .table(table_name)
6134 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
6135 let sql = self.insert_row_sql(full_table, table);
6136 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
6137 let params = row
6138 .iter()
6139 .map(RowParam::Cell)
6140 .chain(std::iter::once(RowParam::Version(version)));
6141 stmt.execute(rusqlite::params_from_iter(params))
6142 .map_err(|e| e.to_string())?;
6143 Ok(())
6144 }
6145
6146 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
6147 let table = self
6148 .schema
6149 .table(table_name)
6150 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
6151 self.overlay_dirty.set(true);
6152 let sql = format!(
6153 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
6154 base_table(table_name),
6155 quote_ident(&table.primary_key)
6156 );
6157 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
6158 stmt.execute(rusqlite::params![row_id])
6159 .map_err(|e| e.to_string())?;
6160 Ok(())
6161 }
6162
6163 fn rebuild_overlay_if_dirty(&mut self) {
6166 if self.overlay_dirty.get() {
6167 self.rebuild_overlay();
6168 }
6169 }
6170
6171 fn rebuild_overlay(&mut self) {
6175 self.exec("SAVEPOINT syncular_overlay");
6176 for table in self.schema.tables.clone() {
6177 for index in &table.fts_indexes {
6178 let _ = self.drop_fts_triggers(index);
6179 }
6180 let visible = visible_table(&table.name);
6181 let base = base_table(&table.name);
6182 self.exec(&format!("DELETE FROM {visible}"));
6183 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
6184 }
6185 for commit in self.outbox.clone() {
6186 for op in &commit.ops {
6187 let Some(table) = self.schema.table(&op.table).cloned() else {
6188 continue;
6189 };
6190 if op.upsert {
6191 let Some(values) = op.values.as_ref() else {
6192 continue;
6193 };
6194 let mut row: Row = Vec::with_capacity(table.columns.len());
6195 let mut ok = true;
6196 for column in &table.columns {
6197 match json_to_column_value(column, values.get(&column.name)) {
6198 Ok(v) => row.push(v),
6199 Err(_) => {
6200 ok = false;
6201 break;
6202 }
6203 }
6204 }
6205 if ok {
6206 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
6207 }
6208 } else {
6209 let sql = format!(
6210 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
6211 visible_table(&table.name),
6212 quote_ident(&table.primary_key)
6213 );
6214 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
6215 }
6216 }
6217 }
6218 for table in self.schema.tables.clone() {
6219 for index in &table.fts_indexes {
6220 let _ = self.rebuild_fts_projection(&table, index);
6221 let _ = self.create_fts_triggers(&table, index);
6222 }
6223 }
6224 self.exec("RELEASE syncular_overlay");
6225 self.overlay_dirty.set(false);
6226 }
6227
6228 fn exec(&self, sql: &str) {
6229 let _ = self.conn.execute_batch(sql);
6230 }
6231
6232 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
6235 transport
6236 .realtime_connect_for_client(&self.client_id)
6237 .map_err(|e| format!("{}: {}", e.code, e.message))?;
6238 self.realtime_connected = true;
6239 Ok(())
6240 }
6241
6242 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
6243 let _ = transport.realtime_close();
6244 self.realtime_connected = false;
6245 self.presence.clear(); }
6247
6248 pub fn set_presence(
6254 &mut self,
6255 transport: &mut dyn Transport,
6256 scope_key: &str,
6257 doc: Option<&Value>,
6258 ) -> Result<(), String> {
6259 if !self.realtime_connected {
6260 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
6261 }
6262 let text = encode_presence_publish(scope_key, doc);
6263 transport
6264 .realtime_send(&text)
6265 .map_err(|e| format!("{}: {}", e.code, e.message))
6266 }
6267
6268 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
6270 self.presence
6271 .get(scope_key)
6272 .map(|peers| peers.values().cloned().collect())
6273 .unwrap_or_default()
6274 }
6275
6276 fn apply_presence(
6278 &mut self,
6279 scope_key: String,
6280 kind: Option<PresenceKind>,
6281 actor_id: Option<String>,
6282 client_id: Option<String>,
6283 doc: Option<Value>,
6284 error: Option<String>,
6285 ) {
6286 if error.is_some() {
6289 return;
6290 }
6291 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
6292 return;
6293 };
6294 let peer_key = format!("{actor_id} {client_id}");
6295 match kind {
6296 PresenceKind::Leave => {
6297 if let Some(peers) = self.presence.get_mut(&scope_key) {
6298 peers.remove(&peer_key);
6299 if peers.is_empty() {
6300 self.presence.remove(&scope_key);
6301 }
6302 }
6303 }
6304 _ => {
6305 let doc = match doc {
6306 Some(Value::Object(_)) => doc.unwrap(),
6307 _ => return,
6308 };
6309 self.presence.entry(scope_key).or_default().insert(
6310 peer_key,
6311 PresencePeer {
6312 actor_id,
6313 client_id,
6314 doc,
6315 },
6316 );
6317 }
6318 }
6319 }
6320
6321 pub fn on_realtime_text(&mut self, text: &str) {
6323 match parse_control(text) {
6324 Ok(ControlMessage::Hello { requires_sync, .. }) => {
6325 if requires_sync {
6326 self.set_sync_needed(true, true);
6328 }
6329 }
6330 Ok(ControlMessage::Presence {
6331 scope_key,
6332 kind,
6333 actor_id,
6334 client_id,
6335 doc,
6336 error,
6337 ..
6338 }) => {
6339 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
6340 }
6341 Ok(ControlMessage::Wake { .. }) => {
6342 self.set_sync_needed(true, true);
6344 }
6345 _ => {}
6346 }
6347 }
6348
6349 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
6352 if self.stopped {
6353 return;
6354 }
6355 let message = match decode_message(bytes) {
6356 Ok(m) if m.msg_kind == MsgKind::Response => m,
6357 _ => {
6358 self.set_sync_needed(true, true);
6359 return;
6360 }
6361 };
6362 let mut frames = message.frames.into_iter();
6363 let mut applied_cursor: Option<i64> = None;
6364 let mut any_covered = false;
6365 let mut dropped = false;
6366 while let Some(frame) = frames.next() {
6367 let Frame::SubStart {
6368 id,
6369 status,
6370 effective_scopes,
6371 ..
6372 } = frame
6373 else {
6374 continue;
6375 };
6376 let mut body = Vec::new();
6377 let mut next_cursor: Option<i64> = None;
6378 for inner in frames.by_ref() {
6379 match inner {
6380 Frame::SubEnd {
6381 next_cursor: nc, ..
6382 } => {
6383 next_cursor = Some(nc);
6384 break;
6385 }
6386 Frame::Unknown { .. } => {}
6387 other => body.push(other),
6388 }
6389 }
6390 let Some(next_cursor) = next_cursor else {
6391 dropped = true;
6392 break;
6393 };
6394 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
6395 dropped = true;
6396 continue;
6397 };
6398 let sub = &self.subs[sub_index];
6399 if status != SubStatus::Active
6402 || sub.state != SubState::Active
6403 || sub.bootstrap_state.is_some()
6404 || !sub.synced_once
6405 {
6406 dropped = true;
6407 continue;
6408 }
6409 if next_cursor <= sub.cursor {
6410 any_covered = true;
6412 continue;
6413 }
6414 let previous_effective = self.subs[sub_index].effective.clone();
6415 let previous_cursor = self.subs[sub_index].cursor;
6416 if self.begin_observation("syncular_delta").is_err() {
6417 dropped = true;
6418 continue;
6419 }
6420 self.subs[sub_index].effective = Some(effective_scopes);
6421 let mut batch = ChangeAccumulator::default();
6422 let mut failed = false;
6423 for inner in body {
6424 if let Frame::Commit {
6425 tables, changes, ..
6426 } = inner
6427 {
6428 self.record_commit_changes(&mut batch, &tables, &changes);
6429 if self.apply_commit_changes(&tables, &changes).is_err() {
6430 failed = true;
6431 break;
6432 }
6433 }
6434 }
6435 if failed {
6436 self.rollback_observation("syncular_delta");
6437 self.subs[sub_index].effective = previous_effective;
6438 self.subs[sub_index].cursor = previous_cursor;
6439 self.overlay_dirty.set(true);
6440 self.rebuild_overlay();
6441 dropped = true;
6442 continue;
6443 }
6444 let sub = &mut self.subs[sub_index];
6445 sub.cursor = next_cursor;
6446 self.persist_sub(&self.subs[sub_index].clone());
6447 self.rebuild_overlay_if_dirty();
6448 if self.finish_observation("syncular_delta", batch).is_err() {
6449 self.rollback_observation("syncular_delta");
6450 self.subs[sub_index].effective = previous_effective;
6451 self.subs[sub_index].cursor = previous_cursor;
6452 self.overlay_dirty.set(true);
6453 self.rebuild_overlay();
6454 dropped = true;
6455 continue;
6456 }
6457 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
6458 }
6459 if let Some(cursor) = applied_cursor {
6460 self.reconcile_blob_refcounts(false);
6461 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
6463 let _ = transport.realtime_send(&ack);
6464 } else if !any_covered || dropped {
6465 self.set_sync_needed(true, true);
6467 }
6468 }
6469
6470 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
6474 if !self.realtime_connected {
6475 return;
6476 }
6477 let floor = self
6478 .subs
6479 .iter()
6480 .filter(|s| {
6481 s.state == SubState::Active
6482 && s.bootstrap_state.is_none()
6483 && s.synced_once
6484 && s.cursor >= 0
6485 })
6486 .map(|s| s.cursor)
6487 .min();
6488 if let Some(cursor) = floor {
6489 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
6490 let _ = transport.realtime_send(&ack);
6491 }
6492 }
6493}