1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, CommitOperation, CommitOperationOutcome,
27 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
28 ConflictRecord, CoverageSnapshot, LeaseState, Mutation, PresencePeer, QuerySnapshot,
29 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
30 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
31 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
32};
33use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
34use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
35use crate::values::{
36 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
37 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
38 scope_map_to_json, sort_scope_map,
39};
40
41const DEFAULT_ACCEPT: u8 = 0b0111;
46const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
47const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
48const ACCEPT_SQLITE: u8 = 1 << 2;
49const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
50
51const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
53const LOCAL_REVISION_KEY: &str = "localRevision";
54const CLIENT_ID_KEY: &str = "clientId";
55const LEASE_STATE_KEY: &str = "leaseState";
56const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
57const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62enum SubState {
63 Active,
64 Revoked,
65 Failed,
66}
67
68#[cfg(test)]
69mod observation_tests {
70 use super::*;
71 use serde_json::json;
72
73 fn client() -> SyncClient {
74 SyncClient::new(
75 "retry-test".to_owned(),
76 &json!({
77 "version": 1,
78 "tables": [{
79 "name": "tasks",
80 "primaryKey": "id",
81 "columns": [
82 { "name": "id", "type": "string", "nullable": false },
83 { "name": "project_id", "type": "string", "nullable": false }
84 ],
85 "scopes": [{ "pattern": "project:{project_id}" }]
86 }]
87 }),
88 ClientLimits::default(),
89 )
90 .expect("test client")
91 }
92
93 #[test]
94 fn background_retry_deadlines_back_off_and_reset() {
95 let mut client = client();
96 client.schedule_background_retry();
97 assert!(matches!(
98 client.drain_sync_intents().as_slice(),
99 [SyncIntent::Background { delay_ms: 250 }]
100 ));
101 client.schedule_background_retry();
102 assert!(matches!(
103 client.drain_sync_intents().as_slice(),
104 [SyncIntent::Background { delay_ms: 500 }]
105 ));
106 client.reset_background_retry();
107 client.schedule_background_retry();
108 assert!(matches!(
109 client.drain_sync_intents().as_slice(),
110 [SyncIntent::Background { delay_ms: 250 }]
111 ));
112 }
113
114 #[test]
115 fn secondary_unique_collision_preserves_existing_synced_row() {
116 let client = SyncClient::new(
117 "unique-upsert-test".to_owned(),
118 &json!({
119 "version": 1,
120 "tables": [{
121 "name": "tasks",
122 "primaryKey": "id",
123 "columns": [
124 { "name": "id", "type": "string", "nullable": false },
125 { "name": "project_id", "type": "string", "nullable": false },
126 { "name": "title", "type": "string", "nullable": false }
127 ],
128 "scopes": [{ "pattern": "project:{project_id}" }],
129 "indexes": [{
130 "name": "tasks_by_project_title",
131 "columns": ["project_id", "title"],
132 "unique": true
133 }]
134 }]
135 }),
136 ClientLimits::default(),
137 )
138 .expect("test client");
139 let table = client.schema.table("tasks").expect("tasks table");
140 let sql = client.insert_row_sql(&base_table("tasks"), table);
141
142 client
143 .conn
144 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
145 .expect("insert first row");
146 client
147 .conn
148 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
149 .expect("update same primary key");
150 client
151 .conn
152 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
153 .expect("insert second row");
154 assert!(client
155 .conn
156 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
157 .is_err());
158
159 let rows = client
160 .conn
161 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
162 .expect("prepare rows")
163 .query_map([], |row| {
164 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
165 })
166 .expect("query rows")
167 .collect::<Result<Vec<_>, _>>()
168 .expect("collect rows");
169 assert_eq!(
170 rows,
171 vec![
172 ("t1".to_owned(), "updated".to_owned()),
173 ("t2".to_owned(), "original".to_owned())
174 ]
175 );
176 }
177
178 #[test]
179 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
180 let path = std::env::temp_dir().join(format!(
181 "syncular-startup-intent-{}.db",
182 uuid::Uuid::new_v4()
183 ));
184 let schema = json!({
185 "version": 1,
186 "tables": [{
187 "name": "tasks",
188 "primaryKey": "id",
189 "columns": [
190 { "name": "id", "type": "string", "nullable": false },
191 { "name": "project_id", "type": "string", "nullable": false }
192 ],
193 "scopes": [{ "pattern": "project:{project_id}" }]
194 }]
195 });
196
197 {
198 let mut first = SyncClient::open_path_with_identity(
199 None,
200 &schema,
201 ClientLimits::default(),
202 path.to_str().expect("UTF-8 temp path"),
203 )
204 .expect("first open");
205 first
206 .set_window(
207 &WindowBase {
208 table: "tasks".to_owned(),
209 variable: "project_id".to_owned(),
210 fixed_scopes: Vec::new(),
211 params: None,
212 },
213 &["persisted".to_owned()],
214 )
215 .expect("persist window");
216 }
217
218 let mut reopened = SyncClient::open_path_with_identity(
219 None,
220 &schema,
221 ClientLimits::default(),
222 path.to_str().expect("UTF-8 temp path"),
223 )
224 .expect("reopen");
225 assert!(reopened.sync_needed());
226 assert!(matches!(
227 reopened.drain_sync_intents().as_slice(),
228 [SyncIntent::Interactive]
229 ));
230 drop(reopened);
231 std::fs::remove_file(path).expect("remove temp database");
232 }
233
234 #[test]
235 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
236 let path = std::env::temp_dir().join(format!(
237 "syncular-indexed-column-bump-{}.db",
238 uuid::Uuid::new_v4()
239 ));
240 let old_schema = json!({
241 "version": 1,
242 "tables": [
243 {
244 "name": "tasks",
245 "primaryKey": "id",
246 "columns": [
247 { "name": "id", "type": "string", "nullable": false },
248 { "name": "project_id", "type": "string", "nullable": false }
249 ],
250 "scopes": [{ "pattern": "project:{project_id}" }]
251 },
252 {
253 "name": "legacy",
254 "primaryKey": "id",
255 "columns": [
256 { "name": "id", "type": "string", "nullable": false },
257 { "name": "project_id", "type": "string", "nullable": false }
258 ],
259 "scopes": [{ "pattern": "project:{project_id}" }]
260 }
261 ]
262 });
263 {
264 let mut first = SyncClient::open_path_with_identity(
265 None,
266 &old_schema,
267 ClientLimits::default(),
268 path.to_str().expect("UTF-8 temp path"),
269 )
270 .expect("open old schema");
271 first
272 .subscribe(
273 "legacy-sub".to_owned(),
274 "legacy".to_owned(),
275 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
276 None,
277 )
278 .expect("persist legacy subscription");
279 }
280
281 let new_schema = json!({
282 "version": 2,
283 "tables": [{
284 "name": "tasks",
285 "primaryKey": "id",
286 "columns": [
287 { "name": "id", "type": "string", "nullable": false },
288 { "name": "project_id", "type": "string", "nullable": false },
289 { "name": "facility_membership_id", "type": "string", "nullable": true }
290 ],
291 "scopes": [{ "pattern": "project:{project_id}" }],
292 "indexes": [{
293 "name": "tasks_by_membership",
294 "columns": ["project_id", "facility_membership_id"],
295 "unique": false
296 }]
297 }]
298 });
299 let upgraded = SyncClient::open_path_with_identity(
300 None,
301 &new_schema,
302 ClientLimits::default(),
303 path.to_str().expect("UTF-8 temp path"),
304 )
305 .expect("open upgraded schema");
306 let columns = upgraded
307 .conn
308 .prepare("PRAGMA table_info(tasks)")
309 .expect("prepare columns")
310 .query_map([], |row| row.get::<_, String>(1))
311 .expect("query columns")
312 .collect::<Result<Vec<_>, _>>()
313 .expect("collect columns");
314 assert!(columns.contains(&"facility_membership_id".to_owned()));
315 let index_count: i64 = upgraded
316 .conn
317 .query_row(
318 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
319 [],
320 |row| row.get(0),
321 )
322 .expect("query index");
323 assert_eq!(index_count, 1);
324 assert!(upgraded.subscription_state("legacy-sub").is_none());
325 drop(upgraded);
326 std::fs::remove_file(path).expect("remove temp database");
327 }
328
329 #[test]
330 fn migrates_pre_envelope_outcome_journal_additively() {
331 let path = std::env::temp_dir().join(format!(
332 "syncular-outcome-migration-{}.db",
333 uuid::Uuid::new_v4()
334 ));
335 let conn = Connection::open(&path).expect("open legacy database");
336 conn.execute_batch(
337 "CREATE TABLE _syncular_commit_outcomes (
338 seq INTEGER PRIMARY KEY AUTOINCREMENT,
339 client_commit_id TEXT NOT NULL UNIQUE,
340 status TEXT NOT NULL,
341 recorded_at_ms INTEGER NOT NULL,
342 results_json TEXT NOT NULL,
343 resolution TEXT NOT NULL DEFAULT 'active',
344 resolved_at_ms INTEGER,
345 replacement_client_commit_id TEXT);",
346 )
347 .expect("create legacy outcome journal");
348 drop(conn);
349 let schema = json!({
350 "version": 1,
351 "tables": [{
352 "name": "tasks",
353 "primaryKey": "id",
354 "columns": [
355 { "name": "id", "type": "string", "nullable": false },
356 { "name": "project_id", "type": "string", "nullable": false }
357 ],
358 "scopes": [{ "pattern": "project:{project_id}" }]
359 }]
360 });
361 let client = SyncClient::open_path(
362 "migration-native".to_owned(),
363 &schema,
364 ClientLimits::default(),
365 path.to_str().expect("UTF-8 temp path"),
366 )
367 .expect("migrate database");
368 let has_operations = client
369 .conn
370 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
371 .expect("prepare table info")
372 .query_map([], |row| row.get::<_, String>(1))
373 .expect("query table info")
374 .filter_map(Result::ok)
375 .any(|column| column == "operations_json");
376 assert!(has_operations);
377 drop(client);
378 std::fs::remove_file(path).expect("remove temp database");
379 }
380
381 #[test]
382 fn durable_conflict_outcome_and_resolution_survive_reopen() {
383 let path = std::env::temp_dir().join(format!(
384 "syncular-durable-outcome-{}.db",
385 uuid::Uuid::new_v4()
386 ));
387 let schema = json!({
388 "version": 1,
389 "tables": [{
390 "name": "tasks",
391 "primaryKey": "id",
392 "columns": [
393 { "name": "id", "type": "string", "nullable": false },
394 { "name": "project_id", "type": "string", "nullable": false }
395 ],
396 "scopes": [{ "pattern": "project:{project_id}" }]
397 }]
398 });
399 let conflict = ConflictRecord {
400 client_commit_id: "losing-commit".to_owned(),
401 op_index: 0,
402 table: "tasks".to_owned(),
403 row_id: "t1".to_owned(),
404 code: "sync.version_conflict".to_owned(),
405 message: "stale base version".to_owned(),
406 server_version: 2,
407 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
408 operation: Some(CommitOperation {
409 table: "tasks".to_owned(),
410 row_id: "t1".to_owned(),
411 op: "upsert".to_owned(),
412 base_version: Some(1),
413 values: None,
414 changed_fields: None,
415 }),
416 };
417 let failed_operations = vec![
418 OutboxOp {
419 upsert: true,
420 table: "tasks".to_owned(),
421 row_id: "t1".to_owned(),
422 base_version: Some(1),
423 values: None,
424 changed_fields: None,
425 },
426 OutboxOp {
427 upsert: true,
428 table: "tasks".to_owned(),
429 row_id: "status-event-1".to_owned(),
430 base_version: Some(0),
431 values: None,
432 changed_fields: None,
433 },
434 ];
435
436 {
437 let mut first = SyncClient::open_path(
438 "durable-native".to_owned(),
439 &schema,
440 ClientLimits::default(),
441 path.to_str().expect("UTF-8 temp path"),
442 )
443 .expect("first open");
444 first
445 .begin_observation("test_outcome")
446 .expect("begin outcome");
447 first
448 .persist_commit_outcome(
449 "losing-commit",
450 CommitOutcomeStatus::Conflict,
451 &[CommitOperationOutcome::Conflict {
452 conflict: conflict.clone(),
453 }],
454 Some(&failed_operations),
455 )
456 .expect("persist outcome");
457 first.conflicts.push(conflict);
458 first
459 .finish_observation(
460 "test_outcome",
461 ChangeAccumulator {
462 conflicts: true,
463 outcomes: true,
464 ..ChangeAccumulator::default()
465 },
466 )
467 .expect("commit outcome");
468 }
469
470 {
471 let mut reopened = SyncClient::open_path(
472 "durable-native".to_owned(),
473 &schema,
474 ClientLimits::default(),
475 path.to_str().expect("UTF-8 temp path"),
476 )
477 .expect("reopen");
478 assert_eq!(reopened.conflicts().len(), 1);
479 let outcome = reopened
480 .commit_outcome("losing-commit")
481 .expect("read outcome")
482 .expect("outcome");
483 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
484 let operations = outcome.operations.expect("aggregate envelope");
485 assert_eq!(operations.len(), 2);
486 assert_eq!(operations[1].row_id, "status-event-1");
487 let resolved = reopened
488 .resolve_commit_outcome(ResolveCommitOutcomeInput {
489 client_commit_id: "losing-commit".to_owned(),
490 resolution: CommitOutcomeResolution::ResolvedKeepServer,
491 replacement_client_commit_id: None,
492 })
493 .expect("resolve");
494 assert_eq!(
495 resolved.resolution,
496 CommitOutcomeResolution::ResolvedKeepServer
497 );
498 assert!(reopened.conflicts().is_empty());
499 }
500
501 let reopened = SyncClient::open_path(
502 "durable-native".to_owned(),
503 &schema,
504 ClientLimits::default(),
505 path.to_str().expect("UTF-8 temp path"),
506 )
507 .expect("second reopen");
508 assert!(reopened.conflicts().is_empty());
509 assert_eq!(
510 reopened
511 .commit_outcome("losing-commit")
512 .expect("read outcome")
513 .expect("outcome")
514 .resolution,
515 CommitOutcomeResolution::ResolvedKeepServer
516 );
517 drop(reopened);
518 std::fs::remove_file(path).expect("remove temp database");
519 }
520
521 #[test]
522 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
523 let path =
524 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
525 let schema = json!({
526 "version": 1,
527 "tables": [{
528 "name": "tasks",
529 "primaryKey": "id",
530 "columns": [
531 { "name": "id", "type": "string", "nullable": false },
532 { "name": "project_id", "type": "string", "nullable": false }
533 ],
534 "scopes": [{ "pattern": "project:{project_id}" }]
535 }]
536 });
537 let mut client = SyncClient::open_path_with_identity(
538 Some("sidecar-client".to_owned()),
539 &schema,
540 ClientLimits::default(),
541 path.to_str().expect("UTF-8 temp path"),
542 )
543 .expect("open owner");
544 let base = WindowBase {
545 table: "tasks".to_owned(),
546 variable: "project_id".to_owned(),
547 fixed_scopes: Vec::new(),
548 params: None,
549 };
550 client
551 .set_window(&base, &["one".to_owned()])
552 .expect("set window");
553 client
554 .mutate(vec![Mutation::Upsert {
555 table: "tasks".to_owned(),
556 values: Map::from_iter([
557 ("id".to_owned(), Value::from("t1")),
558 ("project_id".to_owned(), Value::from("one")),
559 ]),
560 base_version: None,
561 }])
562 .expect("local mutate");
563
564 let coverage = [WindowCoverage {
565 base,
566 units: vec!["one".to_owned(), "missing".to_owned()],
567 }];
568 let owner = client
569 .query_snapshot(
570 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
571 &[],
572 &coverage,
573 )
574 .expect("owner snapshot");
575 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
576 let sidecar = reader
577 .query_snapshot(
578 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
579 &[],
580 &coverage,
581 )
582 .expect("sidecar snapshot");
583
584 assert_eq!(sidecar.revision, owner.revision);
585 assert_eq!(sidecar.rows, owner.rows);
586 assert_eq!(
587 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
588 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
589 );
590 assert_eq!(sidecar.revision, "2");
591 assert_eq!(sidecar.rows[0]["id"], "t1");
592 assert_eq!(sidecar.rows[0]["server_version"], -1);
593 assert!(!sidecar.coverage.complete);
594 assert_eq!(sidecar.coverage.pending.len(), 1);
595 assert_eq!(sidecar.coverage.missing.len(), 1);
596
597 drop(reader);
598 drop(client);
599 std::fs::remove_file(path).expect("remove temp database");
600 }
601
602 #[test]
603 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
604 let schema = json!({
605 "version": 1,
606 "tables": [{
607 "name": "catalogue_codes",
608 "primaryKey": "id",
609 "columns": [
610 { "name": "id", "type": "string", "nullable": false },
611 { "name": "release_id", "type": "string", "nullable": false },
612 { "name": "code", "type": "string", "nullable": false },
613 { "name": "title", "type": "string", "nullable": false }
614 ],
615 "scopes": [{ "pattern": "release:{release_id}" }],
616 "ftsIndexes": [{
617 "name": "catalogue_codes_fts",
618 "columns": ["code", "title"],
619 "tokenize": "unicode61 remove_diacritics 2"
620 }]
621 }]
622 });
623 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
624 .expect("FTS5 client");
625 let search = |client: &SyncClient, query: &str| {
626 client
627 .query(
628 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
629 &[Value::from(query)],
630 )
631 .expect("FTS query")
632 };
633
634 client
635 .mutate(vec![Mutation::Upsert {
636 table: "catalogue_codes".to_owned(),
637 values: Map::from_iter([
638 ("id".to_owned(), Value::from("c1")),
639 ("release_id".to_owned(), Value::from("r1")),
640 ("code".to_owned(), Value::from("A01")),
641 ("title".to_owned(), Value::from("Cholera")),
642 ]),
643 base_version: None,
644 }])
645 .expect("insert code");
646 assert_eq!(search(&client, "cholera").len(), 1);
647
648 client
649 .mutate(vec![Mutation::Upsert {
650 table: "catalogue_codes".to_owned(),
651 values: Map::from_iter([
652 ("id".to_owned(), Value::from("c1")),
653 ("release_id".to_owned(), Value::from("r1")),
654 ("code".to_owned(), Value::from("A01")),
655 ("title".to_owned(), Value::from("Enteric infection")),
656 ]),
657 base_version: None,
658 }])
659 .expect("update code");
660 assert!(search(&client, "cholera").is_empty());
661 assert_eq!(search(&client, "enteric").len(), 1);
662
663 client
664 .mutate(vec![Mutation::Delete {
665 table: "catalogue_codes".to_owned(),
666 row_id: "c1".to_owned(),
667 base_version: None,
668 }])
669 .expect("delete code");
670 assert!(search(&client, "enteric").is_empty());
671 }
672}
673
674impl SubState {
675 fn name(self) -> &'static str {
676 match self {
677 SubState::Active => "active",
678 SubState::Revoked => "revoked",
679 SubState::Failed => "failed",
680 }
681 }
682
683 fn parse(value: &str) -> Self {
684 match value {
685 "revoked" => Self::Revoked,
686 "failed" => Self::Failed,
687 _ => Self::Active,
688 }
689 }
690}
691
692#[derive(Debug, Clone)]
693struct Subscription {
694 id: String,
695 table: String,
696 requested: Vec<(String, Vec<String>)>,
697 params: Option<String>,
698 cursor: i64,
699 bootstrap_state: Option<String>,
701 state: SubState,
702 reason_code: Option<String>,
703 effective: Option<Vec<(String, Vec<String>)>>,
706 synced_once: bool,
707}
708
709#[derive(Debug, Clone)]
710struct OutboxOp {
711 upsert: bool,
712 table: String,
713 row_id: String,
714 base_version: Option<i64>,
715 values: Option<Map<String, Value>>,
718 changed_fields: Option<Vec<String>>,
720}
721
722impl From<&OutboxOp> for CommitOperation {
723 fn from(operation: &OutboxOp) -> Self {
724 Self {
725 table: operation.table.clone(),
726 row_id: operation.row_id.clone(),
727 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
728 base_version: operation.base_version,
729 values: operation.values.clone(),
730 changed_fields: operation.changed_fields.clone(),
731 }
732 }
733}
734
735#[derive(Debug, Clone)]
736struct OutboxCommit {
737 client_commit_id: String,
738 ops: Vec<OutboxOp>,
739}
740
741struct StoredCommitOutcomeRow {
742 sequence: i64,
743 client_commit_id: String,
744 status: String,
745 recorded_at_ms: i64,
746 results_json: String,
747 operations_json: Option<String>,
748 resolution: String,
749 resolved_at_ms: Option<i64>,
750 replacement_client_commit_id: Option<String>,
751}
752
753enum SectionError {
756 FailClosed,
757 Abort(String, String),
758}
759
760struct RequestMeta {
761 pushed_ids: Vec<String>,
762 fresh: Vec<(String, bool)>,
765 accept: u8,
766 deferred_commits: usize,
770}
771
772#[derive(Default)]
773struct ChangeAccumulator {
774 tables: BTreeMap<String, Option<BTreeSet<String>>>,
776 windows: BTreeMap<(String, String), BTreeSet<String>>,
777 status: bool,
778 conflicts: bool,
779 rejections: bool,
780 outcomes: bool,
781}
782
783impl ChangeAccumulator {
784 fn table(&mut self, table: &str) {
785 self.tables.insert(table.to_owned(), None);
786 }
787
788 fn scope(&mut self, table: &str, key: String) {
789 match self.tables.get_mut(table) {
790 Some(None) => {}
791 Some(Some(keys)) => {
792 keys.insert(key);
793 }
794 None => {
795 self.tables
796 .insert(table.to_owned(), Some(BTreeSet::from([key])));
797 }
798 }
799 }
800
801 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
802 self.windows
803 .entry((base_key.to_owned(), table.to_owned()))
804 .or_default()
805 .insert(unit.to_owned());
806 }
807
808 fn touched(&self) -> bool {
809 !self.tables.is_empty()
810 || !self.windows.is_empty()
811 || self.status
812 || self.conflicts
813 || self.rejections
814 || self.outcomes
815 }
816}
817
818const PUSH_OPS_PER_REQUEST: usize = 500;
824
825pub struct SyncClient {
826 conn: Connection,
827 schema: ClientSchema,
828 client_id: String,
829 limits: ClientLimits,
830 subs: Vec<Subscription>,
831 outbox: Vec<OutboxCommit>,
832 conflicts: Vec<ConflictRecord>,
833 rejections: Vec<RejectionRecord>,
834 schema_floor: Option<SchemaFloor>,
835 lease_state: Option<LeaseState>,
837 stopped: bool,
839 upgrading: bool,
841 sync_needed: bool,
843 realtime_connected: bool,
844 presence: HashMap<String, HashMap<String, PresencePeer>>,
846 now_ms: Option<i64>,
849 encryption: crate::values::EncryptionConfig,
853 insert_sql: RefCell<HashMap<String, String>>,
859 overlay_dirty: Cell<bool>,
863 change_queue: VecDeque<ClientChangeBatch>,
865 sync_intent_queue: VecDeque<SyncIntent>,
866 retry_delay_ms: u64,
868}
869
870fn quote_ident(name: &str) -> String {
871 format!("\"{}\"", name.replace('"', "\"\""))
872}
873
874fn base_table(name: &str) -> String {
875 quote_ident(&format!("_syncular_base_{name}"))
876}
877
878type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
883
884fn window_base_key(base: &WindowBase) -> String {
885 format!(
886 "{}\0{}\0{}",
887 base.table,
888 base.variable,
889 canonical_scope_json(&base.fixed_scopes)
890 )
891}
892
893fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
895 let mut scopes = base.fixed_scopes.clone();
896 scopes.retain(|(k, _)| k != &base.variable);
897 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
898 scopes
899}
900
901fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
905 let canonical = canonical_scope_json(&unit_scopes(base, unit));
906 let digest = Sha256::digest(canonical.as_bytes());
907 let hex = bytes_to_hex(&digest);
908 format!("w:{}:{}", base.table, &hex[..16])
909}
910
911fn visible_table(name: &str) -> String {
912 quote_ident(name)
913}
914
915const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
916
917fn is_synced_table_name(name: &str) -> bool {
920 if name.starts_with("sqlite_") {
921 return false;
922 }
923 if name.starts_with("_syncular_base_") {
924 return true; }
926 !name.starts_with("_syncular_")
928}
929
930fn blob_id_for(bytes: &[u8]) -> String {
932 let digest = Sha256::digest(bytes);
933 format!("sha256:{}", bytes_to_hex(&digest))
934}
935
936enum RowParam<'a> {
940 Cell(&'a Option<ColumnValue>),
941 Version(i64),
942}
943
944impl rusqlite::ToSql for RowParam<'_> {
945 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
946 Ok(match self {
947 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
948 RowParam::Cell(cell) => match cell {
949 None => ToSqlOutput::Owned(SqlValue::Null),
950 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
951 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
952 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
953 Some(ColumnValue::Boolean(b)) => {
954 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
955 }
956 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
957 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
958 }
959 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
961 ToSqlOutput::Borrowed(ValueRef::Blob(b))
962 }
963 },
964 })
965 }
966}
967
968fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
976 use ssp2::segment::ColumnType;
977 let mismatch = || {
978 Err(format!(
979 "image column {:?} holds a value of the wrong type",
980 column.name
981 ))
982 };
983 match value {
984 ValueRef::Null => {
985 if !column.nullable {
986 return Err(format!(
987 "image column {:?} is NULL but not nullable",
988 column.name
989 ));
990 }
991 Ok(ToSqlOutput::Owned(SqlValue::Null))
992 }
993 ValueRef::Integer(i) => match column.ty {
994 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
995 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
996 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
997 _ => mismatch(),
998 },
999 ValueRef::Real(_) => match column.ty {
1000 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1001 _ => mismatch(),
1002 },
1003 ValueRef::Text(t) => {
1004 std::str::from_utf8(t)
1005 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1006 match column.ty {
1007 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1008 Ok(ToSqlOutput::Borrowed(value))
1009 }
1010 _ => mismatch(),
1011 }
1012 }
1013 ValueRef::Blob(_) => match column.ty {
1014 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1016 _ => mismatch(),
1017 },
1018 }
1019}
1020
1021fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1022 use rusqlite::types::ValueRef;
1023 match value {
1024 ValueRef::Null => Value::Null,
1025 ValueRef::Integer(i) => match column.ty {
1026 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1027 ssp2::segment::ColumnType::Float => {
1028 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1029 }
1030 _ => Value::from(i),
1031 },
1032 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1033 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1034 ValueRef::Blob(b) => {
1035 let mut map = Map::new();
1036 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1037 Value::Object(map)
1038 }
1039 }
1040}
1041
1042fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1046 Ok(match value {
1047 Value::Null => SqlValue::Null,
1048 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1049 Value::Number(n) => {
1050 if let Some(i) = n.as_i64() {
1051 SqlValue::Integer(i)
1052 } else if let Some(f) = n.as_f64() {
1053 SqlValue::Real(f)
1054 } else {
1055 return Err(format!("query param number {n} is out of range"));
1056 }
1057 }
1058 Value::String(s) => SqlValue::Text(s.clone()),
1059 Value::Object(_) => {
1060 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1061 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1062 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1063 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1064 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1065 })?)
1066 } else {
1067 return Err(
1068 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1069 .to_owned(),
1070 );
1071 }
1072 }
1073 Value::Array(_) => return Err("query array params are not supported".to_owned()),
1074 })
1075}
1076
1077fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1082 use rusqlite::types::ValueRef;
1083 match value {
1084 ValueRef::Null => Value::Null,
1085 ValueRef::Integer(i) => {
1086 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1090 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1091 Value::from(i)
1092 } else {
1093 let mut map = Map::new();
1094 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1095 Value::Object(map)
1096 }
1097 }
1098 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1099 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1100 ValueRef::Blob(b) => {
1101 let mut map = Map::new();
1102 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1103 Value::Object(map)
1104 }
1105 }
1106}
1107
1108fn query_connection(
1109 conn: &Connection,
1110 sql: &str,
1111 params: &[Value],
1112) -> Result<Vec<Map<String, Value>>, String> {
1113 crate::query_guard::assert_read_only_query(sql)?;
1114 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1115 let bound: Vec<SqlValue> = params
1116 .iter()
1117 .map(json_param_to_sql)
1118 .collect::<Result<_, _>>()?;
1119 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
1120 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1121 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1122 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1123 let mut sql_rows = stmt
1124 .query(bound_refs.as_slice())
1125 .map_err(|e| e.to_string())?;
1126 let mut out = Vec::new();
1127 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1128 let mut record = Map::new();
1129 for (i, name) in column_names.iter().enumerate() {
1130 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1131 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1132 }
1133 out.push(record);
1134 }
1135 Ok(out)
1136}
1137
1138fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1139 let mut stmt = conn
1140 .prepare(
1141 "SELECT windows.unit, subscriptions.state_json
1142 FROM _syncular_windows AS windows
1143 JOIN _syncular_subscriptions AS subscriptions
1144 ON subscriptions.id = windows.sub_id
1145 WHERE windows.base = ?1
1146 ORDER BY windows.unit ASC",
1147 )
1148 .map_err(|error| error.to_string())?;
1149 let rows = stmt
1150 .query_map(rusqlite::params![window_base_key(base)], |row| {
1151 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1152 })
1153 .map_err(|error| error.to_string())?;
1154 let mut units = Vec::new();
1155 let mut pending = Vec::new();
1156 for row in rows {
1157 let (unit, raw) = row.map_err(|error| error.to_string())?;
1158 let state: Value = serde_json::from_str(&raw)
1159 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1160 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1161 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1162 || state
1163 .get("bootstrapState")
1164 .is_some_and(|value| !value.is_null());
1165 if is_pending {
1166 pending.push(unit.clone());
1167 }
1168 units.push(unit);
1169 }
1170 Ok(WindowState { units, pending })
1171}
1172
1173fn snapshot_connection(
1174 conn: &Connection,
1175 sql: &str,
1176 params: &[Value],
1177 coverage: &[WindowCoverage],
1178) -> Result<QuerySnapshot, String> {
1179 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1180 .map_err(|error| error.to_string())?;
1181 let result = (|| {
1182 let revision = conn
1183 .query_row(
1184 "SELECT value FROM _syncular_meta WHERE key = ?1",
1185 rusqlite::params![LOCAL_REVISION_KEY],
1186 |row| row.get::<_, String>(0),
1187 )
1188 .ok()
1189 .and_then(|value| value.parse::<u64>().ok())
1190 .unwrap_or(0);
1191 let rows = query_connection(conn, sql, params)?;
1192 let mut pending = Vec::new();
1193 let mut missing = Vec::new();
1194 for requested in coverage {
1195 let base_key = window_base_key(&requested.base);
1196 let state = persisted_window_state(conn, &requested.base)?;
1197 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1198 let reference = WindowUnitRef {
1199 base_key: base_key.clone(),
1200 unit: unit.clone(),
1201 };
1202 if !state.units.iter().any(|held| held == &unit) {
1203 missing.push(reference);
1204 } else if state.pending.iter().any(|held| held == &unit) {
1205 pending.push(reference);
1206 }
1207 }
1208 }
1209 Ok(QuerySnapshot {
1210 revision: revision.to_string(),
1211 rows,
1212 coverage: CoverageSnapshot {
1213 complete: pending.is_empty() && missing.is_empty(),
1214 pending,
1215 missing,
1216 },
1217 })
1218 })();
1219 match result {
1220 Ok(snapshot) => {
1221 conn.execute_batch("RELEASE syncular_snapshot_read")
1222 .map_err(|error| error.to_string())?;
1223 Ok(snapshot)
1224 }
1225 Err(error) => {
1226 let _ = conn.execute_batch(
1227 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1228 );
1229 Err(error)
1230 }
1231 }
1232}
1233
1234pub struct FileQuerySnapshotReader {
1239 path: String,
1240 conn: Option<Connection>,
1241}
1242
1243impl FileQuerySnapshotReader {
1244 #[must_use]
1245 pub fn new(path: impl Into<String>) -> Self {
1246 Self {
1247 path: path.into(),
1248 conn: None,
1249 }
1250 }
1251
1252 fn connection(&mut self) -> Result<&Connection, String> {
1253 if self.conn.is_none() {
1254 let conn = Connection::open_with_flags(
1255 &self.path,
1256 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1257 )
1258 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1259 conn.busy_timeout(std::time::Duration::from_millis(250))
1260 .map_err(|error| error.to_string())?;
1261 self.conn = Some(conn);
1262 }
1263 self.conn
1264 .as_ref()
1265 .ok_or_else(|| "read sidecar connection missing".to_owned())
1266 }
1267
1268 pub fn query_snapshot(
1269 &mut self,
1270 sql: &str,
1271 params: &[Value],
1272 coverage: &[WindowCoverage],
1273 ) -> Result<QuerySnapshot, String> {
1274 snapshot_connection(self.connection()?, sql, params, coverage)
1275 }
1276}
1277
1278impl SyncClient {
1279 pub fn new_with_identity(
1280 client_id: Option<String>,
1281 schema_json: &Value,
1282 limits: ClientLimits,
1283 ) -> Result<Self, String> {
1284 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1285 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1286 Self::with_connection(resolved, schema_json, limits, conn)
1287 }
1288
1289 pub fn new(
1290 client_id: String,
1291 schema_json: &Value,
1292 limits: ClientLimits,
1293 ) -> Result<Self, String> {
1294 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1295 Self::with_connection(client_id, schema_json, limits, conn)
1296 }
1297
1298 pub fn open_path(
1304 client_id: String,
1305 schema_json: &Value,
1306 limits: ClientLimits,
1307 path: &str,
1308 ) -> Result<Self, String> {
1309 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1310 Self::with_connection(client_id, schema_json, limits, conn)
1311 }
1312
1313 pub fn open_path_with_identity(
1314 client_id: Option<String>,
1315 schema_json: &Value,
1316 limits: ClientLimits,
1317 path: &str,
1318 ) -> Result<Self, String> {
1319 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1320 conn.busy_timeout(std::time::Duration::from_millis(250))
1326 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1327 conn.pragma_update(None, "journal_mode", "WAL")
1328 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
1329 let persisted = conn
1330 .query_row(
1331 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
1332 [],
1333 |row| row.get::<_, String>(0),
1334 )
1335 .ok();
1336 let resolved = persisted
1337 .clone()
1338 .or(client_id.clone())
1339 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1340 if let (Some(existing), Some(requested)) = (persisted, client_id) {
1341 if existing != requested {
1342 return Err(format!(
1343 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
1344 ));
1345 }
1346 }
1347 Self::with_connection(resolved, schema_json, limits, conn)
1348 }
1349
1350 #[must_use]
1351 pub fn client_id(&self) -> &str {
1352 &self.client_id
1353 }
1354
1355 pub fn with_connection(
1362 client_id: String,
1363 schema_json: &Value,
1364 limits: ClientLimits,
1365 conn: Connection,
1366 ) -> Result<Self, String> {
1367 if limits.outcome_retention_max_entries == Some(0) {
1368 return Err(
1369 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
1370 );
1371 }
1372 let schema = parse_schema_json(schema_json)?;
1373 let mut client = SyncClient {
1374 conn,
1375 schema,
1376 client_id,
1377 limits,
1378 subs: Vec::new(),
1379 outbox: Vec::new(),
1380 conflicts: Vec::new(),
1381 rejections: Vec::new(),
1382 schema_floor: None,
1383 lease_state: None,
1384 stopped: false,
1385 upgrading: false,
1386 sync_needed: false,
1387 realtime_connected: false,
1388 presence: HashMap::new(),
1389 now_ms: None,
1390 encryption: crate::values::EncryptionConfig::default(),
1391 insert_sql: RefCell::new(HashMap::new()),
1392 overlay_dirty: Cell::new(false),
1393 change_queue: VecDeque::new(),
1394 sync_intent_queue: VecDeque::new(),
1395 retry_delay_ms: 250,
1396 };
1397 client
1401 .conn
1402 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
1403 client.create_bookkeeping_tables()?;
1407 match client.get_meta(CLIENT_ID_KEY) {
1408 Some(existing) if existing != client.client_id => {
1409 return Err(format!(
1410 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
1411 client.client_id
1412 ));
1413 }
1414 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
1415 _ => {}
1416 }
1417 client.restore_persisted_state()?;
1418 let marker = client
1419 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1420 .and_then(|value| value.parse::<i32>().ok());
1421 match marker {
1422 None => {
1423 client.create_synced_tables()?;
1424 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
1425 }
1426 Some(version) if version == client.schema.version => {
1427 client.create_synced_tables()?;
1428 }
1429 Some(_) => client.run_schema_reset()?,
1430 }
1431 client.prune_unknown_subscriptions()?;
1432 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
1433 client.overlay_dirty.set(true);
1436 client.rebuild_overlay();
1437 }
1438 client.enqueue_startup_sync_if_needed();
1444 Ok(client)
1445 }
1446
1447 pub fn set_now_ms(&mut self, now_ms: i64) {
1450 self.now_ms = Some(now_ms);
1451 }
1452
1453 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
1457 self.encryption = encryption;
1458 }
1459
1460 fn clock_now_ms(&self) -> i64 {
1461 self.now_ms.unwrap_or_else(|| {
1462 std::time::SystemTime::now()
1463 .duration_since(std::time::UNIX_EPOCH)
1464 .map(|d| d.as_millis() as i64)
1465 .unwrap_or(0)
1466 })
1467 }
1468
1469 fn create_bookkeeping_tables(&self) -> Result<(), String> {
1470 self.conn
1472 .execute_batch(
1473 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
1474 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1475 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
1476 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
1477 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1478 client_commit_id TEXT NOT NULL UNIQUE,
1479 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
1480 recorded_at_ms INTEGER NOT NULL,
1481 results_json TEXT NOT NULL,
1482 operations_json TEXT,
1483 resolution TEXT NOT NULL DEFAULT 'active'
1484 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
1485 resolved_at_ms INTEGER,
1486 replacement_client_commit_id TEXT);
1487 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
1488 ON _syncular_commit_outcomes(resolution, seq);
1489 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
1490 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
1491 CREATE TABLE IF NOT EXISTS _syncular_meta (
1492 key TEXT PRIMARY KEY, value TEXT NOT NULL);
1493 CREATE TABLE IF NOT EXISTS _syncular_windows (
1494 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
1495 PRIMARY KEY (base, unit));
1496 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
1497 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
1498 effective_scopes TEXT NOT NULL);",
1499 )
1500 .map_err(|e| e.to_string())?;
1501 let _ = self
1504 .conn
1505 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
1506 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
1507 self.set_meta(LOCAL_REVISION_KEY, "0");
1508 }
1509 if self.schema_has_blobs() {
1514 self.conn
1515 .execute_batch(
1516 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
1517 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
1518 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
1519 created_at_ms INTEGER NOT NULL,
1520 last_used_ms INTEGER NOT NULL DEFAULT 0);
1521 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
1522 media_type TEXT, created_at_ms INTEGER NOT NULL);",
1523 )
1524 .map_err(|e| e.to_string())?;
1525 let _ = self.conn.execute_batch(
1529 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
1530 );
1531 }
1532 Ok(())
1533 }
1534
1535 fn schema_has_blobs(&self) -> bool {
1537 self.schema
1538 .tables
1539 .iter()
1540 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
1541 }
1542
1543 fn get_meta(&self, key: &str) -> Option<String> {
1546 self.conn
1547 .query_row(
1548 "SELECT value FROM _syncular_meta WHERE key = ?1",
1549 rusqlite::params![key],
1550 |row| row.get::<_, String>(0),
1551 )
1552 .ok()
1553 }
1554
1555 fn set_meta(&self, key: &str, value: &str) {
1556 let _ = self.conn.execute(
1557 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
1558 rusqlite::params![key, value],
1559 );
1560 }
1561
1562 fn delete_meta(&self, key: &str) {
1563 let _ = self.conn.execute(
1564 "DELETE FROM _syncular_meta WHERE key = ?1",
1565 rusqlite::params![key],
1566 );
1567 }
1568
1569 fn restore_persisted_state(&mut self) -> Result<(), String> {
1570 self.subs = {
1571 let mut stmt = self
1572 .conn
1573 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
1574 .map_err(|error| error.to_string())?;
1575 let rows = stmt
1576 .query_map([], |row| {
1577 Ok((
1578 row.get::<_, String>(0)?,
1579 row.get::<_, String>(1)?,
1580 row.get::<_, String>(2)?,
1581 ))
1582 })
1583 .map_err(|error| error.to_string())?;
1584 let mut subscriptions = Vec::new();
1585 for row in rows {
1586 let (id, table, raw) = row.map_err(|error| error.to_string())?;
1587 let state: Value = serde_json::from_str(&raw)
1588 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
1589 let requested = json_to_scope_map(
1590 state.get("requested").unwrap_or(&Value::Object(Map::new())),
1591 )?;
1592 let effective = state
1593 .get("effectiveScopes")
1594 .filter(|value| !value.is_null())
1595 .map(json_to_scope_map)
1596 .transpose()?;
1597 subscriptions.push(Subscription {
1598 id,
1599 table,
1600 requested,
1601 params: state
1602 .get("params")
1603 .and_then(Value::as_str)
1604 .map(str::to_owned),
1605 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
1606 bootstrap_state: state
1607 .get("bootstrapState")
1608 .and_then(Value::as_str)
1609 .map(str::to_owned),
1610 state: SubState::parse(
1611 state
1612 .get("status")
1613 .and_then(Value::as_str)
1614 .unwrap_or("active"),
1615 ),
1616 reason_code: state
1617 .get("reasonCode")
1618 .and_then(Value::as_str)
1619 .map(str::to_owned),
1620 effective,
1621 synced_once: state
1622 .get("syncedOnce")
1623 .and_then(Value::as_bool)
1624 .unwrap_or(false),
1625 });
1626 }
1627 subscriptions
1628 };
1629
1630 self.outbox = {
1631 let mut stmt = self
1632 .conn
1633 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
1634 .map_err(|error| error.to_string())?;
1635 let rows = stmt
1636 .query_map([], |row| {
1637 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1638 })
1639 .map_err(|error| error.to_string())?;
1640 let mut commits = Vec::new();
1641 for row in rows {
1642 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
1643 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
1644 format!("invalid persisted outbox {client_commit_id:?}: {error}")
1645 })?;
1646 let mut ops = Vec::with_capacity(entries.len());
1647 for entry in entries {
1648 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
1649 ops.push(OutboxOp {
1650 upsert: op == "upsert",
1651 table: entry
1652 .get("table")
1653 .and_then(Value::as_str)
1654 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
1655 .to_owned(),
1656 row_id: entry
1657 .get("rowId")
1658 .and_then(Value::as_str)
1659 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
1660 .to_owned(),
1661 base_version: entry.get("baseVersion").and_then(Value::as_i64),
1662 values: entry.get("values").and_then(Value::as_object).cloned(),
1663 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
1664 |values| {
1665 values
1666 .iter()
1667 .filter_map(Value::as_str)
1668 .map(str::to_owned)
1669 .collect()
1670 },
1671 ),
1672 });
1673 }
1674 commits.push(OutboxCommit {
1675 client_commit_id,
1676 ops,
1677 });
1678 }
1679 commits
1680 };
1681
1682 self.prune_commit_outcomes()?;
1683 let active = self.commit_outcomes(CommitOutcomeQuery {
1684 active_only: true,
1685 ..CommitOutcomeQuery::default()
1686 })?;
1687 self.conflicts = active
1688 .iter()
1689 .flat_map(|outcome| outcome.results.iter())
1690 .filter_map(|result| match result {
1691 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
1692 _ => None,
1693 })
1694 .collect();
1695 self.rejections = active
1696 .iter()
1697 .flat_map(|outcome| outcome.results.iter())
1698 .filter_map(|result| match result {
1699 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
1700 _ => None,
1701 })
1702 .collect();
1703
1704 self.lease_state = self
1705 .get_meta(LEASE_STATE_KEY)
1706 .map(|raw| serde_json::from_str(&raw))
1707 .transpose()
1708 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
1709 self.schema_floor = self
1710 .get_meta(SCHEMA_FLOOR_KEY)
1711 .map(|raw| serde_json::from_str(&raw))
1712 .transpose()
1713 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
1714 self.stopped = self.schema_floor.is_some();
1715 Ok(())
1716 }
1717
1718 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
1721 let valid_tables: BTreeSet<String> = self
1722 .schema
1723 .tables
1724 .iter()
1725 .map(|table| table.name.clone())
1726 .collect();
1727 let stale_ids: Vec<String> = self
1728 .subs
1729 .iter()
1730 .filter(|sub| !valid_tables.contains(&sub.table))
1731 .map(|sub| sub.id.clone())
1732 .collect();
1733 for id in &stale_ids {
1734 self.conn
1735 .execute(
1736 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
1737 rusqlite::params![id],
1738 )
1739 .map_err(|error| error.to_string())?;
1740 self.conn
1741 .execute(
1742 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
1743 rusqlite::params![id],
1744 )
1745 .map_err(|error| error.to_string())?;
1746 self.conn
1747 .execute(
1748 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
1749 rusqlite::params![id],
1750 )
1751 .map_err(|error| error.to_string())?;
1752 }
1753 self.subs.retain(|sub| valid_tables.contains(&sub.table));
1754 Ok(())
1755 }
1756
1757 #[must_use]
1758 pub fn local_revision(&self) -> u64 {
1759 self.get_meta(LOCAL_REVISION_KEY)
1760 .and_then(|value| value.parse().ok())
1761 .unwrap_or(0)
1762 }
1763
1764 #[must_use]
1765 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
1766 SyncStatusSnapshot {
1767 outbox: self.outbox.len(),
1768 upgrading: self.upgrading,
1769 lease_state: self.lease_state.clone(),
1770 schema_floor: self.schema_floor.clone(),
1771 sync_needed: self.sync_needed,
1772 }
1773 }
1774
1775 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
1776 self.change_queue.drain(..).collect()
1777 }
1778
1779 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
1780 self.sync_intent_queue.drain(..).collect()
1781 }
1782
1783 fn schedule_background_retry(&mut self) {
1784 self.sync_intent_queue.push_back(SyncIntent::Background {
1785 delay_ms: self.retry_delay_ms,
1786 });
1787 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
1788 }
1789
1790 fn reset_background_retry(&mut self) {
1791 self.retry_delay_ms = 250;
1792 }
1793
1794 fn retryable_transport_code(code: &str) -> bool {
1795 code == "transport.failed" || code == "sync.transport_failed"
1796 }
1797
1798 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
1799 if self.sync_needed != value {
1800 if self.begin_observation("syncular_status").is_ok() {
1801 self.sync_needed = value;
1802 let batch = ChangeAccumulator {
1803 status: true,
1804 ..ChangeAccumulator::default()
1805 };
1806 if self.finish_observation("syncular_status", batch).is_err() {
1807 self.rollback_observation("syncular_status");
1808 }
1809 } else {
1810 self.sync_needed = value;
1811 }
1812 }
1813 if value && interactive {
1814 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1815 }
1816 }
1817
1818 fn begin_observation(&self, name: &str) -> Result<(), String> {
1819 self.conn
1820 .execute_batch(&format!("SAVEPOINT {name}"))
1821 .map_err(|error| error.to_string())
1822 }
1823
1824 fn rollback_observation(&self, name: &str) {
1825 let _ = self
1826 .conn
1827 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
1828 }
1829
1830 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
1831 if !batch.touched() {
1832 self.conn
1833 .execute_batch(&format!("RELEASE {name}"))
1834 .map_err(|error| error.to_string())?;
1835 return Ok(());
1836 }
1837 let revision = self
1838 .local_revision()
1839 .checked_add(1)
1840 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
1841 self.conn
1842 .execute(
1843 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
1844 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
1845 )
1846 .map_err(|error| error.to_string())?;
1847 let status = batch.status.then(|| self.status_snapshot());
1848 let event = ClientChangeBatch {
1849 revision: revision.to_string(),
1850 tables: batch
1851 .tables
1852 .into_iter()
1853 .map(|(table, scope_keys)| TableChange {
1854 table,
1855 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
1856 })
1857 .collect(),
1858 windows: batch
1859 .windows
1860 .into_iter()
1861 .map(|((base_key, table), units)| WindowChange {
1862 base_key,
1863 table,
1864 units: units.into_iter().collect(),
1865 })
1866 .collect(),
1867 status,
1868 conflicts_changed: batch.conflicts,
1869 rejections_changed: batch.rejections,
1870 outcomes_changed: batch.outcomes,
1871 };
1872 self.conn
1873 .execute_batch(&format!("RELEASE {name}"))
1874 .map_err(|error| error.to_string())?;
1875 self.change_queue.push_back(event);
1876 Ok(())
1877 }
1878
1879 fn record_scope_map(
1880 &self,
1881 batch: &mut ChangeAccumulator,
1882 table_name: &str,
1883 scopes: &[(String, Vec<String>)],
1884 ) {
1885 let Some(table) = self.schema.table(table_name) else {
1886 return;
1887 };
1888 for (variable, values) in scopes {
1889 let Some(scope) = table
1890 .scope_variables
1891 .iter()
1892 .find(|scope| &scope.variable == variable)
1893 else {
1894 continue;
1895 };
1896 for value in values {
1897 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1898 }
1899 }
1900 }
1901
1902 fn record_row_scopes(
1904 &self,
1905 batch: &mut ChangeAccumulator,
1906 table_name: &str,
1907 row_id: &str,
1908 base: bool,
1909 ) -> bool {
1910 let Some(table) = self.schema.table(table_name) else {
1911 return false;
1912 };
1913 if table.scope_variables.is_empty() {
1914 return false;
1915 }
1916 let columns = table
1917 .scope_variables
1918 .iter()
1919 .map(|scope| quote_ident(&scope.column))
1920 .collect::<Vec<_>>()
1921 .join(", ");
1922 let full_table = if base {
1923 base_table(table_name)
1924 } else {
1925 visible_table(table_name)
1926 };
1927 let sql = format!(
1928 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
1929 quote_ident(&table.primary_key)
1930 );
1931 let Ok(mut stmt) = self.conn.prepare(&sql) else {
1932 return false;
1933 };
1934 let values = stmt.query_row(rusqlite::params![row_id], |row| {
1935 let mut values = Vec::with_capacity(table.scope_variables.len());
1936 for index in 0..table.scope_variables.len() {
1937 values.push(row.get::<_, Option<String>>(index)?);
1938 }
1939 Ok(values)
1940 });
1941 let Ok(values) = values else {
1942 return false;
1943 };
1944 let mut recorded = false;
1945 for (scope, value) in table.scope_variables.iter().zip(values) {
1946 if let Some(value) = value {
1947 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1948 recorded = true;
1949 }
1950 }
1951 recorded
1952 }
1953
1954 fn record_commit_changes(
1955 &self,
1956 batch: &mut ChangeAccumulator,
1957 tables: &[String],
1958 changes: &[ssp2::model::Change],
1959 ) {
1960 for change in changes {
1961 let Some(table_name) = tables.get(change.table_index as usize) else {
1962 continue;
1963 };
1964 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
1965 if let Some(table) = self.schema.table(table_name) {
1966 for (variable, value) in &change.scopes {
1967 if let Some(scope) = table
1968 .scope_variables
1969 .iter()
1970 .find(|scope| &scope.variable == variable)
1971 {
1972 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1973 precise = true;
1974 }
1975 }
1976 }
1977 if !precise {
1978 batch.table(table_name);
1979 }
1980 }
1981 }
1982
1983 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
1984 if effective.is_empty() {
1985 return false;
1986 }
1987 let Some(table) = self.schema.table(table_name) else {
1988 return false;
1989 };
1990 let mut clauses = Vec::new();
1991 let mut params = Vec::new();
1992 for (variable, values) in effective {
1993 let Some(column) = table.scope_column(variable) else {
1994 return false;
1995 };
1996 if values.is_empty() {
1997 return false;
1998 }
1999 let placeholders = values
2000 .iter()
2001 .map(|value| {
2002 params.push(SqlValue::Text(value.clone()));
2003 "?"
2004 })
2005 .collect::<Vec<_>>()
2006 .join(", ");
2007 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
2008 }
2009 let sql = format!(
2010 "SELECT 1 FROM {} WHERE {} LIMIT 1",
2011 base_table(table_name),
2012 clauses.join(" AND ")
2013 );
2014 self.conn
2015 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
2016 .is_ok()
2017 }
2018
2019 pub fn upgrading(&self) -> bool {
2021 self.upgrading
2022 }
2023
2024 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
2030 let new_schema = parse_schema_json(schema_json)?;
2031 let marker: Option<i32> = self
2032 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
2033 .and_then(|v| v.parse().ok());
2034 self.schema = new_schema;
2035 if marker != Some(self.schema.version) {
2036 self.run_schema_reset()?;
2037 }
2038 self.prune_unknown_subscriptions()?;
2039 self.enqueue_startup_sync_if_needed();
2043 Ok(())
2044 }
2045
2046 fn enqueue_startup_sync_if_needed(&mut self) {
2047 let startup_work = !self.stopped
2048 && (!self.outbox.is_empty()
2049 || self.subs.iter().any(|sub| sub.state == SubState::Active));
2050 if startup_work {
2051 self.sync_needed = true;
2052 self.sync_intent_queue.push_back(SyncIntent::Interactive);
2053 }
2054 }
2055
2056 fn run_schema_reset(&mut self) -> Result<(), String> {
2062 self.begin_observation("syncular_schema_reset")?;
2063 let mut batch = ChangeAccumulator::default();
2064 let result = self.run_schema_reset_observed(&mut batch);
2065 if let Err(error) = result {
2066 self.rollback_observation("syncular_schema_reset");
2067 return Err(error);
2068 }
2069 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
2070 self.rollback_observation("syncular_schema_reset");
2071 return Err(error);
2072 }
2073 Ok(())
2074 }
2075
2076 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
2077 self.upgrading = true;
2078 batch.status = true;
2079 for table in &self.schema.tables {
2080 batch.table(&table.name);
2081 }
2082 for (base_key, unit, table) in self.load_registered_window_units() {
2083 batch.window(&base_key, &table, &unit);
2084 }
2085 self.insert_sql.borrow_mut().clear();
2087 self.overlay_dirty.set(true);
2088 let virtual_tables: Vec<String> = {
2096 let mut stmt = self
2097 .conn
2098 .prepare(
2099 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
2100 )
2101 .map_err(|e| e.to_string())?;
2102 let rows = stmt
2103 .query_map([], |row| row.get::<_, String>(0))
2104 .map_err(|e| e.to_string())?;
2105 rows.filter_map(Result::ok)
2106 .filter(|name| is_synced_table_name(name))
2107 .collect()
2108 };
2109 for name in virtual_tables {
2110 self.conn
2111 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2112 .map_err(|e| e.to_string())?;
2113 }
2114 let existing: Vec<String> = {
2115 let mut stmt = self
2116 .conn
2117 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
2118 .map_err(|e| e.to_string())?;
2119 let rows = stmt
2120 .query_map([], |row| row.get::<_, String>(0))
2121 .map_err(|e| e.to_string())?;
2122 rows.filter_map(Result::ok)
2123 .filter(|name| is_synced_table_name(name))
2124 .collect()
2125 };
2126 for name in &existing {
2127 if let Some(table) = name.strip_prefix("_syncular_base_") {
2128 batch.table(table);
2129 } else if !name.starts_with("_syncular_") {
2130 batch.table(name);
2131 }
2132 }
2133 for name in existing {
2134 self.conn
2135 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
2136 .map_err(|e| e.to_string())?;
2137 }
2138 self.create_synced_tables()?;
2140 for sub in &mut self.subs {
2142 sub.cursor = -1;
2143 sub.bootstrap_state = None;
2144 sub.effective = None;
2145 sub.state = SubState::Active;
2146 sub.reason_code = None;
2147 sub.synced_once = false;
2148 }
2149 let subs = self.subs.clone();
2150 for sub in &subs {
2151 self.persist_sub(sub);
2152 }
2153 self.stopped = false;
2155 self.schema_floor = None;
2156 self.delete_meta(SCHEMA_FLOOR_KEY);
2157 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
2159 if self.drop_incompatible_outbox()? {
2163 batch.rejections = true;
2164 batch.status = true;
2165 batch.outcomes = true;
2166 }
2167 self.rebuild_overlay();
2169 Ok(())
2170 }
2171
2172 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
2176 let schema = &self.schema;
2177 let incompatible = self
2178 .outbox
2179 .iter()
2180 .filter(|commit| {
2181 commit.ops.iter().any(|op| {
2182 if !op.upsert {
2183 return false;
2184 }
2185 match schema.table(&op.table) {
2186 None => true,
2187 Some(table) => op.values.as_ref().is_some_and(|values| {
2188 values
2189 .keys()
2190 .any(|key| !table.columns.iter().any(|c| &c.name == key))
2191 }),
2192 }
2193 })
2194 })
2195 .cloned()
2196 .collect::<Vec<_>>();
2197 if incompatible.is_empty() {
2198 return Ok(false);
2199 }
2200 let mut rejections = Vec::new();
2201 for commit in &incompatible {
2202 let results = commit
2203 .ops
2204 .iter()
2205 .enumerate()
2206 .map(|(op_index, operation)| {
2207 let rejection = RejectionRecord {
2208 client_commit_id: commit.client_commit_id.clone(),
2209 op_index: op_index as i32,
2210 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
2211 message: "the persisted commit cannot encode under the current schema"
2212 .to_owned(),
2213 retryable: false,
2214 details: None,
2215 operation: Some(CommitOperation::from(operation)),
2216 };
2217 rejections.push(rejection.clone());
2218 CommitOperationOutcome::Error { rejection }
2219 })
2220 .collect::<Vec<_>>();
2221 self.persist_commit_outcome(
2222 &commit.client_commit_id,
2223 CommitOutcomeStatus::Rejected,
2224 &results,
2225 Some(&commit.ops),
2226 )?;
2227 self.delete_outbox_persisted(&commit.client_commit_id)?;
2228 }
2229 self.prune_commit_outcomes()?;
2230 let incompatible_ids = incompatible
2231 .iter()
2232 .map(|commit| commit.client_commit_id.as_str())
2233 .collect::<BTreeSet<_>>();
2234 self.outbox
2235 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
2236 self.rejections.extend(rejections);
2237 Ok(true)
2238 }
2239
2240 fn create_synced_tables(&self) -> Result<(), String> {
2243 for table in &self.schema.tables {
2244 for (full, index_prefix) in [
2248 (base_table(&table.name), "_syncular_base_"),
2249 (visible_table(&table.name), ""),
2250 ] {
2251 let mut cols: Vec<String> =
2252 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
2253 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
2254 let sql = format!(
2255 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
2256 cols.join(", "),
2257 quote_ident(&table.primary_key)
2258 );
2259 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
2260 for index in &table.indexes {
2264 let unique = if index.unique { "UNIQUE " } else { "" };
2265 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
2266 let cols_sql = index
2267 .columns
2268 .iter()
2269 .map(|c| quote_ident(c))
2270 .collect::<Vec<_>>()
2271 .join(", ");
2272 let index_sql = format!(
2273 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
2274 );
2275 self.conn
2276 .execute(&index_sql, [])
2277 .map_err(|e| e.to_string())?;
2278 }
2279 }
2280 }
2281 for table in &self.schema.tables {
2284 for index in &table.fts_indexes {
2285 self.create_fts_projection(table, index)?;
2286 }
2287 }
2288 Ok(())
2289 }
2290
2291 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
2292 let count: i64 = self
2293 .conn
2294 .query_row(
2295 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
2296 rusqlite::params![index.name],
2297 |row| row.get(0),
2298 )
2299 .map_err(|e| e.to_string())?;
2300 Ok(count > 0)
2301 }
2302
2303 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
2304 for suffix in ["ai", "ad", "au"] {
2305 self.conn
2306 .execute(
2307 &format!(
2308 "DROP TRIGGER IF EXISTS {}",
2309 quote_ident(&format!("{}_{suffix}", index.name))
2310 ),
2311 [],
2312 )
2313 .map_err(|e| e.to_string())?;
2314 }
2315 Ok(())
2316 }
2317
2318 fn create_fts_triggers(
2319 &self,
2320 table: &TableSchema,
2321 index: &FtsIndexSchema,
2322 ) -> Result<(), String> {
2323 let fts = quote_ident(&index.name);
2324 let source = visible_table(&table.name);
2325 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2326 let pk = quote_ident(&table.primary_key);
2327 let projection_columns = std::iter::once(source_id.clone())
2328 .chain(index.columns.iter().map(|column| quote_ident(column)))
2329 .collect::<Vec<_>>()
2330 .join(", ");
2331 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
2332 .chain(
2333 index
2334 .columns
2335 .iter()
2336 .map(|column| format!("new.{}", quote_ident(column))),
2337 )
2338 .collect::<Vec<_>>()
2339 .join(", ");
2340 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
2341 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
2342 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
2343 let sql = format!(
2344 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
2345 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
2346 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
2347 ai = quote_ident(&format!("{}_ai", index.name)),
2348 ad = quote_ident(&format!("{}_ad", index.name)),
2349 au = quote_ident(&format!("{}_au", index.name)),
2350 );
2351 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2352 }
2353
2354 fn rebuild_fts_projection(
2355 &self,
2356 table: &TableSchema,
2357 index: &FtsIndexSchema,
2358 ) -> Result<(), String> {
2359 let fts = quote_ident(&index.name);
2360 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2361 let pk = quote_ident(&table.primary_key);
2362 let indexed_columns = index
2363 .columns
2364 .iter()
2365 .map(|column| quote_ident(column))
2366 .collect::<Vec<_>>();
2367 let projection_columns = std::iter::once(source_id)
2368 .chain(indexed_columns.iter().cloned())
2369 .collect::<Vec<_>>()
2370 .join(", ");
2371 let sql = format!(
2372 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
2373 columns = indexed_columns.join(", "),
2374 source = visible_table(&table.name),
2375 );
2376 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2377 }
2378
2379 fn create_fts_projection(
2380 &self,
2381 table: &TableSchema,
2382 index: &FtsIndexSchema,
2383 ) -> Result<(), String> {
2384 let existed = self.fts_projection_exists(index)?;
2385 let tokenizer = index.tokenize.replace('\'', "''");
2386 let columns = index
2387 .columns
2388 .iter()
2389 .map(|column| quote_ident(column))
2390 .collect::<Vec<_>>()
2391 .join(", ");
2392 let sql = format!(
2393 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
2394 fts = quote_ident(&index.name),
2395 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
2396 );
2397 self.conn.execute(&sql, []).map_err(|error| {
2398 format!(
2399 "cannot create local FTS5 projection {:?}: {error}",
2400 index.name
2401 )
2402 })?;
2403 self.create_fts_triggers(table, index)?;
2404 if !existed {
2405 self.rebuild_fts_projection(table, index)?;
2406 }
2407 Ok(())
2408 }
2409
2410 fn persist_sub(&self, sub: &Subscription) {
2413 let state = serde_json::json!({
2414 "requested": scope_map_to_json(&sub.requested),
2415 "params": sub.params,
2416 "cursor": sub.cursor,
2417 "bootstrapState": sub.bootstrap_state,
2418 "status": sub.state.name(),
2419 "reasonCode": sub.reason_code,
2420 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2421 "syncedOnce": sub.synced_once,
2422 });
2423 let _ = self.conn.execute(
2424 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
2425 rusqlite::params![sub.id, sub.table, state.to_string()],
2426 );
2427 }
2428
2429 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
2430 let ops: Vec<Value> = commit
2431 .ops
2432 .iter()
2433 .map(|op| {
2434 serde_json::json!({
2435 "op": if op.upsert { "upsert" } else { "delete" },
2436 "table": op.table,
2437 "rowId": op.row_id,
2438 "baseVersion": op.base_version,
2439 "values": op.values.clone().map(Value::Object),
2440 "changedFields": op.changed_fields,
2441 })
2442 })
2443 .collect();
2444 let _ = self.conn.execute(
2445 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
2446 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
2447 );
2448 }
2449
2450 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
2451 self.conn
2452 .execute(
2453 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
2454 rusqlite::params![client_commit_id],
2455 )
2456 .map(|_| ())
2457 .map_err(|error| error.to_string())
2458 }
2459
2460 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
2461 match status {
2462 CommitOutcomeStatus::Applied => "applied",
2463 CommitOutcomeStatus::Cached => "cached",
2464 CommitOutcomeStatus::Conflict => "conflict",
2465 CommitOutcomeStatus::Rejected => "rejected",
2466 }
2467 }
2468
2469 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
2470 match resolution {
2471 CommitOutcomeResolution::Active => "active",
2472 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
2473 CommitOutcomeResolution::Superseded => "superseded",
2474 CommitOutcomeResolution::Dismissed => "dismissed",
2475 }
2476 }
2477
2478 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
2479 match value {
2480 "applied" => Ok(CommitOutcomeStatus::Applied),
2481 "cached" => Ok(CommitOutcomeStatus::Cached),
2482 "conflict" => Ok(CommitOutcomeStatus::Conflict),
2483 "rejected" => Ok(CommitOutcomeStatus::Rejected),
2484 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
2485 }
2486 }
2487
2488 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
2489 match value {
2490 "active" => Ok(CommitOutcomeResolution::Active),
2491 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
2492 "superseded" => Ok(CommitOutcomeResolution::Superseded),
2493 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
2494 _ => Err(format!(
2495 "invalid persisted commit outcome resolution {value:?}"
2496 )),
2497 }
2498 }
2499
2500 fn persist_commit_outcome(
2501 &self,
2502 client_commit_id: &str,
2503 status: CommitOutcomeStatus,
2504 results: &[CommitOperationOutcome],
2505 operations: Option<&[OutboxOp]>,
2506 ) -> Result<(), String> {
2507 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
2508 let operations_json = operations
2509 .map(|items| {
2510 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
2511 })
2512 .transpose()
2513 .map_err(|error| error.to_string())?;
2514 self.conn
2515 .execute(
2516 "INSERT INTO _syncular_commit_outcomes (
2517 client_commit_id, status, recorded_at_ms, results_json,
2518 operations_json, resolution
2519 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
2520 rusqlite::params![
2521 client_commit_id,
2522 Self::outcome_status_name(status),
2523 self.clock_now_ms(),
2524 results_json,
2525 operations_json
2526 ],
2527 )
2528 .map(|_| ())
2529 .map_err(|error| error.to_string())
2530 }
2531
2532 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
2533 let StoredCommitOutcomeRow {
2534 sequence,
2535 client_commit_id,
2536 status,
2537 recorded_at_ms,
2538 results_json,
2539 operations_json,
2540 resolution,
2541 resolved_at_ms,
2542 replacement_client_commit_id,
2543 } = row;
2544 Ok(CommitOutcome {
2545 sequence,
2546 client_commit_id,
2547 status: Self::parse_outcome_status(&status)?,
2548 recorded_at_ms,
2549 results: serde_json::from_str(&results_json)
2550 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
2551 operations: operations_json
2552 .map(|value| {
2553 serde_json::from_str(&value).map_err(|error| {
2554 format!("invalid persisted commit outcome operations: {error}")
2555 })
2556 })
2557 .transpose()?,
2558 resolution: Self::parse_outcome_resolution(&resolution)?,
2559 resolved_at_ms,
2560 replacement_client_commit_id,
2561 })
2562 }
2563
2564 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
2565 let row = self
2566 .conn
2567 .query_row(
2568 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2569 resolution, resolved_at_ms, replacement_client_commit_id
2570 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
2571 rusqlite::params![client_commit_id],
2572 |row| {
2573 Ok(StoredCommitOutcomeRow {
2574 sequence: row.get(0)?,
2575 client_commit_id: row.get(1)?,
2576 status: row.get(2)?,
2577 recorded_at_ms: row.get(3)?,
2578 results_json: row.get(4)?,
2579 operations_json: row.get(5)?,
2580 resolution: row.get(6)?,
2581 resolved_at_ms: row.get(7)?,
2582 replacement_client_commit_id: row.get(8)?,
2583 })
2584 },
2585 )
2586 .optional()
2587 .map_err(|error| error.to_string())?;
2588 row.map(Self::outcome_from_row).transpose()
2589 }
2590
2591 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
2592 if query.limit == Some(0) {
2593 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
2594 }
2595 let mut sql = String::from(
2596 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2597 resolution, resolved_at_ms, replacement_client_commit_id
2598 FROM _syncular_commit_outcomes",
2599 );
2600 if query.active_only {
2601 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
2602 }
2603 sql.push_str(" ORDER BY seq DESC");
2604 if let Some(limit) = query.limit {
2605 sql.push_str(&format!(" LIMIT {limit}"));
2606 }
2607 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
2608 let rows = stmt
2609 .query_map([], |row| {
2610 Ok(StoredCommitOutcomeRow {
2611 sequence: row.get(0)?,
2612 client_commit_id: row.get(1)?,
2613 status: row.get(2)?,
2614 recorded_at_ms: row.get(3)?,
2615 results_json: row.get(4)?,
2616 operations_json: row.get(5)?,
2617 resolution: row.get(6)?,
2618 resolved_at_ms: row.get(7)?,
2619 replacement_client_commit_id: row.get(8)?,
2620 })
2621 })
2622 .map_err(|error| error.to_string())?;
2623 let mut outcomes = Vec::new();
2624 for row in rows {
2625 outcomes.push(Self::outcome_from_row(
2626 row.map_err(|error| error.to_string())?,
2627 )?);
2628 }
2629 Ok(outcomes)
2630 }
2631
2632 fn prune_commit_outcomes(&self) -> Result<(), String> {
2633 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
2634 let count = self
2635 .conn
2636 .query_row(
2637 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
2638 [],
2639 |row| row.get::<_, i64>(0),
2640 )
2641 .map_err(|error| error.to_string())? as usize;
2642 let excess = count.saturating_sub(max_entries);
2643 if excess == 0 {
2644 return Ok(());
2645 }
2646 let mut stmt = self
2647 .conn
2648 .prepare(
2649 "SELECT seq FROM _syncular_commit_outcomes
2650 WHERE status IN ('applied', 'cached') OR resolution != 'active'
2651 ORDER BY seq ASC LIMIT ?1",
2652 )
2653 .map_err(|error| error.to_string())?;
2654 let rows = stmt
2655 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
2656 .map_err(|error| error.to_string())?;
2657 let sequences = rows
2658 .collect::<Result<Vec<_>, _>>()
2659 .map_err(|error| error.to_string())?;
2660 drop(stmt);
2661 for sequence in sequences {
2662 self.conn
2663 .execute(
2664 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
2665 rusqlite::params![sequence],
2666 )
2667 .map_err(|error| error.to_string())?;
2668 }
2669 Ok(())
2670 }
2671
2672 pub fn subscribe(
2675 &mut self,
2676 id: String,
2677 table: String,
2678 scopes: Vec<(String, Vec<String>)>,
2679 params: Option<String>,
2680 ) -> Result<(), String> {
2681 if self.schema.table(&table).is_none() {
2682 return Err(format!("unknown table {table:?}"));
2683 }
2684 let sub = Subscription {
2685 id: id.clone(),
2686 table,
2687 requested: scopes,
2688 params,
2689 cursor: -1,
2690 bootstrap_state: None,
2691 state: SubState::Active,
2692 reason_code: None,
2693 effective: None,
2694 synced_once: false,
2695 };
2696 self.persist_sub(&sub);
2697 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
2698 *existing = sub;
2699 } else {
2700 self.subs.push(sub);
2701 }
2702 Ok(())
2703 }
2704
2705 pub fn unsubscribe(&mut self, id: &str) {
2706 self.subs.retain(|s| s.id != id);
2707 let _ = self.conn.execute(
2708 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2709 rusqlite::params![id],
2710 );
2711 }
2712
2713 pub fn set_window(
2721 &mut self,
2722 base: &WindowBase,
2723 units: &[String],
2724 ) -> Result<CommandEffects, String> {
2725 let table = self
2726 .schema
2727 .table(&base.table)
2728 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
2729 if table.scope_column(&base.variable).is_none() {
2730 return Err(format!(
2731 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
2732 base.table, base.variable
2733 ));
2734 }
2735 let base_key = window_base_key(base);
2736 let wanted: std::collections::HashSet<&String> = units.iter().collect();
2737 let live = self.load_window_units(&base_key);
2738 self.begin_observation("syncular_window")?;
2739 let mut batch = ChangeAccumulator::default();
2740 let mut changed = false;
2741
2742 for unit in units {
2744 if live.iter().any(|(u, _)| u == unit) {
2745 continue;
2746 }
2747 let sub_id = derive_sub_id(base, unit);
2748 self.delete_pending_evict(&sub_id);
2749 self.insert_window_unit(&base_key, unit, &sub_id);
2750 self.subscribe(
2751 sub_id,
2752 base.table.clone(),
2753 unit_scopes(base, unit),
2754 base.params.clone(),
2755 )?;
2756 batch.window(&base_key, &base.table, unit);
2757 changed = true;
2758 }
2759
2760 for (unit, sub_id) in live {
2762 if wanted.contains(&unit) {
2763 continue;
2764 }
2765 let effective = self
2766 .subs
2767 .iter()
2768 .find(|sub| sub.id == sub_id)
2769 .and_then(|sub| sub.effective.clone())
2770 .unwrap_or_else(|| unit_scopes(base, &unit));
2771 self.record_scope_map(&mut batch, &base.table, &effective);
2772 batch.window(&base_key, &base.table, &unit);
2773 self.evict_unit(&base_key, base, &unit, &sub_id);
2774 changed = true;
2775 }
2776 if let Err(error) = self.finish_observation("syncular_window", batch) {
2777 self.rollback_observation("syncular_window");
2778 return Err(error);
2779 }
2780 Ok(if changed {
2781 CommandEffects::interactive()
2782 } else {
2783 CommandEffects::none()
2784 })
2785 }
2786
2787 pub fn window_state(&self, base: &WindowBase) -> WindowState {
2792 let mut units = Vec::new();
2793 let mut pending = Vec::new();
2794 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
2795 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
2796 Some(sub) => {
2797 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
2798 }
2799 None => true,
2800 };
2801 if is_pending {
2802 pending.push(unit.clone());
2803 }
2804 units.push(unit);
2805 }
2806 WindowState { units, pending }
2807 }
2808
2809 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
2810 let mut stmt = match self
2811 .conn
2812 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
2813 {
2814 Ok(stmt) => stmt,
2815 Err(_) => return Vec::new(),
2816 };
2817 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
2818 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2819 });
2820 match rows {
2821 Ok(rows) => rows.filter_map(Result::ok).collect(),
2822 Err(_) => Vec::new(),
2823 }
2824 }
2825
2826 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
2827 let mut stmt = match self.conn.prepare(
2828 "SELECT windows.base, windows.unit, subscriptions.tbl
2829 FROM _syncular_windows AS windows
2830 JOIN _syncular_subscriptions AS subscriptions
2831 ON subscriptions.id = windows.sub_id
2832 ORDER BY windows.base, windows.unit",
2833 ) {
2834 Ok(stmt) => stmt,
2835 Err(_) => return Vec::new(),
2836 };
2837 let rows = stmt.query_map([], |row| {
2838 Ok((
2839 row.get::<_, String>(0)?,
2840 row.get::<_, String>(1)?,
2841 row.get::<_, String>(2)?,
2842 ))
2843 });
2844 match rows {
2845 Ok(rows) => rows.filter_map(Result::ok).collect(),
2846 Err(_) => Vec::new(),
2847 }
2848 }
2849
2850 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
2851 self.conn
2852 .query_row(
2853 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
2854 rusqlite::params![sub_id],
2855 |row| Ok((row.get(0)?, row.get(1)?)),
2856 )
2857 .ok()
2858 }
2859
2860 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
2861 let _ = self.conn.execute(
2862 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
2863 rusqlite::params![base_key, unit, sub_id],
2864 );
2865 }
2866
2867 fn delete_window_unit(&self, base_key: &str, unit: &str) {
2868 let _ = self.conn.execute(
2869 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
2870 rusqlite::params![base_key, unit],
2871 );
2872 }
2873
2874 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
2881 let effective = self
2882 .subs
2883 .iter()
2884 .find(|s| s.id == sub_id)
2885 .and_then(|s| s.effective.clone())
2886 .unwrap_or_else(|| unit_scopes(base, unit));
2887 let pinned = self.pinned_row_ids(&base.table);
2888 let deferred = self
2889 .evict_scope_rows(&base.table, &effective, &pinned)
2890 .unwrap_or(false);
2891 self.delete_window_unit(base_key, unit);
2892 self.unsubscribe(sub_id);
2893 if deferred {
2894 self.save_pending_evict(sub_id, &base.table, &effective);
2895 } else {
2896 self.delete_pending_evict(sub_id);
2897 }
2898 self.rebuild_overlay();
2899 }
2900
2901 fn evict_scope_rows(
2905 &mut self,
2906 table_name: &str,
2907 effective: &[(String, Vec<String>)],
2908 pinned: &std::collections::HashSet<String>,
2909 ) -> Result<bool, ()> {
2910 if effective.is_empty() {
2911 return Ok(false);
2912 }
2913 let table = self.schema.table(table_name).ok_or(())?.clone();
2914 let mut clauses = Vec::new();
2915 let mut params: Vec<SqlValue> = Vec::new();
2916 for (variable, values) in effective {
2917 let column = table.scope_column(variable).ok_or(())?;
2918 let placeholders: Vec<String> = values
2919 .iter()
2920 .map(|v| {
2921 params.push(SqlValue::Text(v.clone()));
2922 "?".to_owned()
2923 })
2924 .collect();
2925 clauses.push(format!(
2926 "{} IN ({})",
2927 quote_ident(column),
2928 placeholders.join(", ")
2929 ));
2930 }
2931 let mut sql = format!(
2932 "DELETE FROM {} WHERE {}",
2933 base_table(table_name),
2934 clauses.join(" AND ")
2935 );
2936 if !pinned.is_empty() {
2937 let pk = quote_ident(&table.primary_key);
2938 let holes: Vec<String> = pinned
2939 .iter()
2940 .map(|id| {
2941 params.push(SqlValue::Text(id.clone()));
2942 "?".to_owned()
2943 })
2944 .collect();
2945 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
2946 }
2947 self.overlay_dirty.set(true);
2948 self.conn
2949 .execute(&sql, rusqlite::params_from_iter(params))
2950 .map_err(|_| ())?;
2951 if pinned.is_empty() {
2952 return Ok(false);
2953 }
2954 let mut where_params: Vec<SqlValue> = Vec::new();
2957 let mut where_clauses = Vec::new();
2958 for (variable, values) in effective {
2959 let column = table.scope_column(variable).ok_or(())?;
2960 let placeholders: Vec<String> = values
2961 .iter()
2962 .map(|v| {
2963 where_params.push(SqlValue::Text(v.clone()));
2964 "?".to_owned()
2965 })
2966 .collect();
2967 where_clauses.push(format!(
2968 "{} IN ({})",
2969 quote_ident(column),
2970 placeholders.join(", ")
2971 ));
2972 }
2973 let pk = quote_ident(&table.primary_key);
2974 let select = format!(
2975 "SELECT {} FROM {} WHERE {}",
2976 pk,
2977 base_table(table_name),
2978 where_clauses.join(" AND ")
2979 );
2980 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
2981 let survivors: Vec<String> = stmt
2982 .query_map(rusqlite::params_from_iter(where_params), |row| {
2983 row.get::<_, String>(0)
2984 })
2985 .map_err(|_| ())?
2986 .filter_map(Result::ok)
2987 .collect();
2988 Ok(survivors.iter().any(|id| pinned.contains(id)))
2989 }
2990
2991 fn drain_pending_evictions(&mut self) {
2995 let pending = self.load_pending_evictions();
2996 if pending.is_empty() {
2997 return;
2998 }
2999 for (sub_id, table_name, effective) in pending {
3000 if self.schema.table(&table_name).is_none() {
3001 self.delete_pending_evict(&sub_id);
3002 continue;
3003 }
3004 let pinned = self.pinned_row_ids(&table_name);
3005 let deferred = self
3006 .evict_scope_rows(&table_name, &effective, &pinned)
3007 .unwrap_or(false);
3008 if !deferred {
3009 self.delete_pending_evict(&sub_id);
3010 }
3011 }
3012 self.rebuild_overlay_if_dirty();
3013 }
3014
3015 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
3018 let mut pinned = std::collections::HashSet::new();
3019 for commit in &self.outbox {
3020 for op in &commit.ops {
3021 if op.table == table {
3022 pinned.insert(op.row_id.clone());
3023 }
3024 }
3025 }
3026 pinned
3027 }
3028
3029 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
3030 let _ = self.conn.execute(
3031 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
3032 VALUES (?1, ?2, ?3)",
3033 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
3034 );
3035 }
3036
3037 fn delete_pending_evict(&self, sub_id: &str) {
3038 let _ = self.conn.execute(
3039 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
3040 rusqlite::params![sub_id],
3041 );
3042 }
3043
3044 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
3045 let mut stmt = match self
3046 .conn
3047 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
3048 {
3049 Ok(stmt) => stmt,
3050 Err(_) => return Vec::new(),
3051 };
3052 let rows = stmt.query_map([], |row| {
3053 Ok((
3054 row.get::<_, String>(0)?,
3055 row.get::<_, String>(1)?,
3056 row.get::<_, String>(2)?,
3057 ))
3058 });
3059 let mut out = Vec::new();
3060 if let Ok(rows) = rows {
3061 for entry in rows.filter_map(Result::ok) {
3062 let (sub_id, table, json) = entry;
3063 if let Ok(value) = serde_json::from_str::<Value>(&json) {
3064 if let Ok(effective) = json_to_scope_map(&value) {
3065 out.push((sub_id, table, effective));
3066 }
3067 }
3068 }
3069 }
3070 out
3071 }
3072
3073 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
3075 if mutations.is_empty() {
3076 return Err("a commit must contain at least one operation (§6.1)".to_owned());
3077 }
3078 let mut ops = Vec::with_capacity(mutations.len());
3079 for mutation in mutations {
3080 match mutation {
3081 Mutation::Upsert {
3082 table,
3083 values,
3084 base_version,
3085 } => {
3086 let schema_table = self
3087 .schema
3088 .table(&table)
3089 .ok_or_else(|| format!("unknown table {table:?}"))?;
3090 let values = normalize_values_casing(schema_table, values)?;
3094 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
3095 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
3098 ops.push(OutboxOp {
3099 upsert: true,
3100 table,
3101 row_id,
3102 base_version,
3103 values: Some(values),
3104 changed_fields: None,
3105 });
3106 }
3107 Mutation::Delete {
3108 table,
3109 row_id,
3110 base_version,
3111 } => {
3112 if self.schema.table(&table).is_none() {
3113 return Err(format!("unknown table {table:?}"));
3114 }
3115 ops.push(OutboxOp {
3116 upsert: false,
3117 table,
3118 row_id,
3119 base_version,
3120 values: None,
3121 changed_fields: None,
3122 });
3123 }
3124 }
3125 }
3126 self.record_outbox_commit(ops)
3127 }
3128
3129 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
3130 let commit = OutboxCommit {
3131 client_commit_id: uuid::Uuid::new_v4().to_string(),
3132 ops,
3133 };
3134 self.begin_observation("syncular_mutation")?;
3135 let mut batch = ChangeAccumulator::default();
3136 for op in &commit.ops {
3137 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
3138 if let Some(values) = &op.values {
3139 if let Some(table) = self.schema.table(&op.table) {
3140 for scope in &table.scope_variables {
3141 if let Some(Value::String(value)) = values.get(&scope.column) {
3142 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
3143 precise = true;
3144 }
3145 }
3146 }
3147 }
3148 if !precise {
3149 batch.table(&op.table);
3150 }
3151 }
3152 self.persist_outbox_insert(&commit);
3153 let id = commit.client_commit_id.clone();
3154 self.outbox.push(commit);
3155 self.overlay_dirty.set(true);
3156 self.rebuild_overlay();
3157 batch.status = true;
3158 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
3159 self.rollback_observation("syncular_mutation");
3160 return Err(error);
3161 }
3162 Ok(id)
3163 }
3164
3165 pub fn patch(
3170 &mut self,
3171 table: &str,
3172 row_id: &str,
3173 partial: Map<String, Value>,
3174 base_version: Option<i64>,
3175 ) -> Result<String, String> {
3176 let schema_table = self
3177 .schema
3178 .table(table)
3179 .ok_or_else(|| format!("unknown table {table:?}"))?;
3180 let partial = normalize_values_casing(schema_table, partial)?;
3181 let mut values = self
3182 .read_rows(table)?
3183 .into_iter()
3184 .find(|row| row.row_id == row_id)
3185 .map(|row| row.values)
3186 .ok_or_else(|| {
3187 format!(
3188 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
3189 )
3190 })?;
3191 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
3192 changed_fields.sort();
3193 values.extend(partial);
3194 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
3195 if row_id_from_values != row_id {
3196 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
3197 }
3198 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
3199 self.record_outbox_commit(vec![OutboxOp {
3200 upsert: true,
3201 table: table.to_owned(),
3202 row_id: row_id.to_owned(),
3203 base_version,
3204 values: Some(values),
3205 changed_fields: Some(changed_fields),
3206 }])
3207 }
3208
3209 pub fn pending_commit_ids(&self) -> Vec<String> {
3210 self.outbox
3211 .iter()
3212 .map(|c| c.client_commit_id.clone())
3213 .collect()
3214 }
3215
3216 pub fn conflicts(&self) -> &[ConflictRecord] {
3217 &self.conflicts
3218 }
3219
3220 pub fn rejections(&self) -> &[RejectionRecord] {
3221 &self.rejections
3222 }
3223
3224 pub fn resolve_commit_outcome(
3225 &mut self,
3226 input: ResolveCommitOutcomeInput,
3227 ) -> Result<CommitOutcome, String> {
3228 let current = self
3229 .commit_outcome(&input.client_commit_id)?
3230 .ok_or_else(|| {
3231 format!(
3232 "sync.outcome_not_found: no durable outcome exists for {:?}",
3233 input.client_commit_id
3234 )
3235 })?;
3236 if current.resolution != CommitOutcomeResolution::Active {
3237 return Ok(current);
3238 }
3239 if input.resolution == CommitOutcomeResolution::Active {
3240 return Err("sync.invalid_request: resolution must leave active state".to_owned());
3241 }
3242 match input.resolution {
3243 CommitOutcomeResolution::Superseded => {
3244 let replacement = input
3245 .replacement_client_commit_id
3246 .as_deref()
3247 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
3248 .ok_or_else(|| {
3249 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
3250 .to_owned()
3251 })?;
3252 let _ = replacement;
3253 }
3254 _ if input.replacement_client_commit_id.is_some() => {
3255 return Err(
3256 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
3257 .to_owned(),
3258 );
3259 }
3260 _ => {}
3261 }
3262 let allowed = match current.status {
3263 CommitOutcomeStatus::Conflict => matches!(
3264 input.resolution,
3265 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
3266 ),
3267 CommitOutcomeStatus::Rejected => {
3268 input.resolution == CommitOutcomeResolution::Superseded
3269 }
3270 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
3271 input.resolution == CommitOutcomeResolution::Dismissed
3272 }
3273 };
3274 if !allowed {
3275 return Err(format!(
3276 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
3277 input.resolution, current.status
3278 ));
3279 }
3280
3281 self.begin_observation("syncular_outcome_resolution")?;
3282 let result = (|| {
3283 self.conn
3284 .execute(
3285 "UPDATE _syncular_commit_outcomes
3286 SET resolution = ?1, resolved_at_ms = ?2,
3287 replacement_client_commit_id = ?3
3288 WHERE client_commit_id = ?4 AND resolution = 'active'",
3289 rusqlite::params![
3290 Self::outcome_resolution_name(input.resolution),
3291 self.clock_now_ms(),
3292 input.replacement_client_commit_id,
3293 input.client_commit_id
3294 ],
3295 )
3296 .map_err(|error| error.to_string())?;
3297 let resolved = self
3298 .commit_outcome(¤t.client_commit_id)?
3299 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
3300 self.prune_commit_outcomes()?;
3301 Ok(resolved)
3302 })();
3303 let resolved = match result {
3304 Ok(outcome) => outcome,
3305 Err(error) => {
3306 self.rollback_observation("syncular_outcome_resolution");
3307 return Err(error);
3308 }
3309 };
3310 self.conflicts
3311 .retain(|record| record.client_commit_id != current.client_commit_id);
3312 self.rejections
3313 .retain(|record| record.client_commit_id != current.client_commit_id);
3314 let batch = ChangeAccumulator {
3315 conflicts: current.status == CommitOutcomeStatus::Conflict,
3316 rejections: current.status == CommitOutcomeStatus::Rejected,
3317 outcomes: true,
3318 ..ChangeAccumulator::default()
3319 };
3320 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
3321 self.rollback_observation("syncular_outcome_resolution");
3322 return Err(error);
3323 }
3324 Ok(resolved)
3325 }
3326
3327 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
3328 self.schema_floor.as_ref()
3329 }
3330
3331 pub fn lease_state(&self) -> Option<&LeaseState> {
3333 self.lease_state.as_ref()
3334 }
3335
3336 fn record_lease_error(&mut self, code: &str) {
3339 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
3340 return;
3341 }
3342 let mut next = self.lease_state.clone().unwrap_or_default();
3343 next.error_code = Some(code.to_owned());
3344 self.set_lease_state(Some(next));
3345 }
3346
3347 fn set_lease_state(&mut self, next: Option<LeaseState>) {
3348 if self.lease_state == next {
3349 return;
3350 }
3351 if self.begin_observation("syncular_lease").is_err() {
3352 return;
3353 }
3354 self.lease_state = next;
3355 if let Some(lease) = &self.lease_state {
3356 if let Ok(json) = serde_json::to_string(lease) {
3357 self.set_meta(LEASE_STATE_KEY, &json);
3358 }
3359 } else {
3360 self.delete_meta(LEASE_STATE_KEY);
3361 }
3362 let batch = ChangeAccumulator {
3363 status: true,
3364 ..ChangeAccumulator::default()
3365 };
3366 if self.finish_observation("syncular_lease", batch).is_err() {
3367 self.rollback_observation("syncular_lease");
3368 }
3369 }
3370
3371 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
3372 if self.schema_floor == next {
3373 return;
3374 }
3375 if self.begin_observation("syncular_schema_floor").is_err() {
3376 return;
3377 }
3378 self.schema_floor = next;
3379 self.stopped = self.schema_floor.is_some();
3380 if let Some(floor) = &self.schema_floor {
3381 if let Ok(json) = serde_json::to_string(floor) {
3382 self.set_meta(SCHEMA_FLOOR_KEY, &json);
3383 }
3384 } else {
3385 self.delete_meta(SCHEMA_FLOOR_KEY);
3386 }
3387 let batch = ChangeAccumulator {
3388 status: true,
3389 ..ChangeAccumulator::default()
3390 };
3391 if self
3392 .finish_observation("syncular_schema_floor", batch)
3393 .is_err()
3394 {
3395 self.rollback_observation("syncular_schema_floor");
3396 }
3397 }
3398
3399 fn set_upgrading(&mut self, value: bool) {
3400 if self.upgrading == value {
3401 return;
3402 }
3403 if self.begin_observation("syncular_upgrading").is_err() {
3404 return;
3405 }
3406 self.upgrading = value;
3407 let batch = ChangeAccumulator {
3408 status: true,
3409 ..ChangeAccumulator::default()
3410 };
3411 if self
3412 .finish_observation("syncular_upgrading", batch)
3413 .is_err()
3414 {
3415 self.rollback_observation("syncular_upgrading");
3416 }
3417 }
3418
3419 pub fn sync_needed(&self) -> bool {
3420 self.sync_needed
3421 }
3422
3423 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
3424 let sub = self.subs.iter().find(|s| s.id == id)?;
3425 Some(SubscriptionStateView {
3426 id: sub.id.clone(),
3427 table: sub.table.clone(),
3428 status: sub.state.name().to_owned(),
3429 cursor: sub.cursor,
3430 has_resume_token: sub.bootstrap_state.is_some(),
3431 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3432 reason_code: sub.reason_code.clone(),
3433 })
3434 }
3435
3436 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
3437 let schema_table = self
3438 .schema
3439 .table(table)
3440 .ok_or_else(|| format!("unknown table {table:?}"))?;
3441 let sql = format!(
3442 "SELECT * FROM {} ORDER BY {} ASC",
3443 visible_table(table),
3444 quote_ident(&schema_table.primary_key)
3445 );
3446 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
3447 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
3448 let mut out = Vec::new();
3449 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
3450 let mut values = Map::new();
3451 for (i, column) in schema_table.columns.iter().enumerate() {
3452 let value = row.get_ref(i).map_err(|e| e.to_string())?;
3453 values.insert(column.name.clone(), sql_ref_to_json(column, value));
3454 }
3455 let version: i64 = row
3456 .get(schema_table.columns.len())
3457 .map_err(|e| e.to_string())?;
3458 let row_id = match values.get(&schema_table.primary_key) {
3459 Some(Value::String(s)) => s.clone(),
3460 Some(Value::Number(n)) => n.to_string(),
3461 Some(Value::Bool(b)) => b.to_string(),
3462 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
3463 };
3464 out.push(RowState {
3465 row_id,
3466 version,
3467 values,
3468 });
3469 }
3470 Ok(out)
3471 }
3472
3473 #[cfg(feature = "crdt-yjs")]
3489 fn crdt_column_bytes(
3490 &self,
3491 table: &str,
3492 row_id: &str,
3493 column: &str,
3494 ) -> Result<Option<Vec<u8>>, String> {
3495 let schema_table = self
3496 .schema
3497 .table(table)
3498 .ok_or_else(|| format!("unknown table {table:?}"))?;
3499 let col = schema_table
3500 .columns
3501 .iter()
3502 .find(|c| c.name == column)
3503 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
3504 if col.ty != ColumnType::Crdt {
3505 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
3506 }
3507 let sql = format!(
3508 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
3509 quote_ident(column),
3510 visible_table(table),
3511 quote_ident(&schema_table.primary_key)
3512 );
3513 let bytes: Option<Vec<u8>> = self
3514 .conn
3515 .query_row(&sql, rusqlite::params![row_id], |row| {
3516 row.get::<_, Option<Vec<u8>>>(0)
3517 })
3518 .map_err(|e| match e {
3519 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
3520 other => other.to_string(),
3521 })?;
3522 Ok(bytes)
3523 }
3524
3525 #[cfg(feature = "crdt-yjs")]
3529 pub fn crdt_text(
3530 &self,
3531 table: &str,
3532 row_id: &str,
3533 column: &str,
3534 name: &str,
3535 ) -> Result<String, String> {
3536 let bytes = self
3537 .crdt_column_bytes(table, row_id, column)?
3538 .unwrap_or_default();
3539 crate::crdt::text(&bytes, name)
3540 }
3541
3542 #[cfg(feature = "crdt-yjs")]
3546 pub fn crdt_insert_text(
3547 &mut self,
3548 table: &str,
3549 row_id: &str,
3550 column: &str,
3551 name: &str,
3552 index: u32,
3553 value: &str,
3554 ) -> Result<String, String> {
3555 let current = self
3556 .crdt_column_bytes(table, row_id, column)?
3557 .unwrap_or_default();
3558 let update = crate::crdt::insert_text(¤t, name, index, value)?;
3559 self.crdt_push_update(table, row_id, column, &update)
3560 }
3561
3562 #[cfg(feature = "crdt-yjs")]
3565 pub fn crdt_delete_text(
3566 &mut self,
3567 table: &str,
3568 row_id: &str,
3569 column: &str,
3570 name: &str,
3571 index: u32,
3572 len: u32,
3573 ) -> Result<String, String> {
3574 let current = self
3575 .crdt_column_bytes(table, row_id, column)?
3576 .unwrap_or_default();
3577 let update = crate::crdt::delete_text(¤t, name, index, len)?;
3578 self.crdt_push_update(table, row_id, column, &update)
3579 }
3580
3581 #[cfg(feature = "crdt-yjs")]
3586 pub fn crdt_apply_update(
3587 &mut self,
3588 table: &str,
3589 row_id: &str,
3590 column: &str,
3591 update: &[u8],
3592 ) -> Result<String, String> {
3593 let current = self
3594 .crdt_column_bytes(table, row_id, column)?
3595 .unwrap_or_default();
3596 let next = crate::crdt::apply_update(¤t, update)?;
3597 self.crdt_push_update(table, row_id, column, &next)
3598 }
3599
3600 #[cfg(feature = "crdt-yjs")]
3606 fn crdt_push_update(
3607 &mut self,
3608 table: &str,
3609 row_id: &str,
3610 column: &str,
3611 crdt_bytes: &[u8],
3612 ) -> Result<String, String> {
3613 let schema_table = self
3614 .schema
3615 .table(table)
3616 .ok_or_else(|| format!("unknown table {table:?}"))?
3617 .clone();
3618 let mut values: Map<String, Value> = self
3621 .read_rows(table)?
3622 .into_iter()
3623 .find(|r| r.row_id == row_id)
3624 .map(|r| r.values)
3625 .unwrap_or_else(|| {
3626 let mut map = Map::new();
3627 map.insert(
3628 schema_table.primary_key.clone(),
3629 Value::from(row_id.to_owned()),
3630 );
3631 map
3632 });
3633 let mut bytes_obj = Map::new();
3635 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
3636 values.insert(column.to_owned(), Value::Object(bytes_obj));
3637 self.mutate(vec![Mutation::Upsert {
3638 table: table.to_owned(),
3639 values,
3640 base_version: None,
3641 }])
3642 }
3643
3644 pub fn query(&self, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {
3659 query_connection(&self.conn, sql, params)
3660 }
3661
3662 pub fn query_snapshot(
3664 &mut self,
3665 sql: &str,
3666 params: &[Value],
3667 coverage: &[WindowCoverage],
3668 ) -> Result<QuerySnapshot, String> {
3669 snapshot_connection(&self.conn, sql, params, coverage)
3670 }
3671
3672 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
3675 let mut frames = vec![Frame::ReqHeader {
3676 client_id: self.client_id.clone(),
3677 schema_version: self.schema.version,
3678 }];
3679 let mut pushed_ids = Vec::new();
3680 let mut ops_in_request = 0usize;
3681 let mut deferred_commits = 0usize;
3682 for (index, commit) in self.outbox.iter().enumerate() {
3683 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
3688 deferred_commits = self.outbox.len() - index;
3689 break;
3690 }
3691 ops_in_request += commit.ops.len();
3692 let operations = commit
3693 .ops
3694 .iter()
3695 .map(|op| {
3696 let payload = op.values.as_ref().and_then(|values| {
3697 let table = self.schema.table(&op.table)?;
3698 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
3702 });
3703 ssp2::model::Operation {
3704 table: op.table.clone(),
3705 row_id: op.row_id.clone(),
3706 op: if op.upsert { Op::Upsert } else { Op::Delete },
3707 base_version: op.base_version,
3708 payload,
3709 }
3710 })
3711 .collect();
3712 frames.push(Frame::PushCommit {
3713 client_commit_id: commit.client_commit_id.clone(),
3714 operations,
3715 });
3716 pushed_ids.push(commit.client_commit_id.clone());
3717 }
3718 let accept = self.limits.accept.unwrap_or(if url_capable {
3721 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
3722 } else {
3723 DEFAULT_ACCEPT
3724 });
3725 frames.push(Frame::PullHeader {
3726 limit_commits: self.limits.limit_commits.unwrap_or(0),
3727 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
3728 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
3729 accept,
3730 });
3731 let mut fresh = Vec::new();
3732 for sub in &self.subs {
3733 if sub.state != SubState::Active {
3734 continue;
3735 }
3736 let mut scopes = sub.requested.clone();
3737 sort_scope_map(&mut scopes);
3738 frames.push(Frame::Subscription {
3739 id: sub.id.clone(),
3740 table: sub.table.clone(),
3741 scopes,
3742 params: sub.params.clone().map(RawJson),
3743 cursor: sub.cursor,
3744 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
3745 });
3746 fresh.push((
3747 sub.id.clone(),
3748 sub.cursor < 0 && sub.bootstrap_state.is_none(),
3749 ));
3750 }
3751 let message = Message {
3752 msg_kind: MsgKind::Request,
3753 frames,
3754 };
3755 (
3756 message,
3757 RequestMeta {
3758 pushed_ids,
3759 fresh,
3760 accept,
3761 deferred_commits,
3762 },
3763 )
3764 }
3765
3766 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
3769 if self.stopped {
3770 return SyncOutcome::Ok(SyncReport {
3773 schema_floor: self.schema_floor.clone(),
3774 ..SyncReport::default()
3775 });
3776 }
3777 self.set_sync_needed(false, false);
3780 if self.schema_has_blobs() {
3783 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
3784 if Self::retryable_transport_code(&code) {
3785 self.schedule_background_retry();
3786 }
3787 return SyncOutcome::Failed {
3788 error_code: code,
3789 message,
3790 };
3791 }
3792 }
3793 let (message, meta) = self.build_request(transport.supports_url_fetch());
3794 let request_bytes = encode_message(&message);
3795 let round = if self.realtime_connected {
3799 transport.realtime_sync(&request_bytes)
3800 } else {
3801 transport.sync(&request_bytes)
3802 };
3803 let response_bytes = match round {
3804 Ok(bytes) => bytes,
3805 Err(TransportError { code, message }) => {
3806 self.record_lease_error(&code);
3809 if Self::retryable_transport_code(&code) {
3810 self.schedule_background_retry();
3811 }
3812 return SyncOutcome::Failed {
3813 error_code: code,
3814 message,
3815 };
3816 }
3817 };
3818 let response = match decode_message(&response_bytes) {
3819 Ok(message) => message,
3820 Err(error) => {
3821 return SyncOutcome::Failed {
3824 error_code: error.code.as_str().to_owned(),
3825 message: error.detail,
3826 };
3827 }
3828 };
3829 if response.msg_kind != MsgKind::Response {
3830 return SyncOutcome::Failed {
3831 error_code: "sync.invalid_request".to_owned(),
3832 message: "expected a response message".to_owned(),
3833 };
3834 }
3835 let outcome = self.process_response(transport, response, &meta);
3836 match &outcome {
3837 SyncOutcome::Ok(_) => self.reset_background_retry(),
3838 SyncOutcome::Failed { error_code, .. }
3839 if Self::retryable_transport_code(error_code) =>
3840 {
3841 self.schedule_background_retry();
3842 }
3843 SyncOutcome::Failed { .. } => {}
3844 }
3845 if meta.deferred_commits > 0 {
3846 self.set_sync_needed(true, true);
3849 }
3850 outcome
3851 }
3852
3853 pub fn sync_until_idle(
3854 &mut self,
3855 transport: &mut dyn Transport,
3856 max_rounds: Option<u32>,
3857 ) -> SyncOutcome {
3858 let rounds = max_rounds.unwrap_or(12).max(1);
3859 let mut aggregate = SyncReport::default();
3860 for _ in 0..rounds {
3861 match self.sync(transport) {
3862 SyncOutcome::Failed {
3863 error_code,
3864 message,
3865 } => {
3866 return SyncOutcome::Failed {
3867 error_code,
3868 message,
3869 };
3870 }
3871 SyncOutcome::Ok(report) => {
3872 aggregate.pushed += report.pushed;
3873 aggregate.applied.extend(report.applied.iter().cloned());
3874 aggregate.rejected.extend(report.rejected.iter().cloned());
3875 aggregate.retryable.extend(report.retryable.iter().cloned());
3876 aggregate.conflicts += report.conflicts;
3877 aggregate.commits_applied += report.commits_applied;
3878 aggregate.segment_rows_applied += report.segment_rows_applied;
3879 aggregate.bootstrapping = report.bootstrapping.clone();
3880 aggregate.resets.extend(report.resets.iter().cloned());
3881 aggregate.revoked.extend(report.revoked.iter().cloned());
3882 aggregate.failed.extend(report.failed.iter().cloned());
3883 if report.schema_floor.is_some() {
3884 aggregate.schema_floor = report.schema_floor.clone();
3885 }
3886 let more = !report.bootstrapping.is_empty()
3892 || report.commits_applied > 0
3893 || report.segment_rows_applied > 0
3894 || !report.resets.is_empty()
3895 || self.sync_needed;
3896 if !more {
3897 break;
3898 }
3899 }
3900 }
3901 }
3902 SyncOutcome::Ok(aggregate)
3903 }
3904
3905 fn process_response(
3906 &mut self,
3907 transport: &mut dyn Transport,
3908 response: Message,
3909 meta: &RequestMeta,
3910 ) -> SyncOutcome {
3911 let mut report = SyncReport {
3912 pushed: meta.pushed_ids.len() as u32,
3913 ..SyncReport::default()
3914 };
3915 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
3916 HashMap::new();
3917 for frame in &response.frames {
3918 if let Frame::PushResultDetails {
3919 client_commit_id,
3920 entries,
3921 } = frame
3922 {
3923 let details = rejection_details_by_commit
3924 .entry(client_commit_id.clone())
3925 .or_default();
3926 for entry in entries {
3927 let parsed = match RejectionDetails::parse(&entry.details.0) {
3928 Ok(value) => value,
3929 Err(message) => {
3930 return SyncOutcome::Failed {
3931 error_code: "sync.invalid_request".to_owned(),
3932 message,
3933 };
3934 }
3935 };
3936 details.insert(entry.op_index, parsed);
3937 }
3938 }
3939 }
3940 let mut frames = response.frames.into_iter();
3941 match frames.next() {
3942 Some(Frame::RespHeader {
3943 required_schema_version,
3944 latest_schema_version,
3945 }) => {
3946 if let Some(required) = required_schema_version {
3947 let floor = SchemaFloor {
3950 required_schema_version: Some(required),
3951 latest_schema_version,
3952 };
3953 self.set_schema_floor(Some(floor.clone()));
3954 report.schema_floor = Some(floor);
3955 return SyncOutcome::Ok(report);
3956 }
3957 }
3958 _ => {
3959 return SyncOutcome::Failed {
3960 error_code: "sync.invalid_request".to_owned(),
3961 message: "response does not start with RESP_HEADER".to_owned(),
3962 };
3963 }
3964 }
3965
3966 let mut failure: Option<(String, String)> = None;
3967 while let Some(frame) = frames.next() {
3968 match frame {
3969 Frame::PushResult {
3970 client_commit_id,
3971 status,
3972 commit_seq: _,
3973 results,
3974 } => {
3975 self.handle_push_result(
3976 &client_commit_id,
3977 status,
3978 &results,
3979 rejection_details_by_commit.get(&client_commit_id),
3980 &mut report,
3981 );
3982 }
3983 Frame::PushResultDetails { .. } => {}
3984 Frame::SubStart {
3985 id,
3986 status,
3987 reason_code,
3988 effective_scopes,
3989 bootstrap: _,
3990 } => {
3991 let mut body = Vec::new();
3992 let mut sub_end: Option<(i64, Option<String>)> = None;
3993 for inner in frames.by_ref() {
3994 match inner {
3995 Frame::SubEnd {
3996 next_cursor,
3997 bootstrap_state,
3998 } => {
3999 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
4000 break;
4001 }
4002 Frame::Unknown { .. } => {}
4003 other => body.push(other),
4004 }
4005 }
4006 let Some((next_cursor, bootstrap_state)) = sub_end else {
4007 failure = Some((
4008 "sync.invalid_request".to_owned(),
4009 "subscription section without SUB_END".to_owned(),
4010 ));
4011 break;
4012 };
4013 if let Err(SectionError::Abort(code, message)) = self.process_section(
4014 transport,
4015 &id,
4016 status,
4017 &reason_code,
4018 effective_scopes,
4019 body,
4020 next_cursor,
4021 bootstrap_state,
4022 meta,
4023 &mut report,
4024 ) {
4025 failure = Some((code, message));
4026 break;
4027 }
4028 }
4029 Frame::Lease {
4030 lease_id,
4031 expires_at_ms,
4032 } => {
4033 self.set_lease_state(Some(LeaseState {
4036 lease_id: Some(lease_id),
4037 expires_at_ms: Some(expires_at_ms),
4038 error_code: None,
4039 }));
4040 }
4041 Frame::Error { code, message, .. } => {
4042 failure = Some((code, message));
4045 break;
4046 }
4047 Frame::Unknown { .. } => {}
4048 _ => {
4049 failure = Some((
4050 "sync.invalid_request".to_owned(),
4051 "unexpected frame in response".to_owned(),
4052 ));
4053 break;
4054 }
4055 }
4056 }
4057
4058 if self.overlay_dirty.get() {
4061 self.rebuild_overlay();
4062 }
4063 self.reconcile_blob_refcounts(false);
4070
4071 if let Some((error_code, message)) = failure {
4072 return SyncOutcome::Failed {
4073 error_code,
4074 message,
4075 };
4076 }
4077 self.drain_pending_evictions();
4080 self.ack_after_pull(transport);
4081 if self.upgrading && report.bootstrapping.is_empty() {
4084 self.set_upgrading(false);
4085 }
4086 SyncOutcome::Ok(report)
4087 }
4088
4089 fn handle_push_result(
4092 &mut self,
4093 client_commit_id: &str,
4094 status: PushStatus,
4095 results: &[OpResult],
4096 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
4097 report: &mut SyncReport,
4098 ) {
4099 let Some(index) = self
4100 .outbox
4101 .iter()
4102 .position(|c| c.client_commit_id == client_commit_id)
4103 else {
4104 return;
4105 };
4106 if self.begin_observation("syncular_push_result").is_err() {
4107 return;
4108 }
4109 let mut batch = ChangeAccumulator::default();
4110 let operations = self.outbox[index].ops.clone();
4111 match status {
4112 PushStatus::Applied | PushStatus::Cached => {
4113 let journal_results = results
4116 .iter()
4117 .map(|result| {
4118 let op_index = match result {
4119 OpResult::Applied { op_index }
4120 | OpResult::Conflict { op_index, .. }
4121 | OpResult::Error { op_index, .. } => *op_index,
4122 };
4123 CommitOperationOutcome::Applied { op_index }
4124 })
4125 .collect::<Vec<_>>();
4126 let outcome_status = if status == PushStatus::Applied {
4127 CommitOutcomeStatus::Applied
4128 } else {
4129 CommitOutcomeStatus::Cached
4130 };
4131 if self
4132 .persist_commit_outcome(
4133 client_commit_id,
4134 outcome_status,
4135 &journal_results,
4136 None,
4137 )
4138 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4139 .and_then(|()| self.prune_commit_outcomes())
4140 .is_err()
4141 {
4142 self.rollback_observation("syncular_push_result");
4143 return;
4144 }
4145 report.applied.push(client_commit_id.to_owned());
4146 self.outbox.remove(index);
4147 self.overlay_dirty.set(true);
4148 batch.status = true;
4149 batch.outcomes = true;
4150 }
4151 PushStatus::Rejected => {
4152 if results.iter().any(|result| {
4153 matches!(
4154 result,
4155 OpResult::Error {
4156 code,
4157 retryable: true,
4158 ..
4159 } if code == "sync.idempotency_cache_miss"
4160 )
4161 }) {
4162 report.retryable.push(client_commit_id.to_owned());
4165 if self
4166 .finish_observation("syncular_push_result", batch)
4167 .is_err()
4168 {
4169 self.rollback_observation("syncular_push_result");
4170 }
4171 return;
4172 }
4173
4174 let mut journal_results = Vec::with_capacity(results.len());
4175 let mut conflicts = Vec::new();
4176 let mut rejections = Vec::new();
4177 for result in results {
4178 match result {
4179 OpResult::Applied { op_index } => {
4180 journal_results.push(CommitOperationOutcome::Applied {
4181 op_index: *op_index,
4182 });
4183 }
4184 OpResult::Conflict {
4185 op_index,
4186 code,
4187 message,
4188 server_version,
4189 server_row,
4190 } => {
4191 let operation = operations
4192 .get(*op_index as usize)
4193 .map(CommitOperation::from);
4194 let (table, row_id) = operation
4195 .as_ref()
4196 .map(|op| (op.table.clone(), op.row_id.clone()))
4197 .unwrap_or_default();
4198 let server_row_json = self
4199 .schema
4200 .table(&table)
4201 .and_then(|t| {
4202 decode_row_bytes(t, server_row, &self.encryption)
4203 .ok()
4204 .map(|row| (t, row))
4205 })
4206 .map(|(t, row)| {
4207 let mut map = Map::new();
4208 for (i, column) in t.columns.iter().enumerate() {
4209 map.insert(
4210 column.name.clone(),
4211 column_value_to_json(row.get(i).unwrap_or(&None)),
4212 );
4213 }
4214 map
4215 })
4216 .unwrap_or_default();
4217 let conflict = ConflictRecord {
4218 client_commit_id: client_commit_id.to_owned(),
4219 op_index: *op_index,
4220 table,
4221 row_id,
4222 code: code.clone(),
4223 message: message.clone(),
4224 server_version: *server_version,
4225 server_row: server_row_json,
4226 operation,
4227 };
4228 journal_results.push(CommitOperationOutcome::Conflict {
4229 conflict: conflict.clone(),
4230 });
4231 conflicts.push(conflict);
4232 }
4233 OpResult::Error {
4234 op_index,
4235 code,
4236 message,
4237 retryable,
4238 } => {
4239 let rejection = RejectionRecord {
4240 client_commit_id: client_commit_id.to_owned(),
4241 op_index: *op_index,
4242 code: code.clone(),
4243 message: message.clone(),
4244 retryable: *retryable,
4245 details: rejection_details
4246 .and_then(|details| details.get(op_index))
4247 .cloned(),
4248 operation: operations
4249 .get(*op_index as usize)
4250 .map(CommitOperation::from),
4251 };
4252 journal_results.push(CommitOperationOutcome::Error {
4253 rejection: rejection.clone(),
4254 });
4255 rejections.push(rejection);
4256 }
4257 }
4258 }
4259 let outcome_status = if conflicts.is_empty() {
4260 CommitOutcomeStatus::Rejected
4261 } else {
4262 CommitOutcomeStatus::Conflict
4263 };
4264 if self
4265 .persist_commit_outcome(
4266 client_commit_id,
4267 outcome_status,
4268 &journal_results,
4269 Some(&operations),
4270 )
4271 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4272 .and_then(|()| self.prune_commit_outcomes())
4273 .is_err()
4274 {
4275 self.rollback_observation("syncular_push_result");
4276 return;
4277 }
4278 report.conflicts += conflicts.len() as u32;
4279 report.rejected.push(client_commit_id.to_owned());
4280 batch.conflicts = !conflicts.is_empty();
4281 batch.rejections = !rejections.is_empty();
4282 batch.status = true;
4283 batch.outcomes = true;
4284 self.conflicts.extend(conflicts);
4285 self.rejections.extend(rejections);
4286 self.outbox.remove(index);
4287 self.overlay_dirty.set(true);
4288 }
4289 }
4290 if batch.status {
4291 for operation in &operations {
4292 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
4293 batch.table(&operation.table);
4294 }
4295 }
4296 self.rebuild_overlay_if_dirty();
4297 }
4298 if self
4299 .finish_observation("syncular_push_result", batch)
4300 .is_err()
4301 {
4302 self.rollback_observation("syncular_push_result");
4303 }
4304 }
4305
4306 #[allow(clippy::too_many_arguments)]
4309 fn process_section(
4310 &mut self,
4311 transport: &mut dyn Transport,
4312 id: &str,
4313 status: SubStatus,
4314 reason_code: &str,
4315 effective_scopes: Vec<(String, Vec<String>)>,
4316 body: Vec<Frame>,
4317 next_cursor: i64,
4318 bootstrap_state: Option<String>,
4319 meta: &RequestMeta,
4320 report: &mut SyncReport,
4321 ) -> Result<(), SectionError> {
4322 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4323 return Ok(()); };
4325 match status {
4326 SubStatus::Revoked => {
4327 self.begin_observation("syncular_revocation")
4328 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4329 let mut batch = ChangeAccumulator::default();
4330 let registered = self.window_unit_by_sub_id(id);
4331 let (table, effective) = {
4333 let sub = &self.subs[sub_index];
4334 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4335 };
4336 let purged = self.purge_scope_rows(&table, &effective);
4337 match purged {
4338 Ok(()) => {
4339 self.record_scope_map(&mut batch, &table, &effective);
4340 let sub = &mut self.subs[sub_index];
4341 sub.state = SubState::Revoked;
4342 sub.reason_code = Some(if reason_code.is_empty() {
4343 "sync.scope_revoked".to_owned()
4344 } else {
4345 reason_code.to_owned()
4346 });
4347 report.revoked.push(id.to_owned());
4348 let doomed_effective = effective;
4349 let sub_table = table;
4350 self.persist_sub(&self.subs[sub_index].clone());
4351 let dropped = self
4352 .drop_doomed_outbox(&sub_table, &doomed_effective)
4353 .map_err(|message| {
4354 SectionError::Abort("storage.failed".to_owned(), message)
4355 })?;
4356 if dropped {
4357 batch.status = true;
4358 batch.rejections = true;
4359 batch.outcomes = true;
4360 }
4361 self.reconcile_blob_refcounts(true);
4364 }
4365 Err(()) => {
4366 let sub = &mut self.subs[sub_index];
4369 sub.state = SubState::Failed;
4370 sub.reason_code = Some("sync.scope_revoked".to_owned());
4371 report.failed.push(id.to_owned());
4372 self.persist_sub(&self.subs[sub_index].clone());
4373 }
4374 }
4375 if let Some((base_key, unit)) = registered {
4376 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4377 }
4378 self.rebuild_overlay_if_dirty();
4379 self.finish_observation("syncular_revocation", batch)
4380 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4381 Ok(())
4382 }
4383 SubStatus::Reset => {
4384 self.begin_observation("syncular_reset")
4385 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4386 let mut batch = ChangeAccumulator::default();
4387 let registered = self.window_unit_by_sub_id(id);
4388 let sub = &mut self.subs[sub_index];
4391 sub.cursor = -1;
4392 sub.bootstrap_state = None;
4393 report.resets.push(id.to_owned());
4394 self.persist_sub(&self.subs[sub_index].clone());
4395 if let Some((base_key, unit)) = registered {
4396 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4397 }
4398 self.finish_observation("syncular_reset", batch)
4399 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4400 Ok(())
4401 }
4402 SubStatus::Active => {
4403 let fresh = meta
4404 .fresh
4405 .iter()
4406 .find(|(fid, _)| fid == id)
4407 .map(|(_, f)| *f)
4408 .unwrap_or(false);
4409 let was_pending = self.subs[sub_index].cursor < 0
4410 || self.subs[sub_index].bootstrap_state.is_some();
4411 let registered = self.window_unit_by_sub_id(id);
4412 self.subs[sub_index].effective = Some(effective_scopes);
4414 self.begin_observation("syncular_section")
4415 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4416 let mut batch = ChangeAccumulator::default();
4417 let outcome = self.apply_section_body(
4418 transport, sub_index, body, fresh, meta, report, &mut batch,
4419 );
4420 match outcome {
4421 Ok(()) => {
4422 let sub = &mut self.subs[sub_index];
4423 sub.cursor = next_cursor;
4426 sub.bootstrap_state = bootstrap_state;
4427 sub.synced_once = true;
4428 if sub.bootstrap_state.is_some() {
4429 report.bootstrapping.push(id.to_owned());
4430 }
4431 let completed =
4432 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
4433 self.persist_sub(&self.subs[sub_index].clone());
4434 if completed {
4435 if let Some((base_key, unit)) = registered.clone() {
4436 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4437 }
4438 }
4439 self.rebuild_overlay_if_dirty();
4440 self.finish_observation("syncular_section", batch)
4441 .map_err(|message| {
4442 SectionError::Abort("storage.failed".to_owned(), message)
4443 })?;
4444 Ok(())
4445 }
4446 Err(SectionError::FailClosed) => {
4447 self.rollback_observation("syncular_section");
4450 self.begin_observation("syncular_section_failure")
4451 .map_err(|message| {
4452 SectionError::Abort("storage.failed".to_owned(), message)
4453 })?;
4454 let mut failure_batch = ChangeAccumulator::default();
4455 let sub = &mut self.subs[sub_index];
4456 sub.state = SubState::Failed;
4457 sub.reason_code = Some("sync.scope_revoked".to_owned());
4458 report.failed.push(id.to_owned());
4459 self.persist_sub(&self.subs[sub_index].clone());
4460 if let Some((base_key, unit)) = registered {
4461 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
4462 }
4463 self.finish_observation("syncular_section_failure", failure_batch)
4464 .map_err(|message| {
4465 SectionError::Abort("storage.failed".to_owned(), message)
4466 })?;
4467 Ok(())
4468 }
4469 Err(SectionError::Abort(code, message)) => {
4470 self.rollback_observation("syncular_section");
4473 Err(SectionError::Abort(code, message))
4474 }
4475 }
4476 }
4477 }
4478 }
4479
4480 #[allow(clippy::too_many_arguments)]
4484 fn apply_section_body(
4485 &mut self,
4486 transport: &mut dyn Transport,
4487 sub_index: usize,
4488 body: Vec<Frame>,
4489 fresh: bool,
4490 meta: &RequestMeta,
4491 report: &mut SyncReport,
4492 batch: &mut ChangeAccumulator,
4493 ) -> Result<(), SectionError> {
4494 let mut saw_segment = false;
4495 for frame in body {
4496 match frame {
4497 Frame::Commit {
4498 tables, changes, ..
4499 } => {
4500 self.record_commit_changes(batch, &tables, &changes);
4501 self.apply_commit_changes(&tables, &changes)
4502 .map_err(|(c, m)| SectionError::Abort(c, m))?;
4503 report.commits_applied += 1;
4504 }
4505 Frame::SegmentInline { payload } => {
4506 let segment = decode_rows_segment(&payload)
4507 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
4508 let first = !saw_segment;
4509 saw_segment = true;
4510 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4511 let cleared =
4512 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4513 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4514 if applied > 0 || cleared {
4515 batch.table(&segment.table);
4516 }
4517 report.segment_rows_applied += applied;
4518 }
4519 Frame::SegmentRef {
4520 segment_id,
4521 media_type,
4522 table,
4523 row_count,
4524 as_of_commit_seq,
4525 scope_digest,
4526 row_cursor,
4527 next_row_cursor,
4528 url,
4529 url_expires_at_ms,
4530 ..
4531 } => {
4532 let advertised = match media_type {
4535 MediaType::Rows => {
4536 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
4537 || meta.accept & ACCEPT_INLINE_ROWS != 0
4538 }
4539 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
4540 };
4541 if !advertised {
4542 return Err(SectionError::Abort(
4543 "sync.invalid_request".to_owned(),
4544 format!(
4545 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
4546 media_type.name()
4547 ),
4548 ));
4549 }
4550 let bytes = if let Some(url) = url {
4551 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
4556 return Err(SectionError::Abort(
4557 "sync.invalid_request".to_owned(),
4558 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
4559 .to_owned(),
4560 ));
4561 }
4562 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
4564 return Err(SectionError::Abort(
4565 "sync.segment_expired".to_owned(),
4566 format!(
4567 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
4568 ),
4569 ));
4570 }
4571 transport
4572 .fetch_url(&url)
4573 .map_err(|e| SectionError::Abort(e.code, e.message))?
4574 } else {
4575 let requested_scopes_json =
4576 canonical_scope_json(&self.subs[sub_index].requested);
4577 transport
4578 .download_segment(&SegmentRequest {
4579 segment_id: segment_id.clone(),
4580 table,
4581 requested_scopes_json,
4582 })
4583 .map_err(|e| SectionError::Abort(e.code, e.message))?
4584 };
4585 let digest = Sha256::digest(&bytes);
4587 let expected = segment_id
4588 .strip_prefix("sha256:")
4589 .unwrap_or(segment_id.as_str());
4590 if bytes_to_hex(&digest) != expected {
4591 return Err(SectionError::Abort(
4592 "sync.invalid_request".to_owned(),
4593 "segment bytes do not match the content address (§5.1)".to_owned(),
4594 ));
4595 }
4596 if media_type == MediaType::Sqlite {
4597 if row_cursor.is_some() || next_row_cursor.is_some() {
4600 return Err(SectionError::Abort(
4601 "sync.invalid_request".to_owned(),
4602 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
4603 .to_owned(),
4604 ));
4605 }
4606 let first = !saw_segment;
4607 saw_segment = true;
4608 let sub_table = self.subs[sub_index].table.clone();
4609 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4610 let cleared =
4611 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
4612 let applied = self.apply_sqlite_segment(
4613 sub_index,
4614 &bytes,
4615 fresh && first,
4616 row_count,
4617 as_of_commit_seq,
4618 &scope_digest,
4619 )?;
4620 if applied > 0 || cleared {
4621 batch.table(&sub_table);
4622 }
4623 report.segment_rows_applied += applied;
4624 } else {
4625 let segment = decode_rows_segment(&bytes).map_err(|e| {
4626 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
4627 })?;
4628 let first = row_cursor.is_none();
4629 saw_segment = true;
4630 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4631 let cleared =
4632 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4633 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4634 if applied > 0 || cleared {
4635 batch.table(&segment.table);
4636 }
4637 report.segment_rows_applied += applied;
4638 }
4639 }
4640 Frame::Unknown { .. } => {}
4641 _ => {
4642 return Err(SectionError::Abort(
4643 "sync.invalid_request".to_owned(),
4644 "unexpected frame inside a subscription section".to_owned(),
4645 ));
4646 }
4647 }
4648 }
4649 Ok(())
4650 }
4651
4652 fn apply_commit_changes(
4653 &mut self,
4654 tables: &[String],
4655 changes: &[ssp2::model::Change],
4656 ) -> Result<(), (String, String)> {
4657 for change in changes {
4658 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
4659 (
4660 "sync.invalid_request".to_owned(),
4661 "change tableIndex out of range".to_owned(),
4662 )
4663 })?;
4664 let table = self.schema.table(table_name).ok_or_else(|| {
4665 (
4666 "sync.schema_mismatch".to_owned(),
4667 format!("change targets unknown table {table_name:?}"),
4668 )
4669 })?;
4670 match change.op {
4671 Op::Upsert => {
4672 let payload = change.row.as_ref().ok_or_else(|| {
4673 (
4674 "sync.invalid_request".to_owned(),
4675 "upsert change without row payload".to_owned(),
4676 )
4677 })?;
4678 let row = decode_row_bytes(table, payload, &self.encryption)
4680 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4681 let version = change.row_version.unwrap_or(0);
4682 let table_name = table.name.clone();
4683 self.write_base_row(&table_name, &row, version)
4684 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4685 }
4686 Op::Delete => {
4687 self.delete_base_row(table_name, &change.row_id)
4688 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4689 }
4690 }
4691 }
4692 Ok(())
4693 }
4694
4695 fn apply_segment(
4700 &mut self,
4701 sub_index: usize,
4702 segment: &RowsSegment,
4703 first_fresh_page: bool,
4704 ) -> Result<u32, SectionError> {
4705 let (sub_table, effective) = {
4706 let sub = &self.subs[sub_index];
4707 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4708 };
4709 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
4710 SectionError::Abort(
4711 "sync.schema_mismatch".to_owned(),
4712 format!("subscription table {sub_table:?} is not in the client schema"),
4713 )
4714 })?;
4715 let matches = segment.table == table.name
4720 && segment.schema_version == self.schema.version
4721 && segment.columns.len() == table.wire_columns.len()
4722 && segment
4723 .columns
4724 .iter()
4725 .zip(table.wire_columns.iter())
4726 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
4727 if !matches {
4728 return Err(SectionError::Abort(
4729 "sync.schema_mismatch".to_owned(),
4730 "segment column table does not match the generated schema (§5.2)".to_owned(),
4731 ));
4732 }
4733 if first_fresh_page {
4734 self.purge_scope_rows(&table.name, &effective)
4738 .map_err(|()| SectionError::FailClosed)?;
4739 }
4740 let mut applied = 0u32;
4741 for block in &segment.blocks {
4742 for row in block {
4743 let decrypted;
4748 let values = if table.has_encrypted_columns() {
4749 let mut values = row.values.clone();
4750 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
4751 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
4752 decrypted = values;
4753 &decrypted
4754 } else {
4755 &row.values
4756 };
4757 self.write_base_row(&table.name, values, row.server_version)
4760 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
4761 applied += 1;
4762 }
4763 }
4764 Ok(applied)
4765 }
4766
4767 fn apply_sqlite_segment(
4775 &mut self,
4776 sub_index: usize,
4777 bytes: &[u8],
4778 first_fresh_page: bool,
4779 row_count: i64,
4780 as_of_commit_seq: i64,
4781 scope_digest: &str,
4782 ) -> Result<u32, SectionError> {
4783 let invalid = |detail: &str| {
4784 SectionError::Abort(
4785 "sync.invalid_request".to_owned(),
4786 format!("sqlite segment rejected: {detail} (§5.3)"),
4787 )
4788 };
4789 let (sub_table, effective) = {
4790 let sub = &self.subs[sub_index];
4791 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4792 };
4793 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
4794 SectionError::Abort(
4795 "sync.schema_mismatch".to_owned(),
4796 format!("subscription table {sub_table:?} is not in the client schema"),
4797 )
4798 })?;
4799
4800 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
4801 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
4802 let img = match rusqlite::Connection::open_with_flags(
4803 &path,
4804 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
4805 ) {
4806 Ok(conn) => conn,
4807 Err(_) => {
4808 let _ = std::fs::remove_file(&path);
4809 return Err(invalid("bytes do not open as a SQLite database"));
4810 }
4811 };
4812 let outcome = self.apply_sqlite_image(
4813 &img,
4814 &table,
4815 first_fresh_page,
4816 &effective,
4817 row_count,
4818 as_of_commit_seq,
4819 scope_digest,
4820 );
4821 drop(img);
4822 let _ = std::fs::remove_file(&path);
4823 outcome
4824 }
4825
4826 #[allow(clippy::too_many_arguments)]
4827 fn apply_sqlite_image(
4828 &mut self,
4829 img: &rusqlite::Connection,
4830 table: &crate::schema::TableSchema,
4831 first_fresh_page: bool,
4832 effective: &[(String, Vec<String>)],
4833 row_count: i64,
4834 as_of_commit_seq: i64,
4835 scope_digest: &str,
4836 ) -> Result<u32, SectionError> {
4837 let invalid = |detail: String| {
4838 SectionError::Abort(
4839 "sync.invalid_request".to_owned(),
4840 format!("sqlite segment rejected: {detail} (§5.3)"),
4841 )
4842 };
4843
4844 type MetaRow = (i64, String, i64, i64, String, i64, i64);
4847 let meta: MetaRow = img
4848 .query_row(
4849 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
4850 \"scopeDigest\", \"rowCount\",
4851 (SELECT count(*) FROM _syncular_segment)
4852 FROM _syncular_segment",
4853 [],
4854 |row| {
4855 Ok((
4856 row.get(0)?,
4857 row.get(1)?,
4858 row.get(2)?,
4859 row.get(3)?,
4860 row.get(4)?,
4861 row.get(5)?,
4862 row.get(6)?,
4863 ))
4864 },
4865 )
4866 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
4867 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
4868 if meta_count != 1 {
4869 return Err(invalid(format!(
4870 "_syncular_segment must contain exactly one row, found {meta_count}"
4871 )));
4872 }
4873 if format != 1 {
4874 return Err(invalid(format!("format {format}")));
4875 }
4876 if meta_table != table.name {
4877 return Err(invalid(format!("image table {meta_table:?}")));
4878 }
4879 if schema_version != i64::from(self.schema.version) {
4880 return Err(invalid(format!("schemaVersion {schema_version}")));
4881 }
4882 if pin != as_of_commit_seq {
4883 return Err(invalid(format!("asOfCommitSeq {pin}")));
4884 }
4885 if digest != scope_digest {
4886 return Err(invalid("scopeDigest mismatch".to_owned()));
4887 }
4888 if meta_rows != row_count {
4889 return Err(invalid(format!("rowCount {meta_rows}")));
4890 }
4891
4892 let mut names: Vec<String> = Vec::new();
4894 {
4895 let mut stmt = img
4896 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
4897 .map_err(|_| invalid("image data table missing".to_owned()))?;
4898 let mut rows = stmt
4899 .query([])
4900 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4901 while let Some(row) = rows
4902 .next()
4903 .map_err(|_| invalid("image data table unreadable".to_owned()))?
4904 {
4905 names.push(
4906 row.get::<_, String>(1)
4907 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
4908 );
4909 }
4910 }
4911 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
4912 expected.push("_syncular_version");
4913 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
4914 return Err(SectionError::Abort(
4915 "sync.schema_mismatch".to_owned(),
4916 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
4917 ));
4918 }
4919
4920 if first_fresh_page {
4923 self.purge_scope_rows(&table.name, effective)
4924 .map_err(|()| SectionError::FailClosed)?;
4925 }
4926 self.overlay_dirty.set(true);
4932 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
4940 table.indexes.iter().filter(|i| !i.unique).collect()
4941 } else {
4942 Vec::new()
4943 };
4944 for index in &bulk_indexes {
4945 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
4946 self.conn
4947 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
4948 .map_err(|e| invalid(e.to_string()))?;
4949 }
4950 let insert = self.insert_row_sql(&base_table(&table.name), table);
4951 let applied = {
4952 let mut ins = self
4953 .conn
4954 .prepare_cached(&insert)
4955 .map_err(|e| invalid(e.to_string()))?;
4956 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
4957 let mut stmt = img
4958 .prepare(&format!(
4959 "SELECT {} FROM {}",
4960 column_list.join(", "),
4961 quote_ident(&table.name)
4962 ))
4963 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4964 let mut rows = stmt
4965 .query([])
4966 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4967 let version_index = table.columns.len();
4968 let mut applied = 0u32;
4969 while let Some(row) = rows
4970 .next()
4971 .map_err(|_| invalid("image row unreadable".to_owned()))?
4972 {
4973 for (i, column) in table.columns.iter().enumerate() {
4974 let cell = row
4975 .get_ref(i)
4976 .map_err(|_| invalid("image row unreadable".to_owned()))?;
4977 let param = image_cell_param(column, cell).map_err(&invalid)?;
4978 ins.raw_bind_parameter(i + 1, param)
4979 .map_err(|e| invalid(e.to_string()))?;
4980 }
4981 let version: i64 = row
4982 .get(version_index)
4983 .map_err(|_| invalid("image row unreadable".to_owned()))?;
4984 if version < 1 {
4985 return Err(invalid(format!(
4986 "row _syncular_version must be >= 1, got {version}"
4987 )));
4988 }
4989 ins.raw_bind_parameter(version_index + 1, version)
4990 .map_err(|e| invalid(e.to_string()))?;
4991 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
4992 applied += 1;
4993 }
4994 applied
4995 };
4996 for index in &bulk_indexes {
4997 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
4998 let cols_sql = index
4999 .columns
5000 .iter()
5001 .map(|c| quote_ident(c))
5002 .collect::<Vec<_>>()
5003 .join(", ");
5004 self.conn
5005 .execute(
5006 &format!(
5007 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
5008 base_table(&table.name)
5009 ),
5010 [],
5011 )
5012 .map_err(|e| invalid(e.to_string()))?;
5013 }
5014 if i64::from(applied) != row_count {
5015 return Err(invalid(format!(
5016 "image holds {applied} rows, descriptor says {row_count}"
5017 )));
5018 }
5019 Ok(applied)
5020 }
5021
5022 fn purge_scope_rows(
5027 &mut self,
5028 table_name: &str,
5029 effective: &[(String, Vec<String>)],
5030 ) -> Result<(), ()> {
5031 if effective.is_empty() {
5032 return Ok(());
5033 }
5034 let table = self.schema.table(table_name).ok_or(())?.clone();
5035 let mut clauses = Vec::new();
5036 let mut params: Vec<SqlValue> = Vec::new();
5037 for (variable, values) in effective {
5038 let column = table.scope_column(variable).ok_or(())?;
5039 let placeholders: Vec<String> = values
5040 .iter()
5041 .map(|v| {
5042 params.push(SqlValue::Text(v.clone()));
5043 "?".to_owned()
5044 })
5045 .collect();
5046 clauses.push(format!(
5047 "{} IN ({})",
5048 quote_ident(column),
5049 placeholders.join(", ")
5050 ));
5051 }
5052 let sql = format!(
5053 "DELETE FROM {} WHERE {}",
5054 base_table(table_name),
5055 clauses.join(" AND ")
5056 );
5057 self.overlay_dirty.set(true);
5058 self.conn
5059 .execute(&sql, rusqlite::params_from_iter(params))
5060 .map_err(|_| ())?;
5061 Ok(())
5062 }
5063
5064 fn drop_doomed_outbox(
5067 &mut self,
5068 table_name: &str,
5069 effective: &[(String, Vec<String>)],
5070 ) -> Result<bool, String> {
5071 if effective.is_empty() {
5072 return Ok(false);
5073 }
5074 let Some(table) = self.schema.table(table_name).cloned() else {
5075 return Ok(false);
5076 };
5077 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
5078 for (variable, values) in effective {
5079 match table.scope_column(variable) {
5080 Some(column) => mappings.push((column, values)),
5081 None => return Ok(false), }
5083 }
5084 let doomed: Vec<OutboxCommit> = self
5085 .outbox
5086 .iter()
5087 .filter(|commit| {
5088 commit.ops.iter().any(|op| {
5089 op.upsert
5090 && op.table == table_name
5091 && op.values.as_ref().is_some_and(|values| {
5092 mappings
5093 .iter()
5094 .all(|(column, allowed)| match values.get(*column) {
5095 Some(Value::String(s)) => allowed.contains(s),
5096 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
5097 _ => false,
5098 })
5099 })
5100 })
5101 })
5102 .cloned()
5103 .collect();
5104 if doomed.is_empty() {
5105 return Ok(false);
5106 }
5107 let mut rejections = Vec::new();
5108 for commit in &doomed {
5109 let results = commit
5110 .ops
5111 .iter()
5112 .enumerate()
5113 .map(|(op_index, operation)| {
5114 let rejection = RejectionRecord {
5115 client_commit_id: commit.client_commit_id.clone(),
5116 op_index: op_index as i32,
5117 code: "sync.scope_revoked".to_owned(),
5118 message: "the commit was dropped because its effective scope was revoked"
5119 .to_owned(),
5120 retryable: false,
5121 details: None,
5122 operation: Some(CommitOperation::from(operation)),
5123 };
5124 rejections.push(rejection.clone());
5125 CommitOperationOutcome::Error { rejection }
5126 })
5127 .collect::<Vec<_>>();
5128 self.persist_commit_outcome(
5129 &commit.client_commit_id,
5130 CommitOutcomeStatus::Rejected,
5131 &results,
5132 Some(&commit.ops),
5133 )?;
5134 self.delete_outbox_persisted(&commit.client_commit_id)?;
5135 }
5136 self.prune_commit_outcomes()?;
5137 let doomed_ids = doomed
5138 .iter()
5139 .map(|commit| commit.client_commit_id.as_str())
5140 .collect::<BTreeSet<_>>();
5141 self.outbox
5142 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
5143 self.rejections.extend(rejections);
5144 self.overlay_dirty.set(true);
5145 Ok(true)
5146 }
5147
5148 pub fn upload_blob(
5155 &mut self,
5156 bytes: &[u8],
5157 media_type: Option<String>,
5158 name: Option<String>,
5159 ) -> Result<Value, String> {
5160 let blob_id = blob_id_for(bytes);
5161 let now = self.clock_now_ms();
5162 self.conn
5163 .execute(
5164 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
5165 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
5166 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
5167 )
5168 .map_err(|e| e.to_string())?;
5169 self.conn
5170 .execute(
5171 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
5172 rusqlite::params![blob_id, media_type, now],
5173 )
5174 .map_err(|e| e.to_string())?;
5175 self.enforce_blob_cache_cap();
5178 let mut obj = Map::new();
5179 obj.insert("blobId".to_owned(), Value::from(blob_id));
5180 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
5181 if let Some(mt) = media_type {
5182 obj.insert("mediaType".to_owned(), Value::from(mt));
5183 }
5184 if let Some(n) = name {
5185 obj.insert("name".to_owned(), Value::from(n));
5186 }
5187 Ok(Value::Object(obj))
5188 }
5189
5190 pub fn fetch_blob(
5194 &mut self,
5195 transport: &mut dyn Transport,
5196 blob_id_or_ref: &str,
5197 ) -> Result<Value, (String, String)> {
5198 let simple = |m: String| ("client.failed".to_owned(), m);
5199 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
5200 blob_id_or_ref.to_owned()
5201 } else {
5202 let value: Value = serde_json::from_str(blob_id_or_ref)
5203 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
5204 value
5205 .get("blobId")
5206 .and_then(Value::as_str)
5207 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
5208 .to_owned()
5209 };
5210 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
5211 return Ok(cached);
5212 }
5213 let bytes = match transport
5219 .blob_download(&blob_id)
5220 .map_err(|e| (e.code, e.message))?
5221 {
5222 BlobDownload::Bytes(bytes) => bytes,
5223 BlobDownload::Url {
5224 url,
5225 url_expires_at_ms,
5226 } => {
5227 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5229 return Err((
5230 "sync.segment_expired".to_owned(),
5231 format!(
5232 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
5233 ),
5234 ));
5235 }
5236 transport
5237 .fetch_blob_url(&url)
5238 .map_err(|e| (e.code, e.message))?
5239 }
5240 };
5241 if blob_id_for(&bytes) != blob_id {
5243 return Err(simple(format!(
5244 "blob content address mismatch for {blob_id}"
5245 )));
5246 }
5247 let now = self.clock_now_ms();
5248 self.conn
5249 .execute(
5250 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
5251 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
5252 )
5253 .map_err(|e| simple(e.to_string()))?;
5254 self.enforce_blob_cache_cap();
5255 self.get_cached_blob(&blob_id)
5256 .map_err(simple)?
5257 .ok_or_else(|| simple("blob cache write failed".to_owned()))
5258 }
5259
5260 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
5261 let _ = self.conn.execute(
5264 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
5265 rusqlite::params![self.clock_now_ms(), blob_id],
5266 );
5267 let mut stmt = self
5268 .conn
5269 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
5270 .map_err(|e| e.to_string())?;
5271 let mut rows = stmt
5272 .query(rusqlite::params![blob_id])
5273 .map_err(|e| e.to_string())?;
5274 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
5275 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
5276 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
5277 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
5278 let mut obj = Map::new();
5279 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
5280 obj.insert("byteLength".to_owned(), Value::from(byte_length));
5281 let mut bytes_obj = Map::new();
5282 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
5283 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
5284 if let Some(mt) = media_type {
5285 obj.insert("mediaType".to_owned(), Value::from(mt));
5286 }
5287 return Ok(Some(Value::Object(obj)));
5288 }
5289 Ok(None)
5290 }
5291
5292 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
5294 let pending: Vec<(String, Option<String>)> = {
5295 let mut stmt = self
5296 .conn
5297 .prepare(
5298 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
5299 )
5300 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5301 let rows = stmt
5302 .query_map([], |row| {
5303 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
5304 })
5305 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5306 rows.filter_map(Result::ok).collect()
5307 };
5308 for (blob_id, media_type) in pending {
5309 let bytes: Option<Vec<u8>> = self
5310 .conn
5311 .query_row(
5312 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
5313 rusqlite::params![blob_id],
5314 |row| row.get(0),
5315 )
5316 .ok();
5317 if let Some(bytes) = bytes {
5318 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
5319 }
5320 let _ = self.conn.execute(
5321 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
5322 rusqlite::params![blob_id],
5323 );
5324 }
5325 Ok(())
5326 }
5327
5328 fn upload_one(
5335 &self,
5336 transport: &mut dyn Transport,
5337 blob_id: &str,
5338 bytes: &[u8],
5339 media_type: Option<&str>,
5340 ) -> Result<(), TransportError> {
5341 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
5342 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
5344 url,
5345 url_expires_at_ms,
5346 } => {
5347 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
5348 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
5349 return Ok(());
5350 }
5351 }
5353 BlobUploadGrant::None => {
5354 }
5356 }
5357 transport.blob_upload(blob_id, bytes, media_type)
5358 }
5359
5360 fn enforce_blob_cache_cap(&self) {
5369 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
5370 return;
5371 };
5372 let mut total: i64 = self
5373 .conn
5374 .query_row(
5375 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
5376 [],
5377 |row| row.get(0),
5378 )
5379 .unwrap_or(0);
5380 if total <= max_bytes {
5381 return;
5382 }
5383 let candidates: Vec<(String, i64)> = {
5384 let Ok(mut stmt) = self.conn.prepare(
5385 "SELECT blob_id, byte_length FROM _syncular_blobs
5386 WHERE refcount = 0
5387 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
5388 ORDER BY last_used_ms ASC, created_at_ms ASC",
5389 ) else {
5390 return;
5391 };
5392 let Ok(rows) = stmt.query_map([], |row| {
5393 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
5394 }) else {
5395 return;
5396 };
5397 rows.filter_map(Result::ok).collect()
5398 };
5399 for (blob_id, byte_length) in candidates {
5400 if total <= max_bytes {
5401 break;
5402 }
5403 let _ = self.conn.execute(
5404 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
5405 rusqlite::params![blob_id],
5406 );
5407 total -= byte_length;
5408 }
5409 }
5410
5411 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
5415 if !self.schema_has_blobs() {
5416 return;
5417 }
5418 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
5419 for table in self.schema.tables.clone() {
5420 let blob_cols: Vec<String> = table
5421 .columns
5422 .iter()
5423 .filter(|c| c.ty == ColumnType::BlobRef)
5424 .map(|c| c.name.clone())
5425 .collect();
5426 for column in blob_cols {
5427 let sql = format!(
5428 "SELECT {} FROM {} WHERE {} IS NOT NULL",
5429 quote_ident(&column),
5430 base_table(&table.name),
5431 quote_ident(&column)
5432 );
5433 let Ok(mut stmt) = self.conn.prepare(&sql) else {
5434 continue;
5435 };
5436 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
5437 continue;
5438 };
5439 for raw in rows.flatten().flatten() {
5440 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
5441 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
5442 *counts.entry(id.to_owned()).or_insert(0) += 1;
5443 }
5444 }
5445 }
5446 }
5447 }
5448 let _ = self
5449 .conn
5450 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
5451 for (blob_id, count) in &counts {
5452 let _ = self.conn.execute(
5453 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
5454 rusqlite::params![count, blob_id],
5455 );
5456 }
5457 if delete_orphans {
5458 let _ = self.conn.execute(
5459 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
5460 [],
5461 );
5462 }
5463 }
5464
5465 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
5470 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
5471 return sql.clone();
5472 }
5473 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
5474 columns.push(quote_ident("_syncular_version"));
5475 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
5476 let primary_key = quote_ident(&table.primary_key);
5477 let updates = columns
5478 .iter()
5479 .filter(|column| **column != primary_key)
5480 .map(|column| format!("{column}=excluded.{column}"))
5481 .collect::<Vec<_>>()
5482 .join(", ");
5483 let sql = format!(
5484 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
5485 columns.join(", "),
5486 placeholders.join(", ")
5487 );
5488 self.insert_sql
5489 .borrow_mut()
5490 .insert(full_table.to_owned(), sql.clone());
5491 sql
5492 }
5493
5494 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
5495 self.overlay_dirty.set(true);
5496 self.write_row(&base_table(table_name), table_name, row, version)
5497 }
5498
5499 fn write_row(
5500 &self,
5501 full_table: &str,
5502 table_name: &str,
5503 row: &Row,
5504 version: i64,
5505 ) -> Result<(), String> {
5506 let table = self
5507 .schema
5508 .table(table_name)
5509 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
5510 let sql = self.insert_row_sql(full_table, table);
5511 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
5512 let params = row
5513 .iter()
5514 .map(RowParam::Cell)
5515 .chain(std::iter::once(RowParam::Version(version)));
5516 stmt.execute(rusqlite::params_from_iter(params))
5517 .map_err(|e| e.to_string())?;
5518 Ok(())
5519 }
5520
5521 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
5522 let table = self
5523 .schema
5524 .table(table_name)
5525 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
5526 self.overlay_dirty.set(true);
5527 let sql = format!(
5528 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
5529 base_table(table_name),
5530 quote_ident(&table.primary_key)
5531 );
5532 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
5533 stmt.execute(rusqlite::params![row_id])
5534 .map_err(|e| e.to_string())?;
5535 Ok(())
5536 }
5537
5538 fn rebuild_overlay_if_dirty(&mut self) {
5541 if self.overlay_dirty.get() {
5542 self.rebuild_overlay();
5543 }
5544 }
5545
5546 fn rebuild_overlay(&mut self) {
5550 self.exec("SAVEPOINT syncular_overlay");
5551 for table in self.schema.tables.clone() {
5552 for index in &table.fts_indexes {
5553 let _ = self.drop_fts_triggers(index);
5554 }
5555 let visible = visible_table(&table.name);
5556 let base = base_table(&table.name);
5557 self.exec(&format!("DELETE FROM {visible}"));
5558 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
5559 }
5560 for commit in self.outbox.clone() {
5561 for op in &commit.ops {
5562 let Some(table) = self.schema.table(&op.table).cloned() else {
5563 continue;
5564 };
5565 if op.upsert {
5566 let Some(values) = op.values.as_ref() else {
5567 continue;
5568 };
5569 let mut row: Row = Vec::with_capacity(table.columns.len());
5570 let mut ok = true;
5571 for column in &table.columns {
5572 match json_to_column_value(column, values.get(&column.name)) {
5573 Ok(v) => row.push(v),
5574 Err(_) => {
5575 ok = false;
5576 break;
5577 }
5578 }
5579 }
5580 if ok {
5581 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
5582 }
5583 } else {
5584 let sql = format!(
5585 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
5586 visible_table(&table.name),
5587 quote_ident(&table.primary_key)
5588 );
5589 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
5590 }
5591 }
5592 }
5593 for table in self.schema.tables.clone() {
5594 for index in &table.fts_indexes {
5595 let _ = self.rebuild_fts_projection(&table, index);
5596 let _ = self.create_fts_triggers(&table, index);
5597 }
5598 }
5599 self.exec("RELEASE syncular_overlay");
5600 self.overlay_dirty.set(false);
5601 }
5602
5603 fn exec(&self, sql: &str) {
5604 let _ = self.conn.execute_batch(sql);
5605 }
5606
5607 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
5610 transport
5611 .realtime_connect_for_client(&self.client_id)
5612 .map_err(|e| format!("{}: {}", e.code, e.message))?;
5613 self.realtime_connected = true;
5614 Ok(())
5615 }
5616
5617 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
5618 let _ = transport.realtime_close();
5619 self.realtime_connected = false;
5620 self.presence.clear(); }
5622
5623 pub fn set_presence(
5629 &mut self,
5630 transport: &mut dyn Transport,
5631 scope_key: &str,
5632 doc: Option<&Value>,
5633 ) -> Result<(), String> {
5634 if !self.realtime_connected {
5635 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
5636 }
5637 let text = encode_presence_publish(scope_key, doc);
5638 transport
5639 .realtime_send(&text)
5640 .map_err(|e| format!("{}: {}", e.code, e.message))
5641 }
5642
5643 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
5645 self.presence
5646 .get(scope_key)
5647 .map(|peers| peers.values().cloned().collect())
5648 .unwrap_or_default()
5649 }
5650
5651 fn apply_presence(
5653 &mut self,
5654 scope_key: String,
5655 kind: Option<PresenceKind>,
5656 actor_id: Option<String>,
5657 client_id: Option<String>,
5658 doc: Option<Value>,
5659 error: Option<String>,
5660 ) {
5661 if error.is_some() {
5664 return;
5665 }
5666 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
5667 return;
5668 };
5669 let peer_key = format!("{actor_id} {client_id}");
5670 match kind {
5671 PresenceKind::Leave => {
5672 if let Some(peers) = self.presence.get_mut(&scope_key) {
5673 peers.remove(&peer_key);
5674 if peers.is_empty() {
5675 self.presence.remove(&scope_key);
5676 }
5677 }
5678 }
5679 _ => {
5680 let doc = match doc {
5681 Some(Value::Object(_)) => doc.unwrap(),
5682 _ => return,
5683 };
5684 self.presence.entry(scope_key).or_default().insert(
5685 peer_key,
5686 PresencePeer {
5687 actor_id,
5688 client_id,
5689 doc,
5690 },
5691 );
5692 }
5693 }
5694 }
5695
5696 pub fn on_realtime_text(&mut self, text: &str) {
5698 match parse_control(text) {
5699 Ok(ControlMessage::Hello { requires_sync, .. }) => {
5700 if requires_sync {
5701 self.set_sync_needed(true, true);
5703 }
5704 }
5705 Ok(ControlMessage::Presence {
5706 scope_key,
5707 kind,
5708 actor_id,
5709 client_id,
5710 doc,
5711 error,
5712 ..
5713 }) => {
5714 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
5715 }
5716 Ok(ControlMessage::Wake { .. }) => {
5717 self.set_sync_needed(true, true);
5719 }
5720 _ => {}
5721 }
5722 }
5723
5724 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
5727 if self.stopped {
5728 return;
5729 }
5730 let message = match decode_message(bytes) {
5731 Ok(m) if m.msg_kind == MsgKind::Response => m,
5732 _ => {
5733 self.set_sync_needed(true, true);
5734 return;
5735 }
5736 };
5737 let mut frames = message.frames.into_iter();
5738 let mut applied_cursor: Option<i64> = None;
5739 let mut any_covered = false;
5740 let mut dropped = false;
5741 while let Some(frame) = frames.next() {
5742 let Frame::SubStart {
5743 id,
5744 status,
5745 effective_scopes,
5746 ..
5747 } = frame
5748 else {
5749 continue;
5750 };
5751 let mut body = Vec::new();
5752 let mut next_cursor: Option<i64> = None;
5753 for inner in frames.by_ref() {
5754 match inner {
5755 Frame::SubEnd {
5756 next_cursor: nc, ..
5757 } => {
5758 next_cursor = Some(nc);
5759 break;
5760 }
5761 Frame::Unknown { .. } => {}
5762 other => body.push(other),
5763 }
5764 }
5765 let Some(next_cursor) = next_cursor else {
5766 dropped = true;
5767 break;
5768 };
5769 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
5770 dropped = true;
5771 continue;
5772 };
5773 let sub = &self.subs[sub_index];
5774 if status != SubStatus::Active
5777 || sub.state != SubState::Active
5778 || sub.bootstrap_state.is_some()
5779 || !sub.synced_once
5780 {
5781 dropped = true;
5782 continue;
5783 }
5784 if next_cursor <= sub.cursor {
5785 any_covered = true;
5787 continue;
5788 }
5789 let previous_effective = self.subs[sub_index].effective.clone();
5790 let previous_cursor = self.subs[sub_index].cursor;
5791 if self.begin_observation("syncular_delta").is_err() {
5792 dropped = true;
5793 continue;
5794 }
5795 self.subs[sub_index].effective = Some(effective_scopes);
5796 let mut batch = ChangeAccumulator::default();
5797 let mut failed = false;
5798 for inner in body {
5799 if let Frame::Commit {
5800 tables, changes, ..
5801 } = inner
5802 {
5803 self.record_commit_changes(&mut batch, &tables, &changes);
5804 if self.apply_commit_changes(&tables, &changes).is_err() {
5805 failed = true;
5806 break;
5807 }
5808 }
5809 }
5810 if failed {
5811 self.rollback_observation("syncular_delta");
5812 self.subs[sub_index].effective = previous_effective;
5813 self.subs[sub_index].cursor = previous_cursor;
5814 self.overlay_dirty.set(true);
5815 self.rebuild_overlay();
5816 dropped = true;
5817 continue;
5818 }
5819 let sub = &mut self.subs[sub_index];
5820 sub.cursor = next_cursor;
5821 self.persist_sub(&self.subs[sub_index].clone());
5822 self.rebuild_overlay_if_dirty();
5823 if self.finish_observation("syncular_delta", batch).is_err() {
5824 self.rollback_observation("syncular_delta");
5825 self.subs[sub_index].effective = previous_effective;
5826 self.subs[sub_index].cursor = previous_cursor;
5827 self.overlay_dirty.set(true);
5828 self.rebuild_overlay();
5829 dropped = true;
5830 continue;
5831 }
5832 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
5833 }
5834 if let Some(cursor) = applied_cursor {
5835 self.reconcile_blob_refcounts(false);
5836 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
5838 let _ = transport.realtime_send(&ack);
5839 } else if !any_covered || dropped {
5840 self.set_sync_needed(true, true);
5842 }
5843 }
5844
5845 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
5849 if !self.realtime_connected {
5850 return;
5851 }
5852 let floor = self
5853 .subs
5854 .iter()
5855 .filter(|s| {
5856 s.state == SubState::Active
5857 && s.bootstrap_state.is_none()
5858 && s.synced_once
5859 && s.cursor >= 0
5860 })
5861 .map(|s| s.cursor)
5862 .min();
5863 if let Some(cursor) = floor {
5864 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
5865 let _ = transport.realtime_send(&ack);
5866 }
5867 }
5868}