1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, CommitOperation, CommitOperationOutcome,
27 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
28 ConflictRecord, CoverageSnapshot, LeaseState, Mutation, PresencePeer, QuerySnapshot,
29 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
30 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
31 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
32};
33use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
34use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
35use crate::values::{
36 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
37 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
38 scope_map_to_json, sort_scope_map,
39};
40
41const DEFAULT_ACCEPT: u8 = 0b0111;
46const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
47const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
48const ACCEPT_SQLITE: u8 = 1 << 2;
49const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
50
51const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
53const LOCAL_REVISION_KEY: &str = "localRevision";
54const CLIENT_ID_KEY: &str = "clientId";
55const LEASE_STATE_KEY: &str = "leaseState";
56const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
57const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62enum SubState {
63 Active,
64 Revoked,
65 Failed,
66}
67
68#[cfg(test)]
69mod observation_tests {
70 use super::*;
71 use serde_json::json;
72
73 fn client() -> SyncClient {
74 SyncClient::new(
75 "retry-test".to_owned(),
76 &json!({
77 "version": 1,
78 "tables": [{
79 "name": "tasks",
80 "primaryKey": "id",
81 "columns": [
82 { "name": "id", "type": "string", "nullable": false },
83 { "name": "project_id", "type": "string", "nullable": false }
84 ],
85 "scopes": [{ "pattern": "project:{project_id}" }]
86 }]
87 }),
88 ClientLimits::default(),
89 )
90 .expect("test client")
91 }
92
93 #[test]
94 fn background_retry_deadlines_back_off_and_reset() {
95 let mut client = client();
96 client.schedule_background_retry();
97 assert!(matches!(
98 client.drain_sync_intents().as_slice(),
99 [SyncIntent::Background { delay_ms: 250 }]
100 ));
101 client.schedule_background_retry();
102 assert!(matches!(
103 client.drain_sync_intents().as_slice(),
104 [SyncIntent::Background { delay_ms: 500 }]
105 ));
106 client.reset_background_retry();
107 client.schedule_background_retry();
108 assert!(matches!(
109 client.drain_sync_intents().as_slice(),
110 [SyncIntent::Background { delay_ms: 250 }]
111 ));
112 }
113
114 #[test]
115 fn secondary_unique_collision_preserves_existing_synced_row() {
116 let client = SyncClient::new(
117 "unique-upsert-test".to_owned(),
118 &json!({
119 "version": 1,
120 "tables": [{
121 "name": "tasks",
122 "primaryKey": "id",
123 "columns": [
124 { "name": "id", "type": "string", "nullable": false },
125 { "name": "project_id", "type": "string", "nullable": false },
126 { "name": "title", "type": "string", "nullable": false }
127 ],
128 "scopes": [{ "pattern": "project:{project_id}" }],
129 "indexes": [{
130 "name": "tasks_by_project_title",
131 "columns": ["project_id", "title"],
132 "unique": true
133 }]
134 }]
135 }),
136 ClientLimits::default(),
137 )
138 .expect("test client");
139 let table = client.schema.table("tasks").expect("tasks table");
140 let sql = client.insert_row_sql(&base_table("tasks"), table);
141
142 client
143 .conn
144 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
145 .expect("insert first row");
146 client
147 .conn
148 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
149 .expect("update same primary key");
150 client
151 .conn
152 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
153 .expect("insert second row");
154 assert!(client
155 .conn
156 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
157 .is_err());
158
159 let rows = client
160 .conn
161 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
162 .expect("prepare rows")
163 .query_map([], |row| {
164 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
165 })
166 .expect("query rows")
167 .collect::<Result<Vec<_>, _>>()
168 .expect("collect rows");
169 assert_eq!(
170 rows,
171 vec![
172 ("t1".to_owned(), "updated".to_owned()),
173 ("t2".to_owned(), "original".to_owned())
174 ]
175 );
176 }
177
178 #[test]
179 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
180 let path = std::env::temp_dir().join(format!(
181 "syncular-startup-intent-{}.db",
182 uuid::Uuid::new_v4()
183 ));
184 let schema = json!({
185 "version": 1,
186 "tables": [{
187 "name": "tasks",
188 "primaryKey": "id",
189 "columns": [
190 { "name": "id", "type": "string", "nullable": false },
191 { "name": "project_id", "type": "string", "nullable": false }
192 ],
193 "scopes": [{ "pattern": "project:{project_id}" }]
194 }]
195 });
196
197 {
198 let mut first = SyncClient::open_path_with_identity(
199 None,
200 &schema,
201 ClientLimits::default(),
202 path.to_str().expect("UTF-8 temp path"),
203 )
204 .expect("first open");
205 first
206 .set_window(
207 &WindowBase {
208 table: "tasks".to_owned(),
209 variable: "project_id".to_owned(),
210 fixed_scopes: Vec::new(),
211 params: None,
212 },
213 &["persisted".to_owned()],
214 )
215 .expect("persist window");
216 }
217
218 let mut reopened = SyncClient::open_path_with_identity(
219 None,
220 &schema,
221 ClientLimits::default(),
222 path.to_str().expect("UTF-8 temp path"),
223 )
224 .expect("reopen");
225 assert!(reopened.sync_needed());
226 assert!(matches!(
227 reopened.drain_sync_intents().as_slice(),
228 [SyncIntent::Interactive]
229 ));
230 drop(reopened);
231 std::fs::remove_file(path).expect("remove temp database");
232 }
233
234 #[test]
235 fn migrates_pre_envelope_outcome_journal_additively() {
236 let path = std::env::temp_dir().join(format!(
237 "syncular-outcome-migration-{}.db",
238 uuid::Uuid::new_v4()
239 ));
240 let conn = Connection::open(&path).expect("open legacy database");
241 conn.execute_batch(
242 "CREATE TABLE _syncular_commit_outcomes (
243 seq INTEGER PRIMARY KEY AUTOINCREMENT,
244 client_commit_id TEXT NOT NULL UNIQUE,
245 status TEXT NOT NULL,
246 recorded_at_ms INTEGER NOT NULL,
247 results_json TEXT NOT NULL,
248 resolution TEXT NOT NULL DEFAULT 'active',
249 resolved_at_ms INTEGER,
250 replacement_client_commit_id TEXT);",
251 )
252 .expect("create legacy outcome journal");
253 drop(conn);
254 let schema = json!({
255 "version": 1,
256 "tables": [{
257 "name": "tasks",
258 "primaryKey": "id",
259 "columns": [
260 { "name": "id", "type": "string", "nullable": false },
261 { "name": "project_id", "type": "string", "nullable": false }
262 ],
263 "scopes": [{ "pattern": "project:{project_id}" }]
264 }]
265 });
266 let client = SyncClient::open_path(
267 "migration-native".to_owned(),
268 &schema,
269 ClientLimits::default(),
270 path.to_str().expect("UTF-8 temp path"),
271 )
272 .expect("migrate database");
273 let has_operations = client
274 .conn
275 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
276 .expect("prepare table info")
277 .query_map([], |row| row.get::<_, String>(1))
278 .expect("query table info")
279 .filter_map(Result::ok)
280 .any(|column| column == "operations_json");
281 assert!(has_operations);
282 drop(client);
283 std::fs::remove_file(path).expect("remove temp database");
284 }
285
286 #[test]
287 fn durable_conflict_outcome_and_resolution_survive_reopen() {
288 let path = std::env::temp_dir().join(format!(
289 "syncular-durable-outcome-{}.db",
290 uuid::Uuid::new_v4()
291 ));
292 let schema = json!({
293 "version": 1,
294 "tables": [{
295 "name": "tasks",
296 "primaryKey": "id",
297 "columns": [
298 { "name": "id", "type": "string", "nullable": false },
299 { "name": "project_id", "type": "string", "nullable": false }
300 ],
301 "scopes": [{ "pattern": "project:{project_id}" }]
302 }]
303 });
304 let conflict = ConflictRecord {
305 client_commit_id: "losing-commit".to_owned(),
306 op_index: 0,
307 table: "tasks".to_owned(),
308 row_id: "t1".to_owned(),
309 code: "sync.version_conflict".to_owned(),
310 message: "stale base version".to_owned(),
311 server_version: 2,
312 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
313 operation: Some(CommitOperation {
314 table: "tasks".to_owned(),
315 row_id: "t1".to_owned(),
316 op: "upsert".to_owned(),
317 base_version: Some(1),
318 values: None,
319 changed_fields: None,
320 }),
321 };
322 let failed_operations = vec![
323 OutboxOp {
324 upsert: true,
325 table: "tasks".to_owned(),
326 row_id: "t1".to_owned(),
327 base_version: Some(1),
328 values: None,
329 changed_fields: None,
330 },
331 OutboxOp {
332 upsert: true,
333 table: "tasks".to_owned(),
334 row_id: "status-event-1".to_owned(),
335 base_version: Some(0),
336 values: None,
337 changed_fields: None,
338 },
339 ];
340
341 {
342 let mut first = SyncClient::open_path(
343 "durable-native".to_owned(),
344 &schema,
345 ClientLimits::default(),
346 path.to_str().expect("UTF-8 temp path"),
347 )
348 .expect("first open");
349 first
350 .begin_observation("test_outcome")
351 .expect("begin outcome");
352 first
353 .persist_commit_outcome(
354 "losing-commit",
355 CommitOutcomeStatus::Conflict,
356 &[CommitOperationOutcome::Conflict {
357 conflict: conflict.clone(),
358 }],
359 Some(&failed_operations),
360 )
361 .expect("persist outcome");
362 first.conflicts.push(conflict);
363 first
364 .finish_observation(
365 "test_outcome",
366 ChangeAccumulator {
367 conflicts: true,
368 outcomes: true,
369 ..ChangeAccumulator::default()
370 },
371 )
372 .expect("commit outcome");
373 }
374
375 {
376 let mut reopened = SyncClient::open_path(
377 "durable-native".to_owned(),
378 &schema,
379 ClientLimits::default(),
380 path.to_str().expect("UTF-8 temp path"),
381 )
382 .expect("reopen");
383 assert_eq!(reopened.conflicts().len(), 1);
384 let outcome = reopened
385 .commit_outcome("losing-commit")
386 .expect("read outcome")
387 .expect("outcome");
388 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
389 let operations = outcome.operations.expect("aggregate envelope");
390 assert_eq!(operations.len(), 2);
391 assert_eq!(operations[1].row_id, "status-event-1");
392 let resolved = reopened
393 .resolve_commit_outcome(ResolveCommitOutcomeInput {
394 client_commit_id: "losing-commit".to_owned(),
395 resolution: CommitOutcomeResolution::ResolvedKeepServer,
396 replacement_client_commit_id: None,
397 })
398 .expect("resolve");
399 assert_eq!(
400 resolved.resolution,
401 CommitOutcomeResolution::ResolvedKeepServer
402 );
403 assert!(reopened.conflicts().is_empty());
404 }
405
406 let reopened = SyncClient::open_path(
407 "durable-native".to_owned(),
408 &schema,
409 ClientLimits::default(),
410 path.to_str().expect("UTF-8 temp path"),
411 )
412 .expect("second reopen");
413 assert!(reopened.conflicts().is_empty());
414 assert_eq!(
415 reopened
416 .commit_outcome("losing-commit")
417 .expect("read outcome")
418 .expect("outcome")
419 .resolution,
420 CommitOutcomeResolution::ResolvedKeepServer
421 );
422 drop(reopened);
423 std::fs::remove_file(path).expect("remove temp database");
424 }
425
426 #[test]
427 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
428 let path =
429 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
430 let schema = json!({
431 "version": 1,
432 "tables": [{
433 "name": "tasks",
434 "primaryKey": "id",
435 "columns": [
436 { "name": "id", "type": "string", "nullable": false },
437 { "name": "project_id", "type": "string", "nullable": false }
438 ],
439 "scopes": [{ "pattern": "project:{project_id}" }]
440 }]
441 });
442 let mut client = SyncClient::open_path_with_identity(
443 Some("sidecar-client".to_owned()),
444 &schema,
445 ClientLimits::default(),
446 path.to_str().expect("UTF-8 temp path"),
447 )
448 .expect("open owner");
449 let base = WindowBase {
450 table: "tasks".to_owned(),
451 variable: "project_id".to_owned(),
452 fixed_scopes: Vec::new(),
453 params: None,
454 };
455 client
456 .set_window(&base, &["one".to_owned()])
457 .expect("set window");
458 client
459 .mutate(vec![Mutation::Upsert {
460 table: "tasks".to_owned(),
461 values: Map::from_iter([
462 ("id".to_owned(), Value::from("t1")),
463 ("project_id".to_owned(), Value::from("one")),
464 ]),
465 base_version: None,
466 }])
467 .expect("local mutate");
468
469 let coverage = [WindowCoverage {
470 base,
471 units: vec!["one".to_owned(), "missing".to_owned()],
472 }];
473 let owner = client
474 .query_snapshot(
475 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
476 &[],
477 &coverage,
478 )
479 .expect("owner snapshot");
480 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
481 let sidecar = reader
482 .query_snapshot(
483 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
484 &[],
485 &coverage,
486 )
487 .expect("sidecar snapshot");
488
489 assert_eq!(sidecar.revision, owner.revision);
490 assert_eq!(sidecar.rows, owner.rows);
491 assert_eq!(
492 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
493 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
494 );
495 assert_eq!(sidecar.revision, "2");
496 assert_eq!(sidecar.rows[0]["id"], "t1");
497 assert_eq!(sidecar.rows[0]["server_version"], -1);
498 assert!(!sidecar.coverage.complete);
499 assert_eq!(sidecar.coverage.pending.len(), 1);
500 assert_eq!(sidecar.coverage.missing.len(), 1);
501
502 drop(reader);
503 drop(client);
504 std::fs::remove_file(path).expect("remove temp database");
505 }
506
507 #[test]
508 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
509 let schema = json!({
510 "version": 1,
511 "tables": [{
512 "name": "catalogue_codes",
513 "primaryKey": "id",
514 "columns": [
515 { "name": "id", "type": "string", "nullable": false },
516 { "name": "release_id", "type": "string", "nullable": false },
517 { "name": "code", "type": "string", "nullable": false },
518 { "name": "title", "type": "string", "nullable": false }
519 ],
520 "scopes": [{ "pattern": "release:{release_id}" }],
521 "ftsIndexes": [{
522 "name": "catalogue_codes_fts",
523 "columns": ["code", "title"],
524 "tokenize": "unicode61 remove_diacritics 2"
525 }]
526 }]
527 });
528 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
529 .expect("FTS5 client");
530 let search = |client: &SyncClient, query: &str| {
531 client
532 .query(
533 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
534 &[Value::from(query)],
535 )
536 .expect("FTS query")
537 };
538
539 client
540 .mutate(vec![Mutation::Upsert {
541 table: "catalogue_codes".to_owned(),
542 values: Map::from_iter([
543 ("id".to_owned(), Value::from("c1")),
544 ("release_id".to_owned(), Value::from("r1")),
545 ("code".to_owned(), Value::from("A01")),
546 ("title".to_owned(), Value::from("Cholera")),
547 ]),
548 base_version: None,
549 }])
550 .expect("insert code");
551 assert_eq!(search(&client, "cholera").len(), 1);
552
553 client
554 .mutate(vec![Mutation::Upsert {
555 table: "catalogue_codes".to_owned(),
556 values: Map::from_iter([
557 ("id".to_owned(), Value::from("c1")),
558 ("release_id".to_owned(), Value::from("r1")),
559 ("code".to_owned(), Value::from("A01")),
560 ("title".to_owned(), Value::from("Enteric infection")),
561 ]),
562 base_version: None,
563 }])
564 .expect("update code");
565 assert!(search(&client, "cholera").is_empty());
566 assert_eq!(search(&client, "enteric").len(), 1);
567
568 client
569 .mutate(vec![Mutation::Delete {
570 table: "catalogue_codes".to_owned(),
571 row_id: "c1".to_owned(),
572 base_version: None,
573 }])
574 .expect("delete code");
575 assert!(search(&client, "enteric").is_empty());
576 }
577}
578
579impl SubState {
580 fn name(self) -> &'static str {
581 match self {
582 SubState::Active => "active",
583 SubState::Revoked => "revoked",
584 SubState::Failed => "failed",
585 }
586 }
587
588 fn parse(value: &str) -> Self {
589 match value {
590 "revoked" => Self::Revoked,
591 "failed" => Self::Failed,
592 _ => Self::Active,
593 }
594 }
595}
596
597#[derive(Debug, Clone)]
598struct Subscription {
599 id: String,
600 table: String,
601 requested: Vec<(String, Vec<String>)>,
602 params: Option<String>,
603 cursor: i64,
604 bootstrap_state: Option<String>,
606 state: SubState,
607 reason_code: Option<String>,
608 effective: Option<Vec<(String, Vec<String>)>>,
611 synced_once: bool,
612}
613
614#[derive(Debug, Clone)]
615struct OutboxOp {
616 upsert: bool,
617 table: String,
618 row_id: String,
619 base_version: Option<i64>,
620 values: Option<Map<String, Value>>,
623 changed_fields: Option<Vec<String>>,
625}
626
627impl From<&OutboxOp> for CommitOperation {
628 fn from(operation: &OutboxOp) -> Self {
629 Self {
630 table: operation.table.clone(),
631 row_id: operation.row_id.clone(),
632 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
633 base_version: operation.base_version,
634 values: operation.values.clone(),
635 changed_fields: operation.changed_fields.clone(),
636 }
637 }
638}
639
640#[derive(Debug, Clone)]
641struct OutboxCommit {
642 client_commit_id: String,
643 ops: Vec<OutboxOp>,
644}
645
646struct StoredCommitOutcomeRow {
647 sequence: i64,
648 client_commit_id: String,
649 status: String,
650 recorded_at_ms: i64,
651 results_json: String,
652 operations_json: Option<String>,
653 resolution: String,
654 resolved_at_ms: Option<i64>,
655 replacement_client_commit_id: Option<String>,
656}
657
658enum SectionError {
661 FailClosed,
662 Abort(String, String),
663}
664
665struct RequestMeta {
666 pushed_ids: Vec<String>,
667 fresh: Vec<(String, bool)>,
670 accept: u8,
671 deferred_commits: usize,
675}
676
677#[derive(Default)]
678struct ChangeAccumulator {
679 tables: BTreeMap<String, Option<BTreeSet<String>>>,
681 windows: BTreeMap<(String, String), BTreeSet<String>>,
682 status: bool,
683 conflicts: bool,
684 rejections: bool,
685 outcomes: bool,
686}
687
688impl ChangeAccumulator {
689 fn table(&mut self, table: &str) {
690 self.tables.insert(table.to_owned(), None);
691 }
692
693 fn scope(&mut self, table: &str, key: String) {
694 match self.tables.get_mut(table) {
695 Some(None) => {}
696 Some(Some(keys)) => {
697 keys.insert(key);
698 }
699 None => {
700 self.tables
701 .insert(table.to_owned(), Some(BTreeSet::from([key])));
702 }
703 }
704 }
705
706 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
707 self.windows
708 .entry((base_key.to_owned(), table.to_owned()))
709 .or_default()
710 .insert(unit.to_owned());
711 }
712
713 fn touched(&self) -> bool {
714 !self.tables.is_empty()
715 || !self.windows.is_empty()
716 || self.status
717 || self.conflicts
718 || self.rejections
719 || self.outcomes
720 }
721}
722
723const PUSH_OPS_PER_REQUEST: usize = 500;
729
730pub struct SyncClient {
731 conn: Connection,
732 schema: ClientSchema,
733 client_id: String,
734 limits: ClientLimits,
735 subs: Vec<Subscription>,
736 outbox: Vec<OutboxCommit>,
737 conflicts: Vec<ConflictRecord>,
738 rejections: Vec<RejectionRecord>,
739 schema_floor: Option<SchemaFloor>,
740 lease_state: Option<LeaseState>,
742 stopped: bool,
744 upgrading: bool,
746 sync_needed: bool,
748 realtime_connected: bool,
749 presence: HashMap<String, HashMap<String, PresencePeer>>,
751 now_ms: Option<i64>,
754 encryption: crate::values::EncryptionConfig,
758 insert_sql: RefCell<HashMap<String, String>>,
764 overlay_dirty: Cell<bool>,
768 change_queue: VecDeque<ClientChangeBatch>,
770 sync_intent_queue: VecDeque<SyncIntent>,
771 retry_delay_ms: u64,
773}
774
775fn quote_ident(name: &str) -> String {
776 format!("\"{}\"", name.replace('"', "\"\""))
777}
778
779fn base_table(name: &str) -> String {
780 quote_ident(&format!("_syncular_base_{name}"))
781}
782
783type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
788
789fn window_base_key(base: &WindowBase) -> String {
790 format!(
791 "{}\0{}\0{}",
792 base.table,
793 base.variable,
794 canonical_scope_json(&base.fixed_scopes)
795 )
796}
797
798fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
800 let mut scopes = base.fixed_scopes.clone();
801 scopes.retain(|(k, _)| k != &base.variable);
802 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
803 scopes
804}
805
806fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
810 let canonical = canonical_scope_json(&unit_scopes(base, unit));
811 let digest = Sha256::digest(canonical.as_bytes());
812 let hex = bytes_to_hex(&digest);
813 format!("w:{}:{}", base.table, &hex[..16])
814}
815
816fn visible_table(name: &str) -> String {
817 quote_ident(name)
818}
819
820const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
821
822fn is_synced_table_name(name: &str) -> bool {
825 if name.starts_with("sqlite_") {
826 return false;
827 }
828 if name.starts_with("_syncular_base_") {
829 return true; }
831 !name.starts_with("_syncular_")
833}
834
835fn blob_id_for(bytes: &[u8]) -> String {
837 let digest = Sha256::digest(bytes);
838 format!("sha256:{}", bytes_to_hex(&digest))
839}
840
841enum RowParam<'a> {
845 Cell(&'a Option<ColumnValue>),
846 Version(i64),
847}
848
849impl rusqlite::ToSql for RowParam<'_> {
850 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
851 Ok(match self {
852 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
853 RowParam::Cell(cell) => match cell {
854 None => ToSqlOutput::Owned(SqlValue::Null),
855 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
856 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
857 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
858 Some(ColumnValue::Boolean(b)) => {
859 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
860 }
861 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
862 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
863 }
864 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
866 ToSqlOutput::Borrowed(ValueRef::Blob(b))
867 }
868 },
869 })
870 }
871}
872
873fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
881 use ssp2::segment::ColumnType;
882 let mismatch = || {
883 Err(format!(
884 "image column {:?} holds a value of the wrong type",
885 column.name
886 ))
887 };
888 match value {
889 ValueRef::Null => {
890 if !column.nullable {
891 return Err(format!(
892 "image column {:?} is NULL but not nullable",
893 column.name
894 ));
895 }
896 Ok(ToSqlOutput::Owned(SqlValue::Null))
897 }
898 ValueRef::Integer(i) => match column.ty {
899 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
900 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
901 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
902 _ => mismatch(),
903 },
904 ValueRef::Real(_) => match column.ty {
905 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
906 _ => mismatch(),
907 },
908 ValueRef::Text(t) => {
909 std::str::from_utf8(t)
910 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
911 match column.ty {
912 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
913 Ok(ToSqlOutput::Borrowed(value))
914 }
915 _ => mismatch(),
916 }
917 }
918 ValueRef::Blob(_) => match column.ty {
919 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
921 _ => mismatch(),
922 },
923 }
924}
925
926fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
927 use rusqlite::types::ValueRef;
928 match value {
929 ValueRef::Null => Value::Null,
930 ValueRef::Integer(i) => match column.ty {
931 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
932 ssp2::segment::ColumnType::Float => {
933 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
934 }
935 _ => Value::from(i),
936 },
937 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
938 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
939 ValueRef::Blob(b) => {
940 let mut map = Map::new();
941 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
942 Value::Object(map)
943 }
944 }
945}
946
947fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
951 Ok(match value {
952 Value::Null => SqlValue::Null,
953 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
954 Value::Number(n) => {
955 if let Some(i) = n.as_i64() {
956 SqlValue::Integer(i)
957 } else if let Some(f) = n.as_f64() {
958 SqlValue::Real(f)
959 } else {
960 return Err(format!("query param number {n} is out of range"));
961 }
962 }
963 Value::String(s) => SqlValue::Text(s.clone()),
964 Value::Object(_) => {
965 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
966 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
967 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
968 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
969 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
970 })?)
971 } else {
972 return Err(
973 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
974 .to_owned(),
975 );
976 }
977 }
978 Value::Array(_) => return Err("query array params are not supported".to_owned()),
979 })
980}
981
982fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
987 use rusqlite::types::ValueRef;
988 match value {
989 ValueRef::Null => Value::Null,
990 ValueRef::Integer(i) => {
991 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
995 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
996 Value::from(i)
997 } else {
998 let mut map = Map::new();
999 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1000 Value::Object(map)
1001 }
1002 }
1003 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1004 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1005 ValueRef::Blob(b) => {
1006 let mut map = Map::new();
1007 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1008 Value::Object(map)
1009 }
1010 }
1011}
1012
1013fn query_connection(
1014 conn: &Connection,
1015 sql: &str,
1016 params: &[Value],
1017) -> Result<Vec<Map<String, Value>>, String> {
1018 crate::query_guard::assert_read_only_query(sql)?;
1019 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1020 let bound: Vec<SqlValue> = params
1021 .iter()
1022 .map(json_param_to_sql)
1023 .collect::<Result<_, _>>()?;
1024 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
1025 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1026 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1027 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1028 let mut sql_rows = stmt
1029 .query(bound_refs.as_slice())
1030 .map_err(|e| e.to_string())?;
1031 let mut out = Vec::new();
1032 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1033 let mut record = Map::new();
1034 for (i, name) in column_names.iter().enumerate() {
1035 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1036 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1037 }
1038 out.push(record);
1039 }
1040 Ok(out)
1041}
1042
1043fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1044 let mut stmt = conn
1045 .prepare(
1046 "SELECT windows.unit, subscriptions.state_json
1047 FROM _syncular_windows AS windows
1048 JOIN _syncular_subscriptions AS subscriptions
1049 ON subscriptions.id = windows.sub_id
1050 WHERE windows.base = ?1
1051 ORDER BY windows.unit ASC",
1052 )
1053 .map_err(|error| error.to_string())?;
1054 let rows = stmt
1055 .query_map(rusqlite::params![window_base_key(base)], |row| {
1056 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1057 })
1058 .map_err(|error| error.to_string())?;
1059 let mut units = Vec::new();
1060 let mut pending = Vec::new();
1061 for row in rows {
1062 let (unit, raw) = row.map_err(|error| error.to_string())?;
1063 let state: Value = serde_json::from_str(&raw)
1064 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1065 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1066 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1067 || state
1068 .get("bootstrapState")
1069 .is_some_and(|value| !value.is_null());
1070 if is_pending {
1071 pending.push(unit.clone());
1072 }
1073 units.push(unit);
1074 }
1075 Ok(WindowState { units, pending })
1076}
1077
1078fn snapshot_connection(
1079 conn: &Connection,
1080 sql: &str,
1081 params: &[Value],
1082 coverage: &[WindowCoverage],
1083) -> Result<QuerySnapshot, String> {
1084 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1085 .map_err(|error| error.to_string())?;
1086 let result = (|| {
1087 let revision = conn
1088 .query_row(
1089 "SELECT value FROM _syncular_meta WHERE key = ?1",
1090 rusqlite::params![LOCAL_REVISION_KEY],
1091 |row| row.get::<_, String>(0),
1092 )
1093 .ok()
1094 .and_then(|value| value.parse::<u64>().ok())
1095 .unwrap_or(0);
1096 let rows = query_connection(conn, sql, params)?;
1097 let mut pending = Vec::new();
1098 let mut missing = Vec::new();
1099 for requested in coverage {
1100 let base_key = window_base_key(&requested.base);
1101 let state = persisted_window_state(conn, &requested.base)?;
1102 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1103 let reference = WindowUnitRef {
1104 base_key: base_key.clone(),
1105 unit: unit.clone(),
1106 };
1107 if !state.units.iter().any(|held| held == &unit) {
1108 missing.push(reference);
1109 } else if state.pending.iter().any(|held| held == &unit) {
1110 pending.push(reference);
1111 }
1112 }
1113 }
1114 Ok(QuerySnapshot {
1115 revision: revision.to_string(),
1116 rows,
1117 coverage: CoverageSnapshot {
1118 complete: pending.is_empty() && missing.is_empty(),
1119 pending,
1120 missing,
1121 },
1122 })
1123 })();
1124 match result {
1125 Ok(snapshot) => {
1126 conn.execute_batch("RELEASE syncular_snapshot_read")
1127 .map_err(|error| error.to_string())?;
1128 Ok(snapshot)
1129 }
1130 Err(error) => {
1131 let _ = conn.execute_batch(
1132 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1133 );
1134 Err(error)
1135 }
1136 }
1137}
1138
1139pub struct FileQuerySnapshotReader {
1144 path: String,
1145 conn: Option<Connection>,
1146}
1147
1148impl FileQuerySnapshotReader {
1149 #[must_use]
1150 pub fn new(path: impl Into<String>) -> Self {
1151 Self {
1152 path: path.into(),
1153 conn: None,
1154 }
1155 }
1156
1157 fn connection(&mut self) -> Result<&Connection, String> {
1158 if self.conn.is_none() {
1159 let conn = Connection::open_with_flags(
1160 &self.path,
1161 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1162 )
1163 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1164 conn.busy_timeout(std::time::Duration::from_millis(250))
1165 .map_err(|error| error.to_string())?;
1166 self.conn = Some(conn);
1167 }
1168 self.conn
1169 .as_ref()
1170 .ok_or_else(|| "read sidecar connection missing".to_owned())
1171 }
1172
1173 pub fn query_snapshot(
1174 &mut self,
1175 sql: &str,
1176 params: &[Value],
1177 coverage: &[WindowCoverage],
1178 ) -> Result<QuerySnapshot, String> {
1179 snapshot_connection(self.connection()?, sql, params, coverage)
1180 }
1181}
1182
1183impl SyncClient {
1184 pub fn new_with_identity(
1185 client_id: Option<String>,
1186 schema_json: &Value,
1187 limits: ClientLimits,
1188 ) -> Result<Self, String> {
1189 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1190 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1191 Self::with_connection(resolved, schema_json, limits, conn)
1192 }
1193
1194 pub fn new(
1195 client_id: String,
1196 schema_json: &Value,
1197 limits: ClientLimits,
1198 ) -> Result<Self, String> {
1199 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1200 Self::with_connection(client_id, schema_json, limits, conn)
1201 }
1202
1203 pub fn open_path(
1209 client_id: String,
1210 schema_json: &Value,
1211 limits: ClientLimits,
1212 path: &str,
1213 ) -> Result<Self, String> {
1214 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1215 Self::with_connection(client_id, schema_json, limits, conn)
1216 }
1217
1218 pub fn open_path_with_identity(
1219 client_id: Option<String>,
1220 schema_json: &Value,
1221 limits: ClientLimits,
1222 path: &str,
1223 ) -> Result<Self, String> {
1224 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1225 conn.busy_timeout(std::time::Duration::from_millis(250))
1231 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1232 conn.pragma_update(None, "journal_mode", "WAL")
1233 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
1234 let persisted = conn
1235 .query_row(
1236 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
1237 [],
1238 |row| row.get::<_, String>(0),
1239 )
1240 .ok();
1241 let resolved = persisted
1242 .clone()
1243 .or(client_id.clone())
1244 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1245 if let (Some(existing), Some(requested)) = (persisted, client_id) {
1246 if existing != requested {
1247 return Err(format!(
1248 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
1249 ));
1250 }
1251 }
1252 Self::with_connection(resolved, schema_json, limits, conn)
1253 }
1254
1255 #[must_use]
1256 pub fn client_id(&self) -> &str {
1257 &self.client_id
1258 }
1259
1260 pub fn with_connection(
1267 client_id: String,
1268 schema_json: &Value,
1269 limits: ClientLimits,
1270 conn: Connection,
1271 ) -> Result<Self, String> {
1272 if limits.outcome_retention_max_entries == Some(0) {
1273 return Err(
1274 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
1275 );
1276 }
1277 let schema = parse_schema_json(schema_json)?;
1278 let mut client = SyncClient {
1279 conn,
1280 schema,
1281 client_id,
1282 limits,
1283 subs: Vec::new(),
1284 outbox: Vec::new(),
1285 conflicts: Vec::new(),
1286 rejections: Vec::new(),
1287 schema_floor: None,
1288 lease_state: None,
1289 stopped: false,
1290 upgrading: false,
1291 sync_needed: false,
1292 realtime_connected: false,
1293 presence: HashMap::new(),
1294 now_ms: None,
1295 encryption: crate::values::EncryptionConfig::default(),
1296 insert_sql: RefCell::new(HashMap::new()),
1297 overlay_dirty: Cell::new(false),
1298 change_queue: VecDeque::new(),
1299 sync_intent_queue: VecDeque::new(),
1300 retry_delay_ms: 250,
1301 };
1302 client
1306 .conn
1307 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
1308 client.create_tables()?;
1309 match client.get_meta(CLIENT_ID_KEY) {
1310 Some(existing) if existing != client.client_id => {
1311 return Err(format!(
1312 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
1313 client.client_id
1314 ));
1315 }
1316 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
1317 _ => {}
1318 }
1319 client.restore_persisted_state()?;
1320 let marker = client
1321 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1322 .and_then(|value| value.parse::<i32>().ok());
1323 if marker != Some(client.schema.version) {
1324 client.run_schema_reset()?;
1325 } else if !client.outbox.is_empty() {
1326 client.overlay_dirty.set(true);
1329 client.rebuild_overlay();
1330 }
1331 client.enqueue_startup_sync_if_needed();
1337 Ok(client)
1338 }
1339
1340 pub fn set_now_ms(&mut self, now_ms: i64) {
1343 self.now_ms = Some(now_ms);
1344 }
1345
1346 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
1350 self.encryption = encryption;
1351 }
1352
1353 fn clock_now_ms(&self) -> i64 {
1354 self.now_ms.unwrap_or_else(|| {
1355 std::time::SystemTime::now()
1356 .duration_since(std::time::UNIX_EPOCH)
1357 .map(|d| d.as_millis() as i64)
1358 .unwrap_or(0)
1359 })
1360 }
1361
1362 fn create_tables(&self) -> Result<(), String> {
1363 self.create_synced_tables()?;
1364 self.conn
1366 .execute_batch(
1367 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
1368 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1369 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
1370 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
1371 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1372 client_commit_id TEXT NOT NULL UNIQUE,
1373 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
1374 recorded_at_ms INTEGER NOT NULL,
1375 results_json TEXT NOT NULL,
1376 operations_json TEXT,
1377 resolution TEXT NOT NULL DEFAULT 'active'
1378 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
1379 resolved_at_ms INTEGER,
1380 replacement_client_commit_id TEXT);
1381 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
1382 ON _syncular_commit_outcomes(resolution, seq);
1383 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
1384 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
1385 CREATE TABLE IF NOT EXISTS _syncular_meta (
1386 key TEXT PRIMARY KEY, value TEXT NOT NULL);
1387 CREATE TABLE IF NOT EXISTS _syncular_windows (
1388 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
1389 PRIMARY KEY (base, unit));
1390 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
1391 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
1392 effective_scopes TEXT NOT NULL);",
1393 )
1394 .map_err(|e| e.to_string())?;
1395 let _ = self
1398 .conn
1399 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
1400 if self.get_meta(LOCAL_SCHEMA_VERSION_KEY).is_none() {
1403 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
1404 }
1405 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
1406 self.set_meta(LOCAL_REVISION_KEY, "0");
1407 }
1408 if self.schema_has_blobs() {
1413 self.conn
1414 .execute_batch(
1415 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
1416 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
1417 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
1418 created_at_ms INTEGER NOT NULL,
1419 last_used_ms INTEGER NOT NULL DEFAULT 0);
1420 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
1421 media_type TEXT, created_at_ms INTEGER NOT NULL);",
1422 )
1423 .map_err(|e| e.to_string())?;
1424 let _ = self.conn.execute_batch(
1428 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
1429 );
1430 }
1431 Ok(())
1432 }
1433
1434 fn schema_has_blobs(&self) -> bool {
1436 self.schema
1437 .tables
1438 .iter()
1439 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
1440 }
1441
1442 fn get_meta(&self, key: &str) -> Option<String> {
1445 self.conn
1446 .query_row(
1447 "SELECT value FROM _syncular_meta WHERE key = ?1",
1448 rusqlite::params![key],
1449 |row| row.get::<_, String>(0),
1450 )
1451 .ok()
1452 }
1453
1454 fn set_meta(&self, key: &str, value: &str) {
1455 let _ = self.conn.execute(
1456 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
1457 rusqlite::params![key, value],
1458 );
1459 }
1460
1461 fn delete_meta(&self, key: &str) {
1462 let _ = self.conn.execute(
1463 "DELETE FROM _syncular_meta WHERE key = ?1",
1464 rusqlite::params![key],
1465 );
1466 }
1467
1468 fn restore_persisted_state(&mut self) -> Result<(), String> {
1469 self.subs = {
1470 let mut stmt = self
1471 .conn
1472 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
1473 .map_err(|error| error.to_string())?;
1474 let rows = stmt
1475 .query_map([], |row| {
1476 Ok((
1477 row.get::<_, String>(0)?,
1478 row.get::<_, String>(1)?,
1479 row.get::<_, String>(2)?,
1480 ))
1481 })
1482 .map_err(|error| error.to_string())?;
1483 let mut subscriptions = Vec::new();
1484 for row in rows {
1485 let (id, table, raw) = row.map_err(|error| error.to_string())?;
1486 let state: Value = serde_json::from_str(&raw)
1487 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
1488 let requested = json_to_scope_map(
1489 state.get("requested").unwrap_or(&Value::Object(Map::new())),
1490 )?;
1491 let effective = state
1492 .get("effectiveScopes")
1493 .filter(|value| !value.is_null())
1494 .map(json_to_scope_map)
1495 .transpose()?;
1496 subscriptions.push(Subscription {
1497 id,
1498 table,
1499 requested,
1500 params: state
1501 .get("params")
1502 .and_then(Value::as_str)
1503 .map(str::to_owned),
1504 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
1505 bootstrap_state: state
1506 .get("bootstrapState")
1507 .and_then(Value::as_str)
1508 .map(str::to_owned),
1509 state: SubState::parse(
1510 state
1511 .get("status")
1512 .and_then(Value::as_str)
1513 .unwrap_or("active"),
1514 ),
1515 reason_code: state
1516 .get("reasonCode")
1517 .and_then(Value::as_str)
1518 .map(str::to_owned),
1519 effective,
1520 synced_once: state
1521 .get("syncedOnce")
1522 .and_then(Value::as_bool)
1523 .unwrap_or(false),
1524 });
1525 }
1526 subscriptions
1527 };
1528
1529 self.outbox = {
1530 let mut stmt = self
1531 .conn
1532 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
1533 .map_err(|error| error.to_string())?;
1534 let rows = stmt
1535 .query_map([], |row| {
1536 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1537 })
1538 .map_err(|error| error.to_string())?;
1539 let mut commits = Vec::new();
1540 for row in rows {
1541 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
1542 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
1543 format!("invalid persisted outbox {client_commit_id:?}: {error}")
1544 })?;
1545 let mut ops = Vec::with_capacity(entries.len());
1546 for entry in entries {
1547 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
1548 ops.push(OutboxOp {
1549 upsert: op == "upsert",
1550 table: entry
1551 .get("table")
1552 .and_then(Value::as_str)
1553 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
1554 .to_owned(),
1555 row_id: entry
1556 .get("rowId")
1557 .and_then(Value::as_str)
1558 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
1559 .to_owned(),
1560 base_version: entry.get("baseVersion").and_then(Value::as_i64),
1561 values: entry.get("values").and_then(Value::as_object).cloned(),
1562 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
1563 |values| {
1564 values
1565 .iter()
1566 .filter_map(Value::as_str)
1567 .map(str::to_owned)
1568 .collect()
1569 },
1570 ),
1571 });
1572 }
1573 commits.push(OutboxCommit {
1574 client_commit_id,
1575 ops,
1576 });
1577 }
1578 commits
1579 };
1580
1581 self.prune_commit_outcomes()?;
1582 let active = self.commit_outcomes(CommitOutcomeQuery {
1583 active_only: true,
1584 ..CommitOutcomeQuery::default()
1585 })?;
1586 self.conflicts = active
1587 .iter()
1588 .flat_map(|outcome| outcome.results.iter())
1589 .filter_map(|result| match result {
1590 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
1591 _ => None,
1592 })
1593 .collect();
1594 self.rejections = active
1595 .iter()
1596 .flat_map(|outcome| outcome.results.iter())
1597 .filter_map(|result| match result {
1598 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
1599 _ => None,
1600 })
1601 .collect();
1602
1603 self.lease_state = self
1604 .get_meta(LEASE_STATE_KEY)
1605 .map(|raw| serde_json::from_str(&raw))
1606 .transpose()
1607 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
1608 self.schema_floor = self
1609 .get_meta(SCHEMA_FLOOR_KEY)
1610 .map(|raw| serde_json::from_str(&raw))
1611 .transpose()
1612 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
1613 self.stopped = self.schema_floor.is_some();
1614 Ok(())
1615 }
1616
1617 #[must_use]
1618 pub fn local_revision(&self) -> u64 {
1619 self.get_meta(LOCAL_REVISION_KEY)
1620 .and_then(|value| value.parse().ok())
1621 .unwrap_or(0)
1622 }
1623
1624 #[must_use]
1625 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
1626 SyncStatusSnapshot {
1627 outbox: self.outbox.len(),
1628 upgrading: self.upgrading,
1629 lease_state: self.lease_state.clone(),
1630 schema_floor: self.schema_floor.clone(),
1631 sync_needed: self.sync_needed,
1632 }
1633 }
1634
1635 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
1636 self.change_queue.drain(..).collect()
1637 }
1638
1639 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
1640 self.sync_intent_queue.drain(..).collect()
1641 }
1642
1643 fn schedule_background_retry(&mut self) {
1644 self.sync_intent_queue.push_back(SyncIntent::Background {
1645 delay_ms: self.retry_delay_ms,
1646 });
1647 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
1648 }
1649
1650 fn reset_background_retry(&mut self) {
1651 self.retry_delay_ms = 250;
1652 }
1653
1654 fn retryable_transport_code(code: &str) -> bool {
1655 code == "transport.failed" || code == "sync.transport_failed"
1656 }
1657
1658 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
1659 if self.sync_needed != value {
1660 if self.begin_observation("syncular_status").is_ok() {
1661 self.sync_needed = value;
1662 let batch = ChangeAccumulator {
1663 status: true,
1664 ..ChangeAccumulator::default()
1665 };
1666 if self.finish_observation("syncular_status", batch).is_err() {
1667 self.rollback_observation("syncular_status");
1668 }
1669 } else {
1670 self.sync_needed = value;
1671 }
1672 }
1673 if value && interactive {
1674 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1675 }
1676 }
1677
1678 fn begin_observation(&self, name: &str) -> Result<(), String> {
1679 self.conn
1680 .execute_batch(&format!("SAVEPOINT {name}"))
1681 .map_err(|error| error.to_string())
1682 }
1683
1684 fn rollback_observation(&self, name: &str) {
1685 let _ = self
1686 .conn
1687 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
1688 }
1689
1690 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
1691 if !batch.touched() {
1692 self.conn
1693 .execute_batch(&format!("RELEASE {name}"))
1694 .map_err(|error| error.to_string())?;
1695 return Ok(());
1696 }
1697 let revision = self
1698 .local_revision()
1699 .checked_add(1)
1700 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
1701 self.conn
1702 .execute(
1703 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
1704 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
1705 )
1706 .map_err(|error| error.to_string())?;
1707 let status = batch.status.then(|| self.status_snapshot());
1708 let event = ClientChangeBatch {
1709 revision: revision.to_string(),
1710 tables: batch
1711 .tables
1712 .into_iter()
1713 .map(|(table, scope_keys)| TableChange {
1714 table,
1715 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
1716 })
1717 .collect(),
1718 windows: batch
1719 .windows
1720 .into_iter()
1721 .map(|((base_key, table), units)| WindowChange {
1722 base_key,
1723 table,
1724 units: units.into_iter().collect(),
1725 })
1726 .collect(),
1727 status,
1728 conflicts_changed: batch.conflicts,
1729 rejections_changed: batch.rejections,
1730 outcomes_changed: batch.outcomes,
1731 };
1732 self.conn
1733 .execute_batch(&format!("RELEASE {name}"))
1734 .map_err(|error| error.to_string())?;
1735 self.change_queue.push_back(event);
1736 Ok(())
1737 }
1738
1739 fn record_scope_map(
1740 &self,
1741 batch: &mut ChangeAccumulator,
1742 table_name: &str,
1743 scopes: &[(String, Vec<String>)],
1744 ) {
1745 let Some(table) = self.schema.table(table_name) else {
1746 return;
1747 };
1748 for (variable, values) in scopes {
1749 let Some(scope) = table
1750 .scope_variables
1751 .iter()
1752 .find(|scope| &scope.variable == variable)
1753 else {
1754 continue;
1755 };
1756 for value in values {
1757 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1758 }
1759 }
1760 }
1761
1762 fn record_row_scopes(
1764 &self,
1765 batch: &mut ChangeAccumulator,
1766 table_name: &str,
1767 row_id: &str,
1768 base: bool,
1769 ) -> bool {
1770 let Some(table) = self.schema.table(table_name) else {
1771 return false;
1772 };
1773 if table.scope_variables.is_empty() {
1774 return false;
1775 }
1776 let columns = table
1777 .scope_variables
1778 .iter()
1779 .map(|scope| quote_ident(&scope.column))
1780 .collect::<Vec<_>>()
1781 .join(", ");
1782 let full_table = if base {
1783 base_table(table_name)
1784 } else {
1785 visible_table(table_name)
1786 };
1787 let sql = format!(
1788 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
1789 quote_ident(&table.primary_key)
1790 );
1791 let Ok(mut stmt) = self.conn.prepare(&sql) else {
1792 return false;
1793 };
1794 let values = stmt.query_row(rusqlite::params![row_id], |row| {
1795 let mut values = Vec::with_capacity(table.scope_variables.len());
1796 for index in 0..table.scope_variables.len() {
1797 values.push(row.get::<_, Option<String>>(index)?);
1798 }
1799 Ok(values)
1800 });
1801 let Ok(values) = values else {
1802 return false;
1803 };
1804 let mut recorded = false;
1805 for (scope, value) in table.scope_variables.iter().zip(values) {
1806 if let Some(value) = value {
1807 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1808 recorded = true;
1809 }
1810 }
1811 recorded
1812 }
1813
1814 fn record_commit_changes(
1815 &self,
1816 batch: &mut ChangeAccumulator,
1817 tables: &[String],
1818 changes: &[ssp2::model::Change],
1819 ) {
1820 for change in changes {
1821 let Some(table_name) = tables.get(change.table_index as usize) else {
1822 continue;
1823 };
1824 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
1825 if let Some(table) = self.schema.table(table_name) {
1826 for (variable, value) in &change.scopes {
1827 if let Some(scope) = table
1828 .scope_variables
1829 .iter()
1830 .find(|scope| &scope.variable == variable)
1831 {
1832 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1833 precise = true;
1834 }
1835 }
1836 }
1837 if !precise {
1838 batch.table(table_name);
1839 }
1840 }
1841 }
1842
1843 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
1844 if effective.is_empty() {
1845 return false;
1846 }
1847 let Some(table) = self.schema.table(table_name) else {
1848 return false;
1849 };
1850 let mut clauses = Vec::new();
1851 let mut params = Vec::new();
1852 for (variable, values) in effective {
1853 let Some(column) = table.scope_column(variable) else {
1854 return false;
1855 };
1856 if values.is_empty() {
1857 return false;
1858 }
1859 let placeholders = values
1860 .iter()
1861 .map(|value| {
1862 params.push(SqlValue::Text(value.clone()));
1863 "?"
1864 })
1865 .collect::<Vec<_>>()
1866 .join(", ");
1867 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
1868 }
1869 let sql = format!(
1870 "SELECT 1 FROM {} WHERE {} LIMIT 1",
1871 base_table(table_name),
1872 clauses.join(" AND ")
1873 );
1874 self.conn
1875 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
1876 .is_ok()
1877 }
1878
1879 pub fn upgrading(&self) -> bool {
1881 self.upgrading
1882 }
1883
1884 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
1890 let new_schema = parse_schema_json(schema_json)?;
1891 let marker: Option<i32> = self
1892 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1893 .and_then(|v| v.parse().ok());
1894 self.schema = new_schema;
1895 if marker != Some(self.schema.version) {
1896 self.run_schema_reset()?;
1897 }
1898 self.enqueue_startup_sync_if_needed();
1902 Ok(())
1903 }
1904
1905 fn enqueue_startup_sync_if_needed(&mut self) {
1906 let startup_work = !self.stopped
1907 && (!self.outbox.is_empty()
1908 || self.subs.iter().any(|sub| sub.state == SubState::Active));
1909 if startup_work {
1910 self.sync_needed = true;
1911 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1912 }
1913 }
1914
1915 fn run_schema_reset(&mut self) -> Result<(), String> {
1921 self.begin_observation("syncular_schema_reset")?;
1922 let mut batch = ChangeAccumulator::default();
1923 let result = self.run_schema_reset_observed(&mut batch);
1924 if let Err(error) = result {
1925 self.rollback_observation("syncular_schema_reset");
1926 return Err(error);
1927 }
1928 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
1929 self.rollback_observation("syncular_schema_reset");
1930 return Err(error);
1931 }
1932 Ok(())
1933 }
1934
1935 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
1936 self.upgrading = true;
1937 batch.status = true;
1938 for table in &self.schema.tables {
1939 batch.table(&table.name);
1940 }
1941 for (base_key, unit, table) in self.load_registered_window_units() {
1942 batch.window(&base_key, &table, &unit);
1943 }
1944 self.insert_sql.borrow_mut().clear();
1946 self.overlay_dirty.set(true);
1947 let virtual_tables: Vec<String> = {
1955 let mut stmt = self
1956 .conn
1957 .prepare(
1958 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
1959 )
1960 .map_err(|e| e.to_string())?;
1961 let rows = stmt
1962 .query_map([], |row| row.get::<_, String>(0))
1963 .map_err(|e| e.to_string())?;
1964 rows.filter_map(Result::ok)
1965 .filter(|name| is_synced_table_name(name))
1966 .collect()
1967 };
1968 for name in virtual_tables {
1969 self.conn
1970 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
1971 .map_err(|e| e.to_string())?;
1972 }
1973 let existing: Vec<String> = {
1974 let mut stmt = self
1975 .conn
1976 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
1977 .map_err(|e| e.to_string())?;
1978 let rows = stmt
1979 .query_map([], |row| row.get::<_, String>(0))
1980 .map_err(|e| e.to_string())?;
1981 rows.filter_map(Result::ok)
1982 .filter(|name| is_synced_table_name(name))
1983 .collect()
1984 };
1985 for name in &existing {
1986 if let Some(table) = name.strip_prefix("_syncular_base_") {
1987 batch.table(table);
1988 } else if !name.starts_with("_syncular_") {
1989 batch.table(name);
1990 }
1991 }
1992 for name in existing {
1993 self.conn
1994 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
1995 .map_err(|e| e.to_string())?;
1996 }
1997 self.create_synced_tables()?;
1999 for sub in &mut self.subs {
2001 sub.cursor = -1;
2002 sub.bootstrap_state = None;
2003 sub.effective = None;
2004 sub.state = SubState::Active;
2005 sub.reason_code = None;
2006 sub.synced_once = false;
2007 }
2008 let subs = self.subs.clone();
2009 for sub in &subs {
2010 self.persist_sub(sub);
2011 }
2012 self.stopped = false;
2014 self.schema_floor = None;
2015 self.delete_meta(SCHEMA_FLOOR_KEY);
2016 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
2018 if self.drop_incompatible_outbox()? {
2022 batch.rejections = true;
2023 batch.status = true;
2024 batch.outcomes = true;
2025 }
2026 self.rebuild_overlay();
2028 Ok(())
2029 }
2030
2031 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
2035 let schema = &self.schema;
2036 let incompatible = self
2037 .outbox
2038 .iter()
2039 .filter(|commit| {
2040 commit.ops.iter().any(|op| {
2041 if !op.upsert {
2042 return false;
2043 }
2044 match schema.table(&op.table) {
2045 None => true,
2046 Some(table) => op.values.as_ref().is_some_and(|values| {
2047 values
2048 .keys()
2049 .any(|key| !table.columns.iter().any(|c| &c.name == key))
2050 }),
2051 }
2052 })
2053 })
2054 .cloned()
2055 .collect::<Vec<_>>();
2056 if incompatible.is_empty() {
2057 return Ok(false);
2058 }
2059 let mut rejections = Vec::new();
2060 for commit in &incompatible {
2061 let results = commit
2062 .ops
2063 .iter()
2064 .enumerate()
2065 .map(|(op_index, operation)| {
2066 let rejection = RejectionRecord {
2067 client_commit_id: commit.client_commit_id.clone(),
2068 op_index: op_index as i32,
2069 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
2070 message: "the persisted commit cannot encode under the current schema"
2071 .to_owned(),
2072 retryable: false,
2073 details: None,
2074 operation: Some(CommitOperation::from(operation)),
2075 };
2076 rejections.push(rejection.clone());
2077 CommitOperationOutcome::Error { rejection }
2078 })
2079 .collect::<Vec<_>>();
2080 self.persist_commit_outcome(
2081 &commit.client_commit_id,
2082 CommitOutcomeStatus::Rejected,
2083 &results,
2084 Some(&commit.ops),
2085 )?;
2086 self.delete_outbox_persisted(&commit.client_commit_id)?;
2087 }
2088 self.prune_commit_outcomes()?;
2089 let incompatible_ids = incompatible
2090 .iter()
2091 .map(|commit| commit.client_commit_id.as_str())
2092 .collect::<BTreeSet<_>>();
2093 self.outbox
2094 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
2095 self.rejections.extend(rejections);
2096 Ok(true)
2097 }
2098
2099 fn create_synced_tables(&self) -> Result<(), String> {
2102 for table in &self.schema.tables {
2103 for (full, index_prefix) in [
2107 (base_table(&table.name), "_syncular_base_"),
2108 (visible_table(&table.name), ""),
2109 ] {
2110 let mut cols: Vec<String> =
2111 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
2112 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
2113 let sql = format!(
2114 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
2115 cols.join(", "),
2116 quote_ident(&table.primary_key)
2117 );
2118 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
2119 for index in &table.indexes {
2123 let unique = if index.unique { "UNIQUE " } else { "" };
2124 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
2125 let cols_sql = index
2126 .columns
2127 .iter()
2128 .map(|c| quote_ident(c))
2129 .collect::<Vec<_>>()
2130 .join(", ");
2131 let index_sql = format!(
2132 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
2133 );
2134 self.conn
2135 .execute(&index_sql, [])
2136 .map_err(|e| e.to_string())?;
2137 }
2138 }
2139 }
2140 for table in &self.schema.tables {
2143 for index in &table.fts_indexes {
2144 self.create_fts_projection(table, index)?;
2145 }
2146 }
2147 Ok(())
2148 }
2149
2150 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
2151 let count: i64 = self
2152 .conn
2153 .query_row(
2154 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
2155 rusqlite::params![index.name],
2156 |row| row.get(0),
2157 )
2158 .map_err(|e| e.to_string())?;
2159 Ok(count > 0)
2160 }
2161
2162 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
2163 for suffix in ["ai", "ad", "au"] {
2164 self.conn
2165 .execute(
2166 &format!(
2167 "DROP TRIGGER IF EXISTS {}",
2168 quote_ident(&format!("{}_{suffix}", index.name))
2169 ),
2170 [],
2171 )
2172 .map_err(|e| e.to_string())?;
2173 }
2174 Ok(())
2175 }
2176
2177 fn create_fts_triggers(
2178 &self,
2179 table: &TableSchema,
2180 index: &FtsIndexSchema,
2181 ) -> Result<(), String> {
2182 let fts = quote_ident(&index.name);
2183 let source = visible_table(&table.name);
2184 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2185 let pk = quote_ident(&table.primary_key);
2186 let projection_columns = std::iter::once(source_id.clone())
2187 .chain(index.columns.iter().map(|column| quote_ident(column)))
2188 .collect::<Vec<_>>()
2189 .join(", ");
2190 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
2191 .chain(
2192 index
2193 .columns
2194 .iter()
2195 .map(|column| format!("new.{}", quote_ident(column))),
2196 )
2197 .collect::<Vec<_>>()
2198 .join(", ");
2199 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
2200 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
2201 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
2202 let sql = format!(
2203 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
2204 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
2205 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
2206 ai = quote_ident(&format!("{}_ai", index.name)),
2207 ad = quote_ident(&format!("{}_ad", index.name)),
2208 au = quote_ident(&format!("{}_au", index.name)),
2209 );
2210 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2211 }
2212
2213 fn rebuild_fts_projection(
2214 &self,
2215 table: &TableSchema,
2216 index: &FtsIndexSchema,
2217 ) -> Result<(), String> {
2218 let fts = quote_ident(&index.name);
2219 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2220 let pk = quote_ident(&table.primary_key);
2221 let indexed_columns = index
2222 .columns
2223 .iter()
2224 .map(|column| quote_ident(column))
2225 .collect::<Vec<_>>();
2226 let projection_columns = std::iter::once(source_id)
2227 .chain(indexed_columns.iter().cloned())
2228 .collect::<Vec<_>>()
2229 .join(", ");
2230 let sql = format!(
2231 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
2232 columns = indexed_columns.join(", "),
2233 source = visible_table(&table.name),
2234 );
2235 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2236 }
2237
2238 fn create_fts_projection(
2239 &self,
2240 table: &TableSchema,
2241 index: &FtsIndexSchema,
2242 ) -> Result<(), String> {
2243 let existed = self.fts_projection_exists(index)?;
2244 let tokenizer = index.tokenize.replace('\'', "''");
2245 let columns = index
2246 .columns
2247 .iter()
2248 .map(|column| quote_ident(column))
2249 .collect::<Vec<_>>()
2250 .join(", ");
2251 let sql = format!(
2252 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
2253 fts = quote_ident(&index.name),
2254 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
2255 );
2256 self.conn.execute(&sql, []).map_err(|error| {
2257 format!(
2258 "cannot create local FTS5 projection {:?}: {error}",
2259 index.name
2260 )
2261 })?;
2262 self.create_fts_triggers(table, index)?;
2263 if !existed {
2264 self.rebuild_fts_projection(table, index)?;
2265 }
2266 Ok(())
2267 }
2268
2269 fn persist_sub(&self, sub: &Subscription) {
2272 let state = serde_json::json!({
2273 "requested": scope_map_to_json(&sub.requested),
2274 "params": sub.params,
2275 "cursor": sub.cursor,
2276 "bootstrapState": sub.bootstrap_state,
2277 "status": sub.state.name(),
2278 "reasonCode": sub.reason_code,
2279 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2280 "syncedOnce": sub.synced_once,
2281 });
2282 let _ = self.conn.execute(
2283 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
2284 rusqlite::params![sub.id, sub.table, state.to_string()],
2285 );
2286 }
2287
2288 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
2289 let ops: Vec<Value> = commit
2290 .ops
2291 .iter()
2292 .map(|op| {
2293 serde_json::json!({
2294 "op": if op.upsert { "upsert" } else { "delete" },
2295 "table": op.table,
2296 "rowId": op.row_id,
2297 "baseVersion": op.base_version,
2298 "values": op.values.clone().map(Value::Object),
2299 "changedFields": op.changed_fields,
2300 })
2301 })
2302 .collect();
2303 let _ = self.conn.execute(
2304 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
2305 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
2306 );
2307 }
2308
2309 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
2310 self.conn
2311 .execute(
2312 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
2313 rusqlite::params![client_commit_id],
2314 )
2315 .map(|_| ())
2316 .map_err(|error| error.to_string())
2317 }
2318
2319 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
2320 match status {
2321 CommitOutcomeStatus::Applied => "applied",
2322 CommitOutcomeStatus::Cached => "cached",
2323 CommitOutcomeStatus::Conflict => "conflict",
2324 CommitOutcomeStatus::Rejected => "rejected",
2325 }
2326 }
2327
2328 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
2329 match resolution {
2330 CommitOutcomeResolution::Active => "active",
2331 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
2332 CommitOutcomeResolution::Superseded => "superseded",
2333 CommitOutcomeResolution::Dismissed => "dismissed",
2334 }
2335 }
2336
2337 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
2338 match value {
2339 "applied" => Ok(CommitOutcomeStatus::Applied),
2340 "cached" => Ok(CommitOutcomeStatus::Cached),
2341 "conflict" => Ok(CommitOutcomeStatus::Conflict),
2342 "rejected" => Ok(CommitOutcomeStatus::Rejected),
2343 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
2344 }
2345 }
2346
2347 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
2348 match value {
2349 "active" => Ok(CommitOutcomeResolution::Active),
2350 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
2351 "superseded" => Ok(CommitOutcomeResolution::Superseded),
2352 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
2353 _ => Err(format!(
2354 "invalid persisted commit outcome resolution {value:?}"
2355 )),
2356 }
2357 }
2358
2359 fn persist_commit_outcome(
2360 &self,
2361 client_commit_id: &str,
2362 status: CommitOutcomeStatus,
2363 results: &[CommitOperationOutcome],
2364 operations: Option<&[OutboxOp]>,
2365 ) -> Result<(), String> {
2366 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
2367 let operations_json = operations
2368 .map(|items| {
2369 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
2370 })
2371 .transpose()
2372 .map_err(|error| error.to_string())?;
2373 self.conn
2374 .execute(
2375 "INSERT INTO _syncular_commit_outcomes (
2376 client_commit_id, status, recorded_at_ms, results_json,
2377 operations_json, resolution
2378 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
2379 rusqlite::params![
2380 client_commit_id,
2381 Self::outcome_status_name(status),
2382 self.clock_now_ms(),
2383 results_json,
2384 operations_json
2385 ],
2386 )
2387 .map(|_| ())
2388 .map_err(|error| error.to_string())
2389 }
2390
2391 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
2392 let StoredCommitOutcomeRow {
2393 sequence,
2394 client_commit_id,
2395 status,
2396 recorded_at_ms,
2397 results_json,
2398 operations_json,
2399 resolution,
2400 resolved_at_ms,
2401 replacement_client_commit_id,
2402 } = row;
2403 Ok(CommitOutcome {
2404 sequence,
2405 client_commit_id,
2406 status: Self::parse_outcome_status(&status)?,
2407 recorded_at_ms,
2408 results: serde_json::from_str(&results_json)
2409 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
2410 operations: operations_json
2411 .map(|value| {
2412 serde_json::from_str(&value).map_err(|error| {
2413 format!("invalid persisted commit outcome operations: {error}")
2414 })
2415 })
2416 .transpose()?,
2417 resolution: Self::parse_outcome_resolution(&resolution)?,
2418 resolved_at_ms,
2419 replacement_client_commit_id,
2420 })
2421 }
2422
2423 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
2424 let row = self
2425 .conn
2426 .query_row(
2427 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2428 resolution, resolved_at_ms, replacement_client_commit_id
2429 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
2430 rusqlite::params![client_commit_id],
2431 |row| {
2432 Ok(StoredCommitOutcomeRow {
2433 sequence: row.get(0)?,
2434 client_commit_id: row.get(1)?,
2435 status: row.get(2)?,
2436 recorded_at_ms: row.get(3)?,
2437 results_json: row.get(4)?,
2438 operations_json: row.get(5)?,
2439 resolution: row.get(6)?,
2440 resolved_at_ms: row.get(7)?,
2441 replacement_client_commit_id: row.get(8)?,
2442 })
2443 },
2444 )
2445 .optional()
2446 .map_err(|error| error.to_string())?;
2447 row.map(Self::outcome_from_row).transpose()
2448 }
2449
2450 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
2451 if query.limit == Some(0) {
2452 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
2453 }
2454 let mut sql = String::from(
2455 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2456 resolution, resolved_at_ms, replacement_client_commit_id
2457 FROM _syncular_commit_outcomes",
2458 );
2459 if query.active_only {
2460 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
2461 }
2462 sql.push_str(" ORDER BY seq DESC");
2463 if let Some(limit) = query.limit {
2464 sql.push_str(&format!(" LIMIT {limit}"));
2465 }
2466 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
2467 let rows = stmt
2468 .query_map([], |row| {
2469 Ok(StoredCommitOutcomeRow {
2470 sequence: row.get(0)?,
2471 client_commit_id: row.get(1)?,
2472 status: row.get(2)?,
2473 recorded_at_ms: row.get(3)?,
2474 results_json: row.get(4)?,
2475 operations_json: row.get(5)?,
2476 resolution: row.get(6)?,
2477 resolved_at_ms: row.get(7)?,
2478 replacement_client_commit_id: row.get(8)?,
2479 })
2480 })
2481 .map_err(|error| error.to_string())?;
2482 let mut outcomes = Vec::new();
2483 for row in rows {
2484 outcomes.push(Self::outcome_from_row(
2485 row.map_err(|error| error.to_string())?,
2486 )?);
2487 }
2488 Ok(outcomes)
2489 }
2490
2491 fn prune_commit_outcomes(&self) -> Result<(), String> {
2492 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
2493 let count = self
2494 .conn
2495 .query_row(
2496 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
2497 [],
2498 |row| row.get::<_, i64>(0),
2499 )
2500 .map_err(|error| error.to_string())? as usize;
2501 let excess = count.saturating_sub(max_entries);
2502 if excess == 0 {
2503 return Ok(());
2504 }
2505 let mut stmt = self
2506 .conn
2507 .prepare(
2508 "SELECT seq FROM _syncular_commit_outcomes
2509 WHERE status IN ('applied', 'cached') OR resolution != 'active'
2510 ORDER BY seq ASC LIMIT ?1",
2511 )
2512 .map_err(|error| error.to_string())?;
2513 let rows = stmt
2514 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
2515 .map_err(|error| error.to_string())?;
2516 let sequences = rows
2517 .collect::<Result<Vec<_>, _>>()
2518 .map_err(|error| error.to_string())?;
2519 drop(stmt);
2520 for sequence in sequences {
2521 self.conn
2522 .execute(
2523 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
2524 rusqlite::params![sequence],
2525 )
2526 .map_err(|error| error.to_string())?;
2527 }
2528 Ok(())
2529 }
2530
2531 pub fn subscribe(
2534 &mut self,
2535 id: String,
2536 table: String,
2537 scopes: Vec<(String, Vec<String>)>,
2538 params: Option<String>,
2539 ) -> Result<(), String> {
2540 if self.schema.table(&table).is_none() {
2541 return Err(format!("unknown table {table:?}"));
2542 }
2543 let sub = Subscription {
2544 id: id.clone(),
2545 table,
2546 requested: scopes,
2547 params,
2548 cursor: -1,
2549 bootstrap_state: None,
2550 state: SubState::Active,
2551 reason_code: None,
2552 effective: None,
2553 synced_once: false,
2554 };
2555 self.persist_sub(&sub);
2556 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
2557 *existing = sub;
2558 } else {
2559 self.subs.push(sub);
2560 }
2561 Ok(())
2562 }
2563
2564 pub fn unsubscribe(&mut self, id: &str) {
2565 self.subs.retain(|s| s.id != id);
2566 let _ = self.conn.execute(
2567 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2568 rusqlite::params![id],
2569 );
2570 }
2571
2572 pub fn set_window(
2580 &mut self,
2581 base: &WindowBase,
2582 units: &[String],
2583 ) -> Result<CommandEffects, String> {
2584 let table = self
2585 .schema
2586 .table(&base.table)
2587 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
2588 if table.scope_column(&base.variable).is_none() {
2589 return Err(format!(
2590 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
2591 base.table, base.variable
2592 ));
2593 }
2594 let base_key = window_base_key(base);
2595 let wanted: std::collections::HashSet<&String> = units.iter().collect();
2596 let live = self.load_window_units(&base_key);
2597 self.begin_observation("syncular_window")?;
2598 let mut batch = ChangeAccumulator::default();
2599 let mut changed = false;
2600
2601 for unit in units {
2603 if live.iter().any(|(u, _)| u == unit) {
2604 continue;
2605 }
2606 let sub_id = derive_sub_id(base, unit);
2607 self.delete_pending_evict(&sub_id);
2608 self.insert_window_unit(&base_key, unit, &sub_id);
2609 self.subscribe(
2610 sub_id,
2611 base.table.clone(),
2612 unit_scopes(base, unit),
2613 base.params.clone(),
2614 )?;
2615 batch.window(&base_key, &base.table, unit);
2616 changed = true;
2617 }
2618
2619 for (unit, sub_id) in live {
2621 if wanted.contains(&unit) {
2622 continue;
2623 }
2624 let effective = self
2625 .subs
2626 .iter()
2627 .find(|sub| sub.id == sub_id)
2628 .and_then(|sub| sub.effective.clone())
2629 .unwrap_or_else(|| unit_scopes(base, &unit));
2630 self.record_scope_map(&mut batch, &base.table, &effective);
2631 batch.window(&base_key, &base.table, &unit);
2632 self.evict_unit(&base_key, base, &unit, &sub_id);
2633 changed = true;
2634 }
2635 if let Err(error) = self.finish_observation("syncular_window", batch) {
2636 self.rollback_observation("syncular_window");
2637 return Err(error);
2638 }
2639 Ok(if changed {
2640 CommandEffects::interactive()
2641 } else {
2642 CommandEffects::none()
2643 })
2644 }
2645
2646 pub fn window_state(&self, base: &WindowBase) -> WindowState {
2651 let mut units = Vec::new();
2652 let mut pending = Vec::new();
2653 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
2654 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
2655 Some(sub) => {
2656 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
2657 }
2658 None => true,
2659 };
2660 if is_pending {
2661 pending.push(unit.clone());
2662 }
2663 units.push(unit);
2664 }
2665 WindowState { units, pending }
2666 }
2667
2668 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
2669 let mut stmt = match self
2670 .conn
2671 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
2672 {
2673 Ok(stmt) => stmt,
2674 Err(_) => return Vec::new(),
2675 };
2676 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
2677 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2678 });
2679 match rows {
2680 Ok(rows) => rows.filter_map(Result::ok).collect(),
2681 Err(_) => Vec::new(),
2682 }
2683 }
2684
2685 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
2686 let mut stmt = match self.conn.prepare(
2687 "SELECT windows.base, windows.unit, subscriptions.tbl
2688 FROM _syncular_windows AS windows
2689 JOIN _syncular_subscriptions AS subscriptions
2690 ON subscriptions.id = windows.sub_id
2691 ORDER BY windows.base, windows.unit",
2692 ) {
2693 Ok(stmt) => stmt,
2694 Err(_) => return Vec::new(),
2695 };
2696 let rows = stmt.query_map([], |row| {
2697 Ok((
2698 row.get::<_, String>(0)?,
2699 row.get::<_, String>(1)?,
2700 row.get::<_, String>(2)?,
2701 ))
2702 });
2703 match rows {
2704 Ok(rows) => rows.filter_map(Result::ok).collect(),
2705 Err(_) => Vec::new(),
2706 }
2707 }
2708
2709 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
2710 self.conn
2711 .query_row(
2712 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
2713 rusqlite::params![sub_id],
2714 |row| Ok((row.get(0)?, row.get(1)?)),
2715 )
2716 .ok()
2717 }
2718
2719 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
2720 let _ = self.conn.execute(
2721 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
2722 rusqlite::params![base_key, unit, sub_id],
2723 );
2724 }
2725
2726 fn delete_window_unit(&self, base_key: &str, unit: &str) {
2727 let _ = self.conn.execute(
2728 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
2729 rusqlite::params![base_key, unit],
2730 );
2731 }
2732
2733 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
2740 let effective = self
2741 .subs
2742 .iter()
2743 .find(|s| s.id == sub_id)
2744 .and_then(|s| s.effective.clone())
2745 .unwrap_or_else(|| unit_scopes(base, unit));
2746 let pinned = self.pinned_row_ids(&base.table);
2747 let deferred = self
2748 .evict_scope_rows(&base.table, &effective, &pinned)
2749 .unwrap_or(false);
2750 self.delete_window_unit(base_key, unit);
2751 self.unsubscribe(sub_id);
2752 if deferred {
2753 self.save_pending_evict(sub_id, &base.table, &effective);
2754 } else {
2755 self.delete_pending_evict(sub_id);
2756 }
2757 self.rebuild_overlay();
2758 }
2759
2760 fn evict_scope_rows(
2764 &mut self,
2765 table_name: &str,
2766 effective: &[(String, Vec<String>)],
2767 pinned: &std::collections::HashSet<String>,
2768 ) -> Result<bool, ()> {
2769 if effective.is_empty() {
2770 return Ok(false);
2771 }
2772 let table = self.schema.table(table_name).ok_or(())?.clone();
2773 let mut clauses = Vec::new();
2774 let mut params: Vec<SqlValue> = Vec::new();
2775 for (variable, values) in effective {
2776 let column = table.scope_column(variable).ok_or(())?;
2777 let placeholders: Vec<String> = values
2778 .iter()
2779 .map(|v| {
2780 params.push(SqlValue::Text(v.clone()));
2781 "?".to_owned()
2782 })
2783 .collect();
2784 clauses.push(format!(
2785 "{} IN ({})",
2786 quote_ident(column),
2787 placeholders.join(", ")
2788 ));
2789 }
2790 let mut sql = format!(
2791 "DELETE FROM {} WHERE {}",
2792 base_table(table_name),
2793 clauses.join(" AND ")
2794 );
2795 if !pinned.is_empty() {
2796 let pk = quote_ident(&table.primary_key);
2797 let holes: Vec<String> = pinned
2798 .iter()
2799 .map(|id| {
2800 params.push(SqlValue::Text(id.clone()));
2801 "?".to_owned()
2802 })
2803 .collect();
2804 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
2805 }
2806 self.overlay_dirty.set(true);
2807 self.conn
2808 .execute(&sql, rusqlite::params_from_iter(params))
2809 .map_err(|_| ())?;
2810 if pinned.is_empty() {
2811 return Ok(false);
2812 }
2813 let mut where_params: Vec<SqlValue> = Vec::new();
2816 let mut where_clauses = Vec::new();
2817 for (variable, values) in effective {
2818 let column = table.scope_column(variable).ok_or(())?;
2819 let placeholders: Vec<String> = values
2820 .iter()
2821 .map(|v| {
2822 where_params.push(SqlValue::Text(v.clone()));
2823 "?".to_owned()
2824 })
2825 .collect();
2826 where_clauses.push(format!(
2827 "{} IN ({})",
2828 quote_ident(column),
2829 placeholders.join(", ")
2830 ));
2831 }
2832 let pk = quote_ident(&table.primary_key);
2833 let select = format!(
2834 "SELECT {} FROM {} WHERE {}",
2835 pk,
2836 base_table(table_name),
2837 where_clauses.join(" AND ")
2838 );
2839 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
2840 let survivors: Vec<String> = stmt
2841 .query_map(rusqlite::params_from_iter(where_params), |row| {
2842 row.get::<_, String>(0)
2843 })
2844 .map_err(|_| ())?
2845 .filter_map(Result::ok)
2846 .collect();
2847 Ok(survivors.iter().any(|id| pinned.contains(id)))
2848 }
2849
2850 fn drain_pending_evictions(&mut self) {
2854 let pending = self.load_pending_evictions();
2855 if pending.is_empty() {
2856 return;
2857 }
2858 for (sub_id, table_name, effective) in pending {
2859 if self.schema.table(&table_name).is_none() {
2860 self.delete_pending_evict(&sub_id);
2861 continue;
2862 }
2863 let pinned = self.pinned_row_ids(&table_name);
2864 let deferred = self
2865 .evict_scope_rows(&table_name, &effective, &pinned)
2866 .unwrap_or(false);
2867 if !deferred {
2868 self.delete_pending_evict(&sub_id);
2869 }
2870 }
2871 self.rebuild_overlay_if_dirty();
2872 }
2873
2874 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
2877 let mut pinned = std::collections::HashSet::new();
2878 for commit in &self.outbox {
2879 for op in &commit.ops {
2880 if op.table == table {
2881 pinned.insert(op.row_id.clone());
2882 }
2883 }
2884 }
2885 pinned
2886 }
2887
2888 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
2889 let _ = self.conn.execute(
2890 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
2891 VALUES (?1, ?2, ?3)",
2892 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
2893 );
2894 }
2895
2896 fn delete_pending_evict(&self, sub_id: &str) {
2897 let _ = self.conn.execute(
2898 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2899 rusqlite::params![sub_id],
2900 );
2901 }
2902
2903 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
2904 let mut stmt = match self
2905 .conn
2906 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
2907 {
2908 Ok(stmt) => stmt,
2909 Err(_) => return Vec::new(),
2910 };
2911 let rows = stmt.query_map([], |row| {
2912 Ok((
2913 row.get::<_, String>(0)?,
2914 row.get::<_, String>(1)?,
2915 row.get::<_, String>(2)?,
2916 ))
2917 });
2918 let mut out = Vec::new();
2919 if let Ok(rows) = rows {
2920 for entry in rows.filter_map(Result::ok) {
2921 let (sub_id, table, json) = entry;
2922 if let Ok(value) = serde_json::from_str::<Value>(&json) {
2923 if let Ok(effective) = json_to_scope_map(&value) {
2924 out.push((sub_id, table, effective));
2925 }
2926 }
2927 }
2928 }
2929 out
2930 }
2931
2932 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
2934 if mutations.is_empty() {
2935 return Err("a commit must contain at least one operation (§6.1)".to_owned());
2936 }
2937 let mut ops = Vec::with_capacity(mutations.len());
2938 for mutation in mutations {
2939 match mutation {
2940 Mutation::Upsert {
2941 table,
2942 values,
2943 base_version,
2944 } => {
2945 let schema_table = self
2946 .schema
2947 .table(&table)
2948 .ok_or_else(|| format!("unknown table {table:?}"))?;
2949 let values = normalize_values_casing(schema_table, values)?;
2953 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
2954 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
2957 ops.push(OutboxOp {
2958 upsert: true,
2959 table,
2960 row_id,
2961 base_version,
2962 values: Some(values),
2963 changed_fields: None,
2964 });
2965 }
2966 Mutation::Delete {
2967 table,
2968 row_id,
2969 base_version,
2970 } => {
2971 if self.schema.table(&table).is_none() {
2972 return Err(format!("unknown table {table:?}"));
2973 }
2974 ops.push(OutboxOp {
2975 upsert: false,
2976 table,
2977 row_id,
2978 base_version,
2979 values: None,
2980 changed_fields: None,
2981 });
2982 }
2983 }
2984 }
2985 self.record_outbox_commit(ops)
2986 }
2987
2988 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
2989 let commit = OutboxCommit {
2990 client_commit_id: uuid::Uuid::new_v4().to_string(),
2991 ops,
2992 };
2993 self.begin_observation("syncular_mutation")?;
2994 let mut batch = ChangeAccumulator::default();
2995 for op in &commit.ops {
2996 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
2997 if let Some(values) = &op.values {
2998 if let Some(table) = self.schema.table(&op.table) {
2999 for scope in &table.scope_variables {
3000 if let Some(Value::String(value)) = values.get(&scope.column) {
3001 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
3002 precise = true;
3003 }
3004 }
3005 }
3006 }
3007 if !precise {
3008 batch.table(&op.table);
3009 }
3010 }
3011 self.persist_outbox_insert(&commit);
3012 let id = commit.client_commit_id.clone();
3013 self.outbox.push(commit);
3014 self.overlay_dirty.set(true);
3015 self.rebuild_overlay();
3016 batch.status = true;
3017 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
3018 self.rollback_observation("syncular_mutation");
3019 return Err(error);
3020 }
3021 Ok(id)
3022 }
3023
3024 pub fn patch(
3029 &mut self,
3030 table: &str,
3031 row_id: &str,
3032 partial: Map<String, Value>,
3033 base_version: Option<i64>,
3034 ) -> Result<String, String> {
3035 let schema_table = self
3036 .schema
3037 .table(table)
3038 .ok_or_else(|| format!("unknown table {table:?}"))?;
3039 let partial = normalize_values_casing(schema_table, partial)?;
3040 let mut values = self
3041 .read_rows(table)?
3042 .into_iter()
3043 .find(|row| row.row_id == row_id)
3044 .map(|row| row.values)
3045 .ok_or_else(|| {
3046 format!(
3047 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
3048 )
3049 })?;
3050 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
3051 changed_fields.sort();
3052 values.extend(partial);
3053 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
3054 if row_id_from_values != row_id {
3055 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
3056 }
3057 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
3058 self.record_outbox_commit(vec![OutboxOp {
3059 upsert: true,
3060 table: table.to_owned(),
3061 row_id: row_id.to_owned(),
3062 base_version,
3063 values: Some(values),
3064 changed_fields: Some(changed_fields),
3065 }])
3066 }
3067
3068 pub fn pending_commit_ids(&self) -> Vec<String> {
3069 self.outbox
3070 .iter()
3071 .map(|c| c.client_commit_id.clone())
3072 .collect()
3073 }
3074
3075 pub fn conflicts(&self) -> &[ConflictRecord] {
3076 &self.conflicts
3077 }
3078
3079 pub fn rejections(&self) -> &[RejectionRecord] {
3080 &self.rejections
3081 }
3082
3083 pub fn resolve_commit_outcome(
3084 &mut self,
3085 input: ResolveCommitOutcomeInput,
3086 ) -> Result<CommitOutcome, String> {
3087 let current = self
3088 .commit_outcome(&input.client_commit_id)?
3089 .ok_or_else(|| {
3090 format!(
3091 "sync.outcome_not_found: no durable outcome exists for {:?}",
3092 input.client_commit_id
3093 )
3094 })?;
3095 if current.resolution != CommitOutcomeResolution::Active {
3096 return Ok(current);
3097 }
3098 if input.resolution == CommitOutcomeResolution::Active {
3099 return Err("sync.invalid_request: resolution must leave active state".to_owned());
3100 }
3101 match input.resolution {
3102 CommitOutcomeResolution::Superseded => {
3103 let replacement = input
3104 .replacement_client_commit_id
3105 .as_deref()
3106 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
3107 .ok_or_else(|| {
3108 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
3109 .to_owned()
3110 })?;
3111 let _ = replacement;
3112 }
3113 _ if input.replacement_client_commit_id.is_some() => {
3114 return Err(
3115 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
3116 .to_owned(),
3117 );
3118 }
3119 _ => {}
3120 }
3121 let allowed = match current.status {
3122 CommitOutcomeStatus::Conflict => matches!(
3123 input.resolution,
3124 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
3125 ),
3126 CommitOutcomeStatus::Rejected => {
3127 input.resolution == CommitOutcomeResolution::Superseded
3128 }
3129 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
3130 input.resolution == CommitOutcomeResolution::Dismissed
3131 }
3132 };
3133 if !allowed {
3134 return Err(format!(
3135 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
3136 input.resolution, current.status
3137 ));
3138 }
3139
3140 self.begin_observation("syncular_outcome_resolution")?;
3141 let result = (|| {
3142 self.conn
3143 .execute(
3144 "UPDATE _syncular_commit_outcomes
3145 SET resolution = ?1, resolved_at_ms = ?2,
3146 replacement_client_commit_id = ?3
3147 WHERE client_commit_id = ?4 AND resolution = 'active'",
3148 rusqlite::params![
3149 Self::outcome_resolution_name(input.resolution),
3150 self.clock_now_ms(),
3151 input.replacement_client_commit_id,
3152 input.client_commit_id
3153 ],
3154 )
3155 .map_err(|error| error.to_string())?;
3156 let resolved = self
3157 .commit_outcome(¤t.client_commit_id)?
3158 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
3159 self.prune_commit_outcomes()?;
3160 Ok(resolved)
3161 })();
3162 let resolved = match result {
3163 Ok(outcome) => outcome,
3164 Err(error) => {
3165 self.rollback_observation("syncular_outcome_resolution");
3166 return Err(error);
3167 }
3168 };
3169 self.conflicts
3170 .retain(|record| record.client_commit_id != current.client_commit_id);
3171 self.rejections
3172 .retain(|record| record.client_commit_id != current.client_commit_id);
3173 let batch = ChangeAccumulator {
3174 conflicts: current.status == CommitOutcomeStatus::Conflict,
3175 rejections: current.status == CommitOutcomeStatus::Rejected,
3176 outcomes: true,
3177 ..ChangeAccumulator::default()
3178 };
3179 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
3180 self.rollback_observation("syncular_outcome_resolution");
3181 return Err(error);
3182 }
3183 Ok(resolved)
3184 }
3185
3186 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
3187 self.schema_floor.as_ref()
3188 }
3189
3190 pub fn lease_state(&self) -> Option<&LeaseState> {
3192 self.lease_state.as_ref()
3193 }
3194
3195 fn record_lease_error(&mut self, code: &str) {
3198 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
3199 return;
3200 }
3201 let mut next = self.lease_state.clone().unwrap_or_default();
3202 next.error_code = Some(code.to_owned());
3203 self.set_lease_state(Some(next));
3204 }
3205
3206 fn set_lease_state(&mut self, next: Option<LeaseState>) {
3207 if self.lease_state == next {
3208 return;
3209 }
3210 if self.begin_observation("syncular_lease").is_err() {
3211 return;
3212 }
3213 self.lease_state = next;
3214 if let Some(lease) = &self.lease_state {
3215 if let Ok(json) = serde_json::to_string(lease) {
3216 self.set_meta(LEASE_STATE_KEY, &json);
3217 }
3218 } else {
3219 self.delete_meta(LEASE_STATE_KEY);
3220 }
3221 let batch = ChangeAccumulator {
3222 status: true,
3223 ..ChangeAccumulator::default()
3224 };
3225 if self.finish_observation("syncular_lease", batch).is_err() {
3226 self.rollback_observation("syncular_lease");
3227 }
3228 }
3229
3230 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
3231 if self.schema_floor == next {
3232 return;
3233 }
3234 if self.begin_observation("syncular_schema_floor").is_err() {
3235 return;
3236 }
3237 self.schema_floor = next;
3238 self.stopped = self.schema_floor.is_some();
3239 if let Some(floor) = &self.schema_floor {
3240 if let Ok(json) = serde_json::to_string(floor) {
3241 self.set_meta(SCHEMA_FLOOR_KEY, &json);
3242 }
3243 } else {
3244 self.delete_meta(SCHEMA_FLOOR_KEY);
3245 }
3246 let batch = ChangeAccumulator {
3247 status: true,
3248 ..ChangeAccumulator::default()
3249 };
3250 if self
3251 .finish_observation("syncular_schema_floor", batch)
3252 .is_err()
3253 {
3254 self.rollback_observation("syncular_schema_floor");
3255 }
3256 }
3257
3258 fn set_upgrading(&mut self, value: bool) {
3259 if self.upgrading == value {
3260 return;
3261 }
3262 if self.begin_observation("syncular_upgrading").is_err() {
3263 return;
3264 }
3265 self.upgrading = value;
3266 let batch = ChangeAccumulator {
3267 status: true,
3268 ..ChangeAccumulator::default()
3269 };
3270 if self
3271 .finish_observation("syncular_upgrading", batch)
3272 .is_err()
3273 {
3274 self.rollback_observation("syncular_upgrading");
3275 }
3276 }
3277
3278 pub fn sync_needed(&self) -> bool {
3279 self.sync_needed
3280 }
3281
3282 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
3283 let sub = self.subs.iter().find(|s| s.id == id)?;
3284 Some(SubscriptionStateView {
3285 id: sub.id.clone(),
3286 table: sub.table.clone(),
3287 status: sub.state.name().to_owned(),
3288 cursor: sub.cursor,
3289 has_resume_token: sub.bootstrap_state.is_some(),
3290 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3291 reason_code: sub.reason_code.clone(),
3292 })
3293 }
3294
3295 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
3296 let schema_table = self
3297 .schema
3298 .table(table)
3299 .ok_or_else(|| format!("unknown table {table:?}"))?;
3300 let sql = format!(
3301 "SELECT * FROM {} ORDER BY {} ASC",
3302 visible_table(table),
3303 quote_ident(&schema_table.primary_key)
3304 );
3305 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
3306 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
3307 let mut out = Vec::new();
3308 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
3309 let mut values = Map::new();
3310 for (i, column) in schema_table.columns.iter().enumerate() {
3311 let value = row.get_ref(i).map_err(|e| e.to_string())?;
3312 values.insert(column.name.clone(), sql_ref_to_json(column, value));
3313 }
3314 let version: i64 = row
3315 .get(schema_table.columns.len())
3316 .map_err(|e| e.to_string())?;
3317 let row_id = match values.get(&schema_table.primary_key) {
3318 Some(Value::String(s)) => s.clone(),
3319 Some(Value::Number(n)) => n.to_string(),
3320 Some(Value::Bool(b)) => b.to_string(),
3321 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
3322 };
3323 out.push(RowState {
3324 row_id,
3325 version,
3326 values,
3327 });
3328 }
3329 Ok(out)
3330 }
3331
3332 #[cfg(feature = "crdt-yjs")]
3348 fn crdt_column_bytes(
3349 &self,
3350 table: &str,
3351 row_id: &str,
3352 column: &str,
3353 ) -> Result<Option<Vec<u8>>, String> {
3354 let schema_table = self
3355 .schema
3356 .table(table)
3357 .ok_or_else(|| format!("unknown table {table:?}"))?;
3358 let col = schema_table
3359 .columns
3360 .iter()
3361 .find(|c| c.name == column)
3362 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
3363 if col.ty != ColumnType::Crdt {
3364 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
3365 }
3366 let sql = format!(
3367 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
3368 quote_ident(column),
3369 visible_table(table),
3370 quote_ident(&schema_table.primary_key)
3371 );
3372 let bytes: Option<Vec<u8>> = self
3373 .conn
3374 .query_row(&sql, rusqlite::params![row_id], |row| {
3375 row.get::<_, Option<Vec<u8>>>(0)
3376 })
3377 .map_err(|e| match e {
3378 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
3379 other => other.to_string(),
3380 })?;
3381 Ok(bytes)
3382 }
3383
3384 #[cfg(feature = "crdt-yjs")]
3388 pub fn crdt_text(
3389 &self,
3390 table: &str,
3391 row_id: &str,
3392 column: &str,
3393 name: &str,
3394 ) -> Result<String, String> {
3395 let bytes = self
3396 .crdt_column_bytes(table, row_id, column)?
3397 .unwrap_or_default();
3398 crate::crdt::text(&bytes, name)
3399 }
3400
3401 #[cfg(feature = "crdt-yjs")]
3405 pub fn crdt_insert_text(
3406 &mut self,
3407 table: &str,
3408 row_id: &str,
3409 column: &str,
3410 name: &str,
3411 index: u32,
3412 value: &str,
3413 ) -> Result<String, String> {
3414 let current = self
3415 .crdt_column_bytes(table, row_id, column)?
3416 .unwrap_or_default();
3417 let update = crate::crdt::insert_text(¤t, name, index, value)?;
3418 self.crdt_push_update(table, row_id, column, &update)
3419 }
3420
3421 #[cfg(feature = "crdt-yjs")]
3424 pub fn crdt_delete_text(
3425 &mut self,
3426 table: &str,
3427 row_id: &str,
3428 column: &str,
3429 name: &str,
3430 index: u32,
3431 len: u32,
3432 ) -> Result<String, String> {
3433 let current = self
3434 .crdt_column_bytes(table, row_id, column)?
3435 .unwrap_or_default();
3436 let update = crate::crdt::delete_text(¤t, name, index, len)?;
3437 self.crdt_push_update(table, row_id, column, &update)
3438 }
3439
3440 #[cfg(feature = "crdt-yjs")]
3445 pub fn crdt_apply_update(
3446 &mut self,
3447 table: &str,
3448 row_id: &str,
3449 column: &str,
3450 update: &[u8],
3451 ) -> Result<String, String> {
3452 let current = self
3453 .crdt_column_bytes(table, row_id, column)?
3454 .unwrap_or_default();
3455 let next = crate::crdt::apply_update(¤t, update)?;
3456 self.crdt_push_update(table, row_id, column, &next)
3457 }
3458
3459 #[cfg(feature = "crdt-yjs")]
3465 fn crdt_push_update(
3466 &mut self,
3467 table: &str,
3468 row_id: &str,
3469 column: &str,
3470 crdt_bytes: &[u8],
3471 ) -> Result<String, String> {
3472 let schema_table = self
3473 .schema
3474 .table(table)
3475 .ok_or_else(|| format!("unknown table {table:?}"))?
3476 .clone();
3477 let mut values: Map<String, Value> = self
3480 .read_rows(table)?
3481 .into_iter()
3482 .find(|r| r.row_id == row_id)
3483 .map(|r| r.values)
3484 .unwrap_or_else(|| {
3485 let mut map = Map::new();
3486 map.insert(
3487 schema_table.primary_key.clone(),
3488 Value::from(row_id.to_owned()),
3489 );
3490 map
3491 });
3492 let mut bytes_obj = Map::new();
3494 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
3495 values.insert(column.to_owned(), Value::Object(bytes_obj));
3496 self.mutate(vec![Mutation::Upsert {
3497 table: table.to_owned(),
3498 values,
3499 base_version: None,
3500 }])
3501 }
3502
3503 pub fn query(&self, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {
3518 query_connection(&self.conn, sql, params)
3519 }
3520
3521 pub fn query_snapshot(
3523 &mut self,
3524 sql: &str,
3525 params: &[Value],
3526 coverage: &[WindowCoverage],
3527 ) -> Result<QuerySnapshot, String> {
3528 snapshot_connection(&self.conn, sql, params, coverage)
3529 }
3530
3531 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
3534 let mut frames = vec![Frame::ReqHeader {
3535 client_id: self.client_id.clone(),
3536 schema_version: self.schema.version,
3537 }];
3538 let mut pushed_ids = Vec::new();
3539 let mut ops_in_request = 0usize;
3540 let mut deferred_commits = 0usize;
3541 for (index, commit) in self.outbox.iter().enumerate() {
3542 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
3547 deferred_commits = self.outbox.len() - index;
3548 break;
3549 }
3550 ops_in_request += commit.ops.len();
3551 let operations = commit
3552 .ops
3553 .iter()
3554 .map(|op| {
3555 let payload = op.values.as_ref().and_then(|values| {
3556 let table = self.schema.table(&op.table)?;
3557 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
3561 });
3562 ssp2::model::Operation {
3563 table: op.table.clone(),
3564 row_id: op.row_id.clone(),
3565 op: if op.upsert { Op::Upsert } else { Op::Delete },
3566 base_version: op.base_version,
3567 payload,
3568 }
3569 })
3570 .collect();
3571 frames.push(Frame::PushCommit {
3572 client_commit_id: commit.client_commit_id.clone(),
3573 operations,
3574 });
3575 pushed_ids.push(commit.client_commit_id.clone());
3576 }
3577 let accept = self.limits.accept.unwrap_or(if url_capable {
3580 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
3581 } else {
3582 DEFAULT_ACCEPT
3583 });
3584 frames.push(Frame::PullHeader {
3585 limit_commits: self.limits.limit_commits.unwrap_or(0),
3586 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
3587 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
3588 accept,
3589 });
3590 let mut fresh = Vec::new();
3591 for sub in &self.subs {
3592 if sub.state != SubState::Active {
3593 continue;
3594 }
3595 let mut scopes = sub.requested.clone();
3596 sort_scope_map(&mut scopes);
3597 frames.push(Frame::Subscription {
3598 id: sub.id.clone(),
3599 table: sub.table.clone(),
3600 scopes,
3601 params: sub.params.clone().map(RawJson),
3602 cursor: sub.cursor,
3603 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
3604 });
3605 fresh.push((
3606 sub.id.clone(),
3607 sub.cursor < 0 && sub.bootstrap_state.is_none(),
3608 ));
3609 }
3610 let message = Message {
3611 msg_kind: MsgKind::Request,
3612 frames,
3613 };
3614 (
3615 message,
3616 RequestMeta {
3617 pushed_ids,
3618 fresh,
3619 accept,
3620 deferred_commits,
3621 },
3622 )
3623 }
3624
3625 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
3628 if self.stopped {
3629 return SyncOutcome::Ok(SyncReport {
3632 schema_floor: self.schema_floor.clone(),
3633 ..SyncReport::default()
3634 });
3635 }
3636 self.set_sync_needed(false, false);
3639 if self.schema_has_blobs() {
3642 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
3643 if Self::retryable_transport_code(&code) {
3644 self.schedule_background_retry();
3645 }
3646 return SyncOutcome::Failed {
3647 error_code: code,
3648 message,
3649 };
3650 }
3651 }
3652 let (message, meta) = self.build_request(transport.supports_url_fetch());
3653 let request_bytes = encode_message(&message);
3654 let round = if self.realtime_connected {
3658 transport.realtime_sync(&request_bytes)
3659 } else {
3660 transport.sync(&request_bytes)
3661 };
3662 let response_bytes = match round {
3663 Ok(bytes) => bytes,
3664 Err(TransportError { code, message }) => {
3665 self.record_lease_error(&code);
3668 if Self::retryable_transport_code(&code) {
3669 self.schedule_background_retry();
3670 }
3671 return SyncOutcome::Failed {
3672 error_code: code,
3673 message,
3674 };
3675 }
3676 };
3677 let response = match decode_message(&response_bytes) {
3678 Ok(message) => message,
3679 Err(error) => {
3680 return SyncOutcome::Failed {
3683 error_code: error.code.as_str().to_owned(),
3684 message: error.detail,
3685 };
3686 }
3687 };
3688 if response.msg_kind != MsgKind::Response {
3689 return SyncOutcome::Failed {
3690 error_code: "sync.invalid_request".to_owned(),
3691 message: "expected a response message".to_owned(),
3692 };
3693 }
3694 let outcome = self.process_response(transport, response, &meta);
3695 match &outcome {
3696 SyncOutcome::Ok(_) => self.reset_background_retry(),
3697 SyncOutcome::Failed { error_code, .. }
3698 if Self::retryable_transport_code(error_code) =>
3699 {
3700 self.schedule_background_retry();
3701 }
3702 SyncOutcome::Failed { .. } => {}
3703 }
3704 if meta.deferred_commits > 0 {
3705 self.set_sync_needed(true, true);
3708 }
3709 outcome
3710 }
3711
3712 pub fn sync_until_idle(
3713 &mut self,
3714 transport: &mut dyn Transport,
3715 max_rounds: Option<u32>,
3716 ) -> SyncOutcome {
3717 let rounds = max_rounds.unwrap_or(12).max(1);
3718 let mut aggregate = SyncReport::default();
3719 for _ in 0..rounds {
3720 match self.sync(transport) {
3721 SyncOutcome::Failed {
3722 error_code,
3723 message,
3724 } => {
3725 return SyncOutcome::Failed {
3726 error_code,
3727 message,
3728 };
3729 }
3730 SyncOutcome::Ok(report) => {
3731 aggregate.pushed += report.pushed;
3732 aggregate.applied.extend(report.applied.iter().cloned());
3733 aggregate.rejected.extend(report.rejected.iter().cloned());
3734 aggregate.retryable.extend(report.retryable.iter().cloned());
3735 aggregate.conflicts += report.conflicts;
3736 aggregate.commits_applied += report.commits_applied;
3737 aggregate.segment_rows_applied += report.segment_rows_applied;
3738 aggregate.bootstrapping = report.bootstrapping.clone();
3739 aggregate.resets.extend(report.resets.iter().cloned());
3740 aggregate.revoked.extend(report.revoked.iter().cloned());
3741 aggregate.failed.extend(report.failed.iter().cloned());
3742 if report.schema_floor.is_some() {
3743 aggregate.schema_floor = report.schema_floor.clone();
3744 }
3745 let more = !report.bootstrapping.is_empty()
3751 || report.commits_applied > 0
3752 || report.segment_rows_applied > 0
3753 || !report.resets.is_empty()
3754 || self.sync_needed;
3755 if !more {
3756 break;
3757 }
3758 }
3759 }
3760 }
3761 SyncOutcome::Ok(aggregate)
3762 }
3763
3764 fn process_response(
3765 &mut self,
3766 transport: &mut dyn Transport,
3767 response: Message,
3768 meta: &RequestMeta,
3769 ) -> SyncOutcome {
3770 let mut report = SyncReport {
3771 pushed: meta.pushed_ids.len() as u32,
3772 ..SyncReport::default()
3773 };
3774 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
3775 HashMap::new();
3776 for frame in &response.frames {
3777 if let Frame::PushResultDetails {
3778 client_commit_id,
3779 entries,
3780 } = frame
3781 {
3782 let details = rejection_details_by_commit
3783 .entry(client_commit_id.clone())
3784 .or_default();
3785 for entry in entries {
3786 let parsed = match RejectionDetails::parse(&entry.details.0) {
3787 Ok(value) => value,
3788 Err(message) => {
3789 return SyncOutcome::Failed {
3790 error_code: "sync.invalid_request".to_owned(),
3791 message,
3792 };
3793 }
3794 };
3795 details.insert(entry.op_index, parsed);
3796 }
3797 }
3798 }
3799 let mut frames = response.frames.into_iter();
3800 match frames.next() {
3801 Some(Frame::RespHeader {
3802 required_schema_version,
3803 latest_schema_version,
3804 }) => {
3805 if let Some(required) = required_schema_version {
3806 let floor = SchemaFloor {
3809 required_schema_version: Some(required),
3810 latest_schema_version,
3811 };
3812 self.set_schema_floor(Some(floor.clone()));
3813 report.schema_floor = Some(floor);
3814 return SyncOutcome::Ok(report);
3815 }
3816 }
3817 _ => {
3818 return SyncOutcome::Failed {
3819 error_code: "sync.invalid_request".to_owned(),
3820 message: "response does not start with RESP_HEADER".to_owned(),
3821 };
3822 }
3823 }
3824
3825 let mut failure: Option<(String, String)> = None;
3826 while let Some(frame) = frames.next() {
3827 match frame {
3828 Frame::PushResult {
3829 client_commit_id,
3830 status,
3831 commit_seq: _,
3832 results,
3833 } => {
3834 self.handle_push_result(
3835 &client_commit_id,
3836 status,
3837 &results,
3838 rejection_details_by_commit.get(&client_commit_id),
3839 &mut report,
3840 );
3841 }
3842 Frame::PushResultDetails { .. } => {}
3843 Frame::SubStart {
3844 id,
3845 status,
3846 reason_code,
3847 effective_scopes,
3848 bootstrap: _,
3849 } => {
3850 let mut body = Vec::new();
3851 let mut sub_end: Option<(i64, Option<String>)> = None;
3852 for inner in frames.by_ref() {
3853 match inner {
3854 Frame::SubEnd {
3855 next_cursor,
3856 bootstrap_state,
3857 } => {
3858 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
3859 break;
3860 }
3861 Frame::Unknown { .. } => {}
3862 other => body.push(other),
3863 }
3864 }
3865 let Some((next_cursor, bootstrap_state)) = sub_end else {
3866 failure = Some((
3867 "sync.invalid_request".to_owned(),
3868 "subscription section without SUB_END".to_owned(),
3869 ));
3870 break;
3871 };
3872 if let Err(SectionError::Abort(code, message)) = self.process_section(
3873 transport,
3874 &id,
3875 status,
3876 &reason_code,
3877 effective_scopes,
3878 body,
3879 next_cursor,
3880 bootstrap_state,
3881 meta,
3882 &mut report,
3883 ) {
3884 failure = Some((code, message));
3885 break;
3886 }
3887 }
3888 Frame::Lease {
3889 lease_id,
3890 expires_at_ms,
3891 } => {
3892 self.set_lease_state(Some(LeaseState {
3895 lease_id: Some(lease_id),
3896 expires_at_ms: Some(expires_at_ms),
3897 error_code: None,
3898 }));
3899 }
3900 Frame::Error { code, message, .. } => {
3901 failure = Some((code, message));
3904 break;
3905 }
3906 Frame::Unknown { .. } => {}
3907 _ => {
3908 failure = Some((
3909 "sync.invalid_request".to_owned(),
3910 "unexpected frame in response".to_owned(),
3911 ));
3912 break;
3913 }
3914 }
3915 }
3916
3917 if self.overlay_dirty.get() {
3920 self.rebuild_overlay();
3921 }
3922 self.reconcile_blob_refcounts(false);
3929
3930 if let Some((error_code, message)) = failure {
3931 return SyncOutcome::Failed {
3932 error_code,
3933 message,
3934 };
3935 }
3936 self.drain_pending_evictions();
3939 self.ack_after_pull(transport);
3940 if self.upgrading && report.bootstrapping.is_empty() {
3943 self.set_upgrading(false);
3944 }
3945 SyncOutcome::Ok(report)
3946 }
3947
3948 fn handle_push_result(
3951 &mut self,
3952 client_commit_id: &str,
3953 status: PushStatus,
3954 results: &[OpResult],
3955 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
3956 report: &mut SyncReport,
3957 ) {
3958 let Some(index) = self
3959 .outbox
3960 .iter()
3961 .position(|c| c.client_commit_id == client_commit_id)
3962 else {
3963 return;
3964 };
3965 if self.begin_observation("syncular_push_result").is_err() {
3966 return;
3967 }
3968 let mut batch = ChangeAccumulator::default();
3969 let operations = self.outbox[index].ops.clone();
3970 match status {
3971 PushStatus::Applied | PushStatus::Cached => {
3972 let journal_results = results
3975 .iter()
3976 .map(|result| {
3977 let op_index = match result {
3978 OpResult::Applied { op_index }
3979 | OpResult::Conflict { op_index, .. }
3980 | OpResult::Error { op_index, .. } => *op_index,
3981 };
3982 CommitOperationOutcome::Applied { op_index }
3983 })
3984 .collect::<Vec<_>>();
3985 let outcome_status = if status == PushStatus::Applied {
3986 CommitOutcomeStatus::Applied
3987 } else {
3988 CommitOutcomeStatus::Cached
3989 };
3990 if self
3991 .persist_commit_outcome(
3992 client_commit_id,
3993 outcome_status,
3994 &journal_results,
3995 None,
3996 )
3997 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
3998 .and_then(|()| self.prune_commit_outcomes())
3999 .is_err()
4000 {
4001 self.rollback_observation("syncular_push_result");
4002 return;
4003 }
4004 report.applied.push(client_commit_id.to_owned());
4005 self.outbox.remove(index);
4006 self.overlay_dirty.set(true);
4007 batch.status = true;
4008 batch.outcomes = true;
4009 }
4010 PushStatus::Rejected => {
4011 if results.iter().any(|result| {
4012 matches!(
4013 result,
4014 OpResult::Error {
4015 code,
4016 retryable: true,
4017 ..
4018 } if code == "sync.idempotency_cache_miss"
4019 )
4020 }) {
4021 report.retryable.push(client_commit_id.to_owned());
4024 if self
4025 .finish_observation("syncular_push_result", batch)
4026 .is_err()
4027 {
4028 self.rollback_observation("syncular_push_result");
4029 }
4030 return;
4031 }
4032
4033 let mut journal_results = Vec::with_capacity(results.len());
4034 let mut conflicts = Vec::new();
4035 let mut rejections = Vec::new();
4036 for result in results {
4037 match result {
4038 OpResult::Applied { op_index } => {
4039 journal_results.push(CommitOperationOutcome::Applied {
4040 op_index: *op_index,
4041 });
4042 }
4043 OpResult::Conflict {
4044 op_index,
4045 code,
4046 message,
4047 server_version,
4048 server_row,
4049 } => {
4050 let operation = operations
4051 .get(*op_index as usize)
4052 .map(CommitOperation::from);
4053 let (table, row_id) = operation
4054 .as_ref()
4055 .map(|op| (op.table.clone(), op.row_id.clone()))
4056 .unwrap_or_default();
4057 let server_row_json = self
4058 .schema
4059 .table(&table)
4060 .and_then(|t| {
4061 decode_row_bytes(t, server_row, &self.encryption)
4062 .ok()
4063 .map(|row| (t, row))
4064 })
4065 .map(|(t, row)| {
4066 let mut map = Map::new();
4067 for (i, column) in t.columns.iter().enumerate() {
4068 map.insert(
4069 column.name.clone(),
4070 column_value_to_json(row.get(i).unwrap_or(&None)),
4071 );
4072 }
4073 map
4074 })
4075 .unwrap_or_default();
4076 let conflict = ConflictRecord {
4077 client_commit_id: client_commit_id.to_owned(),
4078 op_index: *op_index,
4079 table,
4080 row_id,
4081 code: code.clone(),
4082 message: message.clone(),
4083 server_version: *server_version,
4084 server_row: server_row_json,
4085 operation,
4086 };
4087 journal_results.push(CommitOperationOutcome::Conflict {
4088 conflict: conflict.clone(),
4089 });
4090 conflicts.push(conflict);
4091 }
4092 OpResult::Error {
4093 op_index,
4094 code,
4095 message,
4096 retryable,
4097 } => {
4098 let rejection = RejectionRecord {
4099 client_commit_id: client_commit_id.to_owned(),
4100 op_index: *op_index,
4101 code: code.clone(),
4102 message: message.clone(),
4103 retryable: *retryable,
4104 details: rejection_details
4105 .and_then(|details| details.get(op_index))
4106 .cloned(),
4107 operation: operations
4108 .get(*op_index as usize)
4109 .map(CommitOperation::from),
4110 };
4111 journal_results.push(CommitOperationOutcome::Error {
4112 rejection: rejection.clone(),
4113 });
4114 rejections.push(rejection);
4115 }
4116 }
4117 }
4118 let outcome_status = if conflicts.is_empty() {
4119 CommitOutcomeStatus::Rejected
4120 } else {
4121 CommitOutcomeStatus::Conflict
4122 };
4123 if self
4124 .persist_commit_outcome(
4125 client_commit_id,
4126 outcome_status,
4127 &journal_results,
4128 Some(&operations),
4129 )
4130 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4131 .and_then(|()| self.prune_commit_outcomes())
4132 .is_err()
4133 {
4134 self.rollback_observation("syncular_push_result");
4135 return;
4136 }
4137 report.conflicts += conflicts.len() as u32;
4138 report.rejected.push(client_commit_id.to_owned());
4139 batch.conflicts = !conflicts.is_empty();
4140 batch.rejections = !rejections.is_empty();
4141 batch.status = true;
4142 batch.outcomes = true;
4143 self.conflicts.extend(conflicts);
4144 self.rejections.extend(rejections);
4145 self.outbox.remove(index);
4146 self.overlay_dirty.set(true);
4147 }
4148 }
4149 if batch.status {
4150 for operation in &operations {
4151 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
4152 batch.table(&operation.table);
4153 }
4154 }
4155 self.rebuild_overlay_if_dirty();
4156 }
4157 if self
4158 .finish_observation("syncular_push_result", batch)
4159 .is_err()
4160 {
4161 self.rollback_observation("syncular_push_result");
4162 }
4163 }
4164
4165 #[allow(clippy::too_many_arguments)]
4168 fn process_section(
4169 &mut self,
4170 transport: &mut dyn Transport,
4171 id: &str,
4172 status: SubStatus,
4173 reason_code: &str,
4174 effective_scopes: Vec<(String, Vec<String>)>,
4175 body: Vec<Frame>,
4176 next_cursor: i64,
4177 bootstrap_state: Option<String>,
4178 meta: &RequestMeta,
4179 report: &mut SyncReport,
4180 ) -> Result<(), SectionError> {
4181 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4182 return Ok(()); };
4184 match status {
4185 SubStatus::Revoked => {
4186 self.begin_observation("syncular_revocation")
4187 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4188 let mut batch = ChangeAccumulator::default();
4189 let registered = self.window_unit_by_sub_id(id);
4190 let (table, effective) = {
4192 let sub = &self.subs[sub_index];
4193 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4194 };
4195 let purged = self.purge_scope_rows(&table, &effective);
4196 match purged {
4197 Ok(()) => {
4198 self.record_scope_map(&mut batch, &table, &effective);
4199 let sub = &mut self.subs[sub_index];
4200 sub.state = SubState::Revoked;
4201 sub.reason_code = Some(if reason_code.is_empty() {
4202 "sync.scope_revoked".to_owned()
4203 } else {
4204 reason_code.to_owned()
4205 });
4206 report.revoked.push(id.to_owned());
4207 let doomed_effective = effective;
4208 let sub_table = table;
4209 self.persist_sub(&self.subs[sub_index].clone());
4210 let dropped = self
4211 .drop_doomed_outbox(&sub_table, &doomed_effective)
4212 .map_err(|message| {
4213 SectionError::Abort("storage.failed".to_owned(), message)
4214 })?;
4215 if dropped {
4216 batch.status = true;
4217 batch.rejections = true;
4218 batch.outcomes = true;
4219 }
4220 self.reconcile_blob_refcounts(true);
4223 }
4224 Err(()) => {
4225 let sub = &mut self.subs[sub_index];
4228 sub.state = SubState::Failed;
4229 sub.reason_code = Some("sync.scope_revoked".to_owned());
4230 report.failed.push(id.to_owned());
4231 self.persist_sub(&self.subs[sub_index].clone());
4232 }
4233 }
4234 if let Some((base_key, unit)) = registered {
4235 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4236 }
4237 self.rebuild_overlay_if_dirty();
4238 self.finish_observation("syncular_revocation", batch)
4239 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4240 Ok(())
4241 }
4242 SubStatus::Reset => {
4243 self.begin_observation("syncular_reset")
4244 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4245 let mut batch = ChangeAccumulator::default();
4246 let registered = self.window_unit_by_sub_id(id);
4247 let sub = &mut self.subs[sub_index];
4250 sub.cursor = -1;
4251 sub.bootstrap_state = None;
4252 report.resets.push(id.to_owned());
4253 self.persist_sub(&self.subs[sub_index].clone());
4254 if let Some((base_key, unit)) = registered {
4255 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4256 }
4257 self.finish_observation("syncular_reset", batch)
4258 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4259 Ok(())
4260 }
4261 SubStatus::Active => {
4262 let fresh = meta
4263 .fresh
4264 .iter()
4265 .find(|(fid, _)| fid == id)
4266 .map(|(_, f)| *f)
4267 .unwrap_or(false);
4268 let was_pending = self.subs[sub_index].cursor < 0
4269 || self.subs[sub_index].bootstrap_state.is_some();
4270 let registered = self.window_unit_by_sub_id(id);
4271 self.subs[sub_index].effective = Some(effective_scopes);
4273 self.begin_observation("syncular_section")
4274 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4275 let mut batch = ChangeAccumulator::default();
4276 let outcome = self.apply_section_body(
4277 transport, sub_index, body, fresh, meta, report, &mut batch,
4278 );
4279 match outcome {
4280 Ok(()) => {
4281 let sub = &mut self.subs[sub_index];
4282 sub.cursor = next_cursor;
4285 sub.bootstrap_state = bootstrap_state;
4286 sub.synced_once = true;
4287 if sub.bootstrap_state.is_some() {
4288 report.bootstrapping.push(id.to_owned());
4289 }
4290 let completed =
4291 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
4292 self.persist_sub(&self.subs[sub_index].clone());
4293 if completed {
4294 if let Some((base_key, unit)) = registered.clone() {
4295 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4296 }
4297 }
4298 self.rebuild_overlay_if_dirty();
4299 self.finish_observation("syncular_section", batch)
4300 .map_err(|message| {
4301 SectionError::Abort("storage.failed".to_owned(), message)
4302 })?;
4303 Ok(())
4304 }
4305 Err(SectionError::FailClosed) => {
4306 self.rollback_observation("syncular_section");
4309 self.begin_observation("syncular_section_failure")
4310 .map_err(|message| {
4311 SectionError::Abort("storage.failed".to_owned(), message)
4312 })?;
4313 let mut failure_batch = ChangeAccumulator::default();
4314 let sub = &mut self.subs[sub_index];
4315 sub.state = SubState::Failed;
4316 sub.reason_code = Some("sync.scope_revoked".to_owned());
4317 report.failed.push(id.to_owned());
4318 self.persist_sub(&self.subs[sub_index].clone());
4319 if let Some((base_key, unit)) = registered {
4320 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
4321 }
4322 self.finish_observation("syncular_section_failure", failure_batch)
4323 .map_err(|message| {
4324 SectionError::Abort("storage.failed".to_owned(), message)
4325 })?;
4326 Ok(())
4327 }
4328 Err(SectionError::Abort(code, message)) => {
4329 self.rollback_observation("syncular_section");
4332 Err(SectionError::Abort(code, message))
4333 }
4334 }
4335 }
4336 }
4337 }
4338
4339 #[allow(clippy::too_many_arguments)]
4343 fn apply_section_body(
4344 &mut self,
4345 transport: &mut dyn Transport,
4346 sub_index: usize,
4347 body: Vec<Frame>,
4348 fresh: bool,
4349 meta: &RequestMeta,
4350 report: &mut SyncReport,
4351 batch: &mut ChangeAccumulator,
4352 ) -> Result<(), SectionError> {
4353 let mut saw_segment = false;
4354 for frame in body {
4355 match frame {
4356 Frame::Commit {
4357 tables, changes, ..
4358 } => {
4359 self.record_commit_changes(batch, &tables, &changes);
4360 self.apply_commit_changes(&tables, &changes)
4361 .map_err(|(c, m)| SectionError::Abort(c, m))?;
4362 report.commits_applied += 1;
4363 }
4364 Frame::SegmentInline { payload } => {
4365 let segment = decode_rows_segment(&payload)
4366 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
4367 let first = !saw_segment;
4368 saw_segment = true;
4369 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4370 let cleared =
4371 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4372 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4373 if applied > 0 || cleared {
4374 batch.table(&segment.table);
4375 }
4376 report.segment_rows_applied += applied;
4377 }
4378 Frame::SegmentRef {
4379 segment_id,
4380 media_type,
4381 table,
4382 row_count,
4383 as_of_commit_seq,
4384 scope_digest,
4385 row_cursor,
4386 next_row_cursor,
4387 url,
4388 url_expires_at_ms,
4389 ..
4390 } => {
4391 let advertised = match media_type {
4394 MediaType::Rows => {
4395 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
4396 || meta.accept & ACCEPT_INLINE_ROWS != 0
4397 }
4398 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
4399 };
4400 if !advertised {
4401 return Err(SectionError::Abort(
4402 "sync.invalid_request".to_owned(),
4403 format!(
4404 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
4405 media_type.name()
4406 ),
4407 ));
4408 }
4409 let bytes = if let Some(url) = url {
4410 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
4415 return Err(SectionError::Abort(
4416 "sync.invalid_request".to_owned(),
4417 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
4418 .to_owned(),
4419 ));
4420 }
4421 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
4423 return Err(SectionError::Abort(
4424 "sync.segment_expired".to_owned(),
4425 format!(
4426 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
4427 ),
4428 ));
4429 }
4430 transport
4431 .fetch_url(&url)
4432 .map_err(|e| SectionError::Abort(e.code, e.message))?
4433 } else {
4434 let requested_scopes_json =
4435 canonical_scope_json(&self.subs[sub_index].requested);
4436 transport
4437 .download_segment(&SegmentRequest {
4438 segment_id: segment_id.clone(),
4439 table,
4440 requested_scopes_json,
4441 })
4442 .map_err(|e| SectionError::Abort(e.code, e.message))?
4443 };
4444 let digest = Sha256::digest(&bytes);
4446 let expected = segment_id
4447 .strip_prefix("sha256:")
4448 .unwrap_or(segment_id.as_str());
4449 if bytes_to_hex(&digest) != expected {
4450 return Err(SectionError::Abort(
4451 "sync.invalid_request".to_owned(),
4452 "segment bytes do not match the content address (§5.1)".to_owned(),
4453 ));
4454 }
4455 if media_type == MediaType::Sqlite {
4456 if row_cursor.is_some() || next_row_cursor.is_some() {
4459 return Err(SectionError::Abort(
4460 "sync.invalid_request".to_owned(),
4461 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
4462 .to_owned(),
4463 ));
4464 }
4465 let first = !saw_segment;
4466 saw_segment = true;
4467 let sub_table = self.subs[sub_index].table.clone();
4468 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4469 let cleared =
4470 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
4471 let applied = self.apply_sqlite_segment(
4472 sub_index,
4473 &bytes,
4474 fresh && first,
4475 row_count,
4476 as_of_commit_seq,
4477 &scope_digest,
4478 )?;
4479 if applied > 0 || cleared {
4480 batch.table(&sub_table);
4481 }
4482 report.segment_rows_applied += applied;
4483 } else {
4484 let segment = decode_rows_segment(&bytes).map_err(|e| {
4485 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
4486 })?;
4487 let first = row_cursor.is_none();
4488 saw_segment = true;
4489 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4490 let cleared =
4491 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4492 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4493 if applied > 0 || cleared {
4494 batch.table(&segment.table);
4495 }
4496 report.segment_rows_applied += applied;
4497 }
4498 }
4499 Frame::Unknown { .. } => {}
4500 _ => {
4501 return Err(SectionError::Abort(
4502 "sync.invalid_request".to_owned(),
4503 "unexpected frame inside a subscription section".to_owned(),
4504 ));
4505 }
4506 }
4507 }
4508 Ok(())
4509 }
4510
4511 fn apply_commit_changes(
4512 &mut self,
4513 tables: &[String],
4514 changes: &[ssp2::model::Change],
4515 ) -> Result<(), (String, String)> {
4516 for change in changes {
4517 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
4518 (
4519 "sync.invalid_request".to_owned(),
4520 "change tableIndex out of range".to_owned(),
4521 )
4522 })?;
4523 let table = self.schema.table(table_name).ok_or_else(|| {
4524 (
4525 "sync.schema_mismatch".to_owned(),
4526 format!("change targets unknown table {table_name:?}"),
4527 )
4528 })?;
4529 match change.op {
4530 Op::Upsert => {
4531 let payload = change.row.as_ref().ok_or_else(|| {
4532 (
4533 "sync.invalid_request".to_owned(),
4534 "upsert change without row payload".to_owned(),
4535 )
4536 })?;
4537 let row = decode_row_bytes(table, payload, &self.encryption)
4539 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4540 let version = change.row_version.unwrap_or(0);
4541 let table_name = table.name.clone();
4542 self.write_base_row(&table_name, &row, version)
4543 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4544 }
4545 Op::Delete => {
4546 self.delete_base_row(table_name, &change.row_id)
4547 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4548 }
4549 }
4550 }
4551 Ok(())
4552 }
4553
4554 fn apply_segment(
4559 &mut self,
4560 sub_index: usize,
4561 segment: &RowsSegment,
4562 first_fresh_page: bool,
4563 ) -> Result<u32, SectionError> {
4564 let (sub_table, effective) = {
4565 let sub = &self.subs[sub_index];
4566 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4567 };
4568 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
4569 SectionError::Abort(
4570 "sync.schema_mismatch".to_owned(),
4571 format!("subscription table {sub_table:?} is not in the client schema"),
4572 )
4573 })?;
4574 let matches = segment.table == table.name
4579 && segment.schema_version == self.schema.version
4580 && segment.columns.len() == table.wire_columns.len()
4581 && segment
4582 .columns
4583 .iter()
4584 .zip(table.wire_columns.iter())
4585 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
4586 if !matches {
4587 return Err(SectionError::Abort(
4588 "sync.schema_mismatch".to_owned(),
4589 "segment column table does not match the generated schema (§5.2)".to_owned(),
4590 ));
4591 }
4592 if first_fresh_page {
4593 self.purge_scope_rows(&table.name, &effective)
4597 .map_err(|()| SectionError::FailClosed)?;
4598 }
4599 let mut applied = 0u32;
4600 for block in &segment.blocks {
4601 for row in block {
4602 let decrypted;
4607 let values = if table.has_encrypted_columns() {
4608 let mut values = row.values.clone();
4609 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
4610 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
4611 decrypted = values;
4612 &decrypted
4613 } else {
4614 &row.values
4615 };
4616 self.write_base_row(&table.name, values, row.server_version)
4619 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
4620 applied += 1;
4621 }
4622 }
4623 Ok(applied)
4624 }
4625
4626 fn apply_sqlite_segment(
4634 &mut self,
4635 sub_index: usize,
4636 bytes: &[u8],
4637 first_fresh_page: bool,
4638 row_count: i64,
4639 as_of_commit_seq: i64,
4640 scope_digest: &str,
4641 ) -> Result<u32, SectionError> {
4642 let invalid = |detail: &str| {
4643 SectionError::Abort(
4644 "sync.invalid_request".to_owned(),
4645 format!("sqlite segment rejected: {detail} (§5.3)"),
4646 )
4647 };
4648 let (sub_table, effective) = {
4649 let sub = &self.subs[sub_index];
4650 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4651 };
4652 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
4653 SectionError::Abort(
4654 "sync.schema_mismatch".to_owned(),
4655 format!("subscription table {sub_table:?} is not in the client schema"),
4656 )
4657 })?;
4658
4659 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
4660 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
4661 let img = match rusqlite::Connection::open_with_flags(
4662 &path,
4663 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
4664 ) {
4665 Ok(conn) => conn,
4666 Err(_) => {
4667 let _ = std::fs::remove_file(&path);
4668 return Err(invalid("bytes do not open as a SQLite database"));
4669 }
4670 };
4671 let outcome = self.apply_sqlite_image(
4672 &img,
4673 &table,
4674 first_fresh_page,
4675 &effective,
4676 row_count,
4677 as_of_commit_seq,
4678 scope_digest,
4679 );
4680 drop(img);
4681 let _ = std::fs::remove_file(&path);
4682 outcome
4683 }
4684
4685 #[allow(clippy::too_many_arguments)]
4686 fn apply_sqlite_image(
4687 &mut self,
4688 img: &rusqlite::Connection,
4689 table: &crate::schema::TableSchema,
4690 first_fresh_page: bool,
4691 effective: &[(String, Vec<String>)],
4692 row_count: i64,
4693 as_of_commit_seq: i64,
4694 scope_digest: &str,
4695 ) -> Result<u32, SectionError> {
4696 let invalid = |detail: String| {
4697 SectionError::Abort(
4698 "sync.invalid_request".to_owned(),
4699 format!("sqlite segment rejected: {detail} (§5.3)"),
4700 )
4701 };
4702
4703 type MetaRow = (i64, String, i64, i64, String, i64, i64);
4706 let meta: MetaRow = img
4707 .query_row(
4708 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
4709 \"scopeDigest\", \"rowCount\",
4710 (SELECT count(*) FROM _syncular_segment)
4711 FROM _syncular_segment",
4712 [],
4713 |row| {
4714 Ok((
4715 row.get(0)?,
4716 row.get(1)?,
4717 row.get(2)?,
4718 row.get(3)?,
4719 row.get(4)?,
4720 row.get(5)?,
4721 row.get(6)?,
4722 ))
4723 },
4724 )
4725 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
4726 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
4727 if meta_count != 1 {
4728 return Err(invalid(format!(
4729 "_syncular_segment must contain exactly one row, found {meta_count}"
4730 )));
4731 }
4732 if format != 1 {
4733 return Err(invalid(format!("format {format}")));
4734 }
4735 if meta_table != table.name {
4736 return Err(invalid(format!("image table {meta_table:?}")));
4737 }
4738 if schema_version != i64::from(self.schema.version) {
4739 return Err(invalid(format!("schemaVersion {schema_version}")));
4740 }
4741 if pin != as_of_commit_seq {
4742 return Err(invalid(format!("asOfCommitSeq {pin}")));
4743 }
4744 if digest != scope_digest {
4745 return Err(invalid("scopeDigest mismatch".to_owned()));
4746 }
4747 if meta_rows != row_count {
4748 return Err(invalid(format!("rowCount {meta_rows}")));
4749 }
4750
4751 let mut names: Vec<String> = Vec::new();
4753 {
4754 let mut stmt = img
4755 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
4756 .map_err(|_| invalid("image data table missing".to_owned()))?;
4757 let mut rows = stmt
4758 .query([])
4759 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4760 while let Some(row) = rows
4761 .next()
4762 .map_err(|_| invalid("image data table unreadable".to_owned()))?
4763 {
4764 names.push(
4765 row.get::<_, String>(1)
4766 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
4767 );
4768 }
4769 }
4770 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
4771 expected.push("_syncular_version");
4772 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
4773 return Err(SectionError::Abort(
4774 "sync.schema_mismatch".to_owned(),
4775 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
4776 ));
4777 }
4778
4779 if first_fresh_page {
4782 self.purge_scope_rows(&table.name, effective)
4783 .map_err(|()| SectionError::FailClosed)?;
4784 }
4785 self.overlay_dirty.set(true);
4791 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
4799 table.indexes.iter().filter(|i| !i.unique).collect()
4800 } else {
4801 Vec::new()
4802 };
4803 for index in &bulk_indexes {
4804 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
4805 self.conn
4806 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
4807 .map_err(|e| invalid(e.to_string()))?;
4808 }
4809 let insert = self.insert_row_sql(&base_table(&table.name), table);
4810 let applied = {
4811 let mut ins = self
4812 .conn
4813 .prepare_cached(&insert)
4814 .map_err(|e| invalid(e.to_string()))?;
4815 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
4816 let mut stmt = img
4817 .prepare(&format!(
4818 "SELECT {} FROM {}",
4819 column_list.join(", "),
4820 quote_ident(&table.name)
4821 ))
4822 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4823 let mut rows = stmt
4824 .query([])
4825 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4826 let version_index = table.columns.len();
4827 let mut applied = 0u32;
4828 while let Some(row) = rows
4829 .next()
4830 .map_err(|_| invalid("image row unreadable".to_owned()))?
4831 {
4832 for (i, column) in table.columns.iter().enumerate() {
4833 let cell = row
4834 .get_ref(i)
4835 .map_err(|_| invalid("image row unreadable".to_owned()))?;
4836 let param = image_cell_param(column, cell).map_err(&invalid)?;
4837 ins.raw_bind_parameter(i + 1, param)
4838 .map_err(|e| invalid(e.to_string()))?;
4839 }
4840 let version: i64 = row
4841 .get(version_index)
4842 .map_err(|_| invalid("image row unreadable".to_owned()))?;
4843 if version < 1 {
4844 return Err(invalid(format!(
4845 "row _syncular_version must be >= 1, got {version}"
4846 )));
4847 }
4848 ins.raw_bind_parameter(version_index + 1, version)
4849 .map_err(|e| invalid(e.to_string()))?;
4850 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
4851 applied += 1;
4852 }
4853 applied
4854 };
4855 for index in &bulk_indexes {
4856 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
4857 let cols_sql = index
4858 .columns
4859 .iter()
4860 .map(|c| quote_ident(c))
4861 .collect::<Vec<_>>()
4862 .join(", ");
4863 self.conn
4864 .execute(
4865 &format!(
4866 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
4867 base_table(&table.name)
4868 ),
4869 [],
4870 )
4871 .map_err(|e| invalid(e.to_string()))?;
4872 }
4873 if i64::from(applied) != row_count {
4874 return Err(invalid(format!(
4875 "image holds {applied} rows, descriptor says {row_count}"
4876 )));
4877 }
4878 Ok(applied)
4879 }
4880
4881 fn purge_scope_rows(
4886 &mut self,
4887 table_name: &str,
4888 effective: &[(String, Vec<String>)],
4889 ) -> Result<(), ()> {
4890 if effective.is_empty() {
4891 return Ok(());
4892 }
4893 let table = self.schema.table(table_name).ok_or(())?.clone();
4894 let mut clauses = Vec::new();
4895 let mut params: Vec<SqlValue> = Vec::new();
4896 for (variable, values) in effective {
4897 let column = table.scope_column(variable).ok_or(())?;
4898 let placeholders: Vec<String> = values
4899 .iter()
4900 .map(|v| {
4901 params.push(SqlValue::Text(v.clone()));
4902 "?".to_owned()
4903 })
4904 .collect();
4905 clauses.push(format!(
4906 "{} IN ({})",
4907 quote_ident(column),
4908 placeholders.join(", ")
4909 ));
4910 }
4911 let sql = format!(
4912 "DELETE FROM {} WHERE {}",
4913 base_table(table_name),
4914 clauses.join(" AND ")
4915 );
4916 self.overlay_dirty.set(true);
4917 self.conn
4918 .execute(&sql, rusqlite::params_from_iter(params))
4919 .map_err(|_| ())?;
4920 Ok(())
4921 }
4922
4923 fn drop_doomed_outbox(
4926 &mut self,
4927 table_name: &str,
4928 effective: &[(String, Vec<String>)],
4929 ) -> Result<bool, String> {
4930 if effective.is_empty() {
4931 return Ok(false);
4932 }
4933 let Some(table) = self.schema.table(table_name).cloned() else {
4934 return Ok(false);
4935 };
4936 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
4937 for (variable, values) in effective {
4938 match table.scope_column(variable) {
4939 Some(column) => mappings.push((column, values)),
4940 None => return Ok(false), }
4942 }
4943 let doomed: Vec<OutboxCommit> = self
4944 .outbox
4945 .iter()
4946 .filter(|commit| {
4947 commit.ops.iter().any(|op| {
4948 op.upsert
4949 && op.table == table_name
4950 && op.values.as_ref().is_some_and(|values| {
4951 mappings
4952 .iter()
4953 .all(|(column, allowed)| match values.get(*column) {
4954 Some(Value::String(s)) => allowed.contains(s),
4955 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
4956 _ => false,
4957 })
4958 })
4959 })
4960 })
4961 .cloned()
4962 .collect();
4963 if doomed.is_empty() {
4964 return Ok(false);
4965 }
4966 let mut rejections = Vec::new();
4967 for commit in &doomed {
4968 let results = commit
4969 .ops
4970 .iter()
4971 .enumerate()
4972 .map(|(op_index, operation)| {
4973 let rejection = RejectionRecord {
4974 client_commit_id: commit.client_commit_id.clone(),
4975 op_index: op_index as i32,
4976 code: "sync.scope_revoked".to_owned(),
4977 message: "the commit was dropped because its effective scope was revoked"
4978 .to_owned(),
4979 retryable: false,
4980 details: None,
4981 operation: Some(CommitOperation::from(operation)),
4982 };
4983 rejections.push(rejection.clone());
4984 CommitOperationOutcome::Error { rejection }
4985 })
4986 .collect::<Vec<_>>();
4987 self.persist_commit_outcome(
4988 &commit.client_commit_id,
4989 CommitOutcomeStatus::Rejected,
4990 &results,
4991 Some(&commit.ops),
4992 )?;
4993 self.delete_outbox_persisted(&commit.client_commit_id)?;
4994 }
4995 self.prune_commit_outcomes()?;
4996 let doomed_ids = doomed
4997 .iter()
4998 .map(|commit| commit.client_commit_id.as_str())
4999 .collect::<BTreeSet<_>>();
5000 self.outbox
5001 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
5002 self.rejections.extend(rejections);
5003 self.overlay_dirty.set(true);
5004 Ok(true)
5005 }
5006
5007 pub fn upload_blob(
5014 &mut self,
5015 bytes: &[u8],
5016 media_type: Option<String>,
5017 name: Option<String>,
5018 ) -> Result<Value, String> {
5019 let blob_id = blob_id_for(bytes);
5020 let now = self.clock_now_ms();
5021 self.conn
5022 .execute(
5023 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
5024 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
5025 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
5026 )
5027 .map_err(|e| e.to_string())?;
5028 self.conn
5029 .execute(
5030 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
5031 rusqlite::params![blob_id, media_type, now],
5032 )
5033 .map_err(|e| e.to_string())?;
5034 self.enforce_blob_cache_cap();
5037 let mut obj = Map::new();
5038 obj.insert("blobId".to_owned(), Value::from(blob_id));
5039 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
5040 if let Some(mt) = media_type {
5041 obj.insert("mediaType".to_owned(), Value::from(mt));
5042 }
5043 if let Some(n) = name {
5044 obj.insert("name".to_owned(), Value::from(n));
5045 }
5046 Ok(Value::Object(obj))
5047 }
5048
5049 pub fn fetch_blob(
5053 &mut self,
5054 transport: &mut dyn Transport,
5055 blob_id_or_ref: &str,
5056 ) -> Result<Value, (String, String)> {
5057 let simple = |m: String| ("client.failed".to_owned(), m);
5058 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
5059 blob_id_or_ref.to_owned()
5060 } else {
5061 let value: Value = serde_json::from_str(blob_id_or_ref)
5062 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
5063 value
5064 .get("blobId")
5065 .and_then(Value::as_str)
5066 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
5067 .to_owned()
5068 };
5069 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
5070 return Ok(cached);
5071 }
5072 let bytes = match transport
5078 .blob_download(&blob_id)
5079 .map_err(|e| (e.code, e.message))?
5080 {
5081 BlobDownload::Bytes(bytes) => bytes,
5082 BlobDownload::Url {
5083 url,
5084 url_expires_at_ms,
5085 } => {
5086 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5088 return Err((
5089 "sync.segment_expired".to_owned(),
5090 format!(
5091 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
5092 ),
5093 ));
5094 }
5095 transport
5096 .fetch_blob_url(&url)
5097 .map_err(|e| (e.code, e.message))?
5098 }
5099 };
5100 if blob_id_for(&bytes) != blob_id {
5102 return Err(simple(format!(
5103 "blob content address mismatch for {blob_id}"
5104 )));
5105 }
5106 let now = self.clock_now_ms();
5107 self.conn
5108 .execute(
5109 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
5110 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
5111 )
5112 .map_err(|e| simple(e.to_string()))?;
5113 self.enforce_blob_cache_cap();
5114 self.get_cached_blob(&blob_id)
5115 .map_err(simple)?
5116 .ok_or_else(|| simple("blob cache write failed".to_owned()))
5117 }
5118
5119 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
5120 let _ = self.conn.execute(
5123 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
5124 rusqlite::params![self.clock_now_ms(), blob_id],
5125 );
5126 let mut stmt = self
5127 .conn
5128 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
5129 .map_err(|e| e.to_string())?;
5130 let mut rows = stmt
5131 .query(rusqlite::params![blob_id])
5132 .map_err(|e| e.to_string())?;
5133 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
5134 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
5135 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
5136 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
5137 let mut obj = Map::new();
5138 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
5139 obj.insert("byteLength".to_owned(), Value::from(byte_length));
5140 let mut bytes_obj = Map::new();
5141 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
5142 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
5143 if let Some(mt) = media_type {
5144 obj.insert("mediaType".to_owned(), Value::from(mt));
5145 }
5146 return Ok(Some(Value::Object(obj)));
5147 }
5148 Ok(None)
5149 }
5150
5151 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
5153 let pending: Vec<(String, Option<String>)> = {
5154 let mut stmt = self
5155 .conn
5156 .prepare(
5157 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
5158 )
5159 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5160 let rows = stmt
5161 .query_map([], |row| {
5162 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
5163 })
5164 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5165 rows.filter_map(Result::ok).collect()
5166 };
5167 for (blob_id, media_type) in pending {
5168 let bytes: Option<Vec<u8>> = self
5169 .conn
5170 .query_row(
5171 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
5172 rusqlite::params![blob_id],
5173 |row| row.get(0),
5174 )
5175 .ok();
5176 if let Some(bytes) = bytes {
5177 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
5178 }
5179 let _ = self.conn.execute(
5180 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
5181 rusqlite::params![blob_id],
5182 );
5183 }
5184 Ok(())
5185 }
5186
5187 fn upload_one(
5194 &self,
5195 transport: &mut dyn Transport,
5196 blob_id: &str,
5197 bytes: &[u8],
5198 media_type: Option<&str>,
5199 ) -> Result<(), TransportError> {
5200 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
5201 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
5203 url,
5204 url_expires_at_ms,
5205 } => {
5206 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
5207 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
5208 return Ok(());
5209 }
5210 }
5212 BlobUploadGrant::None => {
5213 }
5215 }
5216 transport.blob_upload(blob_id, bytes, media_type)
5217 }
5218
5219 fn enforce_blob_cache_cap(&self) {
5228 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
5229 return;
5230 };
5231 let mut total: i64 = self
5232 .conn
5233 .query_row(
5234 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
5235 [],
5236 |row| row.get(0),
5237 )
5238 .unwrap_or(0);
5239 if total <= max_bytes {
5240 return;
5241 }
5242 let candidates: Vec<(String, i64)> = {
5243 let Ok(mut stmt) = self.conn.prepare(
5244 "SELECT blob_id, byte_length FROM _syncular_blobs
5245 WHERE refcount = 0
5246 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
5247 ORDER BY last_used_ms ASC, created_at_ms ASC",
5248 ) else {
5249 return;
5250 };
5251 let Ok(rows) = stmt.query_map([], |row| {
5252 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
5253 }) else {
5254 return;
5255 };
5256 rows.filter_map(Result::ok).collect()
5257 };
5258 for (blob_id, byte_length) in candidates {
5259 if total <= max_bytes {
5260 break;
5261 }
5262 let _ = self.conn.execute(
5263 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
5264 rusqlite::params![blob_id],
5265 );
5266 total -= byte_length;
5267 }
5268 }
5269
5270 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
5274 if !self.schema_has_blobs() {
5275 return;
5276 }
5277 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
5278 for table in self.schema.tables.clone() {
5279 let blob_cols: Vec<String> = table
5280 .columns
5281 .iter()
5282 .filter(|c| c.ty == ColumnType::BlobRef)
5283 .map(|c| c.name.clone())
5284 .collect();
5285 for column in blob_cols {
5286 let sql = format!(
5287 "SELECT {} FROM {} WHERE {} IS NOT NULL",
5288 quote_ident(&column),
5289 base_table(&table.name),
5290 quote_ident(&column)
5291 );
5292 let Ok(mut stmt) = self.conn.prepare(&sql) else {
5293 continue;
5294 };
5295 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
5296 continue;
5297 };
5298 for raw in rows.flatten().flatten() {
5299 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
5300 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
5301 *counts.entry(id.to_owned()).or_insert(0) += 1;
5302 }
5303 }
5304 }
5305 }
5306 }
5307 let _ = self
5308 .conn
5309 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
5310 for (blob_id, count) in &counts {
5311 let _ = self.conn.execute(
5312 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
5313 rusqlite::params![count, blob_id],
5314 );
5315 }
5316 if delete_orphans {
5317 let _ = self.conn.execute(
5318 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
5319 [],
5320 );
5321 }
5322 }
5323
5324 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
5329 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
5330 return sql.clone();
5331 }
5332 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
5333 columns.push(quote_ident("_syncular_version"));
5334 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
5335 let primary_key = quote_ident(&table.primary_key);
5336 let updates = columns
5337 .iter()
5338 .filter(|column| **column != primary_key)
5339 .map(|column| format!("{column}=excluded.{column}"))
5340 .collect::<Vec<_>>()
5341 .join(", ");
5342 let sql = format!(
5343 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
5344 columns.join(", "),
5345 placeholders.join(", ")
5346 );
5347 self.insert_sql
5348 .borrow_mut()
5349 .insert(full_table.to_owned(), sql.clone());
5350 sql
5351 }
5352
5353 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
5354 self.overlay_dirty.set(true);
5355 self.write_row(&base_table(table_name), table_name, row, version)
5356 }
5357
5358 fn write_row(
5359 &self,
5360 full_table: &str,
5361 table_name: &str,
5362 row: &Row,
5363 version: i64,
5364 ) -> Result<(), String> {
5365 let table = self
5366 .schema
5367 .table(table_name)
5368 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
5369 let sql = self.insert_row_sql(full_table, table);
5370 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
5371 let params = row
5372 .iter()
5373 .map(RowParam::Cell)
5374 .chain(std::iter::once(RowParam::Version(version)));
5375 stmt.execute(rusqlite::params_from_iter(params))
5376 .map_err(|e| e.to_string())?;
5377 Ok(())
5378 }
5379
5380 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
5381 let table = self
5382 .schema
5383 .table(table_name)
5384 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
5385 self.overlay_dirty.set(true);
5386 let sql = format!(
5387 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
5388 base_table(table_name),
5389 quote_ident(&table.primary_key)
5390 );
5391 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
5392 stmt.execute(rusqlite::params![row_id])
5393 .map_err(|e| e.to_string())?;
5394 Ok(())
5395 }
5396
5397 fn rebuild_overlay_if_dirty(&mut self) {
5400 if self.overlay_dirty.get() {
5401 self.rebuild_overlay();
5402 }
5403 }
5404
5405 fn rebuild_overlay(&mut self) {
5409 self.exec("SAVEPOINT syncular_overlay");
5410 for table in self.schema.tables.clone() {
5411 for index in &table.fts_indexes {
5412 let _ = self.drop_fts_triggers(index);
5413 }
5414 let visible = visible_table(&table.name);
5415 let base = base_table(&table.name);
5416 self.exec(&format!("DELETE FROM {visible}"));
5417 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
5418 }
5419 for commit in self.outbox.clone() {
5420 for op in &commit.ops {
5421 let Some(table) = self.schema.table(&op.table).cloned() else {
5422 continue;
5423 };
5424 if op.upsert {
5425 let Some(values) = op.values.as_ref() else {
5426 continue;
5427 };
5428 let mut row: Row = Vec::with_capacity(table.columns.len());
5429 let mut ok = true;
5430 for column in &table.columns {
5431 match json_to_column_value(column, values.get(&column.name)) {
5432 Ok(v) => row.push(v),
5433 Err(_) => {
5434 ok = false;
5435 break;
5436 }
5437 }
5438 }
5439 if ok {
5440 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
5441 }
5442 } else {
5443 let sql = format!(
5444 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
5445 visible_table(&table.name),
5446 quote_ident(&table.primary_key)
5447 );
5448 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
5449 }
5450 }
5451 }
5452 for table in self.schema.tables.clone() {
5453 for index in &table.fts_indexes {
5454 let _ = self.rebuild_fts_projection(&table, index);
5455 let _ = self.create_fts_triggers(&table, index);
5456 }
5457 }
5458 self.exec("RELEASE syncular_overlay");
5459 self.overlay_dirty.set(false);
5460 }
5461
5462 fn exec(&self, sql: &str) {
5463 let _ = self.conn.execute_batch(sql);
5464 }
5465
5466 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
5469 transport
5470 .realtime_connect_for_client(&self.client_id)
5471 .map_err(|e| format!("{}: {}", e.code, e.message))?;
5472 self.realtime_connected = true;
5473 Ok(())
5474 }
5475
5476 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
5477 let _ = transport.realtime_close();
5478 self.realtime_connected = false;
5479 self.presence.clear(); }
5481
5482 pub fn set_presence(
5488 &mut self,
5489 transport: &mut dyn Transport,
5490 scope_key: &str,
5491 doc: Option<&Value>,
5492 ) -> Result<(), String> {
5493 if !self.realtime_connected {
5494 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
5495 }
5496 let text = encode_presence_publish(scope_key, doc);
5497 transport
5498 .realtime_send(&text)
5499 .map_err(|e| format!("{}: {}", e.code, e.message))
5500 }
5501
5502 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
5504 self.presence
5505 .get(scope_key)
5506 .map(|peers| peers.values().cloned().collect())
5507 .unwrap_or_default()
5508 }
5509
5510 fn apply_presence(
5512 &mut self,
5513 scope_key: String,
5514 kind: Option<PresenceKind>,
5515 actor_id: Option<String>,
5516 client_id: Option<String>,
5517 doc: Option<Value>,
5518 error: Option<String>,
5519 ) {
5520 if error.is_some() {
5523 return;
5524 }
5525 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
5526 return;
5527 };
5528 let peer_key = format!("{actor_id} {client_id}");
5529 match kind {
5530 PresenceKind::Leave => {
5531 if let Some(peers) = self.presence.get_mut(&scope_key) {
5532 peers.remove(&peer_key);
5533 if peers.is_empty() {
5534 self.presence.remove(&scope_key);
5535 }
5536 }
5537 }
5538 _ => {
5539 let doc = match doc {
5540 Some(Value::Object(_)) => doc.unwrap(),
5541 _ => return,
5542 };
5543 self.presence.entry(scope_key).or_default().insert(
5544 peer_key,
5545 PresencePeer {
5546 actor_id,
5547 client_id,
5548 doc,
5549 },
5550 );
5551 }
5552 }
5553 }
5554
5555 pub fn on_realtime_text(&mut self, text: &str) {
5557 match parse_control(text) {
5558 Ok(ControlMessage::Hello { requires_sync, .. }) => {
5559 if requires_sync {
5560 self.set_sync_needed(true, true);
5562 }
5563 }
5564 Ok(ControlMessage::Presence {
5565 scope_key,
5566 kind,
5567 actor_id,
5568 client_id,
5569 doc,
5570 error,
5571 ..
5572 }) => {
5573 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
5574 }
5575 Ok(ControlMessage::Wake { .. }) => {
5576 self.set_sync_needed(true, true);
5578 }
5579 _ => {}
5580 }
5581 }
5582
5583 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
5586 if self.stopped {
5587 return;
5588 }
5589 let message = match decode_message(bytes) {
5590 Ok(m) if m.msg_kind == MsgKind::Response => m,
5591 _ => {
5592 self.set_sync_needed(true, true);
5593 return;
5594 }
5595 };
5596 let mut frames = message.frames.into_iter();
5597 let mut applied_cursor: Option<i64> = None;
5598 let mut any_covered = false;
5599 let mut dropped = false;
5600 while let Some(frame) = frames.next() {
5601 let Frame::SubStart {
5602 id,
5603 status,
5604 effective_scopes,
5605 ..
5606 } = frame
5607 else {
5608 continue;
5609 };
5610 let mut body = Vec::new();
5611 let mut next_cursor: Option<i64> = None;
5612 for inner in frames.by_ref() {
5613 match inner {
5614 Frame::SubEnd {
5615 next_cursor: nc, ..
5616 } => {
5617 next_cursor = Some(nc);
5618 break;
5619 }
5620 Frame::Unknown { .. } => {}
5621 other => body.push(other),
5622 }
5623 }
5624 let Some(next_cursor) = next_cursor else {
5625 dropped = true;
5626 break;
5627 };
5628 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
5629 dropped = true;
5630 continue;
5631 };
5632 let sub = &self.subs[sub_index];
5633 if status != SubStatus::Active
5636 || sub.state != SubState::Active
5637 || sub.bootstrap_state.is_some()
5638 || !sub.synced_once
5639 {
5640 dropped = true;
5641 continue;
5642 }
5643 if next_cursor <= sub.cursor {
5644 any_covered = true;
5646 continue;
5647 }
5648 let previous_effective = self.subs[sub_index].effective.clone();
5649 let previous_cursor = self.subs[sub_index].cursor;
5650 if self.begin_observation("syncular_delta").is_err() {
5651 dropped = true;
5652 continue;
5653 }
5654 self.subs[sub_index].effective = Some(effective_scopes);
5655 let mut batch = ChangeAccumulator::default();
5656 let mut failed = false;
5657 for inner in body {
5658 if let Frame::Commit {
5659 tables, changes, ..
5660 } = inner
5661 {
5662 self.record_commit_changes(&mut batch, &tables, &changes);
5663 if self.apply_commit_changes(&tables, &changes).is_err() {
5664 failed = true;
5665 break;
5666 }
5667 }
5668 }
5669 if failed {
5670 self.rollback_observation("syncular_delta");
5671 self.subs[sub_index].effective = previous_effective;
5672 self.subs[sub_index].cursor = previous_cursor;
5673 self.overlay_dirty.set(true);
5674 self.rebuild_overlay();
5675 dropped = true;
5676 continue;
5677 }
5678 let sub = &mut self.subs[sub_index];
5679 sub.cursor = next_cursor;
5680 self.persist_sub(&self.subs[sub_index].clone());
5681 self.rebuild_overlay_if_dirty();
5682 if self.finish_observation("syncular_delta", batch).is_err() {
5683 self.rollback_observation("syncular_delta");
5684 self.subs[sub_index].effective = previous_effective;
5685 self.subs[sub_index].cursor = previous_cursor;
5686 self.overlay_dirty.set(true);
5687 self.rebuild_overlay();
5688 dropped = true;
5689 continue;
5690 }
5691 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
5692 }
5693 if let Some(cursor) = applied_cursor {
5694 self.reconcile_blob_refcounts(false);
5695 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
5697 let _ = transport.realtime_send(&ack);
5698 } else if !any_covered || dropped {
5699 self.set_sync_needed(true, true);
5701 }
5702 }
5703
5704 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
5708 if !self.realtime_connected {
5709 return;
5710 }
5711 let floor = self
5712 .subs
5713 .iter()
5714 .filter(|s| {
5715 s.state == SubState::Active
5716 && s.bootstrap_state.is_none()
5717 && s.synced_once
5718 && s.cursor >= 0
5719 })
5720 .map(|s| s.cursor)
5721 .min();
5722 if let Some(cursor) = floor {
5723 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
5724 let _ = transport.realtime_send(&ack);
5725 }
5726 }
5727}