1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, CommitOperation, CommitOperationOutcome,
27 CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution, CommitOutcomeStatus,
28 ConflictRecord, CoverageSnapshot, LeaseState, Mutation, PresencePeer, QuerySnapshot,
29 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
30 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
31 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
32};
33use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
34use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
35use crate::values::{
36 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
37 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
38 scope_map_to_json, sort_scope_map,
39};
40
41const DEFAULT_ACCEPT: u8 = 0b0111;
46const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
47const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
48const ACCEPT_SQLITE: u8 = 1 << 2;
49const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
50
51const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
53const LOCAL_REVISION_KEY: &str = "localRevision";
54const CLIENT_ID_KEY: &str = "clientId";
55const LEASE_STATE_KEY: &str = "leaseState";
56const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
57const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62enum SubState {
63 Active,
64 Revoked,
65 Failed,
66}
67
68#[cfg(test)]
69mod observation_tests {
70 use super::*;
71 use serde_json::json;
72
73 fn client() -> SyncClient {
74 SyncClient::new(
75 "retry-test".to_owned(),
76 &json!({
77 "version": 1,
78 "tables": [{
79 "name": "tasks",
80 "primaryKey": "id",
81 "columns": [
82 { "name": "id", "type": "string", "nullable": false },
83 { "name": "project_id", "type": "string", "nullable": false }
84 ],
85 "scopes": [{ "pattern": "project:{project_id}" }]
86 }]
87 }),
88 ClientLimits::default(),
89 )
90 .expect("test client")
91 }
92
93 #[test]
94 fn background_retry_deadlines_back_off_and_reset() {
95 let mut client = client();
96 client.schedule_background_retry();
97 assert!(matches!(
98 client.drain_sync_intents().as_slice(),
99 [SyncIntent::Background { delay_ms: 250 }]
100 ));
101 client.schedule_background_retry();
102 assert!(matches!(
103 client.drain_sync_intents().as_slice(),
104 [SyncIntent::Background { delay_ms: 500 }]
105 ));
106 client.reset_background_retry();
107 client.schedule_background_retry();
108 assert!(matches!(
109 client.drain_sync_intents().as_slice(),
110 [SyncIntent::Background { delay_ms: 250 }]
111 ));
112 }
113
114 #[test]
115 fn secondary_unique_collision_preserves_existing_synced_row() {
116 let client = SyncClient::new(
117 "unique-upsert-test".to_owned(),
118 &json!({
119 "version": 1,
120 "tables": [{
121 "name": "tasks",
122 "primaryKey": "id",
123 "columns": [
124 { "name": "id", "type": "string", "nullable": false },
125 { "name": "project_id", "type": "string", "nullable": false },
126 { "name": "title", "type": "string", "nullable": false }
127 ],
128 "scopes": [{ "pattern": "project:{project_id}" }],
129 "indexes": [{
130 "name": "tasks_by_project_title",
131 "columns": ["project_id", "title"],
132 "unique": true
133 }]
134 }]
135 }),
136 ClientLimits::default(),
137 )
138 .expect("test client");
139 let table = client.schema.table("tasks").expect("tasks table");
140 let sql = client.insert_row_sql(&base_table("tasks"), table);
141
142 client
143 .conn
144 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
145 .expect("insert first row");
146 client
147 .conn
148 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
149 .expect("update same primary key");
150 client
151 .conn
152 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
153 .expect("insert second row");
154 assert!(client
155 .conn
156 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
157 .is_err());
158
159 let rows = client
160 .conn
161 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
162 .expect("prepare rows")
163 .query_map([], |row| {
164 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
165 })
166 .expect("query rows")
167 .collect::<Result<Vec<_>, _>>()
168 .expect("collect rows");
169 assert_eq!(
170 rows,
171 vec![
172 ("t1".to_owned(), "updated".to_owned()),
173 ("t2".to_owned(), "original".to_owned())
174 ]
175 );
176 }
177
178 #[test]
179 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
180 let path = std::env::temp_dir().join(format!(
181 "syncular-startup-intent-{}.db",
182 uuid::Uuid::new_v4()
183 ));
184 let schema = json!({
185 "version": 1,
186 "tables": [{
187 "name": "tasks",
188 "primaryKey": "id",
189 "columns": [
190 { "name": "id", "type": "string", "nullable": false },
191 { "name": "project_id", "type": "string", "nullable": false }
192 ],
193 "scopes": [{ "pattern": "project:{project_id}" }]
194 }]
195 });
196
197 {
198 let mut first = SyncClient::open_path_with_identity(
199 None,
200 &schema,
201 ClientLimits::default(),
202 path.to_str().expect("UTF-8 temp path"),
203 )
204 .expect("first open");
205 first
206 .set_window(
207 &WindowBase {
208 table: "tasks".to_owned(),
209 variable: "project_id".to_owned(),
210 fixed_scopes: Vec::new(),
211 params: None,
212 },
213 &["persisted".to_owned()],
214 )
215 .expect("persist window");
216 }
217
218 let mut reopened = SyncClient::open_path_with_identity(
219 None,
220 &schema,
221 ClientLimits::default(),
222 path.to_str().expect("UTF-8 temp path"),
223 )
224 .expect("reopen");
225 assert!(reopened.sync_needed());
226 assert!(matches!(
227 reopened.drain_sync_intents().as_slice(),
228 [SyncIntent::Interactive]
229 ));
230 drop(reopened);
231 std::fs::remove_file(path).expect("remove temp database");
232 }
233
234 #[test]
235 fn migrates_pre_envelope_outcome_journal_additively() {
236 let path = std::env::temp_dir().join(format!(
237 "syncular-outcome-migration-{}.db",
238 uuid::Uuid::new_v4()
239 ));
240 let conn = Connection::open(&path).expect("open legacy database");
241 conn.execute_batch(
242 "CREATE TABLE _syncular_commit_outcomes (
243 seq INTEGER PRIMARY KEY AUTOINCREMENT,
244 client_commit_id TEXT NOT NULL UNIQUE,
245 status TEXT NOT NULL,
246 recorded_at_ms INTEGER NOT NULL,
247 results_json TEXT NOT NULL,
248 resolution TEXT NOT NULL DEFAULT 'active',
249 resolved_at_ms INTEGER,
250 replacement_client_commit_id TEXT);",
251 )
252 .expect("create legacy outcome journal");
253 drop(conn);
254 let schema = json!({
255 "version": 1,
256 "tables": [{
257 "name": "tasks",
258 "primaryKey": "id",
259 "columns": [
260 { "name": "id", "type": "string", "nullable": false },
261 { "name": "project_id", "type": "string", "nullable": false }
262 ],
263 "scopes": [{ "pattern": "project:{project_id}" }]
264 }]
265 });
266 let client = SyncClient::open_path(
267 "migration-native".to_owned(),
268 &schema,
269 ClientLimits::default(),
270 path.to_str().expect("UTF-8 temp path"),
271 )
272 .expect("migrate database");
273 let has_operations = client
274 .conn
275 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
276 .expect("prepare table info")
277 .query_map([], |row| row.get::<_, String>(1))
278 .expect("query table info")
279 .filter_map(Result::ok)
280 .any(|column| column == "operations_json");
281 assert!(has_operations);
282 drop(client);
283 std::fs::remove_file(path).expect("remove temp database");
284 }
285
286 #[test]
287 fn durable_conflict_outcome_and_resolution_survive_reopen() {
288 let path = std::env::temp_dir().join(format!(
289 "syncular-durable-outcome-{}.db",
290 uuid::Uuid::new_v4()
291 ));
292 let schema = json!({
293 "version": 1,
294 "tables": [{
295 "name": "tasks",
296 "primaryKey": "id",
297 "columns": [
298 { "name": "id", "type": "string", "nullable": false },
299 { "name": "project_id", "type": "string", "nullable": false }
300 ],
301 "scopes": [{ "pattern": "project:{project_id}" }]
302 }]
303 });
304 let conflict = ConflictRecord {
305 client_commit_id: "losing-commit".to_owned(),
306 op_index: 0,
307 table: "tasks".to_owned(),
308 row_id: "t1".to_owned(),
309 code: "sync.version_conflict".to_owned(),
310 message: "stale base version".to_owned(),
311 server_version: 2,
312 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
313 operation: Some(CommitOperation {
314 table: "tasks".to_owned(),
315 row_id: "t1".to_owned(),
316 op: "upsert".to_owned(),
317 base_version: Some(1),
318 values: None,
319 changed_fields: None,
320 }),
321 };
322 let failed_operations = vec![
323 OutboxOp {
324 upsert: true,
325 table: "tasks".to_owned(),
326 row_id: "t1".to_owned(),
327 base_version: Some(1),
328 values: None,
329 changed_fields: None,
330 },
331 OutboxOp {
332 upsert: true,
333 table: "tasks".to_owned(),
334 row_id: "status-event-1".to_owned(),
335 base_version: Some(0),
336 values: None,
337 changed_fields: None,
338 },
339 ];
340
341 {
342 let mut first = SyncClient::open_path(
343 "durable-native".to_owned(),
344 &schema,
345 ClientLimits::default(),
346 path.to_str().expect("UTF-8 temp path"),
347 )
348 .expect("first open");
349 first
350 .begin_observation("test_outcome")
351 .expect("begin outcome");
352 first
353 .persist_commit_outcome(
354 "losing-commit",
355 CommitOutcomeStatus::Conflict,
356 &[CommitOperationOutcome::Conflict {
357 conflict: conflict.clone(),
358 }],
359 Some(&failed_operations),
360 )
361 .expect("persist outcome");
362 first.conflicts.push(conflict);
363 first
364 .finish_observation(
365 "test_outcome",
366 ChangeAccumulator {
367 conflicts: true,
368 outcomes: true,
369 ..ChangeAccumulator::default()
370 },
371 )
372 .expect("commit outcome");
373 }
374
375 {
376 let mut reopened = SyncClient::open_path(
377 "durable-native".to_owned(),
378 &schema,
379 ClientLimits::default(),
380 path.to_str().expect("UTF-8 temp path"),
381 )
382 .expect("reopen");
383 assert_eq!(reopened.conflicts().len(), 1);
384 let outcome = reopened
385 .commit_outcome("losing-commit")
386 .expect("read outcome")
387 .expect("outcome");
388 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
389 let operations = outcome.operations.expect("aggregate envelope");
390 assert_eq!(operations.len(), 2);
391 assert_eq!(operations[1].row_id, "status-event-1");
392 let resolved = reopened
393 .resolve_commit_outcome(ResolveCommitOutcomeInput {
394 client_commit_id: "losing-commit".to_owned(),
395 resolution: CommitOutcomeResolution::ResolvedKeepServer,
396 replacement_client_commit_id: None,
397 })
398 .expect("resolve");
399 assert_eq!(
400 resolved.resolution,
401 CommitOutcomeResolution::ResolvedKeepServer
402 );
403 assert!(reopened.conflicts().is_empty());
404 }
405
406 let reopened = SyncClient::open_path(
407 "durable-native".to_owned(),
408 &schema,
409 ClientLimits::default(),
410 path.to_str().expect("UTF-8 temp path"),
411 )
412 .expect("second reopen");
413 assert!(reopened.conflicts().is_empty());
414 assert_eq!(
415 reopened
416 .commit_outcome("losing-commit")
417 .expect("read outcome")
418 .expect("outcome")
419 .resolution,
420 CommitOutcomeResolution::ResolvedKeepServer
421 );
422 drop(reopened);
423 std::fs::remove_file(path).expect("remove temp database");
424 }
425
426 #[test]
427 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
428 let path =
429 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
430 let schema = json!({
431 "version": 1,
432 "tables": [{
433 "name": "tasks",
434 "primaryKey": "id",
435 "columns": [
436 { "name": "id", "type": "string", "nullable": false },
437 { "name": "project_id", "type": "string", "nullable": false }
438 ],
439 "scopes": [{ "pattern": "project:{project_id}" }]
440 }]
441 });
442 let mut client = SyncClient::open_path_with_identity(
443 Some("sidecar-client".to_owned()),
444 &schema,
445 ClientLimits::default(),
446 path.to_str().expect("UTF-8 temp path"),
447 )
448 .expect("open owner");
449 let base = WindowBase {
450 table: "tasks".to_owned(),
451 variable: "project_id".to_owned(),
452 fixed_scopes: Vec::new(),
453 params: None,
454 };
455 client
456 .set_window(&base, &["one".to_owned()])
457 .expect("set window");
458 client
459 .mutate(vec![Mutation::Upsert {
460 table: "tasks".to_owned(),
461 values: Map::from_iter([
462 ("id".to_owned(), Value::from("t1")),
463 ("project_id".to_owned(), Value::from("one")),
464 ]),
465 base_version: None,
466 }])
467 .expect("local mutate");
468
469 let coverage = [WindowCoverage {
470 base,
471 units: vec!["one".to_owned(), "missing".to_owned()],
472 }];
473 let owner = client
474 .query_snapshot(
475 "SELECT id, project_id FROM tasks ORDER BY id",
476 &[],
477 &coverage,
478 )
479 .expect("owner snapshot");
480 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
481 let sidecar = reader
482 .query_snapshot(
483 "SELECT id, project_id FROM tasks ORDER BY id",
484 &[],
485 &coverage,
486 )
487 .expect("sidecar snapshot");
488
489 assert_eq!(sidecar.revision, owner.revision);
490 assert_eq!(sidecar.rows, owner.rows);
491 assert_eq!(
492 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
493 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
494 );
495 assert_eq!(sidecar.revision, "2");
496 assert_eq!(sidecar.rows[0]["id"], "t1");
497 assert!(!sidecar.coverage.complete);
498 assert_eq!(sidecar.coverage.pending.len(), 1);
499 assert_eq!(sidecar.coverage.missing.len(), 1);
500
501 drop(reader);
502 drop(client);
503 std::fs::remove_file(path).expect("remove temp database");
504 }
505
506 #[test]
507 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
508 let schema = json!({
509 "version": 1,
510 "tables": [{
511 "name": "catalogue_codes",
512 "primaryKey": "id",
513 "columns": [
514 { "name": "id", "type": "string", "nullable": false },
515 { "name": "release_id", "type": "string", "nullable": false },
516 { "name": "code", "type": "string", "nullable": false },
517 { "name": "title", "type": "string", "nullable": false }
518 ],
519 "scopes": [{ "pattern": "release:{release_id}" }],
520 "ftsIndexes": [{
521 "name": "catalogue_codes_fts",
522 "columns": ["code", "title"],
523 "tokenize": "unicode61 remove_diacritics 2"
524 }]
525 }]
526 });
527 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
528 .expect("FTS5 client");
529 let search = |client: &SyncClient, query: &str| {
530 client
531 .query(
532 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
533 &[Value::from(query)],
534 )
535 .expect("FTS query")
536 };
537
538 client
539 .mutate(vec![Mutation::Upsert {
540 table: "catalogue_codes".to_owned(),
541 values: Map::from_iter([
542 ("id".to_owned(), Value::from("c1")),
543 ("release_id".to_owned(), Value::from("r1")),
544 ("code".to_owned(), Value::from("A01")),
545 ("title".to_owned(), Value::from("Cholera")),
546 ]),
547 base_version: None,
548 }])
549 .expect("insert code");
550 assert_eq!(search(&client, "cholera").len(), 1);
551
552 client
553 .mutate(vec![Mutation::Upsert {
554 table: "catalogue_codes".to_owned(),
555 values: Map::from_iter([
556 ("id".to_owned(), Value::from("c1")),
557 ("release_id".to_owned(), Value::from("r1")),
558 ("code".to_owned(), Value::from("A01")),
559 ("title".to_owned(), Value::from("Enteric infection")),
560 ]),
561 base_version: None,
562 }])
563 .expect("update code");
564 assert!(search(&client, "cholera").is_empty());
565 assert_eq!(search(&client, "enteric").len(), 1);
566
567 client
568 .mutate(vec![Mutation::Delete {
569 table: "catalogue_codes".to_owned(),
570 row_id: "c1".to_owned(),
571 base_version: None,
572 }])
573 .expect("delete code");
574 assert!(search(&client, "enteric").is_empty());
575 }
576}
577
578impl SubState {
579 fn name(self) -> &'static str {
580 match self {
581 SubState::Active => "active",
582 SubState::Revoked => "revoked",
583 SubState::Failed => "failed",
584 }
585 }
586
587 fn parse(value: &str) -> Self {
588 match value {
589 "revoked" => Self::Revoked,
590 "failed" => Self::Failed,
591 _ => Self::Active,
592 }
593 }
594}
595
596#[derive(Debug, Clone)]
597struct Subscription {
598 id: String,
599 table: String,
600 requested: Vec<(String, Vec<String>)>,
601 params: Option<String>,
602 cursor: i64,
603 bootstrap_state: Option<String>,
605 state: SubState,
606 reason_code: Option<String>,
607 effective: Option<Vec<(String, Vec<String>)>>,
610 synced_once: bool,
611}
612
613#[derive(Debug, Clone)]
614struct OutboxOp {
615 upsert: bool,
616 table: String,
617 row_id: String,
618 base_version: Option<i64>,
619 values: Option<Map<String, Value>>,
622 changed_fields: Option<Vec<String>>,
624}
625
626impl From<&OutboxOp> for CommitOperation {
627 fn from(operation: &OutboxOp) -> Self {
628 Self {
629 table: operation.table.clone(),
630 row_id: operation.row_id.clone(),
631 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
632 base_version: operation.base_version,
633 values: operation.values.clone(),
634 changed_fields: operation.changed_fields.clone(),
635 }
636 }
637}
638
639#[derive(Debug, Clone)]
640struct OutboxCommit {
641 client_commit_id: String,
642 ops: Vec<OutboxOp>,
643}
644
645struct StoredCommitOutcomeRow {
646 sequence: i64,
647 client_commit_id: String,
648 status: String,
649 recorded_at_ms: i64,
650 results_json: String,
651 operations_json: Option<String>,
652 resolution: String,
653 resolved_at_ms: Option<i64>,
654 replacement_client_commit_id: Option<String>,
655}
656
657enum SectionError {
660 FailClosed,
661 Abort(String, String),
662}
663
664struct RequestMeta {
665 pushed_ids: Vec<String>,
666 fresh: Vec<(String, bool)>,
669 accept: u8,
670 deferred_commits: usize,
674}
675
676#[derive(Default)]
677struct ChangeAccumulator {
678 tables: BTreeMap<String, Option<BTreeSet<String>>>,
680 windows: BTreeMap<(String, String), BTreeSet<String>>,
681 status: bool,
682 conflicts: bool,
683 rejections: bool,
684 outcomes: bool,
685}
686
687impl ChangeAccumulator {
688 fn table(&mut self, table: &str) {
689 self.tables.insert(table.to_owned(), None);
690 }
691
692 fn scope(&mut self, table: &str, key: String) {
693 match self.tables.get_mut(table) {
694 Some(None) => {}
695 Some(Some(keys)) => {
696 keys.insert(key);
697 }
698 None => {
699 self.tables
700 .insert(table.to_owned(), Some(BTreeSet::from([key])));
701 }
702 }
703 }
704
705 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
706 self.windows
707 .entry((base_key.to_owned(), table.to_owned()))
708 .or_default()
709 .insert(unit.to_owned());
710 }
711
712 fn touched(&self) -> bool {
713 !self.tables.is_empty()
714 || !self.windows.is_empty()
715 || self.status
716 || self.conflicts
717 || self.rejections
718 || self.outcomes
719 }
720}
721
722const PUSH_OPS_PER_REQUEST: usize = 500;
728
729pub struct SyncClient {
730 conn: Connection,
731 schema: ClientSchema,
732 client_id: String,
733 limits: ClientLimits,
734 subs: Vec<Subscription>,
735 outbox: Vec<OutboxCommit>,
736 conflicts: Vec<ConflictRecord>,
737 rejections: Vec<RejectionRecord>,
738 schema_floor: Option<SchemaFloor>,
739 lease_state: Option<LeaseState>,
741 stopped: bool,
743 upgrading: bool,
745 sync_needed: bool,
747 realtime_connected: bool,
748 presence: HashMap<String, HashMap<String, PresencePeer>>,
750 now_ms: Option<i64>,
753 encryption: crate::values::EncryptionConfig,
757 insert_sql: RefCell<HashMap<String, String>>,
763 overlay_dirty: Cell<bool>,
767 change_queue: VecDeque<ClientChangeBatch>,
769 sync_intent_queue: VecDeque<SyncIntent>,
770 retry_delay_ms: u64,
772}
773
774fn quote_ident(name: &str) -> String {
775 format!("\"{}\"", name.replace('"', "\"\""))
776}
777
778fn base_table(name: &str) -> String {
779 quote_ident(&format!("_syncular_base_{name}"))
780}
781
782type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
787
788fn window_base_key(base: &WindowBase) -> String {
789 format!(
790 "{}\0{}\0{}",
791 base.table,
792 base.variable,
793 canonical_scope_json(&base.fixed_scopes)
794 )
795}
796
797fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
799 let mut scopes = base.fixed_scopes.clone();
800 scopes.retain(|(k, _)| k != &base.variable);
801 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
802 scopes
803}
804
805fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
809 let canonical = canonical_scope_json(&unit_scopes(base, unit));
810 let digest = Sha256::digest(canonical.as_bytes());
811 let hex = bytes_to_hex(&digest);
812 format!("w:{}:{}", base.table, &hex[..16])
813}
814
815fn visible_table(name: &str) -> String {
816 quote_ident(name)
817}
818
819const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
820
821fn is_synced_table_name(name: &str) -> bool {
824 if name.starts_with("sqlite_") {
825 return false;
826 }
827 if name.starts_with("_syncular_base_") {
828 return true; }
830 !name.starts_with("_syncular_")
832}
833
834fn blob_id_for(bytes: &[u8]) -> String {
836 let digest = Sha256::digest(bytes);
837 format!("sha256:{}", bytes_to_hex(&digest))
838}
839
840enum RowParam<'a> {
844 Cell(&'a Option<ColumnValue>),
845 Version(i64),
846}
847
848impl rusqlite::ToSql for RowParam<'_> {
849 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
850 Ok(match self {
851 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
852 RowParam::Cell(cell) => match cell {
853 None => ToSqlOutput::Owned(SqlValue::Null),
854 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
855 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
856 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
857 Some(ColumnValue::Boolean(b)) => {
858 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
859 }
860 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
861 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
862 }
863 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
865 ToSqlOutput::Borrowed(ValueRef::Blob(b))
866 }
867 },
868 })
869 }
870}
871
872fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
880 use ssp2::segment::ColumnType;
881 let mismatch = || {
882 Err(format!(
883 "image column {:?} holds a value of the wrong type",
884 column.name
885 ))
886 };
887 match value {
888 ValueRef::Null => {
889 if !column.nullable {
890 return Err(format!(
891 "image column {:?} is NULL but not nullable",
892 column.name
893 ));
894 }
895 Ok(ToSqlOutput::Owned(SqlValue::Null))
896 }
897 ValueRef::Integer(i) => match column.ty {
898 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
899 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
900 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
901 _ => mismatch(),
902 },
903 ValueRef::Real(_) => match column.ty {
904 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
905 _ => mismatch(),
906 },
907 ValueRef::Text(t) => {
908 std::str::from_utf8(t)
909 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
910 match column.ty {
911 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
912 Ok(ToSqlOutput::Borrowed(value))
913 }
914 _ => mismatch(),
915 }
916 }
917 ValueRef::Blob(_) => match column.ty {
918 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
920 _ => mismatch(),
921 },
922 }
923}
924
925fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
926 use rusqlite::types::ValueRef;
927 match value {
928 ValueRef::Null => Value::Null,
929 ValueRef::Integer(i) => match column.ty {
930 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
931 ssp2::segment::ColumnType::Float => {
932 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
933 }
934 _ => Value::from(i),
935 },
936 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
937 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
938 ValueRef::Blob(b) => {
939 let mut map = Map::new();
940 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
941 Value::Object(map)
942 }
943 }
944}
945
946fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
950 Ok(match value {
951 Value::Null => SqlValue::Null,
952 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
953 Value::Number(n) => {
954 if let Some(i) = n.as_i64() {
955 SqlValue::Integer(i)
956 } else if let Some(f) = n.as_f64() {
957 SqlValue::Real(f)
958 } else {
959 return Err(format!("query param number {n} is out of range"));
960 }
961 }
962 Value::String(s) => SqlValue::Text(s.clone()),
963 Value::Object(_) => {
964 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
965 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
966 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
967 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
968 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
969 })?)
970 } else {
971 return Err(
972 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
973 .to_owned(),
974 );
975 }
976 }
977 Value::Array(_) => return Err("query array params are not supported".to_owned()),
978 })
979}
980
981fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
986 use rusqlite::types::ValueRef;
987 match value {
988 ValueRef::Null => Value::Null,
989 ValueRef::Integer(i) => {
990 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
994 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
995 Value::from(i)
996 } else {
997 let mut map = Map::new();
998 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
999 Value::Object(map)
1000 }
1001 }
1002 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1003 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1004 ValueRef::Blob(b) => {
1005 let mut map = Map::new();
1006 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1007 Value::Object(map)
1008 }
1009 }
1010}
1011
1012fn query_connection(
1013 conn: &Connection,
1014 sql: &str,
1015 params: &[Value],
1016) -> Result<Vec<Map<String, Value>>, String> {
1017 crate::query_guard::assert_read_only_query(sql)?;
1018 let bound: Vec<SqlValue> = params
1019 .iter()
1020 .map(json_param_to_sql)
1021 .collect::<Result<_, _>>()?;
1022 let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?;
1023 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
1024 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1025 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1026 let mut sql_rows = stmt
1027 .query(bound_refs.as_slice())
1028 .map_err(|e| e.to_string())?;
1029 let mut out = Vec::new();
1030 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
1031 let mut record = Map::new();
1032 for (i, name) in column_names.iter().enumerate() {
1033 let value = row.get_ref(i).map_err(|e| e.to_string())?;
1034 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1035 }
1036 out.push(record);
1037 }
1038 Ok(out)
1039}
1040
1041fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
1042 let mut stmt = conn
1043 .prepare(
1044 "SELECT windows.unit, subscriptions.state_json
1045 FROM _syncular_windows AS windows
1046 JOIN _syncular_subscriptions AS subscriptions
1047 ON subscriptions.id = windows.sub_id
1048 WHERE windows.base = ?1
1049 ORDER BY windows.unit ASC",
1050 )
1051 .map_err(|error| error.to_string())?;
1052 let rows = stmt
1053 .query_map(rusqlite::params![window_base_key(base)], |row| {
1054 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1055 })
1056 .map_err(|error| error.to_string())?;
1057 let mut units = Vec::new();
1058 let mut pending = Vec::new();
1059 for row in rows {
1060 let (unit, raw) = row.map_err(|error| error.to_string())?;
1061 let state: Value = serde_json::from_str(&raw)
1062 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
1063 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
1064 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
1065 || state
1066 .get("bootstrapState")
1067 .is_some_and(|value| !value.is_null());
1068 if is_pending {
1069 pending.push(unit.clone());
1070 }
1071 units.push(unit);
1072 }
1073 Ok(WindowState { units, pending })
1074}
1075
1076fn snapshot_connection(
1077 conn: &Connection,
1078 sql: &str,
1079 params: &[Value],
1080 coverage: &[WindowCoverage],
1081) -> Result<QuerySnapshot, String> {
1082 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
1083 .map_err(|error| error.to_string())?;
1084 let result = (|| {
1085 let revision = conn
1086 .query_row(
1087 "SELECT value FROM _syncular_meta WHERE key = ?1",
1088 rusqlite::params![LOCAL_REVISION_KEY],
1089 |row| row.get::<_, String>(0),
1090 )
1091 .ok()
1092 .and_then(|value| value.parse::<u64>().ok())
1093 .unwrap_or(0);
1094 let rows = query_connection(conn, sql, params)?;
1095 let mut pending = Vec::new();
1096 let mut missing = Vec::new();
1097 for requested in coverage {
1098 let base_key = window_base_key(&requested.base);
1099 let state = persisted_window_state(conn, &requested.base)?;
1100 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1101 let reference = WindowUnitRef {
1102 base_key: base_key.clone(),
1103 unit: unit.clone(),
1104 };
1105 if !state.units.iter().any(|held| held == &unit) {
1106 missing.push(reference);
1107 } else if state.pending.iter().any(|held| held == &unit) {
1108 pending.push(reference);
1109 }
1110 }
1111 }
1112 Ok(QuerySnapshot {
1113 revision: revision.to_string(),
1114 rows,
1115 coverage: CoverageSnapshot {
1116 complete: pending.is_empty() && missing.is_empty(),
1117 pending,
1118 missing,
1119 },
1120 })
1121 })();
1122 match result {
1123 Ok(snapshot) => {
1124 conn.execute_batch("RELEASE syncular_snapshot_read")
1125 .map_err(|error| error.to_string())?;
1126 Ok(snapshot)
1127 }
1128 Err(error) => {
1129 let _ = conn.execute_batch(
1130 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
1131 );
1132 Err(error)
1133 }
1134 }
1135}
1136
1137pub struct FileQuerySnapshotReader {
1142 path: String,
1143 conn: Option<Connection>,
1144}
1145
1146impl FileQuerySnapshotReader {
1147 #[must_use]
1148 pub fn new(path: impl Into<String>) -> Self {
1149 Self {
1150 path: path.into(),
1151 conn: None,
1152 }
1153 }
1154
1155 fn connection(&mut self) -> Result<&Connection, String> {
1156 if self.conn.is_none() {
1157 let conn = Connection::open_with_flags(
1158 &self.path,
1159 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1160 )
1161 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
1162 conn.busy_timeout(std::time::Duration::from_millis(250))
1163 .map_err(|error| error.to_string())?;
1164 self.conn = Some(conn);
1165 }
1166 self.conn
1167 .as_ref()
1168 .ok_or_else(|| "read sidecar connection missing".to_owned())
1169 }
1170
1171 pub fn query_snapshot(
1172 &mut self,
1173 sql: &str,
1174 params: &[Value],
1175 coverage: &[WindowCoverage],
1176 ) -> Result<QuerySnapshot, String> {
1177 snapshot_connection(self.connection()?, sql, params, coverage)
1178 }
1179}
1180
1181impl SyncClient {
1182 pub fn new_with_identity(
1183 client_id: Option<String>,
1184 schema_json: &Value,
1185 limits: ClientLimits,
1186 ) -> Result<Self, String> {
1187 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1188 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1189 Self::with_connection(resolved, schema_json, limits, conn)
1190 }
1191
1192 pub fn new(
1193 client_id: String,
1194 schema_json: &Value,
1195 limits: ClientLimits,
1196 ) -> Result<Self, String> {
1197 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1198 Self::with_connection(client_id, schema_json, limits, conn)
1199 }
1200
1201 pub fn open_path(
1207 client_id: String,
1208 schema_json: &Value,
1209 limits: ClientLimits,
1210 path: &str,
1211 ) -> Result<Self, String> {
1212 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1213 Self::with_connection(client_id, schema_json, limits, conn)
1214 }
1215
1216 pub fn open_path_with_identity(
1217 client_id: Option<String>,
1218 schema_json: &Value,
1219 limits: ClientLimits,
1220 path: &str,
1221 ) -> Result<Self, String> {
1222 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1223 conn.busy_timeout(std::time::Duration::from_millis(250))
1229 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1230 conn.pragma_update(None, "journal_mode", "WAL")
1231 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
1232 let persisted = conn
1233 .query_row(
1234 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
1235 [],
1236 |row| row.get::<_, String>(0),
1237 )
1238 .ok();
1239 let resolved = persisted
1240 .clone()
1241 .or(client_id.clone())
1242 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1243 if let (Some(existing), Some(requested)) = (persisted, client_id) {
1244 if existing != requested {
1245 return Err(format!(
1246 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
1247 ));
1248 }
1249 }
1250 Self::with_connection(resolved, schema_json, limits, conn)
1251 }
1252
1253 #[must_use]
1254 pub fn client_id(&self) -> &str {
1255 &self.client_id
1256 }
1257
1258 pub fn with_connection(
1265 client_id: String,
1266 schema_json: &Value,
1267 limits: ClientLimits,
1268 conn: Connection,
1269 ) -> Result<Self, String> {
1270 if limits.outcome_retention_max_entries == Some(0) {
1271 return Err(
1272 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
1273 );
1274 }
1275 let schema = parse_schema_json(schema_json)?;
1276 let mut client = SyncClient {
1277 conn,
1278 schema,
1279 client_id,
1280 limits,
1281 subs: Vec::new(),
1282 outbox: Vec::new(),
1283 conflicts: Vec::new(),
1284 rejections: Vec::new(),
1285 schema_floor: None,
1286 lease_state: None,
1287 stopped: false,
1288 upgrading: false,
1289 sync_needed: false,
1290 realtime_connected: false,
1291 presence: HashMap::new(),
1292 now_ms: None,
1293 encryption: crate::values::EncryptionConfig::default(),
1294 insert_sql: RefCell::new(HashMap::new()),
1295 overlay_dirty: Cell::new(false),
1296 change_queue: VecDeque::new(),
1297 sync_intent_queue: VecDeque::new(),
1298 retry_delay_ms: 250,
1299 };
1300 client
1304 .conn
1305 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
1306 client.create_tables()?;
1307 match client.get_meta(CLIENT_ID_KEY) {
1308 Some(existing) if existing != client.client_id => {
1309 return Err(format!(
1310 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
1311 client.client_id
1312 ));
1313 }
1314 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
1315 _ => {}
1316 }
1317 client.restore_persisted_state()?;
1318 let marker = client
1319 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1320 .and_then(|value| value.parse::<i32>().ok());
1321 if marker != Some(client.schema.version) {
1322 client.run_schema_reset()?;
1323 } else if !client.outbox.is_empty() {
1324 client.overlay_dirty.set(true);
1327 client.rebuild_overlay();
1328 }
1329 client.enqueue_startup_sync_if_needed();
1335 Ok(client)
1336 }
1337
1338 pub fn set_now_ms(&mut self, now_ms: i64) {
1341 self.now_ms = Some(now_ms);
1342 }
1343
1344 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
1348 self.encryption = encryption;
1349 }
1350
1351 fn clock_now_ms(&self) -> i64 {
1352 self.now_ms.unwrap_or_else(|| {
1353 std::time::SystemTime::now()
1354 .duration_since(std::time::UNIX_EPOCH)
1355 .map(|d| d.as_millis() as i64)
1356 .unwrap_or(0)
1357 })
1358 }
1359
1360 fn create_tables(&self) -> Result<(), String> {
1361 self.create_synced_tables()?;
1362 self.conn
1364 .execute_batch(
1365 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
1366 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1367 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
1368 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
1369 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1370 client_commit_id TEXT NOT NULL UNIQUE,
1371 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
1372 recorded_at_ms INTEGER NOT NULL,
1373 results_json TEXT NOT NULL,
1374 operations_json TEXT,
1375 resolution TEXT NOT NULL DEFAULT 'active'
1376 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
1377 resolved_at_ms INTEGER,
1378 replacement_client_commit_id TEXT);
1379 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
1380 ON _syncular_commit_outcomes(resolution, seq);
1381 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
1382 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
1383 CREATE TABLE IF NOT EXISTS _syncular_meta (
1384 key TEXT PRIMARY KEY, value TEXT NOT NULL);
1385 CREATE TABLE IF NOT EXISTS _syncular_windows (
1386 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
1387 PRIMARY KEY (base, unit));
1388 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
1389 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
1390 effective_scopes TEXT NOT NULL);",
1391 )
1392 .map_err(|e| e.to_string())?;
1393 let _ = self
1396 .conn
1397 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
1398 if self.get_meta(LOCAL_SCHEMA_VERSION_KEY).is_none() {
1401 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
1402 }
1403 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
1404 self.set_meta(LOCAL_REVISION_KEY, "0");
1405 }
1406 if self.schema_has_blobs() {
1411 self.conn
1412 .execute_batch(
1413 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
1414 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
1415 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
1416 created_at_ms INTEGER NOT NULL,
1417 last_used_ms INTEGER NOT NULL DEFAULT 0);
1418 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
1419 media_type TEXT, created_at_ms INTEGER NOT NULL);",
1420 )
1421 .map_err(|e| e.to_string())?;
1422 let _ = self.conn.execute_batch(
1426 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
1427 );
1428 }
1429 Ok(())
1430 }
1431
1432 fn schema_has_blobs(&self) -> bool {
1434 self.schema
1435 .tables
1436 .iter()
1437 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
1438 }
1439
1440 fn get_meta(&self, key: &str) -> Option<String> {
1443 self.conn
1444 .query_row(
1445 "SELECT value FROM _syncular_meta WHERE key = ?1",
1446 rusqlite::params![key],
1447 |row| row.get::<_, String>(0),
1448 )
1449 .ok()
1450 }
1451
1452 fn set_meta(&self, key: &str, value: &str) {
1453 let _ = self.conn.execute(
1454 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
1455 rusqlite::params![key, value],
1456 );
1457 }
1458
1459 fn delete_meta(&self, key: &str) {
1460 let _ = self.conn.execute(
1461 "DELETE FROM _syncular_meta WHERE key = ?1",
1462 rusqlite::params![key],
1463 );
1464 }
1465
1466 fn restore_persisted_state(&mut self) -> Result<(), String> {
1467 self.subs = {
1468 let mut stmt = self
1469 .conn
1470 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
1471 .map_err(|error| error.to_string())?;
1472 let rows = stmt
1473 .query_map([], |row| {
1474 Ok((
1475 row.get::<_, String>(0)?,
1476 row.get::<_, String>(1)?,
1477 row.get::<_, String>(2)?,
1478 ))
1479 })
1480 .map_err(|error| error.to_string())?;
1481 let mut subscriptions = Vec::new();
1482 for row in rows {
1483 let (id, table, raw) = row.map_err(|error| error.to_string())?;
1484 let state: Value = serde_json::from_str(&raw)
1485 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
1486 let requested = json_to_scope_map(
1487 state.get("requested").unwrap_or(&Value::Object(Map::new())),
1488 )?;
1489 let effective = state
1490 .get("effectiveScopes")
1491 .filter(|value| !value.is_null())
1492 .map(json_to_scope_map)
1493 .transpose()?;
1494 subscriptions.push(Subscription {
1495 id,
1496 table,
1497 requested,
1498 params: state
1499 .get("params")
1500 .and_then(Value::as_str)
1501 .map(str::to_owned),
1502 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
1503 bootstrap_state: state
1504 .get("bootstrapState")
1505 .and_then(Value::as_str)
1506 .map(str::to_owned),
1507 state: SubState::parse(
1508 state
1509 .get("status")
1510 .and_then(Value::as_str)
1511 .unwrap_or("active"),
1512 ),
1513 reason_code: state
1514 .get("reasonCode")
1515 .and_then(Value::as_str)
1516 .map(str::to_owned),
1517 effective,
1518 synced_once: state
1519 .get("syncedOnce")
1520 .and_then(Value::as_bool)
1521 .unwrap_or(false),
1522 });
1523 }
1524 subscriptions
1525 };
1526
1527 self.outbox = {
1528 let mut stmt = self
1529 .conn
1530 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
1531 .map_err(|error| error.to_string())?;
1532 let rows = stmt
1533 .query_map([], |row| {
1534 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1535 })
1536 .map_err(|error| error.to_string())?;
1537 let mut commits = Vec::new();
1538 for row in rows {
1539 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
1540 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
1541 format!("invalid persisted outbox {client_commit_id:?}: {error}")
1542 })?;
1543 let mut ops = Vec::with_capacity(entries.len());
1544 for entry in entries {
1545 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
1546 ops.push(OutboxOp {
1547 upsert: op == "upsert",
1548 table: entry
1549 .get("table")
1550 .and_then(Value::as_str)
1551 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
1552 .to_owned(),
1553 row_id: entry
1554 .get("rowId")
1555 .and_then(Value::as_str)
1556 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
1557 .to_owned(),
1558 base_version: entry.get("baseVersion").and_then(Value::as_i64),
1559 values: entry.get("values").and_then(Value::as_object).cloned(),
1560 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
1561 |values| {
1562 values
1563 .iter()
1564 .filter_map(Value::as_str)
1565 .map(str::to_owned)
1566 .collect()
1567 },
1568 ),
1569 });
1570 }
1571 commits.push(OutboxCommit {
1572 client_commit_id,
1573 ops,
1574 });
1575 }
1576 commits
1577 };
1578
1579 self.prune_commit_outcomes()?;
1580 let active = self.commit_outcomes(CommitOutcomeQuery {
1581 active_only: true,
1582 ..CommitOutcomeQuery::default()
1583 })?;
1584 self.conflicts = active
1585 .iter()
1586 .flat_map(|outcome| outcome.results.iter())
1587 .filter_map(|result| match result {
1588 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
1589 _ => None,
1590 })
1591 .collect();
1592 self.rejections = active
1593 .iter()
1594 .flat_map(|outcome| outcome.results.iter())
1595 .filter_map(|result| match result {
1596 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
1597 _ => None,
1598 })
1599 .collect();
1600
1601 self.lease_state = self
1602 .get_meta(LEASE_STATE_KEY)
1603 .map(|raw| serde_json::from_str(&raw))
1604 .transpose()
1605 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
1606 self.schema_floor = self
1607 .get_meta(SCHEMA_FLOOR_KEY)
1608 .map(|raw| serde_json::from_str(&raw))
1609 .transpose()
1610 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
1611 self.stopped = self.schema_floor.is_some();
1612 Ok(())
1613 }
1614
1615 #[must_use]
1616 pub fn local_revision(&self) -> u64 {
1617 self.get_meta(LOCAL_REVISION_KEY)
1618 .and_then(|value| value.parse().ok())
1619 .unwrap_or(0)
1620 }
1621
1622 #[must_use]
1623 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
1624 SyncStatusSnapshot {
1625 outbox: self.outbox.len(),
1626 upgrading: self.upgrading,
1627 lease_state: self.lease_state.clone(),
1628 schema_floor: self.schema_floor.clone(),
1629 sync_needed: self.sync_needed,
1630 }
1631 }
1632
1633 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
1634 self.change_queue.drain(..).collect()
1635 }
1636
1637 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
1638 self.sync_intent_queue.drain(..).collect()
1639 }
1640
1641 fn schedule_background_retry(&mut self) {
1642 self.sync_intent_queue.push_back(SyncIntent::Background {
1643 delay_ms: self.retry_delay_ms,
1644 });
1645 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
1646 }
1647
1648 fn reset_background_retry(&mut self) {
1649 self.retry_delay_ms = 250;
1650 }
1651
1652 fn retryable_transport_code(code: &str) -> bool {
1653 code == "transport.failed" || code == "sync.transport_failed"
1654 }
1655
1656 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
1657 if self.sync_needed != value {
1658 if self.begin_observation("syncular_status").is_ok() {
1659 self.sync_needed = value;
1660 let batch = ChangeAccumulator {
1661 status: true,
1662 ..ChangeAccumulator::default()
1663 };
1664 if self.finish_observation("syncular_status", batch).is_err() {
1665 self.rollback_observation("syncular_status");
1666 }
1667 } else {
1668 self.sync_needed = value;
1669 }
1670 }
1671 if value && interactive {
1672 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1673 }
1674 }
1675
1676 fn begin_observation(&self, name: &str) -> Result<(), String> {
1677 self.conn
1678 .execute_batch(&format!("SAVEPOINT {name}"))
1679 .map_err(|error| error.to_string())
1680 }
1681
1682 fn rollback_observation(&self, name: &str) {
1683 let _ = self
1684 .conn
1685 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
1686 }
1687
1688 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
1689 if !batch.touched() {
1690 self.conn
1691 .execute_batch(&format!("RELEASE {name}"))
1692 .map_err(|error| error.to_string())?;
1693 return Ok(());
1694 }
1695 let revision = self
1696 .local_revision()
1697 .checked_add(1)
1698 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
1699 self.conn
1700 .execute(
1701 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
1702 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
1703 )
1704 .map_err(|error| error.to_string())?;
1705 let status = batch.status.then(|| self.status_snapshot());
1706 let event = ClientChangeBatch {
1707 revision: revision.to_string(),
1708 tables: batch
1709 .tables
1710 .into_iter()
1711 .map(|(table, scope_keys)| TableChange {
1712 table,
1713 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
1714 })
1715 .collect(),
1716 windows: batch
1717 .windows
1718 .into_iter()
1719 .map(|((base_key, table), units)| WindowChange {
1720 base_key,
1721 table,
1722 units: units.into_iter().collect(),
1723 })
1724 .collect(),
1725 status,
1726 conflicts_changed: batch.conflicts,
1727 rejections_changed: batch.rejections,
1728 outcomes_changed: batch.outcomes,
1729 };
1730 self.conn
1731 .execute_batch(&format!("RELEASE {name}"))
1732 .map_err(|error| error.to_string())?;
1733 self.change_queue.push_back(event);
1734 Ok(())
1735 }
1736
1737 fn record_scope_map(
1738 &self,
1739 batch: &mut ChangeAccumulator,
1740 table_name: &str,
1741 scopes: &[(String, Vec<String>)],
1742 ) {
1743 let Some(table) = self.schema.table(table_name) else {
1744 return;
1745 };
1746 for (variable, values) in scopes {
1747 let Some(scope) = table
1748 .scope_variables
1749 .iter()
1750 .find(|scope| &scope.variable == variable)
1751 else {
1752 continue;
1753 };
1754 for value in values {
1755 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1756 }
1757 }
1758 }
1759
1760 fn record_row_scopes(
1762 &self,
1763 batch: &mut ChangeAccumulator,
1764 table_name: &str,
1765 row_id: &str,
1766 base: bool,
1767 ) -> bool {
1768 let Some(table) = self.schema.table(table_name) else {
1769 return false;
1770 };
1771 if table.scope_variables.is_empty() {
1772 return false;
1773 }
1774 let columns = table
1775 .scope_variables
1776 .iter()
1777 .map(|scope| quote_ident(&scope.column))
1778 .collect::<Vec<_>>()
1779 .join(", ");
1780 let full_table = if base {
1781 base_table(table_name)
1782 } else {
1783 visible_table(table_name)
1784 };
1785 let sql = format!(
1786 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
1787 quote_ident(&table.primary_key)
1788 );
1789 let Ok(mut stmt) = self.conn.prepare(&sql) else {
1790 return false;
1791 };
1792 let values = stmt.query_row(rusqlite::params![row_id], |row| {
1793 let mut values = Vec::with_capacity(table.scope_variables.len());
1794 for index in 0..table.scope_variables.len() {
1795 values.push(row.get::<_, Option<String>>(index)?);
1796 }
1797 Ok(values)
1798 });
1799 let Ok(values) = values else {
1800 return false;
1801 };
1802 let mut recorded = false;
1803 for (scope, value) in table.scope_variables.iter().zip(values) {
1804 if let Some(value) = value {
1805 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1806 recorded = true;
1807 }
1808 }
1809 recorded
1810 }
1811
1812 fn record_commit_changes(
1813 &self,
1814 batch: &mut ChangeAccumulator,
1815 tables: &[String],
1816 changes: &[ssp2::model::Change],
1817 ) {
1818 for change in changes {
1819 let Some(table_name) = tables.get(change.table_index as usize) else {
1820 continue;
1821 };
1822 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
1823 if let Some(table) = self.schema.table(table_name) {
1824 for (variable, value) in &change.scopes {
1825 if let Some(scope) = table
1826 .scope_variables
1827 .iter()
1828 .find(|scope| &scope.variable == variable)
1829 {
1830 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1831 precise = true;
1832 }
1833 }
1834 }
1835 if !precise {
1836 batch.table(table_name);
1837 }
1838 }
1839 }
1840
1841 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
1842 if effective.is_empty() {
1843 return false;
1844 }
1845 let Some(table) = self.schema.table(table_name) else {
1846 return false;
1847 };
1848 let mut clauses = Vec::new();
1849 let mut params = Vec::new();
1850 for (variable, values) in effective {
1851 let Some(column) = table.scope_column(variable) else {
1852 return false;
1853 };
1854 if values.is_empty() {
1855 return false;
1856 }
1857 let placeholders = values
1858 .iter()
1859 .map(|value| {
1860 params.push(SqlValue::Text(value.clone()));
1861 "?"
1862 })
1863 .collect::<Vec<_>>()
1864 .join(", ");
1865 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
1866 }
1867 let sql = format!(
1868 "SELECT 1 FROM {} WHERE {} LIMIT 1",
1869 base_table(table_name),
1870 clauses.join(" AND ")
1871 );
1872 self.conn
1873 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
1874 .is_ok()
1875 }
1876
1877 pub fn upgrading(&self) -> bool {
1879 self.upgrading
1880 }
1881
1882 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
1888 let new_schema = parse_schema_json(schema_json)?;
1889 let marker: Option<i32> = self
1890 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1891 .and_then(|v| v.parse().ok());
1892 self.schema = new_schema;
1893 if marker != Some(self.schema.version) {
1894 self.run_schema_reset()?;
1895 }
1896 self.enqueue_startup_sync_if_needed();
1900 Ok(())
1901 }
1902
1903 fn enqueue_startup_sync_if_needed(&mut self) {
1904 let startup_work = !self.stopped
1905 && (!self.outbox.is_empty()
1906 || self.subs.iter().any(|sub| sub.state == SubState::Active));
1907 if startup_work {
1908 self.sync_needed = true;
1909 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1910 }
1911 }
1912
1913 fn run_schema_reset(&mut self) -> Result<(), String> {
1919 self.begin_observation("syncular_schema_reset")?;
1920 let mut batch = ChangeAccumulator::default();
1921 let result = self.run_schema_reset_observed(&mut batch);
1922 if let Err(error) = result {
1923 self.rollback_observation("syncular_schema_reset");
1924 return Err(error);
1925 }
1926 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
1927 self.rollback_observation("syncular_schema_reset");
1928 return Err(error);
1929 }
1930 Ok(())
1931 }
1932
1933 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
1934 self.upgrading = true;
1935 batch.status = true;
1936 for table in &self.schema.tables {
1937 batch.table(&table.name);
1938 }
1939 for (base_key, unit, table) in self.load_registered_window_units() {
1940 batch.window(&base_key, &table, &unit);
1941 }
1942 self.insert_sql.borrow_mut().clear();
1944 self.overlay_dirty.set(true);
1945 let virtual_tables: Vec<String> = {
1953 let mut stmt = self
1954 .conn
1955 .prepare(
1956 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
1957 )
1958 .map_err(|e| e.to_string())?;
1959 let rows = stmt
1960 .query_map([], |row| row.get::<_, String>(0))
1961 .map_err(|e| e.to_string())?;
1962 rows.filter_map(Result::ok)
1963 .filter(|name| is_synced_table_name(name))
1964 .collect()
1965 };
1966 for name in virtual_tables {
1967 self.conn
1968 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
1969 .map_err(|e| e.to_string())?;
1970 }
1971 let existing: Vec<String> = {
1972 let mut stmt = self
1973 .conn
1974 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
1975 .map_err(|e| e.to_string())?;
1976 let rows = stmt
1977 .query_map([], |row| row.get::<_, String>(0))
1978 .map_err(|e| e.to_string())?;
1979 rows.filter_map(Result::ok)
1980 .filter(|name| is_synced_table_name(name))
1981 .collect()
1982 };
1983 for name in &existing {
1984 if let Some(table) = name.strip_prefix("_syncular_base_") {
1985 batch.table(table);
1986 } else if !name.starts_with("_syncular_") {
1987 batch.table(name);
1988 }
1989 }
1990 for name in existing {
1991 self.conn
1992 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
1993 .map_err(|e| e.to_string())?;
1994 }
1995 self.create_synced_tables()?;
1997 for sub in &mut self.subs {
1999 sub.cursor = -1;
2000 sub.bootstrap_state = None;
2001 sub.effective = None;
2002 sub.state = SubState::Active;
2003 sub.reason_code = None;
2004 sub.synced_once = false;
2005 }
2006 let subs = self.subs.clone();
2007 for sub in &subs {
2008 self.persist_sub(sub);
2009 }
2010 self.stopped = false;
2012 self.schema_floor = None;
2013 self.delete_meta(SCHEMA_FLOOR_KEY);
2014 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
2016 if self.drop_incompatible_outbox()? {
2020 batch.rejections = true;
2021 batch.status = true;
2022 batch.outcomes = true;
2023 }
2024 self.rebuild_overlay();
2026 Ok(())
2027 }
2028
2029 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
2033 let schema = &self.schema;
2034 let incompatible = self
2035 .outbox
2036 .iter()
2037 .filter(|commit| {
2038 commit.ops.iter().any(|op| {
2039 if !op.upsert {
2040 return false;
2041 }
2042 match schema.table(&op.table) {
2043 None => true,
2044 Some(table) => op.values.as_ref().is_some_and(|values| {
2045 values
2046 .keys()
2047 .any(|key| !table.columns.iter().any(|c| &c.name == key))
2048 }),
2049 }
2050 })
2051 })
2052 .cloned()
2053 .collect::<Vec<_>>();
2054 if incompatible.is_empty() {
2055 return Ok(false);
2056 }
2057 let mut rejections = Vec::new();
2058 for commit in &incompatible {
2059 let results = commit
2060 .ops
2061 .iter()
2062 .enumerate()
2063 .map(|(op_index, operation)| {
2064 let rejection = RejectionRecord {
2065 client_commit_id: commit.client_commit_id.clone(),
2066 op_index: op_index as i32,
2067 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
2068 message: "the persisted commit cannot encode under the current schema"
2069 .to_owned(),
2070 retryable: false,
2071 details: None,
2072 operation: Some(CommitOperation::from(operation)),
2073 };
2074 rejections.push(rejection.clone());
2075 CommitOperationOutcome::Error { rejection }
2076 })
2077 .collect::<Vec<_>>();
2078 self.persist_commit_outcome(
2079 &commit.client_commit_id,
2080 CommitOutcomeStatus::Rejected,
2081 &results,
2082 Some(&commit.ops),
2083 )?;
2084 self.delete_outbox_persisted(&commit.client_commit_id)?;
2085 }
2086 self.prune_commit_outcomes()?;
2087 let incompatible_ids = incompatible
2088 .iter()
2089 .map(|commit| commit.client_commit_id.as_str())
2090 .collect::<BTreeSet<_>>();
2091 self.outbox
2092 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
2093 self.rejections.extend(rejections);
2094 Ok(true)
2095 }
2096
2097 fn create_synced_tables(&self) -> Result<(), String> {
2100 for table in &self.schema.tables {
2101 for (full, index_prefix) in [
2105 (base_table(&table.name), "_syncular_base_"),
2106 (visible_table(&table.name), ""),
2107 ] {
2108 let mut cols: Vec<String> =
2109 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
2110 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
2111 let sql = format!(
2112 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
2113 cols.join(", "),
2114 quote_ident(&table.primary_key)
2115 );
2116 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
2117 for index in &table.indexes {
2121 let unique = if index.unique { "UNIQUE " } else { "" };
2122 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
2123 let cols_sql = index
2124 .columns
2125 .iter()
2126 .map(|c| quote_ident(c))
2127 .collect::<Vec<_>>()
2128 .join(", ");
2129 let index_sql = format!(
2130 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
2131 );
2132 self.conn
2133 .execute(&index_sql, [])
2134 .map_err(|e| e.to_string())?;
2135 }
2136 }
2137 }
2138 for table in &self.schema.tables {
2141 for index in &table.fts_indexes {
2142 self.create_fts_projection(table, index)?;
2143 }
2144 }
2145 Ok(())
2146 }
2147
2148 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
2149 let count: i64 = self
2150 .conn
2151 .query_row(
2152 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
2153 rusqlite::params![index.name],
2154 |row| row.get(0),
2155 )
2156 .map_err(|e| e.to_string())?;
2157 Ok(count > 0)
2158 }
2159
2160 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
2161 for suffix in ["ai", "ad", "au"] {
2162 self.conn
2163 .execute(
2164 &format!(
2165 "DROP TRIGGER IF EXISTS {}",
2166 quote_ident(&format!("{}_{suffix}", index.name))
2167 ),
2168 [],
2169 )
2170 .map_err(|e| e.to_string())?;
2171 }
2172 Ok(())
2173 }
2174
2175 fn create_fts_triggers(
2176 &self,
2177 table: &TableSchema,
2178 index: &FtsIndexSchema,
2179 ) -> Result<(), String> {
2180 let fts = quote_ident(&index.name);
2181 let source = visible_table(&table.name);
2182 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2183 let pk = quote_ident(&table.primary_key);
2184 let projection_columns = std::iter::once(source_id.clone())
2185 .chain(index.columns.iter().map(|column| quote_ident(column)))
2186 .collect::<Vec<_>>()
2187 .join(", ");
2188 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
2189 .chain(
2190 index
2191 .columns
2192 .iter()
2193 .map(|column| format!("new.{}", quote_ident(column))),
2194 )
2195 .collect::<Vec<_>>()
2196 .join(", ");
2197 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
2198 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
2199 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
2200 let sql = format!(
2201 "CREATE TRIGGER IF NOT EXISTS {ai} AFTER INSERT ON {source} BEGIN {delete_new}; {insert_new}; END;
2202 CREATE TRIGGER IF NOT EXISTS {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
2203 CREATE TRIGGER IF NOT EXISTS {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
2204 ai = quote_ident(&format!("{}_ai", index.name)),
2205 ad = quote_ident(&format!("{}_ad", index.name)),
2206 au = quote_ident(&format!("{}_au", index.name)),
2207 );
2208 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2209 }
2210
2211 fn rebuild_fts_projection(
2212 &self,
2213 table: &TableSchema,
2214 index: &FtsIndexSchema,
2215 ) -> Result<(), String> {
2216 let fts = quote_ident(&index.name);
2217 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
2218 let pk = quote_ident(&table.primary_key);
2219 let indexed_columns = index
2220 .columns
2221 .iter()
2222 .map(|column| quote_ident(column))
2223 .collect::<Vec<_>>();
2224 let projection_columns = std::iter::once(source_id)
2225 .chain(indexed_columns.iter().cloned())
2226 .collect::<Vec<_>>()
2227 .join(", ");
2228 let sql = format!(
2229 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
2230 columns = indexed_columns.join(", "),
2231 source = visible_table(&table.name),
2232 );
2233 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
2234 }
2235
2236 fn create_fts_projection(
2237 &self,
2238 table: &TableSchema,
2239 index: &FtsIndexSchema,
2240 ) -> Result<(), String> {
2241 let existed = self.fts_projection_exists(index)?;
2242 let tokenizer = index.tokenize.replace('\'', "''");
2243 let columns = index
2244 .columns
2245 .iter()
2246 .map(|column| quote_ident(column))
2247 .collect::<Vec<_>>()
2248 .join(", ");
2249 let sql = format!(
2250 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
2251 fts = quote_ident(&index.name),
2252 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
2253 );
2254 self.conn.execute(&sql, []).map_err(|error| {
2255 format!(
2256 "cannot create local FTS5 projection {:?}: {error}",
2257 index.name
2258 )
2259 })?;
2260 self.create_fts_triggers(table, index)?;
2261 if !existed {
2262 self.rebuild_fts_projection(table, index)?;
2263 }
2264 Ok(())
2265 }
2266
2267 fn persist_sub(&self, sub: &Subscription) {
2270 let state = serde_json::json!({
2271 "requested": scope_map_to_json(&sub.requested),
2272 "params": sub.params,
2273 "cursor": sub.cursor,
2274 "bootstrapState": sub.bootstrap_state,
2275 "status": sub.state.name(),
2276 "reasonCode": sub.reason_code,
2277 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2278 "syncedOnce": sub.synced_once,
2279 });
2280 let _ = self.conn.execute(
2281 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
2282 rusqlite::params![sub.id, sub.table, state.to_string()],
2283 );
2284 }
2285
2286 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
2287 let ops: Vec<Value> = commit
2288 .ops
2289 .iter()
2290 .map(|op| {
2291 serde_json::json!({
2292 "op": if op.upsert { "upsert" } else { "delete" },
2293 "table": op.table,
2294 "rowId": op.row_id,
2295 "baseVersion": op.base_version,
2296 "values": op.values.clone().map(Value::Object),
2297 "changedFields": op.changed_fields,
2298 })
2299 })
2300 .collect();
2301 let _ = self.conn.execute(
2302 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
2303 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
2304 );
2305 }
2306
2307 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
2308 self.conn
2309 .execute(
2310 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
2311 rusqlite::params![client_commit_id],
2312 )
2313 .map(|_| ())
2314 .map_err(|error| error.to_string())
2315 }
2316
2317 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
2318 match status {
2319 CommitOutcomeStatus::Applied => "applied",
2320 CommitOutcomeStatus::Cached => "cached",
2321 CommitOutcomeStatus::Conflict => "conflict",
2322 CommitOutcomeStatus::Rejected => "rejected",
2323 }
2324 }
2325
2326 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
2327 match resolution {
2328 CommitOutcomeResolution::Active => "active",
2329 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
2330 CommitOutcomeResolution::Superseded => "superseded",
2331 CommitOutcomeResolution::Dismissed => "dismissed",
2332 }
2333 }
2334
2335 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
2336 match value {
2337 "applied" => Ok(CommitOutcomeStatus::Applied),
2338 "cached" => Ok(CommitOutcomeStatus::Cached),
2339 "conflict" => Ok(CommitOutcomeStatus::Conflict),
2340 "rejected" => Ok(CommitOutcomeStatus::Rejected),
2341 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
2342 }
2343 }
2344
2345 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
2346 match value {
2347 "active" => Ok(CommitOutcomeResolution::Active),
2348 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
2349 "superseded" => Ok(CommitOutcomeResolution::Superseded),
2350 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
2351 _ => Err(format!(
2352 "invalid persisted commit outcome resolution {value:?}"
2353 )),
2354 }
2355 }
2356
2357 fn persist_commit_outcome(
2358 &self,
2359 client_commit_id: &str,
2360 status: CommitOutcomeStatus,
2361 results: &[CommitOperationOutcome],
2362 operations: Option<&[OutboxOp]>,
2363 ) -> Result<(), String> {
2364 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
2365 let operations_json = operations
2366 .map(|items| {
2367 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
2368 })
2369 .transpose()
2370 .map_err(|error| error.to_string())?;
2371 self.conn
2372 .execute(
2373 "INSERT INTO _syncular_commit_outcomes (
2374 client_commit_id, status, recorded_at_ms, results_json,
2375 operations_json, resolution
2376 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
2377 rusqlite::params![
2378 client_commit_id,
2379 Self::outcome_status_name(status),
2380 self.clock_now_ms(),
2381 results_json,
2382 operations_json
2383 ],
2384 )
2385 .map(|_| ())
2386 .map_err(|error| error.to_string())
2387 }
2388
2389 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
2390 let StoredCommitOutcomeRow {
2391 sequence,
2392 client_commit_id,
2393 status,
2394 recorded_at_ms,
2395 results_json,
2396 operations_json,
2397 resolution,
2398 resolved_at_ms,
2399 replacement_client_commit_id,
2400 } = row;
2401 Ok(CommitOutcome {
2402 sequence,
2403 client_commit_id,
2404 status: Self::parse_outcome_status(&status)?,
2405 recorded_at_ms,
2406 results: serde_json::from_str(&results_json)
2407 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
2408 operations: operations_json
2409 .map(|value| {
2410 serde_json::from_str(&value).map_err(|error| {
2411 format!("invalid persisted commit outcome operations: {error}")
2412 })
2413 })
2414 .transpose()?,
2415 resolution: Self::parse_outcome_resolution(&resolution)?,
2416 resolved_at_ms,
2417 replacement_client_commit_id,
2418 })
2419 }
2420
2421 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
2422 let row = self
2423 .conn
2424 .query_row(
2425 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2426 resolution, resolved_at_ms, replacement_client_commit_id
2427 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
2428 rusqlite::params![client_commit_id],
2429 |row| {
2430 Ok(StoredCommitOutcomeRow {
2431 sequence: row.get(0)?,
2432 client_commit_id: row.get(1)?,
2433 status: row.get(2)?,
2434 recorded_at_ms: row.get(3)?,
2435 results_json: row.get(4)?,
2436 operations_json: row.get(5)?,
2437 resolution: row.get(6)?,
2438 resolved_at_ms: row.get(7)?,
2439 replacement_client_commit_id: row.get(8)?,
2440 })
2441 },
2442 )
2443 .optional()
2444 .map_err(|error| error.to_string())?;
2445 row.map(Self::outcome_from_row).transpose()
2446 }
2447
2448 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
2449 if query.limit == Some(0) {
2450 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
2451 }
2452 let mut sql = String::from(
2453 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
2454 resolution, resolved_at_ms, replacement_client_commit_id
2455 FROM _syncular_commit_outcomes",
2456 );
2457 if query.active_only {
2458 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
2459 }
2460 sql.push_str(" ORDER BY seq DESC");
2461 if let Some(limit) = query.limit {
2462 sql.push_str(&format!(" LIMIT {limit}"));
2463 }
2464 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
2465 let rows = stmt
2466 .query_map([], |row| {
2467 Ok(StoredCommitOutcomeRow {
2468 sequence: row.get(0)?,
2469 client_commit_id: row.get(1)?,
2470 status: row.get(2)?,
2471 recorded_at_ms: row.get(3)?,
2472 results_json: row.get(4)?,
2473 operations_json: row.get(5)?,
2474 resolution: row.get(6)?,
2475 resolved_at_ms: row.get(7)?,
2476 replacement_client_commit_id: row.get(8)?,
2477 })
2478 })
2479 .map_err(|error| error.to_string())?;
2480 let mut outcomes = Vec::new();
2481 for row in rows {
2482 outcomes.push(Self::outcome_from_row(
2483 row.map_err(|error| error.to_string())?,
2484 )?);
2485 }
2486 Ok(outcomes)
2487 }
2488
2489 fn prune_commit_outcomes(&self) -> Result<(), String> {
2490 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
2491 let count = self
2492 .conn
2493 .query_row(
2494 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
2495 [],
2496 |row| row.get::<_, i64>(0),
2497 )
2498 .map_err(|error| error.to_string())? as usize;
2499 let excess = count.saturating_sub(max_entries);
2500 if excess == 0 {
2501 return Ok(());
2502 }
2503 let mut stmt = self
2504 .conn
2505 .prepare(
2506 "SELECT seq FROM _syncular_commit_outcomes
2507 WHERE status IN ('applied', 'cached') OR resolution != 'active'
2508 ORDER BY seq ASC LIMIT ?1",
2509 )
2510 .map_err(|error| error.to_string())?;
2511 let rows = stmt
2512 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
2513 .map_err(|error| error.to_string())?;
2514 let sequences = rows
2515 .collect::<Result<Vec<_>, _>>()
2516 .map_err(|error| error.to_string())?;
2517 drop(stmt);
2518 for sequence in sequences {
2519 self.conn
2520 .execute(
2521 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
2522 rusqlite::params![sequence],
2523 )
2524 .map_err(|error| error.to_string())?;
2525 }
2526 Ok(())
2527 }
2528
2529 pub fn subscribe(
2532 &mut self,
2533 id: String,
2534 table: String,
2535 scopes: Vec<(String, Vec<String>)>,
2536 params: Option<String>,
2537 ) -> Result<(), String> {
2538 if self.schema.table(&table).is_none() {
2539 return Err(format!("unknown table {table:?}"));
2540 }
2541 let sub = Subscription {
2542 id: id.clone(),
2543 table,
2544 requested: scopes,
2545 params,
2546 cursor: -1,
2547 bootstrap_state: None,
2548 state: SubState::Active,
2549 reason_code: None,
2550 effective: None,
2551 synced_once: false,
2552 };
2553 self.persist_sub(&sub);
2554 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
2555 *existing = sub;
2556 } else {
2557 self.subs.push(sub);
2558 }
2559 Ok(())
2560 }
2561
2562 pub fn unsubscribe(&mut self, id: &str) {
2563 self.subs.retain(|s| s.id != id);
2564 let _ = self.conn.execute(
2565 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2566 rusqlite::params![id],
2567 );
2568 }
2569
2570 pub fn set_window(
2578 &mut self,
2579 base: &WindowBase,
2580 units: &[String],
2581 ) -> Result<CommandEffects, String> {
2582 let table = self
2583 .schema
2584 .table(&base.table)
2585 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
2586 if table.scope_column(&base.variable).is_none() {
2587 return Err(format!(
2588 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
2589 base.table, base.variable
2590 ));
2591 }
2592 let base_key = window_base_key(base);
2593 let wanted: std::collections::HashSet<&String> = units.iter().collect();
2594 let live = self.load_window_units(&base_key);
2595 self.begin_observation("syncular_window")?;
2596 let mut batch = ChangeAccumulator::default();
2597 let mut changed = false;
2598
2599 for unit in units {
2601 if live.iter().any(|(u, _)| u == unit) {
2602 continue;
2603 }
2604 let sub_id = derive_sub_id(base, unit);
2605 self.delete_pending_evict(&sub_id);
2606 self.insert_window_unit(&base_key, unit, &sub_id);
2607 self.subscribe(
2608 sub_id,
2609 base.table.clone(),
2610 unit_scopes(base, unit),
2611 base.params.clone(),
2612 )?;
2613 batch.window(&base_key, &base.table, unit);
2614 changed = true;
2615 }
2616
2617 for (unit, sub_id) in live {
2619 if wanted.contains(&unit) {
2620 continue;
2621 }
2622 let effective = self
2623 .subs
2624 .iter()
2625 .find(|sub| sub.id == sub_id)
2626 .and_then(|sub| sub.effective.clone())
2627 .unwrap_or_else(|| unit_scopes(base, &unit));
2628 self.record_scope_map(&mut batch, &base.table, &effective);
2629 batch.window(&base_key, &base.table, &unit);
2630 self.evict_unit(&base_key, base, &unit, &sub_id);
2631 changed = true;
2632 }
2633 if let Err(error) = self.finish_observation("syncular_window", batch) {
2634 self.rollback_observation("syncular_window");
2635 return Err(error);
2636 }
2637 Ok(if changed {
2638 CommandEffects::interactive()
2639 } else {
2640 CommandEffects::none()
2641 })
2642 }
2643
2644 pub fn window_state(&self, base: &WindowBase) -> WindowState {
2649 let mut units = Vec::new();
2650 let mut pending = Vec::new();
2651 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
2652 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
2653 Some(sub) => {
2654 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
2655 }
2656 None => true,
2657 };
2658 if is_pending {
2659 pending.push(unit.clone());
2660 }
2661 units.push(unit);
2662 }
2663 WindowState { units, pending }
2664 }
2665
2666 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
2667 let mut stmt = match self
2668 .conn
2669 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
2670 {
2671 Ok(stmt) => stmt,
2672 Err(_) => return Vec::new(),
2673 };
2674 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
2675 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2676 });
2677 match rows {
2678 Ok(rows) => rows.filter_map(Result::ok).collect(),
2679 Err(_) => Vec::new(),
2680 }
2681 }
2682
2683 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
2684 let mut stmt = match self.conn.prepare(
2685 "SELECT windows.base, windows.unit, subscriptions.tbl
2686 FROM _syncular_windows AS windows
2687 JOIN _syncular_subscriptions AS subscriptions
2688 ON subscriptions.id = windows.sub_id
2689 ORDER BY windows.base, windows.unit",
2690 ) {
2691 Ok(stmt) => stmt,
2692 Err(_) => return Vec::new(),
2693 };
2694 let rows = stmt.query_map([], |row| {
2695 Ok((
2696 row.get::<_, String>(0)?,
2697 row.get::<_, String>(1)?,
2698 row.get::<_, String>(2)?,
2699 ))
2700 });
2701 match rows {
2702 Ok(rows) => rows.filter_map(Result::ok).collect(),
2703 Err(_) => Vec::new(),
2704 }
2705 }
2706
2707 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
2708 self.conn
2709 .query_row(
2710 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
2711 rusqlite::params![sub_id],
2712 |row| Ok((row.get(0)?, row.get(1)?)),
2713 )
2714 .ok()
2715 }
2716
2717 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
2718 let _ = self.conn.execute(
2719 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
2720 rusqlite::params![base_key, unit, sub_id],
2721 );
2722 }
2723
2724 fn delete_window_unit(&self, base_key: &str, unit: &str) {
2725 let _ = self.conn.execute(
2726 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
2727 rusqlite::params![base_key, unit],
2728 );
2729 }
2730
2731 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
2738 let effective = self
2739 .subs
2740 .iter()
2741 .find(|s| s.id == sub_id)
2742 .and_then(|s| s.effective.clone())
2743 .unwrap_or_else(|| unit_scopes(base, unit));
2744 let pinned = self.pinned_row_ids(&base.table);
2745 let deferred = self
2746 .evict_scope_rows(&base.table, &effective, &pinned)
2747 .unwrap_or(false);
2748 self.delete_window_unit(base_key, unit);
2749 self.unsubscribe(sub_id);
2750 if deferred {
2751 self.save_pending_evict(sub_id, &base.table, &effective);
2752 } else {
2753 self.delete_pending_evict(sub_id);
2754 }
2755 self.rebuild_overlay();
2756 }
2757
2758 fn evict_scope_rows(
2762 &mut self,
2763 table_name: &str,
2764 effective: &[(String, Vec<String>)],
2765 pinned: &std::collections::HashSet<String>,
2766 ) -> Result<bool, ()> {
2767 if effective.is_empty() {
2768 return Ok(false);
2769 }
2770 let table = self.schema.table(table_name).ok_or(())?.clone();
2771 let mut clauses = Vec::new();
2772 let mut params: Vec<SqlValue> = Vec::new();
2773 for (variable, values) in effective {
2774 let column = table.scope_column(variable).ok_or(())?;
2775 let placeholders: Vec<String> = values
2776 .iter()
2777 .map(|v| {
2778 params.push(SqlValue::Text(v.clone()));
2779 "?".to_owned()
2780 })
2781 .collect();
2782 clauses.push(format!(
2783 "{} IN ({})",
2784 quote_ident(column),
2785 placeholders.join(", ")
2786 ));
2787 }
2788 let mut sql = format!(
2789 "DELETE FROM {} WHERE {}",
2790 base_table(table_name),
2791 clauses.join(" AND ")
2792 );
2793 if !pinned.is_empty() {
2794 let pk = quote_ident(&table.primary_key);
2795 let holes: Vec<String> = pinned
2796 .iter()
2797 .map(|id| {
2798 params.push(SqlValue::Text(id.clone()));
2799 "?".to_owned()
2800 })
2801 .collect();
2802 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
2803 }
2804 self.overlay_dirty.set(true);
2805 self.conn
2806 .execute(&sql, rusqlite::params_from_iter(params))
2807 .map_err(|_| ())?;
2808 if pinned.is_empty() {
2809 return Ok(false);
2810 }
2811 let mut where_params: Vec<SqlValue> = Vec::new();
2814 let mut where_clauses = Vec::new();
2815 for (variable, values) in effective {
2816 let column = table.scope_column(variable).ok_or(())?;
2817 let placeholders: Vec<String> = values
2818 .iter()
2819 .map(|v| {
2820 where_params.push(SqlValue::Text(v.clone()));
2821 "?".to_owned()
2822 })
2823 .collect();
2824 where_clauses.push(format!(
2825 "{} IN ({})",
2826 quote_ident(column),
2827 placeholders.join(", ")
2828 ));
2829 }
2830 let pk = quote_ident(&table.primary_key);
2831 let select = format!(
2832 "SELECT {} FROM {} WHERE {}",
2833 pk,
2834 base_table(table_name),
2835 where_clauses.join(" AND ")
2836 );
2837 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
2838 let survivors: Vec<String> = stmt
2839 .query_map(rusqlite::params_from_iter(where_params), |row| {
2840 row.get::<_, String>(0)
2841 })
2842 .map_err(|_| ())?
2843 .filter_map(Result::ok)
2844 .collect();
2845 Ok(survivors.iter().any(|id| pinned.contains(id)))
2846 }
2847
2848 fn drain_pending_evictions(&mut self) {
2852 let pending = self.load_pending_evictions();
2853 if pending.is_empty() {
2854 return;
2855 }
2856 for (sub_id, table_name, effective) in pending {
2857 if self.schema.table(&table_name).is_none() {
2858 self.delete_pending_evict(&sub_id);
2859 continue;
2860 }
2861 let pinned = self.pinned_row_ids(&table_name);
2862 let deferred = self
2863 .evict_scope_rows(&table_name, &effective, &pinned)
2864 .unwrap_or(false);
2865 if !deferred {
2866 self.delete_pending_evict(&sub_id);
2867 }
2868 }
2869 self.rebuild_overlay_if_dirty();
2870 }
2871
2872 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
2875 let mut pinned = std::collections::HashSet::new();
2876 for commit in &self.outbox {
2877 for op in &commit.ops {
2878 if op.table == table {
2879 pinned.insert(op.row_id.clone());
2880 }
2881 }
2882 }
2883 pinned
2884 }
2885
2886 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
2887 let _ = self.conn.execute(
2888 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
2889 VALUES (?1, ?2, ?3)",
2890 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
2891 );
2892 }
2893
2894 fn delete_pending_evict(&self, sub_id: &str) {
2895 let _ = self.conn.execute(
2896 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2897 rusqlite::params![sub_id],
2898 );
2899 }
2900
2901 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
2902 let mut stmt = match self
2903 .conn
2904 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
2905 {
2906 Ok(stmt) => stmt,
2907 Err(_) => return Vec::new(),
2908 };
2909 let rows = stmt.query_map([], |row| {
2910 Ok((
2911 row.get::<_, String>(0)?,
2912 row.get::<_, String>(1)?,
2913 row.get::<_, String>(2)?,
2914 ))
2915 });
2916 let mut out = Vec::new();
2917 if let Ok(rows) = rows {
2918 for entry in rows.filter_map(Result::ok) {
2919 let (sub_id, table, json) = entry;
2920 if let Ok(value) = serde_json::from_str::<Value>(&json) {
2921 if let Ok(effective) = json_to_scope_map(&value) {
2922 out.push((sub_id, table, effective));
2923 }
2924 }
2925 }
2926 }
2927 out
2928 }
2929
2930 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
2932 if mutations.is_empty() {
2933 return Err("a commit must contain at least one operation (§6.1)".to_owned());
2934 }
2935 let mut ops = Vec::with_capacity(mutations.len());
2936 for mutation in mutations {
2937 match mutation {
2938 Mutation::Upsert {
2939 table,
2940 values,
2941 base_version,
2942 } => {
2943 let schema_table = self
2944 .schema
2945 .table(&table)
2946 .ok_or_else(|| format!("unknown table {table:?}"))?;
2947 let values = normalize_values_casing(schema_table, values)?;
2951 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
2952 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
2955 ops.push(OutboxOp {
2956 upsert: true,
2957 table,
2958 row_id,
2959 base_version,
2960 values: Some(values),
2961 changed_fields: None,
2962 });
2963 }
2964 Mutation::Delete {
2965 table,
2966 row_id,
2967 base_version,
2968 } => {
2969 if self.schema.table(&table).is_none() {
2970 return Err(format!("unknown table {table:?}"));
2971 }
2972 ops.push(OutboxOp {
2973 upsert: false,
2974 table,
2975 row_id,
2976 base_version,
2977 values: None,
2978 changed_fields: None,
2979 });
2980 }
2981 }
2982 }
2983 self.record_outbox_commit(ops)
2984 }
2985
2986 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
2987 let commit = OutboxCommit {
2988 client_commit_id: uuid::Uuid::new_v4().to_string(),
2989 ops,
2990 };
2991 self.begin_observation("syncular_mutation")?;
2992 let mut batch = ChangeAccumulator::default();
2993 for op in &commit.ops {
2994 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
2995 if let Some(values) = &op.values {
2996 if let Some(table) = self.schema.table(&op.table) {
2997 for scope in &table.scope_variables {
2998 if let Some(Value::String(value)) = values.get(&scope.column) {
2999 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
3000 precise = true;
3001 }
3002 }
3003 }
3004 }
3005 if !precise {
3006 batch.table(&op.table);
3007 }
3008 }
3009 self.persist_outbox_insert(&commit);
3010 let id = commit.client_commit_id.clone();
3011 self.outbox.push(commit);
3012 self.overlay_dirty.set(true);
3013 self.rebuild_overlay();
3014 batch.status = true;
3015 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
3016 self.rollback_observation("syncular_mutation");
3017 return Err(error);
3018 }
3019 Ok(id)
3020 }
3021
3022 pub fn patch(
3027 &mut self,
3028 table: &str,
3029 row_id: &str,
3030 partial: Map<String, Value>,
3031 base_version: Option<i64>,
3032 ) -> Result<String, String> {
3033 let schema_table = self
3034 .schema
3035 .table(table)
3036 .ok_or_else(|| format!("unknown table {table:?}"))?;
3037 let partial = normalize_values_casing(schema_table, partial)?;
3038 let mut values = self
3039 .read_rows(table)?
3040 .into_iter()
3041 .find(|row| row.row_id == row_id)
3042 .map(|row| row.values)
3043 .ok_or_else(|| {
3044 format!(
3045 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
3046 )
3047 })?;
3048 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
3049 changed_fields.sort();
3050 values.extend(partial);
3051 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
3052 if row_id_from_values != row_id {
3053 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
3054 }
3055 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
3056 self.record_outbox_commit(vec![OutboxOp {
3057 upsert: true,
3058 table: table.to_owned(),
3059 row_id: row_id.to_owned(),
3060 base_version,
3061 values: Some(values),
3062 changed_fields: Some(changed_fields),
3063 }])
3064 }
3065
3066 pub fn pending_commit_ids(&self) -> Vec<String> {
3067 self.outbox
3068 .iter()
3069 .map(|c| c.client_commit_id.clone())
3070 .collect()
3071 }
3072
3073 pub fn conflicts(&self) -> &[ConflictRecord] {
3074 &self.conflicts
3075 }
3076
3077 pub fn rejections(&self) -> &[RejectionRecord] {
3078 &self.rejections
3079 }
3080
3081 pub fn resolve_commit_outcome(
3082 &mut self,
3083 input: ResolveCommitOutcomeInput,
3084 ) -> Result<CommitOutcome, String> {
3085 let current = self
3086 .commit_outcome(&input.client_commit_id)?
3087 .ok_or_else(|| {
3088 format!(
3089 "sync.outcome_not_found: no durable outcome exists for {:?}",
3090 input.client_commit_id
3091 )
3092 })?;
3093 if current.resolution != CommitOutcomeResolution::Active {
3094 return Ok(current);
3095 }
3096 if input.resolution == CommitOutcomeResolution::Active {
3097 return Err("sync.invalid_request: resolution must leave active state".to_owned());
3098 }
3099 match input.resolution {
3100 CommitOutcomeResolution::Superseded => {
3101 let replacement = input
3102 .replacement_client_commit_id
3103 .as_deref()
3104 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
3105 .ok_or_else(|| {
3106 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
3107 .to_owned()
3108 })?;
3109 let _ = replacement;
3110 }
3111 _ if input.replacement_client_commit_id.is_some() => {
3112 return Err(
3113 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
3114 .to_owned(),
3115 );
3116 }
3117 _ => {}
3118 }
3119 let allowed = match current.status {
3120 CommitOutcomeStatus::Conflict => matches!(
3121 input.resolution,
3122 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
3123 ),
3124 CommitOutcomeStatus::Rejected => {
3125 input.resolution == CommitOutcomeResolution::Superseded
3126 }
3127 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
3128 input.resolution == CommitOutcomeResolution::Dismissed
3129 }
3130 };
3131 if !allowed {
3132 return Err(format!(
3133 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
3134 input.resolution, current.status
3135 ));
3136 }
3137
3138 self.begin_observation("syncular_outcome_resolution")?;
3139 let result = (|| {
3140 self.conn
3141 .execute(
3142 "UPDATE _syncular_commit_outcomes
3143 SET resolution = ?1, resolved_at_ms = ?2,
3144 replacement_client_commit_id = ?3
3145 WHERE client_commit_id = ?4 AND resolution = 'active'",
3146 rusqlite::params![
3147 Self::outcome_resolution_name(input.resolution),
3148 self.clock_now_ms(),
3149 input.replacement_client_commit_id,
3150 input.client_commit_id
3151 ],
3152 )
3153 .map_err(|error| error.to_string())?;
3154 let resolved = self
3155 .commit_outcome(¤t.client_commit_id)?
3156 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
3157 self.prune_commit_outcomes()?;
3158 Ok(resolved)
3159 })();
3160 let resolved = match result {
3161 Ok(outcome) => outcome,
3162 Err(error) => {
3163 self.rollback_observation("syncular_outcome_resolution");
3164 return Err(error);
3165 }
3166 };
3167 self.conflicts
3168 .retain(|record| record.client_commit_id != current.client_commit_id);
3169 self.rejections
3170 .retain(|record| record.client_commit_id != current.client_commit_id);
3171 let batch = ChangeAccumulator {
3172 conflicts: current.status == CommitOutcomeStatus::Conflict,
3173 rejections: current.status == CommitOutcomeStatus::Rejected,
3174 outcomes: true,
3175 ..ChangeAccumulator::default()
3176 };
3177 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
3178 self.rollback_observation("syncular_outcome_resolution");
3179 return Err(error);
3180 }
3181 Ok(resolved)
3182 }
3183
3184 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
3185 self.schema_floor.as_ref()
3186 }
3187
3188 pub fn lease_state(&self) -> Option<&LeaseState> {
3190 self.lease_state.as_ref()
3191 }
3192
3193 fn record_lease_error(&mut self, code: &str) {
3196 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
3197 return;
3198 }
3199 let mut next = self.lease_state.clone().unwrap_or_default();
3200 next.error_code = Some(code.to_owned());
3201 self.set_lease_state(Some(next));
3202 }
3203
3204 fn set_lease_state(&mut self, next: Option<LeaseState>) {
3205 if self.lease_state == next {
3206 return;
3207 }
3208 if self.begin_observation("syncular_lease").is_err() {
3209 return;
3210 }
3211 self.lease_state = next;
3212 if let Some(lease) = &self.lease_state {
3213 if let Ok(json) = serde_json::to_string(lease) {
3214 self.set_meta(LEASE_STATE_KEY, &json);
3215 }
3216 } else {
3217 self.delete_meta(LEASE_STATE_KEY);
3218 }
3219 let batch = ChangeAccumulator {
3220 status: true,
3221 ..ChangeAccumulator::default()
3222 };
3223 if self.finish_observation("syncular_lease", batch).is_err() {
3224 self.rollback_observation("syncular_lease");
3225 }
3226 }
3227
3228 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
3229 if self.schema_floor == next {
3230 return;
3231 }
3232 if self.begin_observation("syncular_schema_floor").is_err() {
3233 return;
3234 }
3235 self.schema_floor = next;
3236 self.stopped = self.schema_floor.is_some();
3237 if let Some(floor) = &self.schema_floor {
3238 if let Ok(json) = serde_json::to_string(floor) {
3239 self.set_meta(SCHEMA_FLOOR_KEY, &json);
3240 }
3241 } else {
3242 self.delete_meta(SCHEMA_FLOOR_KEY);
3243 }
3244 let batch = ChangeAccumulator {
3245 status: true,
3246 ..ChangeAccumulator::default()
3247 };
3248 if self
3249 .finish_observation("syncular_schema_floor", batch)
3250 .is_err()
3251 {
3252 self.rollback_observation("syncular_schema_floor");
3253 }
3254 }
3255
3256 fn set_upgrading(&mut self, value: bool) {
3257 if self.upgrading == value {
3258 return;
3259 }
3260 if self.begin_observation("syncular_upgrading").is_err() {
3261 return;
3262 }
3263 self.upgrading = value;
3264 let batch = ChangeAccumulator {
3265 status: true,
3266 ..ChangeAccumulator::default()
3267 };
3268 if self
3269 .finish_observation("syncular_upgrading", batch)
3270 .is_err()
3271 {
3272 self.rollback_observation("syncular_upgrading");
3273 }
3274 }
3275
3276 pub fn sync_needed(&self) -> bool {
3277 self.sync_needed
3278 }
3279
3280 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
3281 let sub = self.subs.iter().find(|s| s.id == id)?;
3282 Some(SubscriptionStateView {
3283 id: sub.id.clone(),
3284 table: sub.table.clone(),
3285 status: sub.state.name().to_owned(),
3286 cursor: sub.cursor,
3287 has_resume_token: sub.bootstrap_state.is_some(),
3288 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3289 reason_code: sub.reason_code.clone(),
3290 })
3291 }
3292
3293 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
3294 let schema_table = self
3295 .schema
3296 .table(table)
3297 .ok_or_else(|| format!("unknown table {table:?}"))?;
3298 let sql = format!(
3299 "SELECT * FROM {} ORDER BY {} ASC",
3300 visible_table(table),
3301 quote_ident(&schema_table.primary_key)
3302 );
3303 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
3304 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
3305 let mut out = Vec::new();
3306 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
3307 let mut values = Map::new();
3308 for (i, column) in schema_table.columns.iter().enumerate() {
3309 let value = row.get_ref(i).map_err(|e| e.to_string())?;
3310 values.insert(column.name.clone(), sql_ref_to_json(column, value));
3311 }
3312 let version: i64 = row
3313 .get(schema_table.columns.len())
3314 .map_err(|e| e.to_string())?;
3315 let row_id = match values.get(&schema_table.primary_key) {
3316 Some(Value::String(s)) => s.clone(),
3317 Some(Value::Number(n)) => n.to_string(),
3318 Some(Value::Bool(b)) => b.to_string(),
3319 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
3320 };
3321 out.push(RowState {
3322 row_id,
3323 version,
3324 values,
3325 });
3326 }
3327 Ok(out)
3328 }
3329
3330 #[cfg(feature = "crdt-yjs")]
3346 fn crdt_column_bytes(
3347 &self,
3348 table: &str,
3349 row_id: &str,
3350 column: &str,
3351 ) -> Result<Option<Vec<u8>>, String> {
3352 let schema_table = self
3353 .schema
3354 .table(table)
3355 .ok_or_else(|| format!("unknown table {table:?}"))?;
3356 let col = schema_table
3357 .columns
3358 .iter()
3359 .find(|c| c.name == column)
3360 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
3361 if col.ty != ColumnType::Crdt {
3362 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
3363 }
3364 let sql = format!(
3365 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
3366 quote_ident(column),
3367 visible_table(table),
3368 quote_ident(&schema_table.primary_key)
3369 );
3370 let bytes: Option<Vec<u8>> = self
3371 .conn
3372 .query_row(&sql, rusqlite::params![row_id], |row| {
3373 row.get::<_, Option<Vec<u8>>>(0)
3374 })
3375 .map_err(|e| match e {
3376 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
3377 other => other.to_string(),
3378 })?;
3379 Ok(bytes)
3380 }
3381
3382 #[cfg(feature = "crdt-yjs")]
3386 pub fn crdt_text(
3387 &self,
3388 table: &str,
3389 row_id: &str,
3390 column: &str,
3391 name: &str,
3392 ) -> Result<String, String> {
3393 let bytes = self
3394 .crdt_column_bytes(table, row_id, column)?
3395 .unwrap_or_default();
3396 crate::crdt::text(&bytes, name)
3397 }
3398
3399 #[cfg(feature = "crdt-yjs")]
3403 pub fn crdt_insert_text(
3404 &mut self,
3405 table: &str,
3406 row_id: &str,
3407 column: &str,
3408 name: &str,
3409 index: u32,
3410 value: &str,
3411 ) -> Result<String, String> {
3412 let current = self
3413 .crdt_column_bytes(table, row_id, column)?
3414 .unwrap_or_default();
3415 let update = crate::crdt::insert_text(¤t, name, index, value)?;
3416 self.crdt_push_update(table, row_id, column, &update)
3417 }
3418
3419 #[cfg(feature = "crdt-yjs")]
3422 pub fn crdt_delete_text(
3423 &mut self,
3424 table: &str,
3425 row_id: &str,
3426 column: &str,
3427 name: &str,
3428 index: u32,
3429 len: u32,
3430 ) -> Result<String, String> {
3431 let current = self
3432 .crdt_column_bytes(table, row_id, column)?
3433 .unwrap_or_default();
3434 let update = crate::crdt::delete_text(¤t, name, index, len)?;
3435 self.crdt_push_update(table, row_id, column, &update)
3436 }
3437
3438 #[cfg(feature = "crdt-yjs")]
3443 pub fn crdt_apply_update(
3444 &mut self,
3445 table: &str,
3446 row_id: &str,
3447 column: &str,
3448 update: &[u8],
3449 ) -> Result<String, String> {
3450 let current = self
3451 .crdt_column_bytes(table, row_id, column)?
3452 .unwrap_or_default();
3453 let next = crate::crdt::apply_update(¤t, update)?;
3454 self.crdt_push_update(table, row_id, column, &next)
3455 }
3456
3457 #[cfg(feature = "crdt-yjs")]
3463 fn crdt_push_update(
3464 &mut self,
3465 table: &str,
3466 row_id: &str,
3467 column: &str,
3468 crdt_bytes: &[u8],
3469 ) -> Result<String, String> {
3470 let schema_table = self
3471 .schema
3472 .table(table)
3473 .ok_or_else(|| format!("unknown table {table:?}"))?
3474 .clone();
3475 let mut values: Map<String, Value> = self
3478 .read_rows(table)?
3479 .into_iter()
3480 .find(|r| r.row_id == row_id)
3481 .map(|r| r.values)
3482 .unwrap_or_else(|| {
3483 let mut map = Map::new();
3484 map.insert(
3485 schema_table.primary_key.clone(),
3486 Value::from(row_id.to_owned()),
3487 );
3488 map
3489 });
3490 let mut bytes_obj = Map::new();
3492 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
3493 values.insert(column.to_owned(), Value::Object(bytes_obj));
3494 self.mutate(vec![Mutation::Upsert {
3495 table: table.to_owned(),
3496 values,
3497 base_version: None,
3498 }])
3499 }
3500
3501 pub fn query(&self, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {
3516 query_connection(&self.conn, sql, params)
3517 }
3518
3519 pub fn query_snapshot(
3521 &mut self,
3522 sql: &str,
3523 params: &[Value],
3524 coverage: &[WindowCoverage],
3525 ) -> Result<QuerySnapshot, String> {
3526 snapshot_connection(&self.conn, sql, params, coverage)
3527 }
3528
3529 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
3532 let mut frames = vec![Frame::ReqHeader {
3533 client_id: self.client_id.clone(),
3534 schema_version: self.schema.version,
3535 }];
3536 let mut pushed_ids = Vec::new();
3537 let mut ops_in_request = 0usize;
3538 let mut deferred_commits = 0usize;
3539 for (index, commit) in self.outbox.iter().enumerate() {
3540 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
3545 deferred_commits = self.outbox.len() - index;
3546 break;
3547 }
3548 ops_in_request += commit.ops.len();
3549 let operations = commit
3550 .ops
3551 .iter()
3552 .map(|op| {
3553 let payload = op.values.as_ref().and_then(|values| {
3554 let table = self.schema.table(&op.table)?;
3555 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
3559 });
3560 ssp2::model::Operation {
3561 table: op.table.clone(),
3562 row_id: op.row_id.clone(),
3563 op: if op.upsert { Op::Upsert } else { Op::Delete },
3564 base_version: op.base_version,
3565 payload,
3566 }
3567 })
3568 .collect();
3569 frames.push(Frame::PushCommit {
3570 client_commit_id: commit.client_commit_id.clone(),
3571 operations,
3572 });
3573 pushed_ids.push(commit.client_commit_id.clone());
3574 }
3575 let accept = self.limits.accept.unwrap_or(if url_capable {
3578 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
3579 } else {
3580 DEFAULT_ACCEPT
3581 });
3582 frames.push(Frame::PullHeader {
3583 limit_commits: self.limits.limit_commits.unwrap_or(0),
3584 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
3585 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
3586 accept,
3587 });
3588 let mut fresh = Vec::new();
3589 for sub in &self.subs {
3590 if sub.state != SubState::Active {
3591 continue;
3592 }
3593 let mut scopes = sub.requested.clone();
3594 sort_scope_map(&mut scopes);
3595 frames.push(Frame::Subscription {
3596 id: sub.id.clone(),
3597 table: sub.table.clone(),
3598 scopes,
3599 params: sub.params.clone().map(RawJson),
3600 cursor: sub.cursor,
3601 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
3602 });
3603 fresh.push((
3604 sub.id.clone(),
3605 sub.cursor < 0 && sub.bootstrap_state.is_none(),
3606 ));
3607 }
3608 let message = Message {
3609 msg_kind: MsgKind::Request,
3610 frames,
3611 };
3612 (
3613 message,
3614 RequestMeta {
3615 pushed_ids,
3616 fresh,
3617 accept,
3618 deferred_commits,
3619 },
3620 )
3621 }
3622
3623 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
3626 if self.stopped {
3627 return SyncOutcome::Ok(SyncReport {
3630 schema_floor: self.schema_floor.clone(),
3631 ..SyncReport::default()
3632 });
3633 }
3634 self.set_sync_needed(false, false);
3637 if self.schema_has_blobs() {
3640 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
3641 if Self::retryable_transport_code(&code) {
3642 self.schedule_background_retry();
3643 }
3644 return SyncOutcome::Failed {
3645 error_code: code,
3646 message,
3647 };
3648 }
3649 }
3650 let (message, meta) = self.build_request(transport.supports_url_fetch());
3651 let request_bytes = encode_message(&message);
3652 let round = if self.realtime_connected {
3656 transport.realtime_sync(&request_bytes)
3657 } else {
3658 transport.sync(&request_bytes)
3659 };
3660 let response_bytes = match round {
3661 Ok(bytes) => bytes,
3662 Err(TransportError { code, message }) => {
3663 self.record_lease_error(&code);
3666 if Self::retryable_transport_code(&code) {
3667 self.schedule_background_retry();
3668 }
3669 return SyncOutcome::Failed {
3670 error_code: code,
3671 message,
3672 };
3673 }
3674 };
3675 let response = match decode_message(&response_bytes) {
3676 Ok(message) => message,
3677 Err(error) => {
3678 return SyncOutcome::Failed {
3681 error_code: error.code.as_str().to_owned(),
3682 message: error.detail,
3683 };
3684 }
3685 };
3686 if response.msg_kind != MsgKind::Response {
3687 return SyncOutcome::Failed {
3688 error_code: "sync.invalid_request".to_owned(),
3689 message: "expected a response message".to_owned(),
3690 };
3691 }
3692 let outcome = self.process_response(transport, response, &meta);
3693 match &outcome {
3694 SyncOutcome::Ok(_) => self.reset_background_retry(),
3695 SyncOutcome::Failed { error_code, .. }
3696 if Self::retryable_transport_code(error_code) =>
3697 {
3698 self.schedule_background_retry();
3699 }
3700 SyncOutcome::Failed { .. } => {}
3701 }
3702 if meta.deferred_commits > 0 {
3703 self.set_sync_needed(true, true);
3706 }
3707 outcome
3708 }
3709
3710 pub fn sync_until_idle(
3711 &mut self,
3712 transport: &mut dyn Transport,
3713 max_rounds: Option<u32>,
3714 ) -> SyncOutcome {
3715 let rounds = max_rounds.unwrap_or(12).max(1);
3716 let mut aggregate = SyncReport::default();
3717 for _ in 0..rounds {
3718 match self.sync(transport) {
3719 SyncOutcome::Failed {
3720 error_code,
3721 message,
3722 } => {
3723 return SyncOutcome::Failed {
3724 error_code,
3725 message,
3726 };
3727 }
3728 SyncOutcome::Ok(report) => {
3729 aggregate.pushed += report.pushed;
3730 aggregate.applied.extend(report.applied.iter().cloned());
3731 aggregate.rejected.extend(report.rejected.iter().cloned());
3732 aggregate.retryable.extend(report.retryable.iter().cloned());
3733 aggregate.conflicts += report.conflicts;
3734 aggregate.commits_applied += report.commits_applied;
3735 aggregate.segment_rows_applied += report.segment_rows_applied;
3736 aggregate.bootstrapping = report.bootstrapping.clone();
3737 aggregate.resets.extend(report.resets.iter().cloned());
3738 aggregate.revoked.extend(report.revoked.iter().cloned());
3739 aggregate.failed.extend(report.failed.iter().cloned());
3740 if report.schema_floor.is_some() {
3741 aggregate.schema_floor = report.schema_floor.clone();
3742 }
3743 let more = !report.bootstrapping.is_empty()
3749 || report.commits_applied > 0
3750 || report.segment_rows_applied > 0
3751 || !report.resets.is_empty()
3752 || self.sync_needed;
3753 if !more {
3754 break;
3755 }
3756 }
3757 }
3758 }
3759 SyncOutcome::Ok(aggregate)
3760 }
3761
3762 fn process_response(
3763 &mut self,
3764 transport: &mut dyn Transport,
3765 response: Message,
3766 meta: &RequestMeta,
3767 ) -> SyncOutcome {
3768 let mut report = SyncReport {
3769 pushed: meta.pushed_ids.len() as u32,
3770 ..SyncReport::default()
3771 };
3772 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
3773 HashMap::new();
3774 for frame in &response.frames {
3775 if let Frame::PushResultDetails {
3776 client_commit_id,
3777 entries,
3778 } = frame
3779 {
3780 let details = rejection_details_by_commit
3781 .entry(client_commit_id.clone())
3782 .or_default();
3783 for entry in entries {
3784 let parsed = match RejectionDetails::parse(&entry.details.0) {
3785 Ok(value) => value,
3786 Err(message) => {
3787 return SyncOutcome::Failed {
3788 error_code: "sync.invalid_request".to_owned(),
3789 message,
3790 };
3791 }
3792 };
3793 details.insert(entry.op_index, parsed);
3794 }
3795 }
3796 }
3797 let mut frames = response.frames.into_iter();
3798 match frames.next() {
3799 Some(Frame::RespHeader {
3800 required_schema_version,
3801 latest_schema_version,
3802 }) => {
3803 if let Some(required) = required_schema_version {
3804 let floor = SchemaFloor {
3807 required_schema_version: Some(required),
3808 latest_schema_version,
3809 };
3810 self.set_schema_floor(Some(floor.clone()));
3811 report.schema_floor = Some(floor);
3812 return SyncOutcome::Ok(report);
3813 }
3814 }
3815 _ => {
3816 return SyncOutcome::Failed {
3817 error_code: "sync.invalid_request".to_owned(),
3818 message: "response does not start with RESP_HEADER".to_owned(),
3819 };
3820 }
3821 }
3822
3823 let mut failure: Option<(String, String)> = None;
3824 while let Some(frame) = frames.next() {
3825 match frame {
3826 Frame::PushResult {
3827 client_commit_id,
3828 status,
3829 commit_seq: _,
3830 results,
3831 } => {
3832 self.handle_push_result(
3833 &client_commit_id,
3834 status,
3835 &results,
3836 rejection_details_by_commit.get(&client_commit_id),
3837 &mut report,
3838 );
3839 }
3840 Frame::PushResultDetails { .. } => {}
3841 Frame::SubStart {
3842 id,
3843 status,
3844 reason_code,
3845 effective_scopes,
3846 bootstrap: _,
3847 } => {
3848 let mut body = Vec::new();
3849 let mut sub_end: Option<(i64, Option<String>)> = None;
3850 for inner in frames.by_ref() {
3851 match inner {
3852 Frame::SubEnd {
3853 next_cursor,
3854 bootstrap_state,
3855 } => {
3856 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
3857 break;
3858 }
3859 Frame::Unknown { .. } => {}
3860 other => body.push(other),
3861 }
3862 }
3863 let Some((next_cursor, bootstrap_state)) = sub_end else {
3864 failure = Some((
3865 "sync.invalid_request".to_owned(),
3866 "subscription section without SUB_END".to_owned(),
3867 ));
3868 break;
3869 };
3870 if let Err(SectionError::Abort(code, message)) = self.process_section(
3871 transport,
3872 &id,
3873 status,
3874 &reason_code,
3875 effective_scopes,
3876 body,
3877 next_cursor,
3878 bootstrap_state,
3879 meta,
3880 &mut report,
3881 ) {
3882 failure = Some((code, message));
3883 break;
3884 }
3885 }
3886 Frame::Lease {
3887 lease_id,
3888 expires_at_ms,
3889 } => {
3890 self.set_lease_state(Some(LeaseState {
3893 lease_id: Some(lease_id),
3894 expires_at_ms: Some(expires_at_ms),
3895 error_code: None,
3896 }));
3897 }
3898 Frame::Error { code, message, .. } => {
3899 failure = Some((code, message));
3902 break;
3903 }
3904 Frame::Unknown { .. } => {}
3905 _ => {
3906 failure = Some((
3907 "sync.invalid_request".to_owned(),
3908 "unexpected frame in response".to_owned(),
3909 ));
3910 break;
3911 }
3912 }
3913 }
3914
3915 if self.overlay_dirty.get() {
3918 self.rebuild_overlay();
3919 }
3920 self.reconcile_blob_refcounts(false);
3927
3928 if let Some((error_code, message)) = failure {
3929 return SyncOutcome::Failed {
3930 error_code,
3931 message,
3932 };
3933 }
3934 self.drain_pending_evictions();
3937 self.ack_after_pull(transport);
3938 if self.upgrading && report.bootstrapping.is_empty() {
3941 self.set_upgrading(false);
3942 }
3943 SyncOutcome::Ok(report)
3944 }
3945
3946 fn handle_push_result(
3949 &mut self,
3950 client_commit_id: &str,
3951 status: PushStatus,
3952 results: &[OpResult],
3953 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
3954 report: &mut SyncReport,
3955 ) {
3956 let Some(index) = self
3957 .outbox
3958 .iter()
3959 .position(|c| c.client_commit_id == client_commit_id)
3960 else {
3961 return;
3962 };
3963 if self.begin_observation("syncular_push_result").is_err() {
3964 return;
3965 }
3966 let mut batch = ChangeAccumulator::default();
3967 let operations = self.outbox[index].ops.clone();
3968 match status {
3969 PushStatus::Applied | PushStatus::Cached => {
3970 let journal_results = results
3973 .iter()
3974 .map(|result| {
3975 let op_index = match result {
3976 OpResult::Applied { op_index }
3977 | OpResult::Conflict { op_index, .. }
3978 | OpResult::Error { op_index, .. } => *op_index,
3979 };
3980 CommitOperationOutcome::Applied { op_index }
3981 })
3982 .collect::<Vec<_>>();
3983 let outcome_status = if status == PushStatus::Applied {
3984 CommitOutcomeStatus::Applied
3985 } else {
3986 CommitOutcomeStatus::Cached
3987 };
3988 if self
3989 .persist_commit_outcome(
3990 client_commit_id,
3991 outcome_status,
3992 &journal_results,
3993 None,
3994 )
3995 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
3996 .and_then(|()| self.prune_commit_outcomes())
3997 .is_err()
3998 {
3999 self.rollback_observation("syncular_push_result");
4000 return;
4001 }
4002 report.applied.push(client_commit_id.to_owned());
4003 self.outbox.remove(index);
4004 self.overlay_dirty.set(true);
4005 batch.status = true;
4006 batch.outcomes = true;
4007 }
4008 PushStatus::Rejected => {
4009 if results.iter().any(|result| {
4010 matches!(
4011 result,
4012 OpResult::Error {
4013 code,
4014 retryable: true,
4015 ..
4016 } if code == "sync.idempotency_cache_miss"
4017 )
4018 }) {
4019 report.retryable.push(client_commit_id.to_owned());
4022 if self
4023 .finish_observation("syncular_push_result", batch)
4024 .is_err()
4025 {
4026 self.rollback_observation("syncular_push_result");
4027 }
4028 return;
4029 }
4030
4031 let mut journal_results = Vec::with_capacity(results.len());
4032 let mut conflicts = Vec::new();
4033 let mut rejections = Vec::new();
4034 for result in results {
4035 match result {
4036 OpResult::Applied { op_index } => {
4037 journal_results.push(CommitOperationOutcome::Applied {
4038 op_index: *op_index,
4039 });
4040 }
4041 OpResult::Conflict {
4042 op_index,
4043 code,
4044 message,
4045 server_version,
4046 server_row,
4047 } => {
4048 let operation = operations
4049 .get(*op_index as usize)
4050 .map(CommitOperation::from);
4051 let (table, row_id) = operation
4052 .as_ref()
4053 .map(|op| (op.table.clone(), op.row_id.clone()))
4054 .unwrap_or_default();
4055 let server_row_json = self
4056 .schema
4057 .table(&table)
4058 .and_then(|t| {
4059 decode_row_bytes(t, server_row, &self.encryption)
4060 .ok()
4061 .map(|row| (t, row))
4062 })
4063 .map(|(t, row)| {
4064 let mut map = Map::new();
4065 for (i, column) in t.columns.iter().enumerate() {
4066 map.insert(
4067 column.name.clone(),
4068 column_value_to_json(row.get(i).unwrap_or(&None)),
4069 );
4070 }
4071 map
4072 })
4073 .unwrap_or_default();
4074 let conflict = ConflictRecord {
4075 client_commit_id: client_commit_id.to_owned(),
4076 op_index: *op_index,
4077 table,
4078 row_id,
4079 code: code.clone(),
4080 message: message.clone(),
4081 server_version: *server_version,
4082 server_row: server_row_json,
4083 operation,
4084 };
4085 journal_results.push(CommitOperationOutcome::Conflict {
4086 conflict: conflict.clone(),
4087 });
4088 conflicts.push(conflict);
4089 }
4090 OpResult::Error {
4091 op_index,
4092 code,
4093 message,
4094 retryable,
4095 } => {
4096 let rejection = RejectionRecord {
4097 client_commit_id: client_commit_id.to_owned(),
4098 op_index: *op_index,
4099 code: code.clone(),
4100 message: message.clone(),
4101 retryable: *retryable,
4102 details: rejection_details
4103 .and_then(|details| details.get(op_index))
4104 .cloned(),
4105 operation: operations
4106 .get(*op_index as usize)
4107 .map(CommitOperation::from),
4108 };
4109 journal_results.push(CommitOperationOutcome::Error {
4110 rejection: rejection.clone(),
4111 });
4112 rejections.push(rejection);
4113 }
4114 }
4115 }
4116 let outcome_status = if conflicts.is_empty() {
4117 CommitOutcomeStatus::Rejected
4118 } else {
4119 CommitOutcomeStatus::Conflict
4120 };
4121 if self
4122 .persist_commit_outcome(
4123 client_commit_id,
4124 outcome_status,
4125 &journal_results,
4126 Some(&operations),
4127 )
4128 .and_then(|()| self.delete_outbox_persisted(client_commit_id))
4129 .and_then(|()| self.prune_commit_outcomes())
4130 .is_err()
4131 {
4132 self.rollback_observation("syncular_push_result");
4133 return;
4134 }
4135 report.conflicts += conflicts.len() as u32;
4136 report.rejected.push(client_commit_id.to_owned());
4137 batch.conflicts = !conflicts.is_empty();
4138 batch.rejections = !rejections.is_empty();
4139 batch.status = true;
4140 batch.outcomes = true;
4141 self.conflicts.extend(conflicts);
4142 self.rejections.extend(rejections);
4143 self.outbox.remove(index);
4144 self.overlay_dirty.set(true);
4145 }
4146 }
4147 if batch.status {
4148 for operation in &operations {
4149 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
4150 batch.table(&operation.table);
4151 }
4152 }
4153 self.rebuild_overlay_if_dirty();
4154 }
4155 if self
4156 .finish_observation("syncular_push_result", batch)
4157 .is_err()
4158 {
4159 self.rollback_observation("syncular_push_result");
4160 }
4161 }
4162
4163 #[allow(clippy::too_many_arguments)]
4166 fn process_section(
4167 &mut self,
4168 transport: &mut dyn Transport,
4169 id: &str,
4170 status: SubStatus,
4171 reason_code: &str,
4172 effective_scopes: Vec<(String, Vec<String>)>,
4173 body: Vec<Frame>,
4174 next_cursor: i64,
4175 bootstrap_state: Option<String>,
4176 meta: &RequestMeta,
4177 report: &mut SyncReport,
4178 ) -> Result<(), SectionError> {
4179 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4180 return Ok(()); };
4182 match status {
4183 SubStatus::Revoked => {
4184 self.begin_observation("syncular_revocation")
4185 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4186 let mut batch = ChangeAccumulator::default();
4187 let registered = self.window_unit_by_sub_id(id);
4188 let (table, effective) = {
4190 let sub = &self.subs[sub_index];
4191 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4192 };
4193 let purged = self.purge_scope_rows(&table, &effective);
4194 match purged {
4195 Ok(()) => {
4196 self.record_scope_map(&mut batch, &table, &effective);
4197 let sub = &mut self.subs[sub_index];
4198 sub.state = SubState::Revoked;
4199 sub.reason_code = Some(if reason_code.is_empty() {
4200 "sync.scope_revoked".to_owned()
4201 } else {
4202 reason_code.to_owned()
4203 });
4204 report.revoked.push(id.to_owned());
4205 let doomed_effective = effective;
4206 let sub_table = table;
4207 self.persist_sub(&self.subs[sub_index].clone());
4208 let dropped = self
4209 .drop_doomed_outbox(&sub_table, &doomed_effective)
4210 .map_err(|message| {
4211 SectionError::Abort("storage.failed".to_owned(), message)
4212 })?;
4213 if dropped {
4214 batch.status = true;
4215 batch.rejections = true;
4216 batch.outcomes = true;
4217 }
4218 self.reconcile_blob_refcounts(true);
4221 }
4222 Err(()) => {
4223 let sub = &mut self.subs[sub_index];
4226 sub.state = SubState::Failed;
4227 sub.reason_code = Some("sync.scope_revoked".to_owned());
4228 report.failed.push(id.to_owned());
4229 self.persist_sub(&self.subs[sub_index].clone());
4230 }
4231 }
4232 if let Some((base_key, unit)) = registered {
4233 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4234 }
4235 self.rebuild_overlay_if_dirty();
4236 self.finish_observation("syncular_revocation", batch)
4237 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4238 Ok(())
4239 }
4240 SubStatus::Reset => {
4241 self.begin_observation("syncular_reset")
4242 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4243 let mut batch = ChangeAccumulator::default();
4244 let registered = self.window_unit_by_sub_id(id);
4245 let sub = &mut self.subs[sub_index];
4248 sub.cursor = -1;
4249 sub.bootstrap_state = None;
4250 report.resets.push(id.to_owned());
4251 self.persist_sub(&self.subs[sub_index].clone());
4252 if let Some((base_key, unit)) = registered {
4253 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4254 }
4255 self.finish_observation("syncular_reset", batch)
4256 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4257 Ok(())
4258 }
4259 SubStatus::Active => {
4260 let fresh = meta
4261 .fresh
4262 .iter()
4263 .find(|(fid, _)| fid == id)
4264 .map(|(_, f)| *f)
4265 .unwrap_or(false);
4266 let was_pending = self.subs[sub_index].cursor < 0
4267 || self.subs[sub_index].bootstrap_state.is_some();
4268 let registered = self.window_unit_by_sub_id(id);
4269 self.subs[sub_index].effective = Some(effective_scopes);
4271 self.begin_observation("syncular_section")
4272 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
4273 let mut batch = ChangeAccumulator::default();
4274 let outcome = self.apply_section_body(
4275 transport, sub_index, body, fresh, meta, report, &mut batch,
4276 );
4277 match outcome {
4278 Ok(()) => {
4279 let sub = &mut self.subs[sub_index];
4280 sub.cursor = next_cursor;
4283 sub.bootstrap_state = bootstrap_state;
4284 sub.synced_once = true;
4285 if sub.bootstrap_state.is_some() {
4286 report.bootstrapping.push(id.to_owned());
4287 }
4288 let completed =
4289 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
4290 self.persist_sub(&self.subs[sub_index].clone());
4291 if completed {
4292 if let Some((base_key, unit)) = registered.clone() {
4293 batch.window(&base_key, &self.subs[sub_index].table, &unit);
4294 }
4295 }
4296 self.rebuild_overlay_if_dirty();
4297 self.finish_observation("syncular_section", batch)
4298 .map_err(|message| {
4299 SectionError::Abort("storage.failed".to_owned(), message)
4300 })?;
4301 Ok(())
4302 }
4303 Err(SectionError::FailClosed) => {
4304 self.rollback_observation("syncular_section");
4307 self.begin_observation("syncular_section_failure")
4308 .map_err(|message| {
4309 SectionError::Abort("storage.failed".to_owned(), message)
4310 })?;
4311 let mut failure_batch = ChangeAccumulator::default();
4312 let sub = &mut self.subs[sub_index];
4313 sub.state = SubState::Failed;
4314 sub.reason_code = Some("sync.scope_revoked".to_owned());
4315 report.failed.push(id.to_owned());
4316 self.persist_sub(&self.subs[sub_index].clone());
4317 if let Some((base_key, unit)) = registered {
4318 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
4319 }
4320 self.finish_observation("syncular_section_failure", failure_batch)
4321 .map_err(|message| {
4322 SectionError::Abort("storage.failed".to_owned(), message)
4323 })?;
4324 Ok(())
4325 }
4326 Err(SectionError::Abort(code, message)) => {
4327 self.rollback_observation("syncular_section");
4330 Err(SectionError::Abort(code, message))
4331 }
4332 }
4333 }
4334 }
4335 }
4336
4337 #[allow(clippy::too_many_arguments)]
4341 fn apply_section_body(
4342 &mut self,
4343 transport: &mut dyn Transport,
4344 sub_index: usize,
4345 body: Vec<Frame>,
4346 fresh: bool,
4347 meta: &RequestMeta,
4348 report: &mut SyncReport,
4349 batch: &mut ChangeAccumulator,
4350 ) -> Result<(), SectionError> {
4351 let mut saw_segment = false;
4352 for frame in body {
4353 match frame {
4354 Frame::Commit {
4355 tables, changes, ..
4356 } => {
4357 self.record_commit_changes(batch, &tables, &changes);
4358 self.apply_commit_changes(&tables, &changes)
4359 .map_err(|(c, m)| SectionError::Abort(c, m))?;
4360 report.commits_applied += 1;
4361 }
4362 Frame::SegmentInline { payload } => {
4363 let segment = decode_rows_segment(&payload)
4364 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
4365 let first = !saw_segment;
4366 saw_segment = true;
4367 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4368 let cleared =
4369 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4370 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4371 if applied > 0 || cleared {
4372 batch.table(&segment.table);
4373 }
4374 report.segment_rows_applied += applied;
4375 }
4376 Frame::SegmentRef {
4377 segment_id,
4378 media_type,
4379 table,
4380 row_count,
4381 as_of_commit_seq,
4382 scope_digest,
4383 row_cursor,
4384 next_row_cursor,
4385 url,
4386 url_expires_at_ms,
4387 ..
4388 } => {
4389 let advertised = match media_type {
4392 MediaType::Rows => {
4393 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
4394 || meta.accept & ACCEPT_INLINE_ROWS != 0
4395 }
4396 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
4397 };
4398 if !advertised {
4399 return Err(SectionError::Abort(
4400 "sync.invalid_request".to_owned(),
4401 format!(
4402 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
4403 media_type.name()
4404 ),
4405 ));
4406 }
4407 let bytes = if let Some(url) = url {
4408 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
4413 return Err(SectionError::Abort(
4414 "sync.invalid_request".to_owned(),
4415 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
4416 .to_owned(),
4417 ));
4418 }
4419 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
4421 return Err(SectionError::Abort(
4422 "sync.segment_expired".to_owned(),
4423 format!(
4424 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
4425 ),
4426 ));
4427 }
4428 transport
4429 .fetch_url(&url)
4430 .map_err(|e| SectionError::Abort(e.code, e.message))?
4431 } else {
4432 let requested_scopes_json =
4433 canonical_scope_json(&self.subs[sub_index].requested);
4434 transport
4435 .download_segment(&SegmentRequest {
4436 segment_id: segment_id.clone(),
4437 table,
4438 requested_scopes_json,
4439 })
4440 .map_err(|e| SectionError::Abort(e.code, e.message))?
4441 };
4442 let digest = Sha256::digest(&bytes);
4444 let expected = segment_id
4445 .strip_prefix("sha256:")
4446 .unwrap_or(segment_id.as_str());
4447 if bytes_to_hex(&digest) != expected {
4448 return Err(SectionError::Abort(
4449 "sync.invalid_request".to_owned(),
4450 "segment bytes do not match the content address (§5.1)".to_owned(),
4451 ));
4452 }
4453 if media_type == MediaType::Sqlite {
4454 if row_cursor.is_some() || next_row_cursor.is_some() {
4457 return Err(SectionError::Abort(
4458 "sync.invalid_request".to_owned(),
4459 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
4460 .to_owned(),
4461 ));
4462 }
4463 let first = !saw_segment;
4464 saw_segment = true;
4465 let sub_table = self.subs[sub_index].table.clone();
4466 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4467 let cleared =
4468 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
4469 let applied = self.apply_sqlite_segment(
4470 sub_index,
4471 &bytes,
4472 fresh && first,
4473 row_count,
4474 as_of_commit_seq,
4475 &scope_digest,
4476 )?;
4477 if applied > 0 || cleared {
4478 batch.table(&sub_table);
4479 }
4480 report.segment_rows_applied += applied;
4481 } else {
4482 let segment = decode_rows_segment(&bytes).map_err(|e| {
4483 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
4484 })?;
4485 let first = row_cursor.is_none();
4486 saw_segment = true;
4487 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
4488 let cleared =
4489 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
4490 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
4491 if applied > 0 || cleared {
4492 batch.table(&segment.table);
4493 }
4494 report.segment_rows_applied += applied;
4495 }
4496 }
4497 Frame::Unknown { .. } => {}
4498 _ => {
4499 return Err(SectionError::Abort(
4500 "sync.invalid_request".to_owned(),
4501 "unexpected frame inside a subscription section".to_owned(),
4502 ));
4503 }
4504 }
4505 }
4506 Ok(())
4507 }
4508
4509 fn apply_commit_changes(
4510 &mut self,
4511 tables: &[String],
4512 changes: &[ssp2::model::Change],
4513 ) -> Result<(), (String, String)> {
4514 for change in changes {
4515 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
4516 (
4517 "sync.invalid_request".to_owned(),
4518 "change tableIndex out of range".to_owned(),
4519 )
4520 })?;
4521 let table = self.schema.table(table_name).ok_or_else(|| {
4522 (
4523 "sync.schema_mismatch".to_owned(),
4524 format!("change targets unknown table {table_name:?}"),
4525 )
4526 })?;
4527 match change.op {
4528 Op::Upsert => {
4529 let payload = change.row.as_ref().ok_or_else(|| {
4530 (
4531 "sync.invalid_request".to_owned(),
4532 "upsert change without row payload".to_owned(),
4533 )
4534 })?;
4535 let row = decode_row_bytes(table, payload, &self.encryption)
4537 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4538 let version = change.row_version.unwrap_or(0);
4539 let table_name = table.name.clone();
4540 self.write_base_row(&table_name, &row, version)
4541 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4542 }
4543 Op::Delete => {
4544 self.delete_base_row(table_name, &change.row_id)
4545 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
4546 }
4547 }
4548 }
4549 Ok(())
4550 }
4551
4552 fn apply_segment(
4557 &mut self,
4558 sub_index: usize,
4559 segment: &RowsSegment,
4560 first_fresh_page: bool,
4561 ) -> Result<u32, SectionError> {
4562 let (sub_table, effective) = {
4563 let sub = &self.subs[sub_index];
4564 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4565 };
4566 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
4567 SectionError::Abort(
4568 "sync.schema_mismatch".to_owned(),
4569 format!("subscription table {sub_table:?} is not in the client schema"),
4570 )
4571 })?;
4572 let matches = segment.table == table.name
4577 && segment.schema_version == self.schema.version
4578 && segment.columns.len() == table.wire_columns.len()
4579 && segment
4580 .columns
4581 .iter()
4582 .zip(table.wire_columns.iter())
4583 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
4584 if !matches {
4585 return Err(SectionError::Abort(
4586 "sync.schema_mismatch".to_owned(),
4587 "segment column table does not match the generated schema (§5.2)".to_owned(),
4588 ));
4589 }
4590 if first_fresh_page {
4591 self.purge_scope_rows(&table.name, &effective)
4595 .map_err(|()| SectionError::FailClosed)?;
4596 }
4597 let mut applied = 0u32;
4598 for block in &segment.blocks {
4599 for row in block {
4600 let decrypted;
4605 let values = if table.has_encrypted_columns() {
4606 let mut values = row.values.clone();
4607 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
4608 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
4609 decrypted = values;
4610 &decrypted
4611 } else {
4612 &row.values
4613 };
4614 self.write_base_row(&table.name, values, row.server_version)
4617 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
4618 applied += 1;
4619 }
4620 }
4621 Ok(applied)
4622 }
4623
4624 fn apply_sqlite_segment(
4632 &mut self,
4633 sub_index: usize,
4634 bytes: &[u8],
4635 first_fresh_page: bool,
4636 row_count: i64,
4637 as_of_commit_seq: i64,
4638 scope_digest: &str,
4639 ) -> Result<u32, SectionError> {
4640 let invalid = |detail: &str| {
4641 SectionError::Abort(
4642 "sync.invalid_request".to_owned(),
4643 format!("sqlite segment rejected: {detail} (§5.3)"),
4644 )
4645 };
4646 let (sub_table, effective) = {
4647 let sub = &self.subs[sub_index];
4648 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
4649 };
4650 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
4651 SectionError::Abort(
4652 "sync.schema_mismatch".to_owned(),
4653 format!("subscription table {sub_table:?} is not in the client schema"),
4654 )
4655 })?;
4656
4657 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
4658 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
4659 let img = match rusqlite::Connection::open_with_flags(
4660 &path,
4661 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
4662 ) {
4663 Ok(conn) => conn,
4664 Err(_) => {
4665 let _ = std::fs::remove_file(&path);
4666 return Err(invalid("bytes do not open as a SQLite database"));
4667 }
4668 };
4669 let outcome = self.apply_sqlite_image(
4670 &img,
4671 &table,
4672 first_fresh_page,
4673 &effective,
4674 row_count,
4675 as_of_commit_seq,
4676 scope_digest,
4677 );
4678 drop(img);
4679 let _ = std::fs::remove_file(&path);
4680 outcome
4681 }
4682
4683 #[allow(clippy::too_many_arguments)]
4684 fn apply_sqlite_image(
4685 &mut self,
4686 img: &rusqlite::Connection,
4687 table: &crate::schema::TableSchema,
4688 first_fresh_page: bool,
4689 effective: &[(String, Vec<String>)],
4690 row_count: i64,
4691 as_of_commit_seq: i64,
4692 scope_digest: &str,
4693 ) -> Result<u32, SectionError> {
4694 let invalid = |detail: String| {
4695 SectionError::Abort(
4696 "sync.invalid_request".to_owned(),
4697 format!("sqlite segment rejected: {detail} (§5.3)"),
4698 )
4699 };
4700
4701 type MetaRow = (i64, String, i64, i64, String, i64, i64);
4704 let meta: MetaRow = img
4705 .query_row(
4706 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
4707 \"scopeDigest\", \"rowCount\",
4708 (SELECT count(*) FROM _syncular_segment)
4709 FROM _syncular_segment",
4710 [],
4711 |row| {
4712 Ok((
4713 row.get(0)?,
4714 row.get(1)?,
4715 row.get(2)?,
4716 row.get(3)?,
4717 row.get(4)?,
4718 row.get(5)?,
4719 row.get(6)?,
4720 ))
4721 },
4722 )
4723 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
4724 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
4725 if meta_count != 1 {
4726 return Err(invalid(format!(
4727 "_syncular_segment must contain exactly one row, found {meta_count}"
4728 )));
4729 }
4730 if format != 1 {
4731 return Err(invalid(format!("format {format}")));
4732 }
4733 if meta_table != table.name {
4734 return Err(invalid(format!("image table {meta_table:?}")));
4735 }
4736 if schema_version != i64::from(self.schema.version) {
4737 return Err(invalid(format!("schemaVersion {schema_version}")));
4738 }
4739 if pin != as_of_commit_seq {
4740 return Err(invalid(format!("asOfCommitSeq {pin}")));
4741 }
4742 if digest != scope_digest {
4743 return Err(invalid("scopeDigest mismatch".to_owned()));
4744 }
4745 if meta_rows != row_count {
4746 return Err(invalid(format!("rowCount {meta_rows}")));
4747 }
4748
4749 let mut names: Vec<String> = Vec::new();
4751 {
4752 let mut stmt = img
4753 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
4754 .map_err(|_| invalid("image data table missing".to_owned()))?;
4755 let mut rows = stmt
4756 .query([])
4757 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4758 while let Some(row) = rows
4759 .next()
4760 .map_err(|_| invalid("image data table unreadable".to_owned()))?
4761 {
4762 names.push(
4763 row.get::<_, String>(1)
4764 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
4765 );
4766 }
4767 }
4768 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
4769 expected.push("_syncular_version");
4770 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
4771 return Err(SectionError::Abort(
4772 "sync.schema_mismatch".to_owned(),
4773 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
4774 ));
4775 }
4776
4777 if first_fresh_page {
4780 self.purge_scope_rows(&table.name, effective)
4781 .map_err(|()| SectionError::FailClosed)?;
4782 }
4783 self.overlay_dirty.set(true);
4789 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
4797 table.indexes.iter().filter(|i| !i.unique).collect()
4798 } else {
4799 Vec::new()
4800 };
4801 for index in &bulk_indexes {
4802 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
4803 self.conn
4804 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
4805 .map_err(|e| invalid(e.to_string()))?;
4806 }
4807 let insert = self.insert_row_sql(&base_table(&table.name), table);
4808 let applied = {
4809 let mut ins = self
4810 .conn
4811 .prepare_cached(&insert)
4812 .map_err(|e| invalid(e.to_string()))?;
4813 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
4814 let mut stmt = img
4815 .prepare(&format!(
4816 "SELECT {} FROM {}",
4817 column_list.join(", "),
4818 quote_ident(&table.name)
4819 ))
4820 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4821 let mut rows = stmt
4822 .query([])
4823 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
4824 let version_index = table.columns.len();
4825 let mut applied = 0u32;
4826 while let Some(row) = rows
4827 .next()
4828 .map_err(|_| invalid("image row unreadable".to_owned()))?
4829 {
4830 for (i, column) in table.columns.iter().enumerate() {
4831 let cell = row
4832 .get_ref(i)
4833 .map_err(|_| invalid("image row unreadable".to_owned()))?;
4834 let param = image_cell_param(column, cell).map_err(&invalid)?;
4835 ins.raw_bind_parameter(i + 1, param)
4836 .map_err(|e| invalid(e.to_string()))?;
4837 }
4838 let version: i64 = row
4839 .get(version_index)
4840 .map_err(|_| invalid("image row unreadable".to_owned()))?;
4841 if version < 1 {
4842 return Err(invalid(format!(
4843 "row _syncular_version must be >= 1, got {version}"
4844 )));
4845 }
4846 ins.raw_bind_parameter(version_index + 1, version)
4847 .map_err(|e| invalid(e.to_string()))?;
4848 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
4849 applied += 1;
4850 }
4851 applied
4852 };
4853 for index in &bulk_indexes {
4854 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
4855 let cols_sql = index
4856 .columns
4857 .iter()
4858 .map(|c| quote_ident(c))
4859 .collect::<Vec<_>>()
4860 .join(", ");
4861 self.conn
4862 .execute(
4863 &format!(
4864 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
4865 base_table(&table.name)
4866 ),
4867 [],
4868 )
4869 .map_err(|e| invalid(e.to_string()))?;
4870 }
4871 if i64::from(applied) != row_count {
4872 return Err(invalid(format!(
4873 "image holds {applied} rows, descriptor says {row_count}"
4874 )));
4875 }
4876 Ok(applied)
4877 }
4878
4879 fn purge_scope_rows(
4884 &mut self,
4885 table_name: &str,
4886 effective: &[(String, Vec<String>)],
4887 ) -> Result<(), ()> {
4888 if effective.is_empty() {
4889 return Ok(());
4890 }
4891 let table = self.schema.table(table_name).ok_or(())?.clone();
4892 let mut clauses = Vec::new();
4893 let mut params: Vec<SqlValue> = Vec::new();
4894 for (variable, values) in effective {
4895 let column = table.scope_column(variable).ok_or(())?;
4896 let placeholders: Vec<String> = values
4897 .iter()
4898 .map(|v| {
4899 params.push(SqlValue::Text(v.clone()));
4900 "?".to_owned()
4901 })
4902 .collect();
4903 clauses.push(format!(
4904 "{} IN ({})",
4905 quote_ident(column),
4906 placeholders.join(", ")
4907 ));
4908 }
4909 let sql = format!(
4910 "DELETE FROM {} WHERE {}",
4911 base_table(table_name),
4912 clauses.join(" AND ")
4913 );
4914 self.overlay_dirty.set(true);
4915 self.conn
4916 .execute(&sql, rusqlite::params_from_iter(params))
4917 .map_err(|_| ())?;
4918 Ok(())
4919 }
4920
4921 fn drop_doomed_outbox(
4924 &mut self,
4925 table_name: &str,
4926 effective: &[(String, Vec<String>)],
4927 ) -> Result<bool, String> {
4928 if effective.is_empty() {
4929 return Ok(false);
4930 }
4931 let Some(table) = self.schema.table(table_name).cloned() else {
4932 return Ok(false);
4933 };
4934 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
4935 for (variable, values) in effective {
4936 match table.scope_column(variable) {
4937 Some(column) => mappings.push((column, values)),
4938 None => return Ok(false), }
4940 }
4941 let doomed: Vec<OutboxCommit> = self
4942 .outbox
4943 .iter()
4944 .filter(|commit| {
4945 commit.ops.iter().any(|op| {
4946 op.upsert
4947 && op.table == table_name
4948 && op.values.as_ref().is_some_and(|values| {
4949 mappings
4950 .iter()
4951 .all(|(column, allowed)| match values.get(*column) {
4952 Some(Value::String(s)) => allowed.contains(s),
4953 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
4954 _ => false,
4955 })
4956 })
4957 })
4958 })
4959 .cloned()
4960 .collect();
4961 if doomed.is_empty() {
4962 return Ok(false);
4963 }
4964 let mut rejections = Vec::new();
4965 for commit in &doomed {
4966 let results = commit
4967 .ops
4968 .iter()
4969 .enumerate()
4970 .map(|(op_index, operation)| {
4971 let rejection = RejectionRecord {
4972 client_commit_id: commit.client_commit_id.clone(),
4973 op_index: op_index as i32,
4974 code: "sync.scope_revoked".to_owned(),
4975 message: "the commit was dropped because its effective scope was revoked"
4976 .to_owned(),
4977 retryable: false,
4978 details: None,
4979 operation: Some(CommitOperation::from(operation)),
4980 };
4981 rejections.push(rejection.clone());
4982 CommitOperationOutcome::Error { rejection }
4983 })
4984 .collect::<Vec<_>>();
4985 self.persist_commit_outcome(
4986 &commit.client_commit_id,
4987 CommitOutcomeStatus::Rejected,
4988 &results,
4989 Some(&commit.ops),
4990 )?;
4991 self.delete_outbox_persisted(&commit.client_commit_id)?;
4992 }
4993 self.prune_commit_outcomes()?;
4994 let doomed_ids = doomed
4995 .iter()
4996 .map(|commit| commit.client_commit_id.as_str())
4997 .collect::<BTreeSet<_>>();
4998 self.outbox
4999 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
5000 self.rejections.extend(rejections);
5001 self.overlay_dirty.set(true);
5002 Ok(true)
5003 }
5004
5005 pub fn upload_blob(
5012 &mut self,
5013 bytes: &[u8],
5014 media_type: Option<String>,
5015 name: Option<String>,
5016 ) -> Result<Value, String> {
5017 let blob_id = blob_id_for(bytes);
5018 let now = self.clock_now_ms();
5019 self.conn
5020 .execute(
5021 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
5022 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
5023 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
5024 )
5025 .map_err(|e| e.to_string())?;
5026 self.conn
5027 .execute(
5028 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
5029 rusqlite::params![blob_id, media_type, now],
5030 )
5031 .map_err(|e| e.to_string())?;
5032 self.enforce_blob_cache_cap();
5035 let mut obj = Map::new();
5036 obj.insert("blobId".to_owned(), Value::from(blob_id));
5037 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
5038 if let Some(mt) = media_type {
5039 obj.insert("mediaType".to_owned(), Value::from(mt));
5040 }
5041 if let Some(n) = name {
5042 obj.insert("name".to_owned(), Value::from(n));
5043 }
5044 Ok(Value::Object(obj))
5045 }
5046
5047 pub fn fetch_blob(
5051 &mut self,
5052 transport: &mut dyn Transport,
5053 blob_id_or_ref: &str,
5054 ) -> Result<Value, (String, String)> {
5055 let simple = |m: String| ("client.failed".to_owned(), m);
5056 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
5057 blob_id_or_ref.to_owned()
5058 } else {
5059 let value: Value = serde_json::from_str(blob_id_or_ref)
5060 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
5061 value
5062 .get("blobId")
5063 .and_then(Value::as_str)
5064 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
5065 .to_owned()
5066 };
5067 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
5068 return Ok(cached);
5069 }
5070 let bytes = match transport
5076 .blob_download(&blob_id)
5077 .map_err(|e| (e.code, e.message))?
5078 {
5079 BlobDownload::Bytes(bytes) => bytes,
5080 BlobDownload::Url {
5081 url,
5082 url_expires_at_ms,
5083 } => {
5084 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
5086 return Err((
5087 "sync.segment_expired".to_owned(),
5088 format!(
5089 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
5090 ),
5091 ));
5092 }
5093 transport
5094 .fetch_blob_url(&url)
5095 .map_err(|e| (e.code, e.message))?
5096 }
5097 };
5098 if blob_id_for(&bytes) != blob_id {
5100 return Err(simple(format!(
5101 "blob content address mismatch for {blob_id}"
5102 )));
5103 }
5104 let now = self.clock_now_ms();
5105 self.conn
5106 .execute(
5107 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
5108 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
5109 )
5110 .map_err(|e| simple(e.to_string()))?;
5111 self.enforce_blob_cache_cap();
5112 self.get_cached_blob(&blob_id)
5113 .map_err(simple)?
5114 .ok_or_else(|| simple("blob cache write failed".to_owned()))
5115 }
5116
5117 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
5118 let _ = self.conn.execute(
5121 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
5122 rusqlite::params![self.clock_now_ms(), blob_id],
5123 );
5124 let mut stmt = self
5125 .conn
5126 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
5127 .map_err(|e| e.to_string())?;
5128 let mut rows = stmt
5129 .query(rusqlite::params![blob_id])
5130 .map_err(|e| e.to_string())?;
5131 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
5132 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
5133 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
5134 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
5135 let mut obj = Map::new();
5136 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
5137 obj.insert("byteLength".to_owned(), Value::from(byte_length));
5138 let mut bytes_obj = Map::new();
5139 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
5140 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
5141 if let Some(mt) = media_type {
5142 obj.insert("mediaType".to_owned(), Value::from(mt));
5143 }
5144 return Ok(Some(Value::Object(obj)));
5145 }
5146 Ok(None)
5147 }
5148
5149 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
5151 let pending: Vec<(String, Option<String>)> = {
5152 let mut stmt = self
5153 .conn
5154 .prepare(
5155 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
5156 )
5157 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5158 let rows = stmt
5159 .query_map([], |row| {
5160 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
5161 })
5162 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
5163 rows.filter_map(Result::ok).collect()
5164 };
5165 for (blob_id, media_type) in pending {
5166 let bytes: Option<Vec<u8>> = self
5167 .conn
5168 .query_row(
5169 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
5170 rusqlite::params![blob_id],
5171 |row| row.get(0),
5172 )
5173 .ok();
5174 if let Some(bytes) = bytes {
5175 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
5176 }
5177 let _ = self.conn.execute(
5178 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
5179 rusqlite::params![blob_id],
5180 );
5181 }
5182 Ok(())
5183 }
5184
5185 fn upload_one(
5192 &self,
5193 transport: &mut dyn Transport,
5194 blob_id: &str,
5195 bytes: &[u8],
5196 media_type: Option<&str>,
5197 ) -> Result<(), TransportError> {
5198 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
5199 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
5201 url,
5202 url_expires_at_ms,
5203 } => {
5204 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
5205 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
5206 return Ok(());
5207 }
5208 }
5210 BlobUploadGrant::None => {
5211 }
5213 }
5214 transport.blob_upload(blob_id, bytes, media_type)
5215 }
5216
5217 fn enforce_blob_cache_cap(&self) {
5226 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
5227 return;
5228 };
5229 let mut total: i64 = self
5230 .conn
5231 .query_row(
5232 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
5233 [],
5234 |row| row.get(0),
5235 )
5236 .unwrap_or(0);
5237 if total <= max_bytes {
5238 return;
5239 }
5240 let candidates: Vec<(String, i64)> = {
5241 let Ok(mut stmt) = self.conn.prepare(
5242 "SELECT blob_id, byte_length FROM _syncular_blobs
5243 WHERE refcount = 0
5244 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
5245 ORDER BY last_used_ms ASC, created_at_ms ASC",
5246 ) else {
5247 return;
5248 };
5249 let Ok(rows) = stmt.query_map([], |row| {
5250 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
5251 }) else {
5252 return;
5253 };
5254 rows.filter_map(Result::ok).collect()
5255 };
5256 for (blob_id, byte_length) in candidates {
5257 if total <= max_bytes {
5258 break;
5259 }
5260 let _ = self.conn.execute(
5261 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
5262 rusqlite::params![blob_id],
5263 );
5264 total -= byte_length;
5265 }
5266 }
5267
5268 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
5272 if !self.schema_has_blobs() {
5273 return;
5274 }
5275 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
5276 for table in self.schema.tables.clone() {
5277 let blob_cols: Vec<String> = table
5278 .columns
5279 .iter()
5280 .filter(|c| c.ty == ColumnType::BlobRef)
5281 .map(|c| c.name.clone())
5282 .collect();
5283 for column in blob_cols {
5284 let sql = format!(
5285 "SELECT {} FROM {} WHERE {} IS NOT NULL",
5286 quote_ident(&column),
5287 base_table(&table.name),
5288 quote_ident(&column)
5289 );
5290 let Ok(mut stmt) = self.conn.prepare(&sql) else {
5291 continue;
5292 };
5293 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
5294 continue;
5295 };
5296 for raw in rows.flatten().flatten() {
5297 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
5298 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
5299 *counts.entry(id.to_owned()).or_insert(0) += 1;
5300 }
5301 }
5302 }
5303 }
5304 }
5305 let _ = self
5306 .conn
5307 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
5308 for (blob_id, count) in &counts {
5309 let _ = self.conn.execute(
5310 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
5311 rusqlite::params![count, blob_id],
5312 );
5313 }
5314 if delete_orphans {
5315 let _ = self.conn.execute(
5316 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
5317 [],
5318 );
5319 }
5320 }
5321
5322 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
5327 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
5328 return sql.clone();
5329 }
5330 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
5331 columns.push(quote_ident("_syncular_version"));
5332 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
5333 let primary_key = quote_ident(&table.primary_key);
5334 let updates = columns
5335 .iter()
5336 .filter(|column| **column != primary_key)
5337 .map(|column| format!("{column}=excluded.{column}"))
5338 .collect::<Vec<_>>()
5339 .join(", ");
5340 let sql = format!(
5341 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
5342 columns.join(", "),
5343 placeholders.join(", ")
5344 );
5345 self.insert_sql
5346 .borrow_mut()
5347 .insert(full_table.to_owned(), sql.clone());
5348 sql
5349 }
5350
5351 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
5352 self.overlay_dirty.set(true);
5353 self.write_row(&base_table(table_name), table_name, row, version)
5354 }
5355
5356 fn write_row(
5357 &self,
5358 full_table: &str,
5359 table_name: &str,
5360 row: &Row,
5361 version: i64,
5362 ) -> Result<(), String> {
5363 let table = self
5364 .schema
5365 .table(table_name)
5366 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
5367 let sql = self.insert_row_sql(full_table, table);
5368 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
5369 let params = row
5370 .iter()
5371 .map(RowParam::Cell)
5372 .chain(std::iter::once(RowParam::Version(version)));
5373 stmt.execute(rusqlite::params_from_iter(params))
5374 .map_err(|e| e.to_string())?;
5375 Ok(())
5376 }
5377
5378 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
5379 let table = self
5380 .schema
5381 .table(table_name)
5382 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
5383 self.overlay_dirty.set(true);
5384 let sql = format!(
5385 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
5386 base_table(table_name),
5387 quote_ident(&table.primary_key)
5388 );
5389 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
5390 stmt.execute(rusqlite::params![row_id])
5391 .map_err(|e| e.to_string())?;
5392 Ok(())
5393 }
5394
5395 fn rebuild_overlay_if_dirty(&mut self) {
5398 if self.overlay_dirty.get() {
5399 self.rebuild_overlay();
5400 }
5401 }
5402
5403 fn rebuild_overlay(&mut self) {
5407 self.exec("SAVEPOINT syncular_overlay");
5408 for table in self.schema.tables.clone() {
5409 for index in &table.fts_indexes {
5410 let _ = self.drop_fts_triggers(index);
5411 }
5412 let visible = visible_table(&table.name);
5413 let base = base_table(&table.name);
5414 self.exec(&format!("DELETE FROM {visible}"));
5415 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
5416 }
5417 for commit in self.outbox.clone() {
5418 for op in &commit.ops {
5419 let Some(table) = self.schema.table(&op.table).cloned() else {
5420 continue;
5421 };
5422 if op.upsert {
5423 let Some(values) = op.values.as_ref() else {
5424 continue;
5425 };
5426 let mut row: Row = Vec::with_capacity(table.columns.len());
5427 let mut ok = true;
5428 for column in &table.columns {
5429 match json_to_column_value(column, values.get(&column.name)) {
5430 Ok(v) => row.push(v),
5431 Err(_) => {
5432 ok = false;
5433 break;
5434 }
5435 }
5436 }
5437 if ok {
5438 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
5439 }
5440 } else {
5441 let sql = format!(
5442 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
5443 visible_table(&table.name),
5444 quote_ident(&table.primary_key)
5445 );
5446 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
5447 }
5448 }
5449 }
5450 for table in self.schema.tables.clone() {
5451 for index in &table.fts_indexes {
5452 let _ = self.rebuild_fts_projection(&table, index);
5453 let _ = self.create_fts_triggers(&table, index);
5454 }
5455 }
5456 self.exec("RELEASE syncular_overlay");
5457 self.overlay_dirty.set(false);
5458 }
5459
5460 fn exec(&self, sql: &str) {
5461 let _ = self.conn.execute_batch(sql);
5462 }
5463
5464 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
5467 transport
5468 .realtime_connect_for_client(&self.client_id)
5469 .map_err(|e| format!("{}: {}", e.code, e.message))?;
5470 self.realtime_connected = true;
5471 Ok(())
5472 }
5473
5474 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
5475 let _ = transport.realtime_close();
5476 self.realtime_connected = false;
5477 self.presence.clear(); }
5479
5480 pub fn set_presence(
5486 &mut self,
5487 transport: &mut dyn Transport,
5488 scope_key: &str,
5489 doc: Option<&Value>,
5490 ) -> Result<(), String> {
5491 if !self.realtime_connected {
5492 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
5493 }
5494 let text = encode_presence_publish(scope_key, doc);
5495 transport
5496 .realtime_send(&text)
5497 .map_err(|e| format!("{}: {}", e.code, e.message))
5498 }
5499
5500 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
5502 self.presence
5503 .get(scope_key)
5504 .map(|peers| peers.values().cloned().collect())
5505 .unwrap_or_default()
5506 }
5507
5508 fn apply_presence(
5510 &mut self,
5511 scope_key: String,
5512 kind: Option<PresenceKind>,
5513 actor_id: Option<String>,
5514 client_id: Option<String>,
5515 doc: Option<Value>,
5516 error: Option<String>,
5517 ) {
5518 if error.is_some() {
5521 return;
5522 }
5523 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
5524 return;
5525 };
5526 let peer_key = format!("{actor_id} {client_id}");
5527 match kind {
5528 PresenceKind::Leave => {
5529 if let Some(peers) = self.presence.get_mut(&scope_key) {
5530 peers.remove(&peer_key);
5531 if peers.is_empty() {
5532 self.presence.remove(&scope_key);
5533 }
5534 }
5535 }
5536 _ => {
5537 let doc = match doc {
5538 Some(Value::Object(_)) => doc.unwrap(),
5539 _ => return,
5540 };
5541 self.presence.entry(scope_key).or_default().insert(
5542 peer_key,
5543 PresencePeer {
5544 actor_id,
5545 client_id,
5546 doc,
5547 },
5548 );
5549 }
5550 }
5551 }
5552
5553 pub fn on_realtime_text(&mut self, text: &str) {
5555 match parse_control(text) {
5556 Ok(ControlMessage::Hello { requires_sync, .. }) => {
5557 if requires_sync {
5558 self.set_sync_needed(true, true);
5560 }
5561 }
5562 Ok(ControlMessage::Presence {
5563 scope_key,
5564 kind,
5565 actor_id,
5566 client_id,
5567 doc,
5568 error,
5569 ..
5570 }) => {
5571 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
5572 }
5573 Ok(ControlMessage::Wake { .. }) => {
5574 self.set_sync_needed(true, true);
5576 }
5577 _ => {}
5578 }
5579 }
5580
5581 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
5584 if self.stopped {
5585 return;
5586 }
5587 let message = match decode_message(bytes) {
5588 Ok(m) if m.msg_kind == MsgKind::Response => m,
5589 _ => {
5590 self.set_sync_needed(true, true);
5591 return;
5592 }
5593 };
5594 let mut frames = message.frames.into_iter();
5595 let mut applied_cursor: Option<i64> = None;
5596 let mut any_covered = false;
5597 let mut dropped = false;
5598 while let Some(frame) = frames.next() {
5599 let Frame::SubStart {
5600 id,
5601 status,
5602 effective_scopes,
5603 ..
5604 } = frame
5605 else {
5606 continue;
5607 };
5608 let mut body = Vec::new();
5609 let mut next_cursor: Option<i64> = None;
5610 for inner in frames.by_ref() {
5611 match inner {
5612 Frame::SubEnd {
5613 next_cursor: nc, ..
5614 } => {
5615 next_cursor = Some(nc);
5616 break;
5617 }
5618 Frame::Unknown { .. } => {}
5619 other => body.push(other),
5620 }
5621 }
5622 let Some(next_cursor) = next_cursor else {
5623 dropped = true;
5624 break;
5625 };
5626 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
5627 dropped = true;
5628 continue;
5629 };
5630 let sub = &self.subs[sub_index];
5631 if status != SubStatus::Active
5634 || sub.state != SubState::Active
5635 || sub.bootstrap_state.is_some()
5636 || !sub.synced_once
5637 {
5638 dropped = true;
5639 continue;
5640 }
5641 if next_cursor <= sub.cursor {
5642 any_covered = true;
5644 continue;
5645 }
5646 let previous_effective = self.subs[sub_index].effective.clone();
5647 let previous_cursor = self.subs[sub_index].cursor;
5648 if self.begin_observation("syncular_delta").is_err() {
5649 dropped = true;
5650 continue;
5651 }
5652 self.subs[sub_index].effective = Some(effective_scopes);
5653 let mut batch = ChangeAccumulator::default();
5654 let mut failed = false;
5655 for inner in body {
5656 if let Frame::Commit {
5657 tables, changes, ..
5658 } = inner
5659 {
5660 self.record_commit_changes(&mut batch, &tables, &changes);
5661 if self.apply_commit_changes(&tables, &changes).is_err() {
5662 failed = true;
5663 break;
5664 }
5665 }
5666 }
5667 if failed {
5668 self.rollback_observation("syncular_delta");
5669 self.subs[sub_index].effective = previous_effective;
5670 self.subs[sub_index].cursor = previous_cursor;
5671 self.overlay_dirty.set(true);
5672 self.rebuild_overlay();
5673 dropped = true;
5674 continue;
5675 }
5676 let sub = &mut self.subs[sub_index];
5677 sub.cursor = next_cursor;
5678 self.persist_sub(&self.subs[sub_index].clone());
5679 self.rebuild_overlay_if_dirty();
5680 if self.finish_observation("syncular_delta", batch).is_err() {
5681 self.rollback_observation("syncular_delta");
5682 self.subs[sub_index].effective = previous_effective;
5683 self.subs[sub_index].cursor = previous_cursor;
5684 self.overlay_dirty.set(true);
5685 self.rebuild_overlay();
5686 dropped = true;
5687 continue;
5688 }
5689 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
5690 }
5691 if let Some(cursor) = applied_cursor {
5692 self.reconcile_blob_refcounts(false);
5693 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
5695 let _ = transport.realtime_send(&ack);
5696 } else if !any_covered || dropped {
5697 self.set_sync_needed(true, true);
5699 }
5700 }
5701
5702 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
5706 if !self.realtime_connected {
5707 return;
5708 }
5709 let floor = self
5710 .subs
5711 .iter()
5712 .filter(|s| {
5713 s.state == SubState::Active
5714 && s.bootstrap_state.is_none()
5715 && s.synced_once
5716 && s.cursor >= 0
5717 })
5718 .map(|s| s.cursor)
5719 .min();
5720 if let Some(cursor) = floor {
5721 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
5722 let _ = transport.realtime_send(&ack);
5723 }
5724 }
5725}