1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::Connection;
15use serde_json::{Map, Value};
16use sha2::{Digest, Sha256};
17use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
18use ssp2::primitives::RawJson;
19use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
20use ssp2::{
21 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
22 PresenceKind,
23};
24
25use crate::api::{
26 ClientChangeBatch, ClientLimits, CommandEffects, ConflictRecord, CoverageSnapshot, LeaseState,
27 Mutation, PresencePeer, QuerySnapshot, RejectionRecord, RowState, SchemaFloor,
28 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
29 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
30};
31use crate::schema::{parse_schema_json, ClientSchema};
32use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
33use crate::values::{
34 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
35 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
36 scope_map_to_json, sort_scope_map,
37};
38
39const DEFAULT_ACCEPT: u8 = 0b0111;
44const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
45const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
46const ACCEPT_SQLITE: u8 = 1 << 2;
47const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
48
49const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
51const LOCAL_REVISION_KEY: &str = "localRevision";
52const CLIENT_ID_KEY: &str = "clientId";
53const LEASE_STATE_KEY: &str = "leaseState";
54const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
55const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
58
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60enum SubState {
61 Active,
62 Revoked,
63 Failed,
64}
65
66#[cfg(test)]
67mod observation_tests {
68 use super::*;
69 use serde_json::json;
70
71 fn client() -> SyncClient {
72 SyncClient::new(
73 "retry-test".to_owned(),
74 &json!({
75 "version": 1,
76 "tables": [{
77 "name": "tasks",
78 "primaryKey": "id",
79 "columns": [
80 { "name": "id", "type": "string", "nullable": false },
81 { "name": "project_id", "type": "string", "nullable": false }
82 ],
83 "scopes": [{ "pattern": "project:{project_id}" }]
84 }]
85 }),
86 ClientLimits::default(),
87 )
88 .expect("test client")
89 }
90
91 #[test]
92 fn background_retry_deadlines_back_off_and_reset() {
93 let mut client = client();
94 client.schedule_background_retry();
95 assert!(matches!(
96 client.drain_sync_intents().as_slice(),
97 [SyncIntent::Background { delay_ms: 250 }]
98 ));
99 client.schedule_background_retry();
100 assert!(matches!(
101 client.drain_sync_intents().as_slice(),
102 [SyncIntent::Background { delay_ms: 500 }]
103 ));
104 client.reset_background_retry();
105 client.schedule_background_retry();
106 assert!(matches!(
107 client.drain_sync_intents().as_slice(),
108 [SyncIntent::Background { delay_ms: 250 }]
109 ));
110 }
111
112 #[test]
113 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
114 let path = std::env::temp_dir().join(format!(
115 "syncular-startup-intent-{}.db",
116 uuid::Uuid::new_v4()
117 ));
118 let schema = json!({
119 "version": 1,
120 "tables": [{
121 "name": "tasks",
122 "primaryKey": "id",
123 "columns": [
124 { "name": "id", "type": "string", "nullable": false },
125 { "name": "project_id", "type": "string", "nullable": false }
126 ],
127 "scopes": [{ "pattern": "project:{project_id}" }]
128 }]
129 });
130
131 {
132 let mut first = SyncClient::open_path_with_identity(
133 None,
134 &schema,
135 ClientLimits::default(),
136 path.to_str().expect("UTF-8 temp path"),
137 )
138 .expect("first open");
139 first
140 .set_window(
141 &WindowBase {
142 table: "tasks".to_owned(),
143 variable: "project_id".to_owned(),
144 fixed_scopes: Vec::new(),
145 params: None,
146 },
147 &["persisted".to_owned()],
148 )
149 .expect("persist window");
150 }
151
152 let mut reopened = SyncClient::open_path_with_identity(
153 None,
154 &schema,
155 ClientLimits::default(),
156 path.to_str().expect("UTF-8 temp path"),
157 )
158 .expect("reopen");
159 assert!(reopened.sync_needed());
160 assert!(matches!(
161 reopened.drain_sync_intents().as_slice(),
162 [SyncIntent::Interactive]
163 ));
164 drop(reopened);
165 std::fs::remove_file(path).expect("remove temp database");
166 }
167}
168
169impl SubState {
170 fn name(self) -> &'static str {
171 match self {
172 SubState::Active => "active",
173 SubState::Revoked => "revoked",
174 SubState::Failed => "failed",
175 }
176 }
177
178 fn parse(value: &str) -> Self {
179 match value {
180 "revoked" => Self::Revoked,
181 "failed" => Self::Failed,
182 _ => Self::Active,
183 }
184 }
185}
186
187#[derive(Debug, Clone)]
188struct Subscription {
189 id: String,
190 table: String,
191 requested: Vec<(String, Vec<String>)>,
192 params: Option<String>,
193 cursor: i64,
194 bootstrap_state: Option<String>,
196 state: SubState,
197 reason_code: Option<String>,
198 effective: Option<Vec<(String, Vec<String>)>>,
201 synced_once: bool,
202}
203
204#[derive(Debug, Clone)]
205struct OutboxOp {
206 upsert: bool,
207 table: String,
208 row_id: String,
209 base_version: Option<i64>,
210 values: Option<Map<String, Value>>,
213}
214
215#[derive(Debug, Clone)]
216struct OutboxCommit {
217 client_commit_id: String,
218 ops: Vec<OutboxOp>,
219}
220
221enum SectionError {
224 FailClosed,
225 Abort(String, String),
226}
227
228struct RequestMeta {
229 pushed_ids: Vec<String>,
230 fresh: Vec<(String, bool)>,
233 accept: u8,
234 deferred_commits: usize,
238}
239
240#[derive(Default)]
241struct ChangeAccumulator {
242 tables: BTreeMap<String, Option<BTreeSet<String>>>,
244 windows: BTreeMap<(String, String), BTreeSet<String>>,
245 status: bool,
246 conflicts: bool,
247 rejections: bool,
248}
249
250impl ChangeAccumulator {
251 fn table(&mut self, table: &str) {
252 self.tables.insert(table.to_owned(), None);
253 }
254
255 fn scope(&mut self, table: &str, key: String) {
256 match self.tables.get_mut(table) {
257 Some(None) => {}
258 Some(Some(keys)) => {
259 keys.insert(key);
260 }
261 None => {
262 self.tables
263 .insert(table.to_owned(), Some(BTreeSet::from([key])));
264 }
265 }
266 }
267
268 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
269 self.windows
270 .entry((base_key.to_owned(), table.to_owned()))
271 .or_default()
272 .insert(unit.to_owned());
273 }
274
275 fn touched(&self) -> bool {
276 !self.tables.is_empty()
277 || !self.windows.is_empty()
278 || self.status
279 || self.conflicts
280 || self.rejections
281 }
282}
283
284const PUSH_OPS_PER_REQUEST: usize = 500;
290
291pub struct SyncClient {
292 conn: Connection,
293 schema: ClientSchema,
294 client_id: String,
295 limits: ClientLimits,
296 subs: Vec<Subscription>,
297 outbox: Vec<OutboxCommit>,
298 conflicts: Vec<ConflictRecord>,
299 rejections: Vec<RejectionRecord>,
300 schema_floor: Option<SchemaFloor>,
301 lease_state: Option<LeaseState>,
303 stopped: bool,
305 upgrading: bool,
307 sync_needed: bool,
309 realtime_connected: bool,
310 presence: HashMap<String, HashMap<String, PresencePeer>>,
312 now_ms: Option<i64>,
315 encryption: crate::values::EncryptionConfig,
319 insert_sql: RefCell<HashMap<String, String>>,
325 overlay_dirty: Cell<bool>,
329 change_queue: VecDeque<ClientChangeBatch>,
331 sync_intent_queue: VecDeque<SyncIntent>,
332 retry_delay_ms: u64,
334}
335
336fn quote_ident(name: &str) -> String {
337 format!("\"{}\"", name.replace('"', "\"\""))
338}
339
340fn base_table(name: &str) -> String {
341 quote_ident(&format!("_syncular_base_{name}"))
342}
343
344type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
349
350fn window_base_key(base: &WindowBase) -> String {
351 format!(
352 "{}\0{}\0{}",
353 base.table,
354 base.variable,
355 canonical_scope_json(&base.fixed_scopes)
356 )
357}
358
359fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
361 let mut scopes = base.fixed_scopes.clone();
362 scopes.retain(|(k, _)| k != &base.variable);
363 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
364 scopes
365}
366
367fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
371 let canonical = canonical_scope_json(&unit_scopes(base, unit));
372 let digest = Sha256::digest(canonical.as_bytes());
373 let hex = bytes_to_hex(&digest);
374 format!("w:{}:{}", base.table, &hex[..16])
375}
376
377fn visible_table(name: &str) -> String {
378 quote_ident(name)
379}
380
381fn is_synced_table_name(name: &str) -> bool {
384 if name.starts_with("sqlite_") {
385 return false;
386 }
387 if name.starts_with("_syncular_base_") {
388 return true; }
390 !name.starts_with("_syncular_")
392}
393
394fn blob_id_for(bytes: &[u8]) -> String {
396 let digest = Sha256::digest(bytes);
397 format!("sha256:{}", bytes_to_hex(&digest))
398}
399
400enum RowParam<'a> {
404 Cell(&'a Option<ColumnValue>),
405 Version(i64),
406}
407
408impl rusqlite::ToSql for RowParam<'_> {
409 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
410 Ok(match self {
411 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
412 RowParam::Cell(cell) => match cell {
413 None => ToSqlOutput::Owned(SqlValue::Null),
414 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
415 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
416 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
417 Some(ColumnValue::Boolean(b)) => {
418 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
419 }
420 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
421 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
422 }
423 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
425 ToSqlOutput::Borrowed(ValueRef::Blob(b))
426 }
427 },
428 })
429 }
430}
431
432fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
440 use ssp2::segment::ColumnType;
441 let mismatch = || {
442 Err(format!(
443 "image column {:?} holds a value of the wrong type",
444 column.name
445 ))
446 };
447 match value {
448 ValueRef::Null => {
449 if !column.nullable {
450 return Err(format!(
451 "image column {:?} is NULL but not nullable",
452 column.name
453 ));
454 }
455 Ok(ToSqlOutput::Owned(SqlValue::Null))
456 }
457 ValueRef::Integer(i) => match column.ty {
458 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
459 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
460 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
461 _ => mismatch(),
462 },
463 ValueRef::Real(_) => match column.ty {
464 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
465 _ => mismatch(),
466 },
467 ValueRef::Text(t) => {
468 std::str::from_utf8(t)
469 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
470 match column.ty {
471 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
472 Ok(ToSqlOutput::Borrowed(value))
473 }
474 _ => mismatch(),
475 }
476 }
477 ValueRef::Blob(_) => match column.ty {
478 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
480 _ => mismatch(),
481 },
482 }
483}
484
485fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
486 use rusqlite::types::ValueRef;
487 match value {
488 ValueRef::Null => Value::Null,
489 ValueRef::Integer(i) => match column.ty {
490 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
491 ssp2::segment::ColumnType::Float => {
492 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
493 }
494 _ => Value::from(i),
495 },
496 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
497 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
498 ValueRef::Blob(b) => {
499 let mut map = Map::new();
500 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
501 Value::Object(map)
502 }
503 }
504}
505
506fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
510 Ok(match value {
511 Value::Null => SqlValue::Null,
512 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
513 Value::Number(n) => {
514 if let Some(i) = n.as_i64() {
515 SqlValue::Integer(i)
516 } else if let Some(f) = n.as_f64() {
517 SqlValue::Real(f)
518 } else {
519 return Err(format!("query param number {n} is out of range"));
520 }
521 }
522 Value::String(s) => SqlValue::Text(s.clone()),
523 Value::Object(_) => {
524 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
525 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
526 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
527 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
528 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
529 })?)
530 } else {
531 return Err(
532 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
533 .to_owned(),
534 );
535 }
536 }
537 Value::Array(_) => return Err("query array params are not supported".to_owned()),
538 })
539}
540
541fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
546 use rusqlite::types::ValueRef;
547 match value {
548 ValueRef::Null => Value::Null,
549 ValueRef::Integer(i) => {
550 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
554 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
555 Value::from(i)
556 } else {
557 let mut map = Map::new();
558 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
559 Value::Object(map)
560 }
561 }
562 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
563 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
564 ValueRef::Blob(b) => {
565 let mut map = Map::new();
566 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
567 Value::Object(map)
568 }
569 }
570}
571
572impl SyncClient {
573 pub fn new_with_identity(
574 client_id: Option<String>,
575 schema_json: &Value,
576 limits: ClientLimits,
577 ) -> Result<Self, String> {
578 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
579 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
580 Self::with_connection(resolved, schema_json, limits, conn)
581 }
582
583 pub fn new(
584 client_id: String,
585 schema_json: &Value,
586 limits: ClientLimits,
587 ) -> Result<Self, String> {
588 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
589 Self::with_connection(client_id, schema_json, limits, conn)
590 }
591
592 pub fn open_path(
598 client_id: String,
599 schema_json: &Value,
600 limits: ClientLimits,
601 path: &str,
602 ) -> Result<Self, String> {
603 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
604 Self::with_connection(client_id, schema_json, limits, conn)
605 }
606
607 pub fn open_path_with_identity(
608 client_id: Option<String>,
609 schema_json: &Value,
610 limits: ClientLimits,
611 path: &str,
612 ) -> Result<Self, String> {
613 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
614 let persisted = conn
615 .query_row(
616 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
617 [],
618 |row| row.get::<_, String>(0),
619 )
620 .ok();
621 let resolved = persisted
622 .clone()
623 .or(client_id.clone())
624 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
625 if let (Some(existing), Some(requested)) = (persisted, client_id) {
626 if existing != requested {
627 return Err(format!(
628 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
629 ));
630 }
631 }
632 Self::with_connection(resolved, schema_json, limits, conn)
633 }
634
635 #[must_use]
636 pub fn client_id(&self) -> &str {
637 &self.client_id
638 }
639
640 pub fn with_connection(
647 client_id: String,
648 schema_json: &Value,
649 limits: ClientLimits,
650 conn: Connection,
651 ) -> Result<Self, String> {
652 let schema = parse_schema_json(schema_json)?;
653 let mut client = SyncClient {
654 conn,
655 schema,
656 client_id,
657 limits,
658 subs: Vec::new(),
659 outbox: Vec::new(),
660 conflicts: Vec::new(),
661 rejections: Vec::new(),
662 schema_floor: None,
663 lease_state: None,
664 stopped: false,
665 upgrading: false,
666 sync_needed: false,
667 realtime_connected: false,
668 presence: HashMap::new(),
669 now_ms: None,
670 encryption: crate::values::EncryptionConfig::default(),
671 insert_sql: RefCell::new(HashMap::new()),
672 overlay_dirty: Cell::new(false),
673 change_queue: VecDeque::new(),
674 sync_intent_queue: VecDeque::new(),
675 retry_delay_ms: 250,
676 };
677 client
681 .conn
682 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
683 client.create_tables()?;
684 match client.get_meta(CLIENT_ID_KEY) {
685 Some(existing) if existing != client.client_id => {
686 return Err(format!(
687 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
688 client.client_id
689 ));
690 }
691 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
692 _ => {}
693 }
694 client.restore_persisted_state()?;
695 let marker = client
696 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
697 .and_then(|value| value.parse::<i32>().ok());
698 if marker != Some(client.schema.version) {
699 client.run_schema_reset()?;
700 } else if !client.outbox.is_empty() {
701 client.overlay_dirty.set(true);
704 client.rebuild_overlay();
705 }
706 client.enqueue_startup_sync_if_needed();
712 Ok(client)
713 }
714
715 pub fn set_now_ms(&mut self, now_ms: i64) {
718 self.now_ms = Some(now_ms);
719 }
720
721 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
725 self.encryption = encryption;
726 }
727
728 fn clock_now_ms(&self) -> i64 {
729 self.now_ms.unwrap_or_else(|| {
730 std::time::SystemTime::now()
731 .duration_since(std::time::UNIX_EPOCH)
732 .map(|d| d.as_millis() as i64)
733 .unwrap_or(0)
734 })
735 }
736
737 fn create_tables(&self) -> Result<(), String> {
738 self.create_synced_tables()?;
739 self.conn
741 .execute_batch(
742 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
743 seq INTEGER PRIMARY KEY AUTOINCREMENT,
744 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
745 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
746 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
747 CREATE TABLE IF NOT EXISTS _syncular_meta (
748 key TEXT PRIMARY KEY, value TEXT NOT NULL);
749 CREATE TABLE IF NOT EXISTS _syncular_windows (
750 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
751 PRIMARY KEY (base, unit));
752 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
753 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
754 effective_scopes TEXT NOT NULL);",
755 )
756 .map_err(|e| e.to_string())?;
757 if self.get_meta(LOCAL_SCHEMA_VERSION_KEY).is_none() {
760 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
761 }
762 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
763 self.set_meta(LOCAL_REVISION_KEY, "0");
764 }
765 if self.schema_has_blobs() {
770 self.conn
771 .execute_batch(
772 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
773 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
774 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
775 created_at_ms INTEGER NOT NULL,
776 last_used_ms INTEGER NOT NULL DEFAULT 0);
777 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
778 media_type TEXT, created_at_ms INTEGER NOT NULL);",
779 )
780 .map_err(|e| e.to_string())?;
781 let _ = self.conn.execute_batch(
785 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
786 );
787 }
788 Ok(())
789 }
790
791 fn schema_has_blobs(&self) -> bool {
793 self.schema
794 .tables
795 .iter()
796 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
797 }
798
799 fn get_meta(&self, key: &str) -> Option<String> {
802 self.conn
803 .query_row(
804 "SELECT value FROM _syncular_meta WHERE key = ?1",
805 rusqlite::params![key],
806 |row| row.get::<_, String>(0),
807 )
808 .ok()
809 }
810
811 fn set_meta(&self, key: &str, value: &str) {
812 let _ = self.conn.execute(
813 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
814 rusqlite::params![key, value],
815 );
816 }
817
818 fn delete_meta(&self, key: &str) {
819 let _ = self.conn.execute(
820 "DELETE FROM _syncular_meta WHERE key = ?1",
821 rusqlite::params![key],
822 );
823 }
824
825 fn restore_persisted_state(&mut self) -> Result<(), String> {
826 self.subs = {
827 let mut stmt = self
828 .conn
829 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
830 .map_err(|error| error.to_string())?;
831 let rows = stmt
832 .query_map([], |row| {
833 Ok((
834 row.get::<_, String>(0)?,
835 row.get::<_, String>(1)?,
836 row.get::<_, String>(2)?,
837 ))
838 })
839 .map_err(|error| error.to_string())?;
840 let mut subscriptions = Vec::new();
841 for row in rows {
842 let (id, table, raw) = row.map_err(|error| error.to_string())?;
843 let state: Value = serde_json::from_str(&raw)
844 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
845 let requested = json_to_scope_map(
846 state.get("requested").unwrap_or(&Value::Object(Map::new())),
847 )?;
848 let effective = state
849 .get("effectiveScopes")
850 .filter(|value| !value.is_null())
851 .map(json_to_scope_map)
852 .transpose()?;
853 subscriptions.push(Subscription {
854 id,
855 table,
856 requested,
857 params: state
858 .get("params")
859 .and_then(Value::as_str)
860 .map(str::to_owned),
861 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
862 bootstrap_state: state
863 .get("bootstrapState")
864 .and_then(Value::as_str)
865 .map(str::to_owned),
866 state: SubState::parse(
867 state
868 .get("status")
869 .and_then(Value::as_str)
870 .unwrap_or("active"),
871 ),
872 reason_code: state
873 .get("reasonCode")
874 .and_then(Value::as_str)
875 .map(str::to_owned),
876 effective,
877 synced_once: state
878 .get("syncedOnce")
879 .and_then(Value::as_bool)
880 .unwrap_or(false),
881 });
882 }
883 subscriptions
884 };
885
886 self.outbox = {
887 let mut stmt = self
888 .conn
889 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
890 .map_err(|error| error.to_string())?;
891 let rows = stmt
892 .query_map([], |row| {
893 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
894 })
895 .map_err(|error| error.to_string())?;
896 let mut commits = Vec::new();
897 for row in rows {
898 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
899 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
900 format!("invalid persisted outbox {client_commit_id:?}: {error}")
901 })?;
902 let mut ops = Vec::with_capacity(entries.len());
903 for entry in entries {
904 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
905 ops.push(OutboxOp {
906 upsert: op == "upsert",
907 table: entry
908 .get("table")
909 .and_then(Value::as_str)
910 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
911 .to_owned(),
912 row_id: entry
913 .get("rowId")
914 .and_then(Value::as_str)
915 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
916 .to_owned(),
917 base_version: entry.get("baseVersion").and_then(Value::as_i64),
918 values: entry.get("values").and_then(Value::as_object).cloned(),
919 });
920 }
921 commits.push(OutboxCommit {
922 client_commit_id,
923 ops,
924 });
925 }
926 commits
927 };
928
929 self.lease_state = self
930 .get_meta(LEASE_STATE_KEY)
931 .map(|raw| serde_json::from_str(&raw))
932 .transpose()
933 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
934 self.schema_floor = self
935 .get_meta(SCHEMA_FLOOR_KEY)
936 .map(|raw| serde_json::from_str(&raw))
937 .transpose()
938 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
939 self.stopped = self.schema_floor.is_some();
940 Ok(())
941 }
942
943 #[must_use]
944 pub fn local_revision(&self) -> u64 {
945 self.get_meta(LOCAL_REVISION_KEY)
946 .and_then(|value| value.parse().ok())
947 .unwrap_or(0)
948 }
949
950 #[must_use]
951 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
952 SyncStatusSnapshot {
953 outbox: self.outbox.len(),
954 upgrading: self.upgrading,
955 lease_state: self.lease_state.clone(),
956 schema_floor: self.schema_floor.clone(),
957 sync_needed: self.sync_needed,
958 }
959 }
960
961 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
962 self.change_queue.drain(..).collect()
963 }
964
965 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
966 self.sync_intent_queue.drain(..).collect()
967 }
968
969 fn schedule_background_retry(&mut self) {
970 self.sync_intent_queue.push_back(SyncIntent::Background {
971 delay_ms: self.retry_delay_ms,
972 });
973 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
974 }
975
976 fn reset_background_retry(&mut self) {
977 self.retry_delay_ms = 250;
978 }
979
980 fn retryable_transport_code(code: &str) -> bool {
981 code == "transport.failed" || code == "sync.transport_failed"
982 }
983
984 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
985 if self.sync_needed != value {
986 if self.begin_observation("syncular_status").is_ok() {
987 self.sync_needed = value;
988 let batch = ChangeAccumulator {
989 status: true,
990 ..ChangeAccumulator::default()
991 };
992 if self.finish_observation("syncular_status", batch).is_err() {
993 self.rollback_observation("syncular_status");
994 }
995 } else {
996 self.sync_needed = value;
997 }
998 }
999 if value && interactive {
1000 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1001 }
1002 }
1003
1004 fn begin_observation(&self, name: &str) -> Result<(), String> {
1005 self.conn
1006 .execute_batch(&format!("SAVEPOINT {name}"))
1007 .map_err(|error| error.to_string())
1008 }
1009
1010 fn rollback_observation(&self, name: &str) {
1011 let _ = self
1012 .conn
1013 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
1014 }
1015
1016 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
1017 if !batch.touched() {
1018 self.conn
1019 .execute_batch(&format!("RELEASE {name}"))
1020 .map_err(|error| error.to_string())?;
1021 return Ok(());
1022 }
1023 let revision = self
1024 .local_revision()
1025 .checked_add(1)
1026 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
1027 self.conn
1028 .execute(
1029 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
1030 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
1031 )
1032 .map_err(|error| error.to_string())?;
1033 let status = batch.status.then(|| self.status_snapshot());
1034 let event = ClientChangeBatch {
1035 revision: revision.to_string(),
1036 tables: batch
1037 .tables
1038 .into_iter()
1039 .map(|(table, scope_keys)| TableChange {
1040 table,
1041 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
1042 })
1043 .collect(),
1044 windows: batch
1045 .windows
1046 .into_iter()
1047 .map(|((base_key, table), units)| WindowChange {
1048 base_key,
1049 table,
1050 units: units.into_iter().collect(),
1051 })
1052 .collect(),
1053 status,
1054 conflicts_changed: batch.conflicts,
1055 rejections_changed: batch.rejections,
1056 };
1057 self.conn
1058 .execute_batch(&format!("RELEASE {name}"))
1059 .map_err(|error| error.to_string())?;
1060 self.change_queue.push_back(event);
1061 Ok(())
1062 }
1063
1064 fn record_scope_map(
1065 &self,
1066 batch: &mut ChangeAccumulator,
1067 table_name: &str,
1068 scopes: &[(String, Vec<String>)],
1069 ) {
1070 let Some(table) = self.schema.table(table_name) else {
1071 return;
1072 };
1073 for (variable, values) in scopes {
1074 let Some(scope) = table
1075 .scope_variables
1076 .iter()
1077 .find(|scope| &scope.variable == variable)
1078 else {
1079 continue;
1080 };
1081 for value in values {
1082 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1083 }
1084 }
1085 }
1086
1087 fn record_row_scopes(
1089 &self,
1090 batch: &mut ChangeAccumulator,
1091 table_name: &str,
1092 row_id: &str,
1093 base: bool,
1094 ) -> bool {
1095 let Some(table) = self.schema.table(table_name) else {
1096 return false;
1097 };
1098 if table.scope_variables.is_empty() {
1099 return false;
1100 }
1101 let columns = table
1102 .scope_variables
1103 .iter()
1104 .map(|scope| quote_ident(&scope.column))
1105 .collect::<Vec<_>>()
1106 .join(", ");
1107 let full_table = if base {
1108 base_table(table_name)
1109 } else {
1110 visible_table(table_name)
1111 };
1112 let sql = format!(
1113 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
1114 quote_ident(&table.primary_key)
1115 );
1116 let Ok(mut stmt) = self.conn.prepare(&sql) else {
1117 return false;
1118 };
1119 let values = stmt.query_row(rusqlite::params![row_id], |row| {
1120 let mut values = Vec::with_capacity(table.scope_variables.len());
1121 for index in 0..table.scope_variables.len() {
1122 values.push(row.get::<_, Option<String>>(index)?);
1123 }
1124 Ok(values)
1125 });
1126 let Ok(values) = values else {
1127 return false;
1128 };
1129 let mut recorded = false;
1130 for (scope, value) in table.scope_variables.iter().zip(values) {
1131 if let Some(value) = value {
1132 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1133 recorded = true;
1134 }
1135 }
1136 recorded
1137 }
1138
1139 fn record_commit_changes(
1140 &self,
1141 batch: &mut ChangeAccumulator,
1142 tables: &[String],
1143 changes: &[ssp2::model::Change],
1144 ) {
1145 for change in changes {
1146 let Some(table_name) = tables.get(change.table_index as usize) else {
1147 continue;
1148 };
1149 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
1150 if let Some(table) = self.schema.table(table_name) {
1151 for (variable, value) in &change.scopes {
1152 if let Some(scope) = table
1153 .scope_variables
1154 .iter()
1155 .find(|scope| &scope.variable == variable)
1156 {
1157 batch.scope(table_name, format!("{}:{value}", scope.prefix));
1158 precise = true;
1159 }
1160 }
1161 }
1162 if !precise {
1163 batch.table(table_name);
1164 }
1165 }
1166 }
1167
1168 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
1169 if effective.is_empty() {
1170 return false;
1171 }
1172 let Some(table) = self.schema.table(table_name) else {
1173 return false;
1174 };
1175 let mut clauses = Vec::new();
1176 let mut params = Vec::new();
1177 for (variable, values) in effective {
1178 let Some(column) = table.scope_column(variable) else {
1179 return false;
1180 };
1181 if values.is_empty() {
1182 return false;
1183 }
1184 let placeholders = values
1185 .iter()
1186 .map(|value| {
1187 params.push(SqlValue::Text(value.clone()));
1188 "?"
1189 })
1190 .collect::<Vec<_>>()
1191 .join(", ");
1192 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
1193 }
1194 let sql = format!(
1195 "SELECT 1 FROM {} WHERE {} LIMIT 1",
1196 base_table(table_name),
1197 clauses.join(" AND ")
1198 );
1199 self.conn
1200 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
1201 .is_ok()
1202 }
1203
1204 pub fn upgrading(&self) -> bool {
1206 self.upgrading
1207 }
1208
1209 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
1215 let new_schema = parse_schema_json(schema_json)?;
1216 let marker: Option<i32> = self
1217 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
1218 .and_then(|v| v.parse().ok());
1219 self.schema = new_schema;
1220 if marker != Some(self.schema.version) {
1221 self.run_schema_reset()?;
1222 }
1223 self.enqueue_startup_sync_if_needed();
1227 Ok(())
1228 }
1229
1230 fn enqueue_startup_sync_if_needed(&mut self) {
1231 let startup_work = !self.stopped
1232 && (!self.outbox.is_empty()
1233 || self.subs.iter().any(|sub| sub.state == SubState::Active));
1234 if startup_work {
1235 self.sync_needed = true;
1236 self.sync_intent_queue.push_back(SyncIntent::Interactive);
1237 }
1238 }
1239
1240 fn run_schema_reset(&mut self) -> Result<(), String> {
1246 self.begin_observation("syncular_schema_reset")?;
1247 let mut batch = ChangeAccumulator::default();
1248 let result = self.run_schema_reset_observed(&mut batch);
1249 if let Err(error) = result {
1250 self.rollback_observation("syncular_schema_reset");
1251 return Err(error);
1252 }
1253 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
1254 self.rollback_observation("syncular_schema_reset");
1255 return Err(error);
1256 }
1257 Ok(())
1258 }
1259
1260 fn run_schema_reset_observed(&mut self, batch: &mut ChangeAccumulator) -> Result<(), String> {
1261 self.upgrading = true;
1262 batch.status = true;
1263 for table in &self.schema.tables {
1264 batch.table(&table.name);
1265 }
1266 for (base_key, unit, table) in self.load_registered_window_units() {
1267 batch.window(&base_key, &table, &unit);
1268 }
1269 self.insert_sql.borrow_mut().clear();
1271 self.overlay_dirty.set(true);
1272 let existing: Vec<String> = {
1278 let mut stmt = self
1279 .conn
1280 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
1281 .map_err(|e| e.to_string())?;
1282 let rows = stmt
1283 .query_map([], |row| row.get::<_, String>(0))
1284 .map_err(|e| e.to_string())?;
1285 rows.filter_map(Result::ok)
1286 .filter(|name| is_synced_table_name(name))
1287 .collect()
1288 };
1289 for name in &existing {
1290 if let Some(table) = name.strip_prefix("_syncular_base_") {
1291 batch.table(table);
1292 } else if !name.starts_with("_syncular_") {
1293 batch.table(name);
1294 }
1295 }
1296 for name in existing {
1297 let _ = self
1298 .conn
1299 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), []);
1300 }
1301 self.create_synced_tables()?;
1303 for sub in &mut self.subs {
1305 sub.cursor = -1;
1306 sub.bootstrap_state = None;
1307 sub.effective = None;
1308 sub.state = SubState::Active;
1309 sub.reason_code = None;
1310 sub.synced_once = false;
1311 }
1312 let subs = self.subs.clone();
1313 for sub in &subs {
1314 self.persist_sub(sub);
1315 }
1316 self.stopped = false;
1318 self.schema_floor = None;
1319 self.delete_meta(SCHEMA_FLOOR_KEY);
1320 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
1322 if self.drop_incompatible_outbox() {
1326 batch.rejections = true;
1327 batch.status = true;
1328 }
1329 self.rebuild_overlay();
1331 Ok(())
1332 }
1333
1334 fn drop_incompatible_outbox(&mut self) -> bool {
1338 let schema = &self.schema;
1339 let mut incompatible: Vec<String> = Vec::new();
1340 self.outbox.retain(|commit| {
1341 let bad = commit.ops.iter().any(|op| {
1342 if !op.upsert {
1343 return false;
1344 }
1345 match schema.table(&op.table) {
1346 None => true,
1347 Some(table) => op.values.as_ref().is_some_and(|values| {
1348 values
1349 .keys()
1350 .any(|key| !table.columns.iter().any(|c| &c.name == key))
1351 }),
1352 }
1353 });
1354 if bad {
1355 incompatible.push(commit.client_commit_id.clone());
1356 }
1357 !bad
1358 });
1359 let changed = !incompatible.is_empty();
1360 for client_commit_id in incompatible {
1361 self.persist_outbox_delete(&client_commit_id);
1362 self.rejections.push(RejectionRecord {
1363 client_commit_id,
1364 op_index: 0,
1365 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
1366 retryable: false,
1367 });
1368 }
1369 changed
1370 }
1371
1372 fn create_synced_tables(&self) -> Result<(), String> {
1375 for table in &self.schema.tables {
1376 for (full, index_prefix) in [
1380 (base_table(&table.name), "_syncular_base_"),
1381 (visible_table(&table.name), ""),
1382 ] {
1383 let mut cols: Vec<String> =
1384 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
1385 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
1386 let sql = format!(
1387 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
1388 cols.join(", "),
1389 quote_ident(&table.primary_key)
1390 );
1391 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
1392 for index in &table.indexes {
1396 let unique = if index.unique { "UNIQUE " } else { "" };
1397 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
1398 let cols_sql = index
1399 .columns
1400 .iter()
1401 .map(|c| quote_ident(c))
1402 .collect::<Vec<_>>()
1403 .join(", ");
1404 let index_sql = format!(
1405 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
1406 );
1407 self.conn
1408 .execute(&index_sql, [])
1409 .map_err(|e| e.to_string())?;
1410 }
1411 }
1412 }
1413 Ok(())
1414 }
1415
1416 fn persist_sub(&self, sub: &Subscription) {
1419 let state = serde_json::json!({
1420 "requested": scope_map_to_json(&sub.requested),
1421 "params": sub.params,
1422 "cursor": sub.cursor,
1423 "bootstrapState": sub.bootstrap_state,
1424 "status": sub.state.name(),
1425 "reasonCode": sub.reason_code,
1426 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
1427 "syncedOnce": sub.synced_once,
1428 });
1429 let _ = self.conn.execute(
1430 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
1431 rusqlite::params![sub.id, sub.table, state.to_string()],
1432 );
1433 }
1434
1435 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
1436 let ops: Vec<Value> = commit
1437 .ops
1438 .iter()
1439 .map(|op| {
1440 serde_json::json!({
1441 "op": if op.upsert { "upsert" } else { "delete" },
1442 "table": op.table,
1443 "rowId": op.row_id,
1444 "baseVersion": op.base_version,
1445 "values": op.values.clone().map(Value::Object),
1446 })
1447 })
1448 .collect();
1449 let _ = self.conn.execute(
1450 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
1451 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
1452 );
1453 }
1454
1455 fn persist_outbox_delete(&self, client_commit_id: &str) {
1456 let _ = self.conn.execute(
1457 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
1458 rusqlite::params![client_commit_id],
1459 );
1460 }
1461
1462 pub fn subscribe(
1465 &mut self,
1466 id: String,
1467 table: String,
1468 scopes: Vec<(String, Vec<String>)>,
1469 params: Option<String>,
1470 ) -> Result<(), String> {
1471 if self.schema.table(&table).is_none() {
1472 return Err(format!("unknown table {table:?}"));
1473 }
1474 let sub = Subscription {
1475 id: id.clone(),
1476 table,
1477 requested: scopes,
1478 params,
1479 cursor: -1,
1480 bootstrap_state: None,
1481 state: SubState::Active,
1482 reason_code: None,
1483 effective: None,
1484 synced_once: false,
1485 };
1486 self.persist_sub(&sub);
1487 if let Some(existing) = self.subs.iter_mut().find(|s| s.id == id) {
1488 *existing = sub;
1489 } else {
1490 self.subs.push(sub);
1491 }
1492 Ok(())
1493 }
1494
1495 pub fn unsubscribe(&mut self, id: &str) {
1496 self.subs.retain(|s| s.id != id);
1497 let _ = self.conn.execute(
1498 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
1499 rusqlite::params![id],
1500 );
1501 }
1502
1503 pub fn set_window(
1511 &mut self,
1512 base: &WindowBase,
1513 units: &[String],
1514 ) -> Result<CommandEffects, String> {
1515 let table = self
1516 .schema
1517 .table(&base.table)
1518 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
1519 if table.scope_column(&base.variable).is_none() {
1520 return Err(format!(
1521 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
1522 base.table, base.variable
1523 ));
1524 }
1525 let base_key = window_base_key(base);
1526 let wanted: std::collections::HashSet<&String> = units.iter().collect();
1527 let live = self.load_window_units(&base_key);
1528 self.begin_observation("syncular_window")?;
1529 let mut batch = ChangeAccumulator::default();
1530 let mut changed = false;
1531
1532 for unit in units {
1534 if live.iter().any(|(u, _)| u == unit) {
1535 continue;
1536 }
1537 let sub_id = derive_sub_id(base, unit);
1538 self.delete_pending_evict(&sub_id);
1539 self.insert_window_unit(&base_key, unit, &sub_id);
1540 self.subscribe(
1541 sub_id,
1542 base.table.clone(),
1543 unit_scopes(base, unit),
1544 base.params.clone(),
1545 )?;
1546 batch.window(&base_key, &base.table, unit);
1547 changed = true;
1548 }
1549
1550 for (unit, sub_id) in live {
1552 if wanted.contains(&unit) {
1553 continue;
1554 }
1555 let effective = self
1556 .subs
1557 .iter()
1558 .find(|sub| sub.id == sub_id)
1559 .and_then(|sub| sub.effective.clone())
1560 .unwrap_or_else(|| unit_scopes(base, &unit));
1561 self.record_scope_map(&mut batch, &base.table, &effective);
1562 batch.window(&base_key, &base.table, &unit);
1563 self.evict_unit(&base_key, base, &unit, &sub_id);
1564 changed = true;
1565 }
1566 if let Err(error) = self.finish_observation("syncular_window", batch) {
1567 self.rollback_observation("syncular_window");
1568 return Err(error);
1569 }
1570 Ok(if changed {
1571 CommandEffects::interactive()
1572 } else {
1573 CommandEffects::none()
1574 })
1575 }
1576
1577 pub fn window_state(&self, base: &WindowBase) -> WindowState {
1582 let mut units = Vec::new();
1583 let mut pending = Vec::new();
1584 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
1585 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
1586 Some(sub) => {
1587 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
1588 }
1589 None => true,
1590 };
1591 if is_pending {
1592 pending.push(unit.clone());
1593 }
1594 units.push(unit);
1595 }
1596 WindowState { units, pending }
1597 }
1598
1599 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
1600 let mut stmt = match self
1601 .conn
1602 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
1603 {
1604 Ok(stmt) => stmt,
1605 Err(_) => return Vec::new(),
1606 };
1607 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
1608 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1609 });
1610 match rows {
1611 Ok(rows) => rows.filter_map(Result::ok).collect(),
1612 Err(_) => Vec::new(),
1613 }
1614 }
1615
1616 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
1617 let mut stmt = match self.conn.prepare(
1618 "SELECT windows.base, windows.unit, subscriptions.tbl
1619 FROM _syncular_windows AS windows
1620 JOIN _syncular_subscriptions AS subscriptions
1621 ON subscriptions.id = windows.sub_id
1622 ORDER BY windows.base, windows.unit",
1623 ) {
1624 Ok(stmt) => stmt,
1625 Err(_) => return Vec::new(),
1626 };
1627 let rows = stmt.query_map([], |row| {
1628 Ok((
1629 row.get::<_, String>(0)?,
1630 row.get::<_, String>(1)?,
1631 row.get::<_, String>(2)?,
1632 ))
1633 });
1634 match rows {
1635 Ok(rows) => rows.filter_map(Result::ok).collect(),
1636 Err(_) => Vec::new(),
1637 }
1638 }
1639
1640 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
1641 self.conn
1642 .query_row(
1643 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
1644 rusqlite::params![sub_id],
1645 |row| Ok((row.get(0)?, row.get(1)?)),
1646 )
1647 .ok()
1648 }
1649
1650 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
1651 let _ = self.conn.execute(
1652 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
1653 rusqlite::params![base_key, unit, sub_id],
1654 );
1655 }
1656
1657 fn delete_window_unit(&self, base_key: &str, unit: &str) {
1658 let _ = self.conn.execute(
1659 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
1660 rusqlite::params![base_key, unit],
1661 );
1662 }
1663
1664 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
1671 let effective = self
1672 .subs
1673 .iter()
1674 .find(|s| s.id == sub_id)
1675 .and_then(|s| s.effective.clone())
1676 .unwrap_or_else(|| unit_scopes(base, unit));
1677 let pinned = self.pinned_row_ids(&base.table);
1678 let deferred = self
1679 .evict_scope_rows(&base.table, &effective, &pinned)
1680 .unwrap_or(false);
1681 self.delete_window_unit(base_key, unit);
1682 self.unsubscribe(sub_id);
1683 if deferred {
1684 self.save_pending_evict(sub_id, &base.table, &effective);
1685 } else {
1686 self.delete_pending_evict(sub_id);
1687 }
1688 self.rebuild_overlay();
1689 }
1690
1691 fn evict_scope_rows(
1695 &mut self,
1696 table_name: &str,
1697 effective: &[(String, Vec<String>)],
1698 pinned: &std::collections::HashSet<String>,
1699 ) -> Result<bool, ()> {
1700 if effective.is_empty() {
1701 return Ok(false);
1702 }
1703 let table = self.schema.table(table_name).ok_or(())?.clone();
1704 let mut clauses = Vec::new();
1705 let mut params: Vec<SqlValue> = Vec::new();
1706 for (variable, values) in effective {
1707 let column = table.scope_column(variable).ok_or(())?;
1708 let placeholders: Vec<String> = values
1709 .iter()
1710 .map(|v| {
1711 params.push(SqlValue::Text(v.clone()));
1712 "?".to_owned()
1713 })
1714 .collect();
1715 clauses.push(format!(
1716 "{} IN ({})",
1717 quote_ident(column),
1718 placeholders.join(", ")
1719 ));
1720 }
1721 let mut sql = format!(
1722 "DELETE FROM {} WHERE {}",
1723 base_table(table_name),
1724 clauses.join(" AND ")
1725 );
1726 if !pinned.is_empty() {
1727 let pk = quote_ident(&table.primary_key);
1728 let holes: Vec<String> = pinned
1729 .iter()
1730 .map(|id| {
1731 params.push(SqlValue::Text(id.clone()));
1732 "?".to_owned()
1733 })
1734 .collect();
1735 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
1736 }
1737 self.overlay_dirty.set(true);
1738 self.conn
1739 .execute(&sql, rusqlite::params_from_iter(params))
1740 .map_err(|_| ())?;
1741 if pinned.is_empty() {
1742 return Ok(false);
1743 }
1744 let mut where_params: Vec<SqlValue> = Vec::new();
1747 let mut where_clauses = Vec::new();
1748 for (variable, values) in effective {
1749 let column = table.scope_column(variable).ok_or(())?;
1750 let placeholders: Vec<String> = values
1751 .iter()
1752 .map(|v| {
1753 where_params.push(SqlValue::Text(v.clone()));
1754 "?".to_owned()
1755 })
1756 .collect();
1757 where_clauses.push(format!(
1758 "{} IN ({})",
1759 quote_ident(column),
1760 placeholders.join(", ")
1761 ));
1762 }
1763 let pk = quote_ident(&table.primary_key);
1764 let select = format!(
1765 "SELECT {} FROM {} WHERE {}",
1766 pk,
1767 base_table(table_name),
1768 where_clauses.join(" AND ")
1769 );
1770 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
1771 let survivors: Vec<String> = stmt
1772 .query_map(rusqlite::params_from_iter(where_params), |row| {
1773 row.get::<_, String>(0)
1774 })
1775 .map_err(|_| ())?
1776 .filter_map(Result::ok)
1777 .collect();
1778 Ok(survivors.iter().any(|id| pinned.contains(id)))
1779 }
1780
1781 fn drain_pending_evictions(&mut self) {
1785 let pending = self.load_pending_evictions();
1786 if pending.is_empty() {
1787 return;
1788 }
1789 for (sub_id, table_name, effective) in pending {
1790 if self.schema.table(&table_name).is_none() {
1791 self.delete_pending_evict(&sub_id);
1792 continue;
1793 }
1794 let pinned = self.pinned_row_ids(&table_name);
1795 let deferred = self
1796 .evict_scope_rows(&table_name, &effective, &pinned)
1797 .unwrap_or(false);
1798 if !deferred {
1799 self.delete_pending_evict(&sub_id);
1800 }
1801 }
1802 self.rebuild_overlay_if_dirty();
1803 }
1804
1805 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
1808 let mut pinned = std::collections::HashSet::new();
1809 for commit in &self.outbox {
1810 for op in &commit.ops {
1811 if op.table == table {
1812 pinned.insert(op.row_id.clone());
1813 }
1814 }
1815 }
1816 pinned
1817 }
1818
1819 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
1820 let _ = self.conn.execute(
1821 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
1822 VALUES (?1, ?2, ?3)",
1823 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
1824 );
1825 }
1826
1827 fn delete_pending_evict(&self, sub_id: &str) {
1828 let _ = self.conn.execute(
1829 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
1830 rusqlite::params![sub_id],
1831 );
1832 }
1833
1834 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
1835 let mut stmt = match self
1836 .conn
1837 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
1838 {
1839 Ok(stmt) => stmt,
1840 Err(_) => return Vec::new(),
1841 };
1842 let rows = stmt.query_map([], |row| {
1843 Ok((
1844 row.get::<_, String>(0)?,
1845 row.get::<_, String>(1)?,
1846 row.get::<_, String>(2)?,
1847 ))
1848 });
1849 let mut out = Vec::new();
1850 if let Ok(rows) = rows {
1851 for entry in rows.filter_map(Result::ok) {
1852 let (sub_id, table, json) = entry;
1853 if let Ok(value) = serde_json::from_str::<Value>(&json) {
1854 if let Ok(effective) = json_to_scope_map(&value) {
1855 out.push((sub_id, table, effective));
1856 }
1857 }
1858 }
1859 }
1860 out
1861 }
1862
1863 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
1865 if mutations.is_empty() {
1866 return Err("a commit must contain at least one operation (§6.1)".to_owned());
1867 }
1868 let mut ops = Vec::with_capacity(mutations.len());
1869 for mutation in mutations {
1870 match mutation {
1871 Mutation::Upsert {
1872 table,
1873 values,
1874 base_version,
1875 } => {
1876 let schema_table = self
1877 .schema
1878 .table(&table)
1879 .ok_or_else(|| format!("unknown table {table:?}"))?;
1880 let values = normalize_values_casing(schema_table, values)?;
1884 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
1885 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
1888 ops.push(OutboxOp {
1889 upsert: true,
1890 table,
1891 row_id,
1892 base_version,
1893 values: Some(values),
1894 });
1895 }
1896 Mutation::Delete {
1897 table,
1898 row_id,
1899 base_version,
1900 } => {
1901 if self.schema.table(&table).is_none() {
1902 return Err(format!("unknown table {table:?}"));
1903 }
1904 ops.push(OutboxOp {
1905 upsert: false,
1906 table,
1907 row_id,
1908 base_version,
1909 values: None,
1910 });
1911 }
1912 }
1913 }
1914 let commit = OutboxCommit {
1915 client_commit_id: uuid::Uuid::new_v4().to_string(),
1916 ops,
1917 };
1918 self.begin_observation("syncular_mutation")?;
1919 let mut batch = ChangeAccumulator::default();
1920 for op in &commit.ops {
1921 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
1922 if let Some(values) = &op.values {
1923 if let Some(table) = self.schema.table(&op.table) {
1924 for scope in &table.scope_variables {
1925 if let Some(Value::String(value)) = values.get(&scope.column) {
1926 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
1927 precise = true;
1928 }
1929 }
1930 }
1931 }
1932 if !precise {
1933 batch.table(&op.table);
1934 }
1935 }
1936 self.persist_outbox_insert(&commit);
1937 let id = commit.client_commit_id.clone();
1938 self.outbox.push(commit);
1939 self.overlay_dirty.set(true);
1940 self.rebuild_overlay();
1941 batch.status = true;
1942 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
1943 self.rollback_observation("syncular_mutation");
1944 return Err(error);
1945 }
1946 Ok(id)
1947 }
1948
1949 pub fn patch(
1954 &mut self,
1955 table: &str,
1956 row_id: &str,
1957 partial: Map<String, Value>,
1958 base_version: Option<i64>,
1959 ) -> Result<String, String> {
1960 let schema_table = self
1961 .schema
1962 .table(table)
1963 .ok_or_else(|| format!("unknown table {table:?}"))?;
1964 let partial = normalize_values_casing(schema_table, partial)?;
1965 let mut values = self
1966 .read_rows(table)?
1967 .into_iter()
1968 .find(|row| row.row_id == row_id)
1969 .map(|row| row.values)
1970 .ok_or_else(|| {
1971 format!(
1972 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
1973 )
1974 })?;
1975 values.extend(partial);
1976 self.mutate(vec![Mutation::Upsert {
1977 table: table.to_owned(),
1978 values,
1979 base_version,
1980 }])
1981 }
1982
1983 pub fn pending_commit_ids(&self) -> Vec<String> {
1984 self.outbox
1985 .iter()
1986 .map(|c| c.client_commit_id.clone())
1987 .collect()
1988 }
1989
1990 pub fn conflicts(&self) -> &[ConflictRecord] {
1991 &self.conflicts
1992 }
1993
1994 pub fn rejections(&self) -> &[RejectionRecord] {
1995 &self.rejections
1996 }
1997
1998 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
1999 self.schema_floor.as_ref()
2000 }
2001
2002 pub fn lease_state(&self) -> Option<&LeaseState> {
2004 self.lease_state.as_ref()
2005 }
2006
2007 fn record_lease_error(&mut self, code: &str) {
2010 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
2011 return;
2012 }
2013 let mut next = self.lease_state.clone().unwrap_or_default();
2014 next.error_code = Some(code.to_owned());
2015 self.set_lease_state(Some(next));
2016 }
2017
2018 fn set_lease_state(&mut self, next: Option<LeaseState>) {
2019 if self.lease_state == next {
2020 return;
2021 }
2022 if self.begin_observation("syncular_lease").is_err() {
2023 return;
2024 }
2025 self.lease_state = next;
2026 if let Some(lease) = &self.lease_state {
2027 if let Ok(json) = serde_json::to_string(lease) {
2028 self.set_meta(LEASE_STATE_KEY, &json);
2029 }
2030 } else {
2031 self.delete_meta(LEASE_STATE_KEY);
2032 }
2033 let batch = ChangeAccumulator {
2034 status: true,
2035 ..ChangeAccumulator::default()
2036 };
2037 if self.finish_observation("syncular_lease", batch).is_err() {
2038 self.rollback_observation("syncular_lease");
2039 }
2040 }
2041
2042 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
2043 if self.schema_floor == next {
2044 return;
2045 }
2046 if self.begin_observation("syncular_schema_floor").is_err() {
2047 return;
2048 }
2049 self.schema_floor = next;
2050 self.stopped = self.schema_floor.is_some();
2051 if let Some(floor) = &self.schema_floor {
2052 if let Ok(json) = serde_json::to_string(floor) {
2053 self.set_meta(SCHEMA_FLOOR_KEY, &json);
2054 }
2055 } else {
2056 self.delete_meta(SCHEMA_FLOOR_KEY);
2057 }
2058 let batch = ChangeAccumulator {
2059 status: true,
2060 ..ChangeAccumulator::default()
2061 };
2062 if self
2063 .finish_observation("syncular_schema_floor", batch)
2064 .is_err()
2065 {
2066 self.rollback_observation("syncular_schema_floor");
2067 }
2068 }
2069
2070 fn set_upgrading(&mut self, value: bool) {
2071 if self.upgrading == value {
2072 return;
2073 }
2074 if self.begin_observation("syncular_upgrading").is_err() {
2075 return;
2076 }
2077 self.upgrading = value;
2078 let batch = ChangeAccumulator {
2079 status: true,
2080 ..ChangeAccumulator::default()
2081 };
2082 if self
2083 .finish_observation("syncular_upgrading", batch)
2084 .is_err()
2085 {
2086 self.rollback_observation("syncular_upgrading");
2087 }
2088 }
2089
2090 pub fn sync_needed(&self) -> bool {
2091 self.sync_needed
2092 }
2093
2094 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
2095 let sub = self.subs.iter().find(|s| s.id == id)?;
2096 Some(SubscriptionStateView {
2097 id: sub.id.clone(),
2098 table: sub.table.clone(),
2099 status: sub.state.name().to_owned(),
2100 cursor: sub.cursor,
2101 has_resume_token: sub.bootstrap_state.is_some(),
2102 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
2103 reason_code: sub.reason_code.clone(),
2104 })
2105 }
2106
2107 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
2108 let schema_table = self
2109 .schema
2110 .table(table)
2111 .ok_or_else(|| format!("unknown table {table:?}"))?;
2112 let sql = format!(
2113 "SELECT * FROM {} ORDER BY {} ASC",
2114 visible_table(table),
2115 quote_ident(&schema_table.primary_key)
2116 );
2117 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
2118 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
2119 let mut out = Vec::new();
2120 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
2121 let mut values = Map::new();
2122 for (i, column) in schema_table.columns.iter().enumerate() {
2123 let value = row.get_ref(i).map_err(|e| e.to_string())?;
2124 values.insert(column.name.clone(), sql_ref_to_json(column, value));
2125 }
2126 let version: i64 = row
2127 .get(schema_table.columns.len())
2128 .map_err(|e| e.to_string())?;
2129 let row_id = match values.get(&schema_table.primary_key) {
2130 Some(Value::String(s)) => s.clone(),
2131 Some(Value::Number(n)) => n.to_string(),
2132 Some(Value::Bool(b)) => b.to_string(),
2133 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
2134 };
2135 out.push(RowState {
2136 row_id,
2137 version,
2138 values,
2139 });
2140 }
2141 Ok(out)
2142 }
2143
2144 #[cfg(feature = "crdt-yjs")]
2160 fn crdt_column_bytes(
2161 &self,
2162 table: &str,
2163 row_id: &str,
2164 column: &str,
2165 ) -> Result<Option<Vec<u8>>, String> {
2166 let schema_table = self
2167 .schema
2168 .table(table)
2169 .ok_or_else(|| format!("unknown table {table:?}"))?;
2170 let col = schema_table
2171 .columns
2172 .iter()
2173 .find(|c| c.name == column)
2174 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
2175 if col.ty != ColumnType::Crdt {
2176 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
2177 }
2178 let sql = format!(
2179 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
2180 quote_ident(column),
2181 visible_table(table),
2182 quote_ident(&schema_table.primary_key)
2183 );
2184 let bytes: Option<Vec<u8>> = self
2185 .conn
2186 .query_row(&sql, rusqlite::params![row_id], |row| {
2187 row.get::<_, Option<Vec<u8>>>(0)
2188 })
2189 .map_err(|e| match e {
2190 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
2191 other => other.to_string(),
2192 })?;
2193 Ok(bytes)
2194 }
2195
2196 #[cfg(feature = "crdt-yjs")]
2200 pub fn crdt_text(
2201 &self,
2202 table: &str,
2203 row_id: &str,
2204 column: &str,
2205 name: &str,
2206 ) -> Result<String, String> {
2207 let bytes = self
2208 .crdt_column_bytes(table, row_id, column)?
2209 .unwrap_or_default();
2210 crate::crdt::text(&bytes, name)
2211 }
2212
2213 #[cfg(feature = "crdt-yjs")]
2217 pub fn crdt_insert_text(
2218 &mut self,
2219 table: &str,
2220 row_id: &str,
2221 column: &str,
2222 name: &str,
2223 index: u32,
2224 value: &str,
2225 ) -> Result<String, String> {
2226 let current = self
2227 .crdt_column_bytes(table, row_id, column)?
2228 .unwrap_or_default();
2229 let update = crate::crdt::insert_text(¤t, name, index, value)?;
2230 self.crdt_push_update(table, row_id, column, &update)
2231 }
2232
2233 #[cfg(feature = "crdt-yjs")]
2236 pub fn crdt_delete_text(
2237 &mut self,
2238 table: &str,
2239 row_id: &str,
2240 column: &str,
2241 name: &str,
2242 index: u32,
2243 len: u32,
2244 ) -> Result<String, String> {
2245 let current = self
2246 .crdt_column_bytes(table, row_id, column)?
2247 .unwrap_or_default();
2248 let update = crate::crdt::delete_text(¤t, name, index, len)?;
2249 self.crdt_push_update(table, row_id, column, &update)
2250 }
2251
2252 #[cfg(feature = "crdt-yjs")]
2257 pub fn crdt_apply_update(
2258 &mut self,
2259 table: &str,
2260 row_id: &str,
2261 column: &str,
2262 update: &[u8],
2263 ) -> Result<String, String> {
2264 let current = self
2265 .crdt_column_bytes(table, row_id, column)?
2266 .unwrap_or_default();
2267 let next = crate::crdt::apply_update(¤t, update)?;
2268 self.crdt_push_update(table, row_id, column, &next)
2269 }
2270
2271 #[cfg(feature = "crdt-yjs")]
2277 fn crdt_push_update(
2278 &mut self,
2279 table: &str,
2280 row_id: &str,
2281 column: &str,
2282 crdt_bytes: &[u8],
2283 ) -> Result<String, String> {
2284 let schema_table = self
2285 .schema
2286 .table(table)
2287 .ok_or_else(|| format!("unknown table {table:?}"))?
2288 .clone();
2289 let mut values: Map<String, Value> = self
2292 .read_rows(table)?
2293 .into_iter()
2294 .find(|r| r.row_id == row_id)
2295 .map(|r| r.values)
2296 .unwrap_or_else(|| {
2297 let mut map = Map::new();
2298 map.insert(
2299 schema_table.primary_key.clone(),
2300 Value::from(row_id.to_owned()),
2301 );
2302 map
2303 });
2304 let mut bytes_obj = Map::new();
2306 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
2307 values.insert(column.to_owned(), Value::Object(bytes_obj));
2308 self.mutate(vec![Mutation::Upsert {
2309 table: table.to_owned(),
2310 values,
2311 base_version: None,
2312 }])
2313 }
2314
2315 pub fn query(&self, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {
2330 crate::query_guard::assert_read_only_query(sql)?;
2331 let bound: Vec<SqlValue> = params
2332 .iter()
2333 .map(json_param_to_sql)
2334 .collect::<Result<_, _>>()?;
2335 let mut stmt = self.conn.prepare(sql).map_err(|e| e.to_string())?;
2336 let column_names: Vec<String> =
2337 stmt.column_names().into_iter().map(str::to_owned).collect();
2338 let bound_refs: Vec<&dyn rusqlite::ToSql> =
2339 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
2340 let mut sql_rows = stmt
2341 .query(bound_refs.as_slice())
2342 .map_err(|e| e.to_string())?;
2343 let mut out = Vec::new();
2344 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
2345 let mut record = Map::new();
2346 for (i, name) in column_names.iter().enumerate() {
2347 let value = row.get_ref(i).map_err(|e| e.to_string())?;
2348 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
2349 }
2350 out.push(record);
2351 }
2352 Ok(out)
2353 }
2354
2355 pub fn query_snapshot(
2357 &mut self,
2358 sql: &str,
2359 params: &[Value],
2360 coverage: &[WindowCoverage],
2361 ) -> Result<QuerySnapshot, String> {
2362 self.conn
2363 .execute_batch("SAVEPOINT syncular_read")
2364 .map_err(|error| error.to_string())?;
2365 let result = (|| {
2366 let revision = self.local_revision();
2367 let rows = self.query(sql, params)?;
2368 let mut pending = Vec::new();
2369 let mut missing = Vec::new();
2370 for requested in coverage {
2371 let base_key = window_base_key(&requested.base);
2372 let state = self.window_state(&requested.base);
2373 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
2374 let reference = WindowUnitRef {
2375 base_key: base_key.clone(),
2376 unit: unit.clone(),
2377 };
2378 if !state.units.iter().any(|held| held == &unit) {
2379 missing.push(reference);
2380 } else if state.pending.iter().any(|held| held == &unit) {
2381 pending.push(reference);
2382 }
2383 }
2384 }
2385 Ok(QuerySnapshot {
2386 revision: revision.to_string(),
2387 rows,
2388 coverage: CoverageSnapshot {
2389 complete: pending.is_empty() && missing.is_empty(),
2390 pending,
2391 missing,
2392 },
2393 })
2394 })();
2395 match result {
2396 Ok(snapshot) => {
2397 self.conn
2398 .execute_batch("RELEASE syncular_read")
2399 .map_err(|error| error.to_string())?;
2400 Ok(snapshot)
2401 }
2402 Err(error) => {
2403 let _ = self
2404 .conn
2405 .execute_batch("ROLLBACK TO syncular_read; RELEASE syncular_read");
2406 Err(error)
2407 }
2408 }
2409 }
2410
2411 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
2414 let mut frames = vec![Frame::ReqHeader {
2415 client_id: self.client_id.clone(),
2416 schema_version: self.schema.version,
2417 }];
2418 let mut pushed_ids = Vec::new();
2419 let mut ops_in_request = 0usize;
2420 let mut deferred_commits = 0usize;
2421 for (index, commit) in self.outbox.iter().enumerate() {
2422 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
2427 deferred_commits = self.outbox.len() - index;
2428 break;
2429 }
2430 ops_in_request += commit.ops.len();
2431 let operations = commit
2432 .ops
2433 .iter()
2434 .map(|op| {
2435 let payload = op.values.as_ref().and_then(|values| {
2436 let table = self.schema.table(&op.table)?;
2437 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
2441 });
2442 ssp2::model::Operation {
2443 table: op.table.clone(),
2444 row_id: op.row_id.clone(),
2445 op: if op.upsert { Op::Upsert } else { Op::Delete },
2446 base_version: op.base_version,
2447 payload,
2448 }
2449 })
2450 .collect();
2451 frames.push(Frame::PushCommit {
2452 client_commit_id: commit.client_commit_id.clone(),
2453 operations,
2454 });
2455 pushed_ids.push(commit.client_commit_id.clone());
2456 }
2457 let accept = self.limits.accept.unwrap_or(if url_capable {
2460 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
2461 } else {
2462 DEFAULT_ACCEPT
2463 });
2464 frames.push(Frame::PullHeader {
2465 limit_commits: self.limits.limit_commits.unwrap_or(0),
2466 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
2467 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
2468 accept,
2469 });
2470 let mut fresh = Vec::new();
2471 for sub in &self.subs {
2472 if sub.state != SubState::Active {
2473 continue;
2474 }
2475 let mut scopes = sub.requested.clone();
2476 sort_scope_map(&mut scopes);
2477 frames.push(Frame::Subscription {
2478 id: sub.id.clone(),
2479 table: sub.table.clone(),
2480 scopes,
2481 params: sub.params.clone().map(RawJson),
2482 cursor: sub.cursor,
2483 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
2484 });
2485 fresh.push((
2486 sub.id.clone(),
2487 sub.cursor < 0 && sub.bootstrap_state.is_none(),
2488 ));
2489 }
2490 let message = Message {
2491 msg_kind: MsgKind::Request,
2492 frames,
2493 };
2494 (
2495 message,
2496 RequestMeta {
2497 pushed_ids,
2498 fresh,
2499 accept,
2500 deferred_commits,
2501 },
2502 )
2503 }
2504
2505 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
2508 if self.stopped {
2509 return SyncOutcome::Ok(SyncReport {
2512 schema_floor: self.schema_floor.clone(),
2513 ..SyncReport::default()
2514 });
2515 }
2516 self.set_sync_needed(false, false);
2519 if self.schema_has_blobs() {
2522 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
2523 if Self::retryable_transport_code(&code) {
2524 self.schedule_background_retry();
2525 }
2526 return SyncOutcome::Failed {
2527 error_code: code,
2528 message,
2529 };
2530 }
2531 }
2532 let (message, meta) = self.build_request(transport.supports_url_fetch());
2533 let request_bytes = encode_message(&message);
2534 let round = if self.realtime_connected {
2538 transport.realtime_sync(&request_bytes)
2539 } else {
2540 transport.sync(&request_bytes)
2541 };
2542 let response_bytes = match round {
2543 Ok(bytes) => bytes,
2544 Err(TransportError { code, message }) => {
2545 self.record_lease_error(&code);
2548 if Self::retryable_transport_code(&code) {
2549 self.schedule_background_retry();
2550 }
2551 return SyncOutcome::Failed {
2552 error_code: code,
2553 message,
2554 };
2555 }
2556 };
2557 let response = match decode_message(&response_bytes) {
2558 Ok(message) => message,
2559 Err(error) => {
2560 return SyncOutcome::Failed {
2563 error_code: error.code.as_str().to_owned(),
2564 message: error.detail,
2565 };
2566 }
2567 };
2568 if response.msg_kind != MsgKind::Response {
2569 return SyncOutcome::Failed {
2570 error_code: "sync.invalid_request".to_owned(),
2571 message: "expected a response message".to_owned(),
2572 };
2573 }
2574 let outcome = self.process_response(transport, response, &meta);
2575 match &outcome {
2576 SyncOutcome::Ok(_) => self.reset_background_retry(),
2577 SyncOutcome::Failed { error_code, .. }
2578 if Self::retryable_transport_code(error_code) =>
2579 {
2580 self.schedule_background_retry();
2581 }
2582 SyncOutcome::Failed { .. } => {}
2583 }
2584 if meta.deferred_commits > 0 {
2585 self.set_sync_needed(true, true);
2588 }
2589 outcome
2590 }
2591
2592 pub fn sync_until_idle(
2593 &mut self,
2594 transport: &mut dyn Transport,
2595 max_rounds: Option<u32>,
2596 ) -> SyncOutcome {
2597 let rounds = max_rounds.unwrap_or(12).max(1);
2598 let mut aggregate = SyncReport::default();
2599 for _ in 0..rounds {
2600 match self.sync(transport) {
2601 SyncOutcome::Failed {
2602 error_code,
2603 message,
2604 } => {
2605 return SyncOutcome::Failed {
2606 error_code,
2607 message,
2608 };
2609 }
2610 SyncOutcome::Ok(report) => {
2611 aggregate.pushed += report.pushed;
2612 aggregate.applied.extend(report.applied.iter().cloned());
2613 aggregate.rejected.extend(report.rejected.iter().cloned());
2614 aggregate.retryable.extend(report.retryable.iter().cloned());
2615 aggregate.conflicts += report.conflicts;
2616 aggregate.commits_applied += report.commits_applied;
2617 aggregate.segment_rows_applied += report.segment_rows_applied;
2618 aggregate.bootstrapping = report.bootstrapping.clone();
2619 aggregate.resets.extend(report.resets.iter().cloned());
2620 aggregate.revoked.extend(report.revoked.iter().cloned());
2621 aggregate.failed.extend(report.failed.iter().cloned());
2622 if report.schema_floor.is_some() {
2623 aggregate.schema_floor = report.schema_floor.clone();
2624 }
2625 let more = !report.bootstrapping.is_empty()
2631 || report.commits_applied > 0
2632 || report.segment_rows_applied > 0
2633 || !report.resets.is_empty()
2634 || self.sync_needed;
2635 if !more {
2636 break;
2637 }
2638 }
2639 }
2640 }
2641 SyncOutcome::Ok(aggregate)
2642 }
2643
2644 fn process_response(
2645 &mut self,
2646 transport: &mut dyn Transport,
2647 response: Message,
2648 meta: &RequestMeta,
2649 ) -> SyncOutcome {
2650 let mut report = SyncReport {
2651 pushed: meta.pushed_ids.len() as u32,
2652 ..SyncReport::default()
2653 };
2654 let mut frames = response.frames.into_iter();
2655 match frames.next() {
2656 Some(Frame::RespHeader {
2657 required_schema_version,
2658 latest_schema_version,
2659 }) => {
2660 if let Some(required) = required_schema_version {
2661 let floor = SchemaFloor {
2664 required_schema_version: Some(required),
2665 latest_schema_version,
2666 };
2667 self.set_schema_floor(Some(floor.clone()));
2668 report.schema_floor = Some(floor);
2669 return SyncOutcome::Ok(report);
2670 }
2671 }
2672 _ => {
2673 return SyncOutcome::Failed {
2674 error_code: "sync.invalid_request".to_owned(),
2675 message: "response does not start with RESP_HEADER".to_owned(),
2676 };
2677 }
2678 }
2679
2680 let mut failure: Option<(String, String)> = None;
2681 while let Some(frame) = frames.next() {
2682 match frame {
2683 Frame::PushResult {
2684 client_commit_id,
2685 status,
2686 commit_seq: _,
2687 results,
2688 } => {
2689 self.handle_push_result(&client_commit_id, status, &results, &mut report);
2690 }
2691 Frame::SubStart {
2692 id,
2693 status,
2694 reason_code,
2695 effective_scopes,
2696 bootstrap: _,
2697 } => {
2698 let mut body = Vec::new();
2699 let mut sub_end: Option<(i64, Option<String>)> = None;
2700 for inner in frames.by_ref() {
2701 match inner {
2702 Frame::SubEnd {
2703 next_cursor,
2704 bootstrap_state,
2705 } => {
2706 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
2707 break;
2708 }
2709 Frame::Unknown { .. } => {}
2710 other => body.push(other),
2711 }
2712 }
2713 let Some((next_cursor, bootstrap_state)) = sub_end else {
2714 failure = Some((
2715 "sync.invalid_request".to_owned(),
2716 "subscription section without SUB_END".to_owned(),
2717 ));
2718 break;
2719 };
2720 if let Err(SectionError::Abort(code, message)) = self.process_section(
2721 transport,
2722 &id,
2723 status,
2724 &reason_code,
2725 effective_scopes,
2726 body,
2727 next_cursor,
2728 bootstrap_state,
2729 meta,
2730 &mut report,
2731 ) {
2732 failure = Some((code, message));
2733 break;
2734 }
2735 }
2736 Frame::Lease {
2737 lease_id,
2738 expires_at_ms,
2739 } => {
2740 self.set_lease_state(Some(LeaseState {
2743 lease_id: Some(lease_id),
2744 expires_at_ms: Some(expires_at_ms),
2745 error_code: None,
2746 }));
2747 }
2748 Frame::Error { code, message, .. } => {
2749 failure = Some((code, message));
2752 break;
2753 }
2754 Frame::Unknown { .. } => {}
2755 _ => {
2756 failure = Some((
2757 "sync.invalid_request".to_owned(),
2758 "unexpected frame in response".to_owned(),
2759 ));
2760 break;
2761 }
2762 }
2763 }
2764
2765 if self.overlay_dirty.get() {
2768 self.rebuild_overlay();
2769 }
2770 self.reconcile_blob_refcounts(false);
2777
2778 if let Some((error_code, message)) = failure {
2779 return SyncOutcome::Failed {
2780 error_code,
2781 message,
2782 };
2783 }
2784 self.drain_pending_evictions();
2787 self.ack_after_pull(transport);
2788 if self.upgrading && report.bootstrapping.is_empty() {
2791 self.set_upgrading(false);
2792 }
2793 SyncOutcome::Ok(report)
2794 }
2795
2796 fn handle_push_result(
2799 &mut self,
2800 client_commit_id: &str,
2801 status: PushStatus,
2802 results: &[OpResult],
2803 report: &mut SyncReport,
2804 ) {
2805 let Some(index) = self
2806 .outbox
2807 .iter()
2808 .position(|c| c.client_commit_id == client_commit_id)
2809 else {
2810 return;
2811 };
2812 if self.begin_observation("syncular_push_result").is_err() {
2813 return;
2814 }
2815 let mut batch = ChangeAccumulator::default();
2816 let operations = self.outbox[index].ops.clone();
2817 self.overlay_dirty.set(true);
2820 match status {
2821 PushStatus::Applied | PushStatus::Cached => {
2822 report.applied.push(client_commit_id.to_owned());
2825 self.outbox.remove(index);
2826 self.persist_outbox_delete(client_commit_id);
2827 batch.status = true;
2828 }
2829 PushStatus::Rejected => {
2830 let terminating = results
2831 .iter()
2832 .find(|r| !matches!(r, OpResult::Applied { .. }));
2833 match terminating {
2834 Some(OpResult::Conflict {
2835 op_index,
2836 code,
2837 message: _,
2838 server_version,
2839 server_row,
2840 }) => {
2841 let commit = &self.outbox[index];
2842 let (table, row_id) = commit
2843 .ops
2844 .get(*op_index as usize)
2845 .map(|op| (op.table.clone(), op.row_id.clone()))
2846 .unwrap_or_default();
2847 let server_row_json = self
2848 .schema
2849 .table(&table)
2850 .and_then(|t| {
2851 decode_row_bytes(t, server_row, &self.encryption)
2852 .ok()
2853 .map(|row| (t, row))
2854 })
2855 .map(|(t, row)| {
2856 let mut map = Map::new();
2857 for (i, column) in t.columns.iter().enumerate() {
2858 map.insert(
2859 column.name.clone(),
2860 column_value_to_json(row.get(i).unwrap_or(&None)),
2861 );
2862 }
2863 map
2864 })
2865 .unwrap_or_default();
2866 self.conflicts.push(ConflictRecord {
2867 client_commit_id: client_commit_id.to_owned(),
2868 op_index: *op_index,
2869 table,
2870 row_id,
2871 code: code.clone(),
2872 server_version: *server_version,
2873 server_row: server_row_json,
2874 });
2875 batch.conflicts = true;
2876 report.conflicts += 1;
2877 report.rejected.push(client_commit_id.to_owned());
2878 self.outbox.remove(index);
2879 self.persist_outbox_delete(client_commit_id);
2880 batch.status = true;
2881 }
2882 Some(OpResult::Error {
2883 op_index,
2884 code,
2885 message: _,
2886 retryable,
2887 }) => {
2888 if code == "sync.idempotency_cache_miss" {
2889 report.retryable.push(client_commit_id.to_owned());
2892 } else {
2893 self.rejections.push(RejectionRecord {
2894 client_commit_id: client_commit_id.to_owned(),
2895 op_index: *op_index,
2896 code: code.clone(),
2897 retryable: *retryable,
2898 });
2899 batch.rejections = true;
2900 report.rejected.push(client_commit_id.to_owned());
2901 self.outbox.remove(index);
2902 self.persist_outbox_delete(client_commit_id);
2903 batch.status = true;
2904 }
2905 }
2906 _ => {
2907 report.rejected.push(client_commit_id.to_owned());
2908 self.outbox.remove(index);
2909 self.persist_outbox_delete(client_commit_id);
2910 batch.status = true;
2911 }
2912 }
2913 }
2914 }
2915 if batch.status {
2916 for operation in &operations {
2917 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
2918 batch.table(&operation.table);
2919 }
2920 }
2921 self.rebuild_overlay_if_dirty();
2922 }
2923 if self
2924 .finish_observation("syncular_push_result", batch)
2925 .is_err()
2926 {
2927 self.rollback_observation("syncular_push_result");
2928 }
2929 }
2930
2931 #[allow(clippy::too_many_arguments)]
2934 fn process_section(
2935 &mut self,
2936 transport: &mut dyn Transport,
2937 id: &str,
2938 status: SubStatus,
2939 reason_code: &str,
2940 effective_scopes: Vec<(String, Vec<String>)>,
2941 body: Vec<Frame>,
2942 next_cursor: i64,
2943 bootstrap_state: Option<String>,
2944 meta: &RequestMeta,
2945 report: &mut SyncReport,
2946 ) -> Result<(), SectionError> {
2947 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
2948 return Ok(()); };
2950 match status {
2951 SubStatus::Revoked => {
2952 self.begin_observation("syncular_revocation")
2953 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
2954 let mut batch = ChangeAccumulator::default();
2955 let registered = self.window_unit_by_sub_id(id);
2956 let (table, effective) = {
2958 let sub = &self.subs[sub_index];
2959 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
2960 };
2961 let purged = self.purge_scope_rows(&table, &effective);
2962 match purged {
2963 Ok(()) => {
2964 self.record_scope_map(&mut batch, &table, &effective);
2965 let sub = &mut self.subs[sub_index];
2966 sub.state = SubState::Revoked;
2967 sub.reason_code = Some(if reason_code.is_empty() {
2968 "sync.scope_revoked".to_owned()
2969 } else {
2970 reason_code.to_owned()
2971 });
2972 report.revoked.push(id.to_owned());
2973 let doomed_effective = effective;
2974 let sub_table = table;
2975 self.persist_sub(&self.subs[sub_index].clone());
2976 let outbox_before = self.outbox.len();
2977 self.drop_doomed_outbox(&sub_table, &doomed_effective);
2978 if self.outbox.len() != outbox_before {
2979 batch.status = true;
2980 }
2981 self.reconcile_blob_refcounts(true);
2984 }
2985 Err(()) => {
2986 let sub = &mut self.subs[sub_index];
2989 sub.state = SubState::Failed;
2990 sub.reason_code = Some("sync.scope_revoked".to_owned());
2991 report.failed.push(id.to_owned());
2992 self.persist_sub(&self.subs[sub_index].clone());
2993 }
2994 }
2995 if let Some((base_key, unit)) = registered {
2996 batch.window(&base_key, &self.subs[sub_index].table, &unit);
2997 }
2998 self.rebuild_overlay_if_dirty();
2999 self.finish_observation("syncular_revocation", batch)
3000 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
3001 Ok(())
3002 }
3003 SubStatus::Reset => {
3004 self.begin_observation("syncular_reset")
3005 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
3006 let mut batch = ChangeAccumulator::default();
3007 let registered = self.window_unit_by_sub_id(id);
3008 let sub = &mut self.subs[sub_index];
3011 sub.cursor = -1;
3012 sub.bootstrap_state = None;
3013 report.resets.push(id.to_owned());
3014 self.persist_sub(&self.subs[sub_index].clone());
3015 if let Some((base_key, unit)) = registered {
3016 batch.window(&base_key, &self.subs[sub_index].table, &unit);
3017 }
3018 self.finish_observation("syncular_reset", batch)
3019 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
3020 Ok(())
3021 }
3022 SubStatus::Active => {
3023 let fresh = meta
3024 .fresh
3025 .iter()
3026 .find(|(fid, _)| fid == id)
3027 .map(|(_, f)| *f)
3028 .unwrap_or(false);
3029 let was_pending = self.subs[sub_index].cursor < 0
3030 || self.subs[sub_index].bootstrap_state.is_some();
3031 let registered = self.window_unit_by_sub_id(id);
3032 self.subs[sub_index].effective = Some(effective_scopes);
3034 self.begin_observation("syncular_section")
3035 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
3036 let mut batch = ChangeAccumulator::default();
3037 let outcome = self.apply_section_body(
3038 transport, sub_index, body, fresh, meta, report, &mut batch,
3039 );
3040 match outcome {
3041 Ok(()) => {
3042 let sub = &mut self.subs[sub_index];
3043 sub.cursor = next_cursor;
3046 sub.bootstrap_state = bootstrap_state;
3047 sub.synced_once = true;
3048 if sub.bootstrap_state.is_some() {
3049 report.bootstrapping.push(id.to_owned());
3050 }
3051 let completed =
3052 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
3053 self.persist_sub(&self.subs[sub_index].clone());
3054 if completed {
3055 if let Some((base_key, unit)) = registered.clone() {
3056 batch.window(&base_key, &self.subs[sub_index].table, &unit);
3057 }
3058 }
3059 self.rebuild_overlay_if_dirty();
3060 self.finish_observation("syncular_section", batch)
3061 .map_err(|message| {
3062 SectionError::Abort("storage.failed".to_owned(), message)
3063 })?;
3064 Ok(())
3065 }
3066 Err(SectionError::FailClosed) => {
3067 self.rollback_observation("syncular_section");
3070 self.begin_observation("syncular_section_failure")
3071 .map_err(|message| {
3072 SectionError::Abort("storage.failed".to_owned(), message)
3073 })?;
3074 let mut failure_batch = ChangeAccumulator::default();
3075 let sub = &mut self.subs[sub_index];
3076 sub.state = SubState::Failed;
3077 sub.reason_code = Some("sync.scope_revoked".to_owned());
3078 report.failed.push(id.to_owned());
3079 self.persist_sub(&self.subs[sub_index].clone());
3080 if let Some((base_key, unit)) = registered {
3081 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
3082 }
3083 self.finish_observation("syncular_section_failure", failure_batch)
3084 .map_err(|message| {
3085 SectionError::Abort("storage.failed".to_owned(), message)
3086 })?;
3087 Ok(())
3088 }
3089 Err(SectionError::Abort(code, message)) => {
3090 self.rollback_observation("syncular_section");
3093 Err(SectionError::Abort(code, message))
3094 }
3095 }
3096 }
3097 }
3098 }
3099
3100 #[allow(clippy::too_many_arguments)]
3104 fn apply_section_body(
3105 &mut self,
3106 transport: &mut dyn Transport,
3107 sub_index: usize,
3108 body: Vec<Frame>,
3109 fresh: bool,
3110 meta: &RequestMeta,
3111 report: &mut SyncReport,
3112 batch: &mut ChangeAccumulator,
3113 ) -> Result<(), SectionError> {
3114 let mut saw_segment = false;
3115 for frame in body {
3116 match frame {
3117 Frame::Commit {
3118 tables, changes, ..
3119 } => {
3120 self.record_commit_changes(batch, &tables, &changes);
3121 self.apply_commit_changes(&tables, &changes)
3122 .map_err(|(c, m)| SectionError::Abort(c, m))?;
3123 report.commits_applied += 1;
3124 }
3125 Frame::SegmentInline { payload } => {
3126 let segment = decode_rows_segment(&payload)
3127 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
3128 let first = !saw_segment;
3129 saw_segment = true;
3130 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
3131 let cleared =
3132 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
3133 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
3134 if applied > 0 || cleared {
3135 batch.table(&segment.table);
3136 }
3137 report.segment_rows_applied += applied;
3138 }
3139 Frame::SegmentRef {
3140 segment_id,
3141 media_type,
3142 table,
3143 row_count,
3144 as_of_commit_seq,
3145 scope_digest,
3146 row_cursor,
3147 next_row_cursor,
3148 url,
3149 url_expires_at_ms,
3150 ..
3151 } => {
3152 let advertised = match media_type {
3155 MediaType::Rows => {
3156 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
3157 || meta.accept & ACCEPT_INLINE_ROWS != 0
3158 }
3159 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
3160 };
3161 if !advertised {
3162 return Err(SectionError::Abort(
3163 "sync.invalid_request".to_owned(),
3164 format!(
3165 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
3166 media_type.name()
3167 ),
3168 ));
3169 }
3170 let bytes = if let Some(url) = url {
3171 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
3176 return Err(SectionError::Abort(
3177 "sync.invalid_request".to_owned(),
3178 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
3179 .to_owned(),
3180 ));
3181 }
3182 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
3184 return Err(SectionError::Abort(
3185 "sync.segment_expired".to_owned(),
3186 format!(
3187 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
3188 ),
3189 ));
3190 }
3191 transport
3192 .fetch_url(&url)
3193 .map_err(|e| SectionError::Abort(e.code, e.message))?
3194 } else {
3195 let requested_scopes_json =
3196 canonical_scope_json(&self.subs[sub_index].requested);
3197 transport
3198 .download_segment(&SegmentRequest {
3199 segment_id: segment_id.clone(),
3200 table,
3201 requested_scopes_json,
3202 })
3203 .map_err(|e| SectionError::Abort(e.code, e.message))?
3204 };
3205 let digest = Sha256::digest(&bytes);
3207 let expected = segment_id
3208 .strip_prefix("sha256:")
3209 .unwrap_or(segment_id.as_str());
3210 if bytes_to_hex(&digest) != expected {
3211 return Err(SectionError::Abort(
3212 "sync.invalid_request".to_owned(),
3213 "segment bytes do not match the content address (§5.1)".to_owned(),
3214 ));
3215 }
3216 if media_type == MediaType::Sqlite {
3217 if row_cursor.is_some() || next_row_cursor.is_some() {
3220 return Err(SectionError::Abort(
3221 "sync.invalid_request".to_owned(),
3222 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
3223 .to_owned(),
3224 ));
3225 }
3226 let first = !saw_segment;
3227 saw_segment = true;
3228 let sub_table = self.subs[sub_index].table.clone();
3229 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
3230 let cleared =
3231 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
3232 let applied = self.apply_sqlite_segment(
3233 sub_index,
3234 &bytes,
3235 fresh && first,
3236 row_count,
3237 as_of_commit_seq,
3238 &scope_digest,
3239 )?;
3240 if applied > 0 || cleared {
3241 batch.table(&sub_table);
3242 }
3243 report.segment_rows_applied += applied;
3244 } else {
3245 let segment = decode_rows_segment(&bytes).map_err(|e| {
3246 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
3247 })?;
3248 let first = row_cursor.is_none();
3249 saw_segment = true;
3250 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
3251 let cleared =
3252 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
3253 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
3254 if applied > 0 || cleared {
3255 batch.table(&segment.table);
3256 }
3257 report.segment_rows_applied += applied;
3258 }
3259 }
3260 Frame::Unknown { .. } => {}
3261 _ => {
3262 return Err(SectionError::Abort(
3263 "sync.invalid_request".to_owned(),
3264 "unexpected frame inside a subscription section".to_owned(),
3265 ));
3266 }
3267 }
3268 }
3269 Ok(())
3270 }
3271
3272 fn apply_commit_changes(
3273 &mut self,
3274 tables: &[String],
3275 changes: &[ssp2::model::Change],
3276 ) -> Result<(), (String, String)> {
3277 for change in changes {
3278 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
3279 (
3280 "sync.invalid_request".to_owned(),
3281 "change tableIndex out of range".to_owned(),
3282 )
3283 })?;
3284 let table = self.schema.table(table_name).ok_or_else(|| {
3285 (
3286 "sync.schema_mismatch".to_owned(),
3287 format!("change targets unknown table {table_name:?}"),
3288 )
3289 })?;
3290 match change.op {
3291 Op::Upsert => {
3292 let payload = change.row.as_ref().ok_or_else(|| {
3293 (
3294 "sync.invalid_request".to_owned(),
3295 "upsert change without row payload".to_owned(),
3296 )
3297 })?;
3298 let row = decode_row_bytes(table, payload, &self.encryption)
3300 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
3301 let version = change.row_version.unwrap_or(0);
3302 let table_name = table.name.clone();
3303 self.write_base_row(&table_name, &row, version)
3304 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
3305 }
3306 Op::Delete => {
3307 self.delete_base_row(table_name, &change.row_id)
3308 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
3309 }
3310 }
3311 }
3312 Ok(())
3313 }
3314
3315 fn apply_segment(
3320 &mut self,
3321 sub_index: usize,
3322 segment: &RowsSegment,
3323 first_fresh_page: bool,
3324 ) -> Result<u32, SectionError> {
3325 let (sub_table, effective) = {
3326 let sub = &self.subs[sub_index];
3327 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
3328 };
3329 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
3330 SectionError::Abort(
3331 "sync.schema_mismatch".to_owned(),
3332 format!("subscription table {sub_table:?} is not in the client schema"),
3333 )
3334 })?;
3335 let matches = segment.table == table.name
3340 && segment.schema_version == self.schema.version
3341 && segment.columns.len() == table.wire_columns.len()
3342 && segment
3343 .columns
3344 .iter()
3345 .zip(table.wire_columns.iter())
3346 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
3347 if !matches {
3348 return Err(SectionError::Abort(
3349 "sync.schema_mismatch".to_owned(),
3350 "segment column table does not match the generated schema (§5.2)".to_owned(),
3351 ));
3352 }
3353 if first_fresh_page {
3354 self.purge_scope_rows(&table.name, &effective)
3358 .map_err(|()| SectionError::FailClosed)?;
3359 }
3360 let mut applied = 0u32;
3361 for block in &segment.blocks {
3362 for row in block {
3363 let decrypted;
3368 let values = if table.has_encrypted_columns() {
3369 let mut values = row.values.clone();
3370 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
3371 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
3372 decrypted = values;
3373 &decrypted
3374 } else {
3375 &row.values
3376 };
3377 self.write_base_row(&table.name, values, row.server_version)
3380 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
3381 applied += 1;
3382 }
3383 }
3384 Ok(applied)
3385 }
3386
3387 fn apply_sqlite_segment(
3395 &mut self,
3396 sub_index: usize,
3397 bytes: &[u8],
3398 first_fresh_page: bool,
3399 row_count: i64,
3400 as_of_commit_seq: i64,
3401 scope_digest: &str,
3402 ) -> Result<u32, SectionError> {
3403 let invalid = |detail: &str| {
3404 SectionError::Abort(
3405 "sync.invalid_request".to_owned(),
3406 format!("sqlite segment rejected: {detail} (§5.3)"),
3407 )
3408 };
3409 let (sub_table, effective) = {
3410 let sub = &self.subs[sub_index];
3411 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
3412 };
3413 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
3414 SectionError::Abort(
3415 "sync.schema_mismatch".to_owned(),
3416 format!("subscription table {sub_table:?} is not in the client schema"),
3417 )
3418 })?;
3419
3420 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
3421 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
3422 let img = match rusqlite::Connection::open_with_flags(
3423 &path,
3424 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
3425 ) {
3426 Ok(conn) => conn,
3427 Err(_) => {
3428 let _ = std::fs::remove_file(&path);
3429 return Err(invalid("bytes do not open as a SQLite database"));
3430 }
3431 };
3432 let outcome = self.apply_sqlite_image(
3433 &img,
3434 &table,
3435 first_fresh_page,
3436 &effective,
3437 row_count,
3438 as_of_commit_seq,
3439 scope_digest,
3440 );
3441 drop(img);
3442 let _ = std::fs::remove_file(&path);
3443 outcome
3444 }
3445
3446 #[allow(clippy::too_many_arguments)]
3447 fn apply_sqlite_image(
3448 &mut self,
3449 img: &rusqlite::Connection,
3450 table: &crate::schema::TableSchema,
3451 first_fresh_page: bool,
3452 effective: &[(String, Vec<String>)],
3453 row_count: i64,
3454 as_of_commit_seq: i64,
3455 scope_digest: &str,
3456 ) -> Result<u32, SectionError> {
3457 let invalid = |detail: String| {
3458 SectionError::Abort(
3459 "sync.invalid_request".to_owned(),
3460 format!("sqlite segment rejected: {detail} (§5.3)"),
3461 )
3462 };
3463
3464 type MetaRow = (i64, String, i64, i64, String, i64, i64);
3467 let meta: MetaRow = img
3468 .query_row(
3469 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
3470 \"scopeDigest\", \"rowCount\",
3471 (SELECT count(*) FROM _syncular_segment)
3472 FROM _syncular_segment",
3473 [],
3474 |row| {
3475 Ok((
3476 row.get(0)?,
3477 row.get(1)?,
3478 row.get(2)?,
3479 row.get(3)?,
3480 row.get(4)?,
3481 row.get(5)?,
3482 row.get(6)?,
3483 ))
3484 },
3485 )
3486 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
3487 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
3488 if meta_count != 1 {
3489 return Err(invalid(format!(
3490 "_syncular_segment must contain exactly one row, found {meta_count}"
3491 )));
3492 }
3493 if format != 1 {
3494 return Err(invalid(format!("format {format}")));
3495 }
3496 if meta_table != table.name {
3497 return Err(invalid(format!("image table {meta_table:?}")));
3498 }
3499 if schema_version != i64::from(self.schema.version) {
3500 return Err(invalid(format!("schemaVersion {schema_version}")));
3501 }
3502 if pin != as_of_commit_seq {
3503 return Err(invalid(format!("asOfCommitSeq {pin}")));
3504 }
3505 if digest != scope_digest {
3506 return Err(invalid("scopeDigest mismatch".to_owned()));
3507 }
3508 if meta_rows != row_count {
3509 return Err(invalid(format!("rowCount {meta_rows}")));
3510 }
3511
3512 let mut names: Vec<String> = Vec::new();
3514 {
3515 let mut stmt = img
3516 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
3517 .map_err(|_| invalid("image data table missing".to_owned()))?;
3518 let mut rows = stmt
3519 .query([])
3520 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
3521 while let Some(row) = rows
3522 .next()
3523 .map_err(|_| invalid("image data table unreadable".to_owned()))?
3524 {
3525 names.push(
3526 row.get::<_, String>(1)
3527 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
3528 );
3529 }
3530 }
3531 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
3532 expected.push("_syncular_version");
3533 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
3534 return Err(SectionError::Abort(
3535 "sync.schema_mismatch".to_owned(),
3536 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
3537 ));
3538 }
3539
3540 if first_fresh_page {
3543 self.purge_scope_rows(&table.name, effective)
3544 .map_err(|()| SectionError::FailClosed)?;
3545 }
3546 self.overlay_dirty.set(true);
3552 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
3559 table.indexes.iter().filter(|i| !i.unique).collect()
3560 } else {
3561 Vec::new()
3562 };
3563 for index in &bulk_indexes {
3564 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
3565 self.conn
3566 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
3567 .map_err(|e| invalid(e.to_string()))?;
3568 }
3569 let insert = self.insert_row_sql(&base_table(&table.name), table);
3570 let applied = {
3571 let mut ins = self
3572 .conn
3573 .prepare_cached(&insert)
3574 .map_err(|e| invalid(e.to_string()))?;
3575 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
3576 let mut stmt = img
3577 .prepare(&format!(
3578 "SELECT {} FROM {}",
3579 column_list.join(", "),
3580 quote_ident(&table.name)
3581 ))
3582 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
3583 let mut rows = stmt
3584 .query([])
3585 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
3586 let version_index = table.columns.len();
3587 let mut applied = 0u32;
3588 while let Some(row) = rows
3589 .next()
3590 .map_err(|_| invalid("image row unreadable".to_owned()))?
3591 {
3592 for (i, column) in table.columns.iter().enumerate() {
3593 let cell = row
3594 .get_ref(i)
3595 .map_err(|_| invalid("image row unreadable".to_owned()))?;
3596 let param = image_cell_param(column, cell).map_err(&invalid)?;
3597 ins.raw_bind_parameter(i + 1, param)
3598 .map_err(|e| invalid(e.to_string()))?;
3599 }
3600 let version: i64 = row
3601 .get(version_index)
3602 .map_err(|_| invalid("image row unreadable".to_owned()))?;
3603 if version < 1 {
3604 return Err(invalid(format!(
3605 "row _syncular_version must be >= 1, got {version}"
3606 )));
3607 }
3608 ins.raw_bind_parameter(version_index + 1, version)
3609 .map_err(|e| invalid(e.to_string()))?;
3610 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
3611 applied += 1;
3612 }
3613 applied
3614 };
3615 for index in &bulk_indexes {
3616 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
3617 let cols_sql = index
3618 .columns
3619 .iter()
3620 .map(|c| quote_ident(c))
3621 .collect::<Vec<_>>()
3622 .join(", ");
3623 self.conn
3624 .execute(
3625 &format!(
3626 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
3627 base_table(&table.name)
3628 ),
3629 [],
3630 )
3631 .map_err(|e| invalid(e.to_string()))?;
3632 }
3633 if i64::from(applied) != row_count {
3634 return Err(invalid(format!(
3635 "image holds {applied} rows, descriptor says {row_count}"
3636 )));
3637 }
3638 Ok(applied)
3639 }
3640
3641 fn purge_scope_rows(
3646 &mut self,
3647 table_name: &str,
3648 effective: &[(String, Vec<String>)],
3649 ) -> Result<(), ()> {
3650 if effective.is_empty() {
3651 return Ok(());
3652 }
3653 let table = self.schema.table(table_name).ok_or(())?.clone();
3654 let mut clauses = Vec::new();
3655 let mut params: Vec<SqlValue> = Vec::new();
3656 for (variable, values) in effective {
3657 let column = table.scope_column(variable).ok_or(())?;
3658 let placeholders: Vec<String> = values
3659 .iter()
3660 .map(|v| {
3661 params.push(SqlValue::Text(v.clone()));
3662 "?".to_owned()
3663 })
3664 .collect();
3665 clauses.push(format!(
3666 "{} IN ({})",
3667 quote_ident(column),
3668 placeholders.join(", ")
3669 ));
3670 }
3671 let sql = format!(
3672 "DELETE FROM {} WHERE {}",
3673 base_table(table_name),
3674 clauses.join(" AND ")
3675 );
3676 self.overlay_dirty.set(true);
3677 self.conn
3678 .execute(&sql, rusqlite::params_from_iter(params))
3679 .map_err(|_| ())?;
3680 Ok(())
3681 }
3682
3683 fn drop_doomed_outbox(&mut self, table_name: &str, effective: &[(String, Vec<String>)]) {
3686 if effective.is_empty() {
3687 return;
3688 }
3689 let Some(table) = self.schema.table(table_name).cloned() else {
3690 return;
3691 };
3692 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
3693 for (variable, values) in effective {
3694 match table.scope_column(variable) {
3695 Some(column) => mappings.push((column, values)),
3696 None => return, }
3698 }
3699 let doomed: Vec<String> = self
3700 .outbox
3701 .iter()
3702 .filter(|commit| {
3703 commit.ops.iter().any(|op| {
3704 op.upsert
3705 && op.table == table_name
3706 && op.values.as_ref().is_some_and(|values| {
3707 mappings
3708 .iter()
3709 .all(|(column, allowed)| match values.get(*column) {
3710 Some(Value::String(s)) => allowed.contains(s),
3711 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
3712 _ => false,
3713 })
3714 })
3715 })
3716 })
3717 .map(|c| c.client_commit_id.clone())
3718 .collect();
3719 for id in doomed {
3720 self.overlay_dirty.set(true);
3721 self.outbox.retain(|c| c.client_commit_id != id);
3722 self.persist_outbox_delete(&id);
3723 }
3724 }
3725
3726 pub fn upload_blob(
3733 &mut self,
3734 bytes: &[u8],
3735 media_type: Option<String>,
3736 name: Option<String>,
3737 ) -> Result<Value, String> {
3738 let blob_id = blob_id_for(bytes);
3739 let now = self.clock_now_ms();
3740 self.conn
3741 .execute(
3742 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
3743 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
3744 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
3745 )
3746 .map_err(|e| e.to_string())?;
3747 self.conn
3748 .execute(
3749 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
3750 rusqlite::params![blob_id, media_type, now],
3751 )
3752 .map_err(|e| e.to_string())?;
3753 self.enforce_blob_cache_cap();
3756 let mut obj = Map::new();
3757 obj.insert("blobId".to_owned(), Value::from(blob_id));
3758 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
3759 if let Some(mt) = media_type {
3760 obj.insert("mediaType".to_owned(), Value::from(mt));
3761 }
3762 if let Some(n) = name {
3763 obj.insert("name".to_owned(), Value::from(n));
3764 }
3765 Ok(Value::Object(obj))
3766 }
3767
3768 pub fn fetch_blob(
3772 &mut self,
3773 transport: &mut dyn Transport,
3774 blob_id_or_ref: &str,
3775 ) -> Result<Value, (String, String)> {
3776 let simple = |m: String| ("client.failed".to_owned(), m);
3777 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
3778 blob_id_or_ref.to_owned()
3779 } else {
3780 let value: Value = serde_json::from_str(blob_id_or_ref)
3781 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
3782 value
3783 .get("blobId")
3784 .and_then(Value::as_str)
3785 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
3786 .to_owned()
3787 };
3788 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
3789 return Ok(cached);
3790 }
3791 let bytes = match transport
3797 .blob_download(&blob_id)
3798 .map_err(|e| (e.code, e.message))?
3799 {
3800 BlobDownload::Bytes(bytes) => bytes,
3801 BlobDownload::Url {
3802 url,
3803 url_expires_at_ms,
3804 } => {
3805 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
3807 return Err((
3808 "sync.segment_expired".to_owned(),
3809 format!(
3810 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
3811 ),
3812 ));
3813 }
3814 transport
3815 .fetch_blob_url(&url)
3816 .map_err(|e| (e.code, e.message))?
3817 }
3818 };
3819 if blob_id_for(&bytes) != blob_id {
3821 return Err(simple(format!(
3822 "blob content address mismatch for {blob_id}"
3823 )));
3824 }
3825 let now = self.clock_now_ms();
3826 self.conn
3827 .execute(
3828 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
3829 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
3830 )
3831 .map_err(|e| simple(e.to_string()))?;
3832 self.enforce_blob_cache_cap();
3833 self.get_cached_blob(&blob_id)
3834 .map_err(simple)?
3835 .ok_or_else(|| simple("blob cache write failed".to_owned()))
3836 }
3837
3838 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
3839 let _ = self.conn.execute(
3842 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
3843 rusqlite::params![self.clock_now_ms(), blob_id],
3844 );
3845 let mut stmt = self
3846 .conn
3847 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
3848 .map_err(|e| e.to_string())?;
3849 let mut rows = stmt
3850 .query(rusqlite::params![blob_id])
3851 .map_err(|e| e.to_string())?;
3852 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
3853 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
3854 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
3855 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
3856 let mut obj = Map::new();
3857 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
3858 obj.insert("byteLength".to_owned(), Value::from(byte_length));
3859 let mut bytes_obj = Map::new();
3860 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
3861 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
3862 if let Some(mt) = media_type {
3863 obj.insert("mediaType".to_owned(), Value::from(mt));
3864 }
3865 return Ok(Some(Value::Object(obj)));
3866 }
3867 Ok(None)
3868 }
3869
3870 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
3872 let pending: Vec<(String, Option<String>)> = {
3873 let mut stmt = self
3874 .conn
3875 .prepare(
3876 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
3877 )
3878 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
3879 let rows = stmt
3880 .query_map([], |row| {
3881 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
3882 })
3883 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
3884 rows.filter_map(Result::ok).collect()
3885 };
3886 for (blob_id, media_type) in pending {
3887 let bytes: Option<Vec<u8>> = self
3888 .conn
3889 .query_row(
3890 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
3891 rusqlite::params![blob_id],
3892 |row| row.get(0),
3893 )
3894 .ok();
3895 if let Some(bytes) = bytes {
3896 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
3897 }
3898 let _ = self.conn.execute(
3899 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
3900 rusqlite::params![blob_id],
3901 );
3902 }
3903 Ok(())
3904 }
3905
3906 fn upload_one(
3913 &self,
3914 transport: &mut dyn Transport,
3915 blob_id: &str,
3916 bytes: &[u8],
3917 media_type: Option<&str>,
3918 ) -> Result<(), TransportError> {
3919 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
3920 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
3922 url,
3923 url_expires_at_ms,
3924 } => {
3925 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
3926 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
3927 return Ok(());
3928 }
3929 }
3931 BlobUploadGrant::None => {
3932 }
3934 }
3935 transport.blob_upload(blob_id, bytes, media_type)
3936 }
3937
3938 fn enforce_blob_cache_cap(&self) {
3947 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
3948 return;
3949 };
3950 let mut total: i64 = self
3951 .conn
3952 .query_row(
3953 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
3954 [],
3955 |row| row.get(0),
3956 )
3957 .unwrap_or(0);
3958 if total <= max_bytes {
3959 return;
3960 }
3961 let candidates: Vec<(String, i64)> = {
3962 let Ok(mut stmt) = self.conn.prepare(
3963 "SELECT blob_id, byte_length FROM _syncular_blobs
3964 WHERE refcount = 0
3965 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
3966 ORDER BY last_used_ms ASC, created_at_ms ASC",
3967 ) else {
3968 return;
3969 };
3970 let Ok(rows) = stmt.query_map([], |row| {
3971 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
3972 }) else {
3973 return;
3974 };
3975 rows.filter_map(Result::ok).collect()
3976 };
3977 for (blob_id, byte_length) in candidates {
3978 if total <= max_bytes {
3979 break;
3980 }
3981 let _ = self.conn.execute(
3982 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
3983 rusqlite::params![blob_id],
3984 );
3985 total -= byte_length;
3986 }
3987 }
3988
3989 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
3993 if !self.schema_has_blobs() {
3994 return;
3995 }
3996 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
3997 for table in self.schema.tables.clone() {
3998 let blob_cols: Vec<String> = table
3999 .columns
4000 .iter()
4001 .filter(|c| c.ty == ColumnType::BlobRef)
4002 .map(|c| c.name.clone())
4003 .collect();
4004 for column in blob_cols {
4005 let sql = format!(
4006 "SELECT {} FROM {} WHERE {} IS NOT NULL",
4007 quote_ident(&column),
4008 base_table(&table.name),
4009 quote_ident(&column)
4010 );
4011 let Ok(mut stmt) = self.conn.prepare(&sql) else {
4012 continue;
4013 };
4014 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
4015 continue;
4016 };
4017 for raw in rows.flatten().flatten() {
4018 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
4019 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
4020 *counts.entry(id.to_owned()).or_insert(0) += 1;
4021 }
4022 }
4023 }
4024 }
4025 }
4026 let _ = self
4027 .conn
4028 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
4029 for (blob_id, count) in &counts {
4030 let _ = self.conn.execute(
4031 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
4032 rusqlite::params![count, blob_id],
4033 );
4034 }
4035 if delete_orphans {
4036 let _ = self.conn.execute(
4037 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
4038 [],
4039 );
4040 }
4041 }
4042
4043 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
4048 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
4049 return sql.clone();
4050 }
4051 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4052 columns.push(quote_ident("_syncular_version"));
4053 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
4054 let sql = format!(
4055 "INSERT OR REPLACE INTO {full_table} ({}) VALUES ({})",
4056 columns.join(", "),
4057 placeholders.join(", ")
4058 );
4059 self.insert_sql
4060 .borrow_mut()
4061 .insert(full_table.to_owned(), sql.clone());
4062 sql
4063 }
4064
4065 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
4066 self.overlay_dirty.set(true);
4067 self.write_row(&base_table(table_name), table_name, row, version)
4068 }
4069
4070 fn write_row(
4071 &self,
4072 full_table: &str,
4073 table_name: &str,
4074 row: &Row,
4075 version: i64,
4076 ) -> Result<(), String> {
4077 let table = self
4078 .schema
4079 .table(table_name)
4080 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
4081 let sql = self.insert_row_sql(full_table, table);
4082 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
4083 let params = row
4084 .iter()
4085 .map(RowParam::Cell)
4086 .chain(std::iter::once(RowParam::Version(version)));
4087 stmt.execute(rusqlite::params_from_iter(params))
4088 .map_err(|e| e.to_string())?;
4089 Ok(())
4090 }
4091
4092 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
4093 let table = self
4094 .schema
4095 .table(table_name)
4096 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
4097 self.overlay_dirty.set(true);
4098 let sql = format!(
4099 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
4100 base_table(table_name),
4101 quote_ident(&table.primary_key)
4102 );
4103 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
4104 stmt.execute(rusqlite::params![row_id])
4105 .map_err(|e| e.to_string())?;
4106 Ok(())
4107 }
4108
4109 fn rebuild_overlay_if_dirty(&mut self) {
4112 if self.overlay_dirty.get() {
4113 self.rebuild_overlay();
4114 }
4115 }
4116
4117 fn rebuild_overlay(&mut self) {
4121 self.exec("SAVEPOINT syncular_overlay");
4122 for table in self.schema.tables.clone() {
4123 let visible = visible_table(&table.name);
4124 let base = base_table(&table.name);
4125 self.exec(&format!("DELETE FROM {visible}"));
4126 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
4127 }
4128 for commit in self.outbox.clone() {
4129 for op in &commit.ops {
4130 let Some(table) = self.schema.table(&op.table).cloned() else {
4131 continue;
4132 };
4133 if op.upsert {
4134 let Some(values) = op.values.as_ref() else {
4135 continue;
4136 };
4137 let mut row: Row = Vec::with_capacity(table.columns.len());
4138 let mut ok = true;
4139 for column in &table.columns {
4140 match json_to_column_value(column, values.get(&column.name)) {
4141 Ok(v) => row.push(v),
4142 Err(_) => {
4143 ok = false;
4144 break;
4145 }
4146 }
4147 }
4148 if ok {
4149 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
4150 }
4151 } else {
4152 let sql = format!(
4153 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
4154 visible_table(&table.name),
4155 quote_ident(&table.primary_key)
4156 );
4157 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
4158 }
4159 }
4160 }
4161 self.exec("RELEASE syncular_overlay");
4162 self.overlay_dirty.set(false);
4163 }
4164
4165 fn exec(&self, sql: &str) {
4166 let _ = self.conn.execute_batch(sql);
4167 }
4168
4169 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
4172 transport
4173 .realtime_connect()
4174 .map_err(|e| format!("{}: {}", e.code, e.message))?;
4175 self.realtime_connected = true;
4176 Ok(())
4177 }
4178
4179 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
4180 let _ = transport.realtime_close();
4181 self.realtime_connected = false;
4182 self.presence.clear(); }
4184
4185 pub fn set_presence(
4191 &mut self,
4192 transport: &mut dyn Transport,
4193 scope_key: &str,
4194 doc: Option<&Value>,
4195 ) -> Result<(), String> {
4196 if !self.realtime_connected {
4197 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
4198 }
4199 let text = encode_presence_publish(scope_key, doc);
4200 transport
4201 .realtime_send(&text)
4202 .map_err(|e| format!("{}: {}", e.code, e.message))
4203 }
4204
4205 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
4207 self.presence
4208 .get(scope_key)
4209 .map(|peers| peers.values().cloned().collect())
4210 .unwrap_or_default()
4211 }
4212
4213 fn apply_presence(
4215 &mut self,
4216 scope_key: String,
4217 kind: Option<PresenceKind>,
4218 actor_id: Option<String>,
4219 client_id: Option<String>,
4220 doc: Option<Value>,
4221 error: Option<String>,
4222 ) {
4223 if error.is_some() {
4226 return;
4227 }
4228 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
4229 return;
4230 };
4231 let peer_key = format!("{actor_id} {client_id}");
4232 match kind {
4233 PresenceKind::Leave => {
4234 if let Some(peers) = self.presence.get_mut(&scope_key) {
4235 peers.remove(&peer_key);
4236 if peers.is_empty() {
4237 self.presence.remove(&scope_key);
4238 }
4239 }
4240 }
4241 _ => {
4242 let doc = match doc {
4243 Some(Value::Object(_)) => doc.unwrap(),
4244 _ => return,
4245 };
4246 self.presence.entry(scope_key).or_default().insert(
4247 peer_key,
4248 PresencePeer {
4249 actor_id,
4250 client_id,
4251 doc,
4252 },
4253 );
4254 }
4255 }
4256 }
4257
4258 pub fn on_realtime_text(&mut self, text: &str) {
4260 match parse_control(text) {
4261 Ok(ControlMessage::Hello { requires_sync, .. }) => {
4262 if requires_sync {
4263 self.set_sync_needed(true, true);
4265 }
4266 }
4267 Ok(ControlMessage::Presence {
4268 scope_key,
4269 kind,
4270 actor_id,
4271 client_id,
4272 doc,
4273 error,
4274 ..
4275 }) => {
4276 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
4277 }
4278 Ok(ControlMessage::Wake { .. }) => {
4279 self.set_sync_needed(true, true);
4281 }
4282 _ => {}
4283 }
4284 }
4285
4286 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
4289 if self.stopped {
4290 return;
4291 }
4292 let message = match decode_message(bytes) {
4293 Ok(m) if m.msg_kind == MsgKind::Response => m,
4294 _ => {
4295 self.set_sync_needed(true, true);
4296 return;
4297 }
4298 };
4299 let mut frames = message.frames.into_iter();
4300 let mut applied_cursor: Option<i64> = None;
4301 let mut any_covered = false;
4302 let mut dropped = false;
4303 while let Some(frame) = frames.next() {
4304 let Frame::SubStart {
4305 id,
4306 status,
4307 effective_scopes,
4308 ..
4309 } = frame
4310 else {
4311 continue;
4312 };
4313 let mut body = Vec::new();
4314 let mut next_cursor: Option<i64> = None;
4315 for inner in frames.by_ref() {
4316 match inner {
4317 Frame::SubEnd {
4318 next_cursor: nc, ..
4319 } => {
4320 next_cursor = Some(nc);
4321 break;
4322 }
4323 Frame::Unknown { .. } => {}
4324 other => body.push(other),
4325 }
4326 }
4327 let Some(next_cursor) = next_cursor else {
4328 dropped = true;
4329 break;
4330 };
4331 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
4332 dropped = true;
4333 continue;
4334 };
4335 let sub = &self.subs[sub_index];
4336 if status != SubStatus::Active
4339 || sub.state != SubState::Active
4340 || sub.bootstrap_state.is_some()
4341 || !sub.synced_once
4342 {
4343 dropped = true;
4344 continue;
4345 }
4346 if next_cursor <= sub.cursor {
4347 any_covered = true;
4349 continue;
4350 }
4351 let previous_effective = self.subs[sub_index].effective.clone();
4352 let previous_cursor = self.subs[sub_index].cursor;
4353 if self.begin_observation("syncular_delta").is_err() {
4354 dropped = true;
4355 continue;
4356 }
4357 self.subs[sub_index].effective = Some(effective_scopes);
4358 let mut batch = ChangeAccumulator::default();
4359 let mut failed = false;
4360 for inner in body {
4361 if let Frame::Commit {
4362 tables, changes, ..
4363 } = inner
4364 {
4365 self.record_commit_changes(&mut batch, &tables, &changes);
4366 if self.apply_commit_changes(&tables, &changes).is_err() {
4367 failed = true;
4368 break;
4369 }
4370 }
4371 }
4372 if failed {
4373 self.rollback_observation("syncular_delta");
4374 self.subs[sub_index].effective = previous_effective;
4375 self.subs[sub_index].cursor = previous_cursor;
4376 self.overlay_dirty.set(true);
4377 self.rebuild_overlay();
4378 dropped = true;
4379 continue;
4380 }
4381 let sub = &mut self.subs[sub_index];
4382 sub.cursor = next_cursor;
4383 self.persist_sub(&self.subs[sub_index].clone());
4384 self.rebuild_overlay_if_dirty();
4385 if self.finish_observation("syncular_delta", batch).is_err() {
4386 self.rollback_observation("syncular_delta");
4387 self.subs[sub_index].effective = previous_effective;
4388 self.subs[sub_index].cursor = previous_cursor;
4389 self.overlay_dirty.set(true);
4390 self.rebuild_overlay();
4391 dropped = true;
4392 continue;
4393 }
4394 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
4395 }
4396 if let Some(cursor) = applied_cursor {
4397 self.reconcile_blob_refcounts(false);
4398 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
4400 let _ = transport.realtime_send(&ack);
4401 } else if !any_covered || dropped {
4402 self.set_sync_needed(true, true);
4404 }
4405 }
4406
4407 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
4411 if !self.realtime_connected {
4412 return;
4413 }
4414 let floor = self
4415 .subs
4416 .iter()
4417 .filter(|s| {
4418 s.state == SubState::Active
4419 && s.bootstrap_state.is_none()
4420 && s.synced_once
4421 && s.cursor >= 0
4422 })
4423 .map(|s| s.cursor)
4424 .min();
4425 if let Some(cursor) = floor {
4426 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
4427 let _ = transport.realtime_send(&ack);
4428 }
4429 }
4430}